def post(self): body = request.get_json() user_id = request.headers.get('userid') if user_id == None: user_id = request.headers.get('x-user-id') pages = body['pages'] file_locale = '' if 'file_locale' in body: file_locale = body['file_locale'] job_id = '' if 'job_id' in body: job_id = body['job_id'] record_id = None if 'record_id' in body: record_id = body['record_id'] src_lang = None if 'src_lang' in body: src_lang = body['src_lang'] tgt_lang = None if 'tgt_lang' in body: tgt_lang = body['tgt_lang'] if 'pages' not in body or user_id is None or record_id == None or src_lang == None or tgt_lang == None: AppContext.addRecordID(record_id) log_info( 'Missing params in FileContentSaveResource {}, user_id:{}'. format(body, user_id), AppContext.getContext()) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 AppContext.addRecordID(record_id) log_info( "FileContentSaveResource record_id ({}) for user ({})".format( record_id, user_id), AppContext.getContext()) try: if fileContentRepo.store(user_id, file_locale, record_id, pages, src_lang, tgt_lang) == False: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 AppContext.addRecordID(record_id) log_info( "FileContentSaveResource record_id ({}) for user ({}) saved". format(record_id, user_id), AppContext.getContext()) res = CustomResponse(Status.SUCCESS.value, None) return res.getres() except Exception as e: AppContext.addRecordID(record_id) log_exception("FileContentSaveResource ", AppContext.getContext(), e) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400
def post(self): body = request.json log_info('received request for WordSaveResource', AppContext.getContext()) if body == None: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 if 'words' not in body: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 for word in body['words']: if word['locale'] != 'en': res = CustomResponse( Status.ERR_ENGLISH_MANDATORY_WHILE_SAVING.value, None) return res.getresjson(), 400 result = wordRepo.store(body['words']) if result == False: res = CustomResponse(Status.ERR_SCHEMA_VALIDATION.value, None) return res.getresjson(), 400 res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def post(self): body = request.get_json() if "keys" not in body or not body["keys"]: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 keys = body["keys"] log_info("Fetching sentences from redis store", AppContext.getContext()) try: result = sentenceRepo.get_sentences_from_store(keys) if result == None: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 res = CustomResponse(Status.SUCCESS.value, result) return res.getres() except Exception as e: log_exception( "Exception while fetching sentences from redis store ", AppContext.getContext(), e) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400
def get(self): parser = reqparse.RequestParser() parser.add_argument( 'start_page', type=int, location='args', help= 'start_page can be 0, set start_page & end_page as 0 to get entire document', required=True) parser.add_argument( 'end_page', type=int, location='args', help= 'end_page can be 0, set start_page & end_page as 0 to get entire document', required=True) parser.add_argument('ad-userid', location='headers', type=str, help='userid cannot be empty', required=True) parser.add_argument('job_id', type=str, location='args', help='Job Id is required', required=False) parser.add_argument('record_id', type=str, location='args', help='record_id is required', required=True) args = parser.parse_args() AppContext.addRecordID(args['record_id']) log_info( "FileContentGetResource record_id {} for user {}".format( args['record_id'], args['ad-userid']), AppContext.getContext()) try: result = fileContentRepo.get(args['ad-userid'], args['record_id'], args['start_page'], args['end_page']) if result == False: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 log_info( "FileContentGetResource record_id {} for user {} has {} pages". format(args['record_id'], args['ad-userid'], result['total']), AppContext.getContext()) res = CustomResponse(Status.SUCCESS.value, result['pages'], result['total']) return res.getres() except Exception as e: log_exception("FileContentGetResource ", AppContext.getContext(), e) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400
def post(self): body = request.get_json() user_id = request.headers.get('userid') if user_id == None: user_id = request.headers.get('x-user-id') if 'sentences' not in body or user_id is None or 'workflowCode' not in body: log_info( 'Missing params in SaveSentenceResource {}, user_id:{}'.format( body, user_id), AppContext.getContext()) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 sentences = body['sentences'] workflowCode = body['workflowCode'] AppContext.addRecordID(None) log_info( "SaveSentenceResource for user {}, number sentences to update {} request {}" .format(user_id, len(sentences), body), AppContext.getContext()) try: result = sentenceRepo.update_sentences(user_id, sentences, workflowCode) if result == False: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 if USER_TRANSLATION_ENABLED: try: result = sentenceRepo.save_sentences(user_id, sentences) except Exception as e: log_exception("SaveSentenceResource", AppContext.getContext(), e) # sentence_ids = [] # for sentence in sentences: # sentence_ids.append(sentence['s_id']) # result = sentenceRepo.get_sentence(user_id, sentence_ids) # if result == False: # res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) # return res.getresjson(), 400 # else: res = CustomResponse(Status.SUCCESS.value, sentences) return res.getres() except Exception as e: log_exception("SaveSentenceResource ", AppContext.getContext(), e) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400
def post(self): body = request.json parser = reqparse.RequestParser() parser.add_argument('dict_fallback', type=int, location='args', help='set 1 to invoke google transalte and 0 to not', required=False,default=1) args = parser.parse_args() dict_fallback=args["dict_fallback"] log_info('received request for WordSearch', AppContext.getContext()) if 'word' not in body or 'word_locale' not in body or 'target_locale' not in body: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 if (body['word_locale'] == 'en') or (body['target_locale'] == 'en'): result = None if body['word_locale'] == 'en': body['word'] = body['word'].lower() result = wordRepo.search_english(body['word'], body['target_locale']) else: result = wordRepo.search_vernacular(body['word'], body['word_locale']) if result == None and DICTIONARY_FALLBACK==True: translate = GoogleTranslate() ''' - call google apis to get the translation - save the translation - return the response ''' log_info('checking google for the searched word ({})'.format(body['word']), AppContext.getContext()) input_word, translated_word, input_locale = translate.translate_text(body['target_locale'], body['word']) log_info('google returned input ({}), translated ({})'.format(input_word, translated_word), AppContext.getContext()) if translated_word == None: res = CustomResponse(Status.SUCCESS.value, None) return res.getres() else: if body['word_locale'] == 'en': result = wordRepo.update(body['word'], 'en', translated_word, body['target_locale']) else: result = wordRepo.update(translated_word, body['target_locale'], body['word'], body['word_locale']) if result == None: res = CustomResponse(Status.SUCCESS.value, None) return res.getres() else: res = CustomResponse(Status.SUCCESS.value, result) return res.getres() else: log_info('returning word search from local database', AppContext.getContext()) res = CustomResponse(Status.SUCCESS.value, result) return res.getres() else: res = CustomResponse(Status.ERR_ENGLISH_MANDATORY.value, None) return res.getresjson(), 400
def post(self): body = request.get_json() user_id = request.headers.get('userid') if user_id == None: user_id = request.headers.get('x-user-id') modifiedSentences = None if 'modifiedSentences' in body: modifiedSentences = body['modifiedSentences'] workflowCode = None record_id = None if 'blocks' not in body or user_id is None: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 if 'workflowCode' in body: workflowCode = body['workflowCode'] if 'record_id' in body: record_id = body['record_id'] blocks = body['blocks'] AppContext.addRecordID(record_id) log_info( "FileContentUpdateResource for user ({}), to update ({}) blocks". format(user_id, len(blocks)), AppContext.getContext()) log_info(str(body), AppContext.getContext()) try: result, updated_blocks = fileContentRepo.update( record_id, user_id, blocks, workflowCode, modifiedSentences) if result == False: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 log_info( "FileContentUpdateResource for user ({}) updated".format( user_id), AppContext.getContext()) response = {'blocks': updated_blocks, 'workflowCode': workflowCode} res = CustomResponse(Status.SUCCESS.value, response, len(updated_blocks)) return res.getres() except Exception as e: log_exception("FileContentUpdateResource ", AppContext.getContext(), e) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400
def get(self, user_id, s_id): AppContext.addRecordID(None) log_info( "SentenceBlockGetResource {} for user {}".format(s_id, user_id), AppContext.getContext()) try: result = SentenceRepositories.get_sentence_block(user_id, s_id) if result == False: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 res = CustomResponse(Status.SUCCESS.value, result) return result, 200 except Exception as e: log_exception("SentenceBlockGetResource ", AppContext.getContext(), e) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400
def post(self): body = request.get_json() user_id = request.headers.get('userid') if user_id == None: user_id = request.headers.get('x-user-id') if 'record_ids' not in body or user_id is None: log_info( 'Missing params in SentenceStatisticsCount {}, user_id:{}'. format(body, user_id), AppContext.getContext()) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 record_ids = body['record_ids'] bleu_return = None if 'bleu_score' in body: bleu_return = body['bleu_score'] else: bleu_return = False AppContext.addRecordID(None) log_info( "SentenceStatisticsCount for user {}, sentence count for record_ids {}" .format(user_id, record_ids), AppContext.getContext()) try: result = sentenceRepo.get_sentences_counts(record_ids, bleu_return) if result == False: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 res = CustomResponse(Status.SUCCESS.value, result) return res.getres() except Exception as e: log_exception("SentenceStatisticsCount ", AppContext.getContext(), e) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400
def post(self): body = request.get_json() user_id = request.headers.get('userid') if user_id == None: user_id = request.headers.get('x-user-id') s_ids = None if 'sentences' in body: s_ids = body['sentences'] if user_id is None or s_ids is None: log_info( 'Missing params in FetchSentenceResource {}, user_id:{}'. format(body, user_id), AppContext.getContext()) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 AppContext.addRecordID(None) log_info( "FetchSentenceResource s_ids {} for user {}".format( len(s_ids), user_id), AppContext.getContext()) try: result = sentenceRepo.get_sentence(user_id, s_ids) if result == False: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 res = CustomResponse(Status.SUCCESS.value, result) return res.getres() except Exception as e: log_exception("FetchSentenceResource ", AppContext.getContext(), e) res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400
def post(self): body = request.json log_info('received request for WordSearch', AppContext.getContext()) if 'word' not in body or 'word_locale' not in body or 'target_locale' not in body: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 if (body['word_locale'] == 'en') or (body['target_locale'] == 'en'): result = None if body['word_locale'] == 'en': body['word'] = body['word'].lower() result = wordRepo.search_english(body['word'], body['target_locale']) else: result = wordRepo.search_vernacular(body['word'], body['word_locale']) if result == None: ''' - call google apis to get the translation - save the translation - return the response ''' log_info( 'checking google for the searched word ({})'.format( body['word']), AppContext.getContext()) input_word, translated_word, input_locale = translate.translate_text( body['target_locale'], body['word']) log_info( 'google returned input ({}), translated ({})'.format( input_word, translated_word), AppContext.getContext()) if translated_word == None: res = CustomResponse(Status.SUCCESS.value, None) return res.getres() else: if body['word_locale'] == 'en': result = wordRepo.update(body['word'], 'en', translated_word, body['target_locale']) else: result = wordRepo.update(translated_word, body['target_locale'], body['word'], body['word_locale']) if result == None: res = CustomResponse(Status.SUCCESS.value, None) return res.getres() else: res = CustomResponse(Status.SUCCESS.value, result) return res.getres() else: log_info('returning word search from local database', AppContext.getContext()) res = CustomResponse(Status.SUCCESS.value, result) return res.getres() else: res = CustomResponse(Status.ERR_ENGLISH_MANDATORY.value, None) return res.getresjson(), 400
def post(self): inputs = request.get_json(force=True) if len(inputs) > 0: log_info("Making performance check API call", MODULE_CONTEXT) try: for i in inputs: if i['mode'] == 0: avg_words_per_sec, target_array = BatchNMTPerformanceService.find_performance(i['input_txt_file'],\ i['model_id'],i['batch_size']) output_file_name = os.path.basename(i['input_txt_file']).split(".")[0] + "_" + str(i['model_id']) + \ "_" + str(i['batch_size']) + "_" + "output" +".txt" with open( os.path.join(str(Path.home()), output_file_name), 'w') as f: for sentence in target_array: f.write("%s\n" % sentence) out = {} out['response_body'] = { "words_per_sec": avg_words_per_sec } out = CustomResponse(Status.SUCCESS.value, out['response_body']) log_info( "out from performance check done: {}".format( out.getresjson()), MODULE_CONTEXT) return out.getres() elif i['mode'] == 1: time_taken_array = BatchNMTPerformanceService.find_performance_pipeline(i['input_txt_file'],\ i['model_id'],i['batch_size'],i['max_batch_size'],i['batch_type']) out = {} out['response_body'] = {"avg_time_loading_per_word": time_taken_array[0] ,\ "avg_time_preprocessing_per_word": time_taken_array[1] ,\ "avg_time_tokenizing_per_word": time_taken_array[2] ,\ "avg_time_encoding_per_word": time_taken_array[3] ,\ "avg_time_translating_per_word": time_taken_array[4] ,\ "avg_time_decoding_per_word": time_taken_array[5] ,\ "avg_time_detokenizing_per_word": time_taken_array[6] ,\ "avg_time_postprocessing_per_word": time_taken_array[7] } out = CustomResponse(Status.SUCCESS.value, out['response_body']) log_info( "out from performance check done: {}".format( out.getresjson()), MODULE_CONTEXT) return out.getres() except Exception as e: status = Status.SYSTEM_ERR.value status['why'] = str(e) out = CustomResponse(status, []) return out.getres() else: log_info("null inputs in request in /v0/performance API", MODULE_CONTEXT) out = CustomResponse(Status.INVALID_API_REQUEST.value, None) return out.getres()
def batch_translator(c_topic): ''' New method for batch translation ''' log_info('KafkaTranslate: batch_translator',MODULE_CONTEXT) out = {} msg_count,msg_sent = 0,0 consumer = get_consumer(c_topic) producer = get_producer() try: for msg in consumer: producer_topic = [topic["producer"] for topic in config.kafka_topic if topic["consumer"] == msg.topic][0] log_info("Producer for current consumer:{} is-{}".format(msg.topic,producer_topic),MODULE_CONTEXT) msg_count +=1 log_info("*******************msg received count: {}; at {} ************".format(msg_count,datetime.datetime.now()),MODULE_CONTEXT) inputs = msg.value translation_batch = {} src_list, response_body = list(), list() if inputs is not None and all(v in inputs for v in ['message','record_id','id']) and len(inputs) is not 0: try: input_time = datetime.datetime.now() log_info("Input for Record Id:{} at {}".format(inputs.get('record_id'),input_time),MODULE_CONTEXT) log_info("Running batch-translation on {}".format(inputs),MODULE_CONTEXT) record_id = inputs.get('record_id') book.logs_book("RecordId",record_id,"Request received") message = inputs.get('message') src_list = [i.get('src') for i in message] translation_batch = {'id':inputs.get('id'),'src_list': src_list} output_batch = NMTTranslateService.batch_translator(translation_batch) log_info("Output of translation batch service at :{}".format(datetime.datetime.now()),MODULE_CONTEXT) output_batch_dict_list = [{'tgt': output_batch['tgt_list'][i], 'tagged_tgt':output_batch['tagged_tgt_list'][i],'tagged_src':output_batch['tagged_src_list'][i]} for i in range(len(message))] for j,k in enumerate(message): k.update(output_batch_dict_list[j]) response_body.append(k) log_info("Record Id:{}; Final response body of current batch translation:{}".format(record_id,response_body),MODULE_CONTEXT) out = CustomResponse(Status.SUCCESS.value,response_body) except Exception as e: status = Status.SYSTEM_ERR.value status['why'] = str(e) log_exception("Exception caught in batch_translator child block: {}".format(e),MODULE_CONTEXT,e) book.logs_book("RecordId",record_id,"Exception") out = CustomResponse(status, inputs.get('message')) out = out.getresjson() out['record_id'] = record_id log_info("Output for Record Id:{} at {}".format(record_id,datetime.datetime.now()),MODULE_CONTEXT) log_info("Total time for processing Record Id:{} is: {}".format(record_id,(datetime.datetime.now()- input_time).total_seconds()),MODULE_CONTEXT) book.logs_book("RecordId",record_id,"Response pushed") else: status = Status.KAFKA_INVALID_REQUEST.value out = CustomResponse(status, inputs.get('message')) out = out.getresjson() if inputs.get('record_id'): out['record_id'] = inputs.get('record_id') log_info("Empty input request or key parameter missing in Batch translation request: batch_translator",MODULE_CONTEXT) book.logs_book("RecordId",record_id,"Invalid Request") producer.send(producer_topic, value={'out':out}) producer.flush() msg_sent += 1 log_info("*******************msg sent count: {}; at {} **************".format(msg_sent,datetime.datetime.now()),MODULE_CONTEXT) except ValueError as e: '''includes simplejson.decoder.JSONDecodeError ''' log_exception("JSON decoding failed in KafkaTranslate-batch_translator method: {}".format(e),MODULE_CONTEXT,e) log_info("Reconnecting kafka c/p after exception handling",MODULE_CONTEXT) KafkaTranslate.batch_translator(c_topic) except Exception as e: log_exception("Exception caught in KafkaTranslate-batch_translator method: {}".format(e),MODULE_CONTEXT,e) log_info("Reconnecting kafka c/p after exception handling",MODULE_CONTEXT) KafkaTranslate.batch_translator(c_topic)