def post(self):
body = request.get_json()
user_id = request.headers.get('userid')
if user_id == None:
user_id = request.headers.get('x-user-id')
pages = body['pages']
file_locale = ''
if 'file_locale' in body:
file_locale = body['file_locale']
job_id = ''
if 'job_id' in body:
job_id = body['job_id']
record_id = None
if 'record_id' in body:
record_id = body['record_id']
src_lang = None
if 'src_lang' in body:
src_lang = body['src_lang']
tgt_lang = None
if 'tgt_lang' in body:
tgt_lang = body['tgt_lang']
if 'pages' not in body or user_id is None or record_id == None or src_lang == None or tgt_lang == None:
AppContext.addRecordID(record_id)
log_info(
'Missing params in FileContentSaveResource {}, user_id:{}'.
format(body, user_id), AppContext.getContext())
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
AppContext.addRecordID(record_id)
log_info(
"FileContentSaveResource record_id ({}) for user ({})".format(
record_id, user_id), AppContext.getContext())
try:
if fileContentRepo.store(user_id, file_locale, record_id, pages,
src_lang, tgt_lang) == False:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
AppContext.addRecordID(record_id)
log_info(
"FileContentSaveResource record_id ({}) for user ({}) saved".
format(record_id, user_id), AppContext.getContext())
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
except Exception as e:
AppContext.addRecordID(record_id)
log_exception("FileContentSaveResource ", AppContext.getContext(),
e)
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
def post(self):
body = request.json
log_info('received request for WordSaveResource',
AppContext.getContext())
if body == None:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
if 'words' not in body:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
for word in body['words']:
if word['locale'] != 'en':
res = CustomResponse(
Status.ERR_ENGLISH_MANDATORY_WHILE_SAVING.value, None)
return res.getresjson(), 400
result = wordRepo.store(body['words'])
if result == False:
res = CustomResponse(Status.ERR_SCHEMA_VALIDATION.value, None)
return res.getresjson(), 400
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def post(self):
body = request.get_json()
if "keys" not in body or not body["keys"]:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
keys = body["keys"]
log_info("Fetching sentences from redis store",
AppContext.getContext())
try:
result = sentenceRepo.get_sentences_from_store(keys)
if result == None:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
res = CustomResponse(Status.SUCCESS.value, result)
return res.getres()
except Exception as e:
log_exception(
"Exception while fetching sentences from redis store ",
AppContext.getContext(), e)
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
def get(self):
parser = reqparse.RequestParser()
parser.add_argument(
'start_page',
type=int,
location='args',
help=
'start_page can be 0, set start_page & end_page as 0 to get entire document',
required=True)
parser.add_argument(
'end_page',
type=int,
location='args',
help=
'end_page can be 0, set start_page & end_page as 0 to get entire document',
required=True)
parser.add_argument('ad-userid',
location='headers',
type=str,
help='userid cannot be empty',
required=True)
parser.add_argument('job_id',
type=str,
location='args',
help='Job Id is required',
required=False)
parser.add_argument('record_id',
type=str,
location='args',
help='record_id is required',
required=True)
args = parser.parse_args()
AppContext.addRecordID(args['record_id'])
log_info(
"FileContentGetResource record_id {} for user {}".format(
args['record_id'], args['ad-userid']), AppContext.getContext())
try:
result = fileContentRepo.get(args['ad-userid'], args['record_id'],
args['start_page'], args['end_page'])
if result == False:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
log_info(
"FileContentGetResource record_id {} for user {} has {} pages".
format(args['record_id'], args['ad-userid'], result['total']),
AppContext.getContext())
res = CustomResponse(Status.SUCCESS.value, result['pages'],
result['total'])
return res.getres()
except Exception as e:
log_exception("FileContentGetResource ", AppContext.getContext(),
e)
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
def post(self):
body = request.get_json()
user_id = request.headers.get('userid')
if user_id == None:
user_id = request.headers.get('x-user-id')
if 'sentences' not in body or user_id is None or 'workflowCode' not in body:
log_info(
'Missing params in SaveSentenceResource {}, user_id:{}'.format(
body, user_id), AppContext.getContext())
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
sentences = body['sentences']
workflowCode = body['workflowCode']
AppContext.addRecordID(None)
log_info(
"SaveSentenceResource for user {}, number sentences to update {} request {}"
.format(user_id, len(sentences), body), AppContext.getContext())
try:
result = sentenceRepo.update_sentences(user_id, sentences,
workflowCode)
if result == False:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
if USER_TRANSLATION_ENABLED:
try:
result = sentenceRepo.save_sentences(user_id, sentences)
except Exception as e:
log_exception("SaveSentenceResource",
AppContext.getContext(), e)
# sentence_ids = []
# for sentence in sentences:
# sentence_ids.append(sentence['s_id'])
# result = sentenceRepo.get_sentence(user_id, sentence_ids)
# if result == False:
# res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
# return res.getresjson(), 400
# else:
res = CustomResponse(Status.SUCCESS.value, sentences)
return res.getres()
except Exception as e:
log_exception("SaveSentenceResource ", AppContext.getContext(), e)
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
def post(self):
body = request.json
parser = reqparse.RequestParser()
parser.add_argument('dict_fallback', type=int, location='args', help='set 1 to invoke google transalte and 0 to not', required=False,default=1)
args = parser.parse_args()
dict_fallback=args["dict_fallback"]
log_info('received request for WordSearch', AppContext.getContext())
if 'word' not in body or 'word_locale' not in body or 'target_locale' not in body:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
if (body['word_locale'] == 'en') or (body['target_locale'] == 'en'):
result = None
if body['word_locale'] == 'en':
body['word'] = body['word'].lower()
result = wordRepo.search_english(body['word'], body['target_locale'])
else:
result = wordRepo.search_vernacular(body['word'], body['word_locale'])
if result == None and DICTIONARY_FALLBACK==True:
translate = GoogleTranslate()
'''
- call google apis to get the translation
- save the translation
- return the response
'''
log_info('checking google for the searched word ({})'.format(body['word']), AppContext.getContext())
input_word, translated_word, input_locale = translate.translate_text(body['target_locale'], body['word'])
log_info('google returned input ({}), translated ({})'.format(input_word, translated_word), AppContext.getContext())
if translated_word == None:
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
else:
if body['word_locale'] == 'en':
result = wordRepo.update(body['word'], 'en', translated_word, body['target_locale'])
else:
result = wordRepo.update(translated_word, body['target_locale'], body['word'], body['word_locale'])
if result == None:
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
else:
res = CustomResponse(Status.SUCCESS.value, result)
return res.getres()
else:
log_info('returning word search from local database', AppContext.getContext())
res = CustomResponse(Status.SUCCESS.value, result)
return res.getres()
else:
res = CustomResponse(Status.ERR_ENGLISH_MANDATORY.value, None)
return res.getresjson(), 400
def post(self):
body = request.get_json()
user_id = request.headers.get('userid')
if user_id == None:
user_id = request.headers.get('x-user-id')
modifiedSentences = None
if 'modifiedSentences' in body:
modifiedSentences = body['modifiedSentences']
workflowCode = None
record_id = None
if 'blocks' not in body or user_id is None:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
if 'workflowCode' in body:
workflowCode = body['workflowCode']
if 'record_id' in body:
record_id = body['record_id']
blocks = body['blocks']
AppContext.addRecordID(record_id)
log_info(
"FileContentUpdateResource for user ({}), to update ({}) blocks".
format(user_id, len(blocks)), AppContext.getContext())
log_info(str(body), AppContext.getContext())
try:
result, updated_blocks = fileContentRepo.update(
record_id, user_id, blocks, workflowCode, modifiedSentences)
if result == False:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
log_info(
"FileContentUpdateResource for user ({}) updated".format(
user_id), AppContext.getContext())
response = {'blocks': updated_blocks, 'workflowCode': workflowCode}
res = CustomResponse(Status.SUCCESS.value, response,
len(updated_blocks))
return res.getres()
except Exception as e:
log_exception("FileContentUpdateResource ",
AppContext.getContext(), e)
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
def get(self, user_id, s_id):
AppContext.addRecordID(None)
log_info(
"SentenceBlockGetResource {} for user {}".format(s_id, user_id),
AppContext.getContext())
try:
result = SentenceRepositories.get_sentence_block(user_id, s_id)
if result == False:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
res = CustomResponse(Status.SUCCESS.value, result)
return result, 200
except Exception as e:
log_exception("SentenceBlockGetResource ", AppContext.getContext(),
e)
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
def post(self):
body = request.get_json()
user_id = request.headers.get('userid')
if user_id == None:
user_id = request.headers.get('x-user-id')
if 'record_ids' not in body or user_id is None:
log_info(
'Missing params in SentenceStatisticsCount {}, user_id:{}'.
format(body, user_id), AppContext.getContext())
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
record_ids = body['record_ids']
bleu_return = None
if 'bleu_score' in body:
bleu_return = body['bleu_score']
else:
bleu_return = False
AppContext.addRecordID(None)
log_info(
"SentenceStatisticsCount for user {}, sentence count for record_ids {}"
.format(user_id, record_ids), AppContext.getContext())
try:
result = sentenceRepo.get_sentences_counts(record_ids, bleu_return)
if result == False:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
res = CustomResponse(Status.SUCCESS.value, result)
return res.getres()
except Exception as e:
log_exception("SentenceStatisticsCount ", AppContext.getContext(),
e)
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
def post(self):
body = request.get_json()
user_id = request.headers.get('userid')
if user_id == None:
user_id = request.headers.get('x-user-id')
s_ids = None
if 'sentences' in body:
s_ids = body['sentences']
if user_id is None or s_ids is None:
log_info(
'Missing params in FetchSentenceResource {}, user_id:{}'.
format(body, user_id), AppContext.getContext())
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
AppContext.addRecordID(None)
log_info(
"FetchSentenceResource s_ids {} for user {}".format(
len(s_ids), user_id), AppContext.getContext())
try:
result = sentenceRepo.get_sentence(user_id, s_ids)
if result == False:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getresjson(), 400
res = CustomResponse(Status.SUCCESS.value, result)
return res.getres()
except Exception as e:
log_exception("FetchSentenceResource ", AppContext.getContext(), e)
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
def post(self):
body = request.json
log_info('received request for WordSearch', AppContext.getContext())
if 'word' not in body or 'word_locale' not in body or 'target_locale' not in body:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
if (body['word_locale'] == 'en') or (body['target_locale'] == 'en'):
result = None
if body['word_locale'] == 'en':
body['word'] = body['word'].lower()
result = wordRepo.search_english(body['word'],
body['target_locale'])
else:
result = wordRepo.search_vernacular(body['word'],
body['word_locale'])
if result == None:
'''
- call google apis to get the translation
- save the translation
- return the response
'''
log_info(
'checking google for the searched word ({})'.format(
body['word']), AppContext.getContext())
input_word, translated_word, input_locale = translate.translate_text(
body['target_locale'], body['word'])
log_info(
'google returned input ({}), translated ({})'.format(
input_word, translated_word), AppContext.getContext())
if translated_word == None:
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
else:
if body['word_locale'] == 'en':
result = wordRepo.update(body['word'], 'en',
translated_word,
body['target_locale'])
else:
result = wordRepo.update(translated_word,
body['target_locale'],
body['word'],
body['word_locale'])
if result == None:
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
else:
res = CustomResponse(Status.SUCCESS.value, result)
return res.getres()
else:
log_info('returning word search from local database',
AppContext.getContext())
res = CustomResponse(Status.SUCCESS.value, result)
return res.getres()
else:
res = CustomResponse(Status.ERR_ENGLISH_MANDATORY.value, None)
return res.getresjson(), 400
def post(self):
inputs = request.get_json(force=True)
if len(inputs) > 0:
log_info("Making performance check API call", MODULE_CONTEXT)
try:
for i in inputs:
if i['mode'] == 0:
avg_words_per_sec, target_array = BatchNMTPerformanceService.find_performance(i['input_txt_file'],\
i['model_id'],i['batch_size'])
output_file_name = os.path.basename(i['input_txt_file']).split(".")[0] + "_" + str(i['model_id']) + \
"_" + str(i['batch_size']) + "_" + "output" +".txt"
with open(
os.path.join(str(Path.home()),
output_file_name), 'w') as f:
for sentence in target_array:
f.write("%s\n" % sentence)
out = {}
out['response_body'] = {
"words_per_sec": avg_words_per_sec
}
out = CustomResponse(Status.SUCCESS.value,
out['response_body'])
log_info(
"out from performance check done: {}".format(
out.getresjson()), MODULE_CONTEXT)
return out.getres()
elif i['mode'] == 1:
time_taken_array = BatchNMTPerformanceService.find_performance_pipeline(i['input_txt_file'],\
i['model_id'],i['batch_size'],i['max_batch_size'],i['batch_type'])
out = {}
out['response_body'] = {"avg_time_loading_per_word": time_taken_array[0] ,\
"avg_time_preprocessing_per_word": time_taken_array[1] ,\
"avg_time_tokenizing_per_word": time_taken_array[2] ,\
"avg_time_encoding_per_word": time_taken_array[3] ,\
"avg_time_translating_per_word": time_taken_array[4] ,\
"avg_time_decoding_per_word": time_taken_array[5] ,\
"avg_time_detokenizing_per_word": time_taken_array[6] ,\
"avg_time_postprocessing_per_word": time_taken_array[7] }
out = CustomResponse(Status.SUCCESS.value,
out['response_body'])
log_info(
"out from performance check done: {}".format(
out.getresjson()), MODULE_CONTEXT)
return out.getres()
except Exception as e:
status = Status.SYSTEM_ERR.value
status['why'] = str(e)
out = CustomResponse(status, [])
return out.getres()
else:
log_info("null inputs in request in /v0/performance API",
MODULE_CONTEXT)
out = CustomResponse(Status.INVALID_API_REQUEST.value, None)
return out.getres()
def batch_translator(c_topic):
''' New method for batch translation '''
log_info('KafkaTranslate: batch_translator',MODULE_CONTEXT)
out = {}
msg_count,msg_sent = 0,0
consumer = get_consumer(c_topic)
producer = get_producer()
try:
for msg in consumer:
producer_topic = [topic["producer"] for topic in config.kafka_topic if topic["consumer"] == msg.topic][0]
log_info("Producer for current consumer:{} is-{}".format(msg.topic,producer_topic),MODULE_CONTEXT)
msg_count +=1
log_info("*******************msg received count: {}; at {} ************".format(msg_count,datetime.datetime.now()),MODULE_CONTEXT)
inputs = msg.value
translation_batch = {}
src_list, response_body = list(), list()
if inputs is not None and all(v in inputs for v in ['message','record_id','id']) and len(inputs) is not 0:
try:
input_time = datetime.datetime.now()
log_info("Input for Record Id:{} at {}".format(inputs.get('record_id'),input_time),MODULE_CONTEXT)
log_info("Running batch-translation on {}".format(inputs),MODULE_CONTEXT)
record_id = inputs.get('record_id')
book.logs_book("RecordId",record_id,"Request received")
message = inputs.get('message')
src_list = [i.get('src') for i in message]
translation_batch = {'id':inputs.get('id'),'src_list': src_list}
output_batch = NMTTranslateService.batch_translator(translation_batch)
log_info("Output of translation batch service at :{}".format(datetime.datetime.now()),MODULE_CONTEXT)
output_batch_dict_list = [{'tgt': output_batch['tgt_list'][i],
'tagged_tgt':output_batch['tagged_tgt_list'][i],'tagged_src':output_batch['tagged_src_list'][i]}
for i in range(len(message))]
for j,k in enumerate(message):
k.update(output_batch_dict_list[j])
response_body.append(k)
log_info("Record Id:{}; Final response body of current batch translation:{}".format(record_id,response_body),MODULE_CONTEXT)
out = CustomResponse(Status.SUCCESS.value,response_body)
except Exception as e:
status = Status.SYSTEM_ERR.value
status['why'] = str(e)
log_exception("Exception caught in batch_translator child block: {}".format(e),MODULE_CONTEXT,e)
book.logs_book("RecordId",record_id,"Exception")
out = CustomResponse(status, inputs.get('message'))
out = out.getresjson()
out['record_id'] = record_id
log_info("Output for Record Id:{} at {}".format(record_id,datetime.datetime.now()),MODULE_CONTEXT)
log_info("Total time for processing Record Id:{} is: {}".format(record_id,(datetime.datetime.now()- input_time).total_seconds()),MODULE_CONTEXT)
book.logs_book("RecordId",record_id,"Response pushed")
else:
status = Status.KAFKA_INVALID_REQUEST.value
out = CustomResponse(status, inputs.get('message'))
out = out.getresjson()
if inputs.get('record_id'): out['record_id'] = inputs.get('record_id')
log_info("Empty input request or key parameter missing in Batch translation request: batch_translator",MODULE_CONTEXT)
book.logs_book("RecordId",record_id,"Invalid Request")
producer.send(producer_topic, value={'out':out})
producer.flush()
msg_sent += 1
log_info("*******************msg sent count: {}; at {} **************".format(msg_sent,datetime.datetime.now()),MODULE_CONTEXT)
except ValueError as e:
'''includes simplejson.decoder.JSONDecodeError '''
log_exception("JSON decoding failed in KafkaTranslate-batch_translator method: {}".format(e),MODULE_CONTEXT,e)
log_info("Reconnecting kafka c/p after exception handling",MODULE_CONTEXT)
KafkaTranslate.batch_translator(c_topic)
except Exception as e:
log_exception("Exception caught in KafkaTranslate-batch_translator method: {}".format(e),MODULE_CONTEXT,e)
log_info("Reconnecting kafka c/p after exception handling",MODULE_CONTEXT)
KafkaTranslate.batch_translator(c_topic)
