def post(self):
body = request.get_json()
upload_id = str(uuid4())
filename = body['filename']
filepath = os.path.join(config.download_folder, filename)
if filename.endswith('.pdf'):
res = CustomResponse(Status.SUCCESS.value, filename)
return res.getres()
try:
result = convert_to(os.path.join(config.download_folder, 'pdf',
upload_id),
filepath,
timeout=60)
copyfile(result,
os.path.join(config.download_folder, upload_id + '.pdf'))
userfile = UserFiles(created_by=request.headers.get('ad-userid'),
filename=upload_id + '.pdf',
created_on=datetime.now())
userfile.save()
except LibreOfficeError as e:
raise InternalServerErrorError(
{'message': 'Error when converting file to PDF'})
except TimeoutExpired:
raise InternalServerErrorError(
{'message': 'Timeout when converting file to PDF'})
res = CustomResponse(Status.SUCCESS.value, upload_id + '.pdf')
return res.getres()
def saveTranslateDocx():
start_time = int(round(time.time() * 1000))
log.info('uploadTranslateDocx: started at ' + str(start_time))
if (request.form.getlist('basename') is None
or not isinstance(request.form.getlist('basename'), list)):
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
basename = request.form.getlist('basename')[0]
current_time = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
f = request.files['file']
filepath = os.path.join(app.config['UPLOAD_FOLDER'], basename + '_u.docx')
index = 0
while (os.path.exists(filepath)):
filepath = os.path.join(app.config['UPLOAD_FOLDER'],
basename + '_' + str(index) + '_u.docx')
index = index + 1
f.save(filepath)
res = CustomResponse(Status.SUCCESS.value,
basename + '_' + str(index) + '_u' + '.docx')
translationProcess = TranslationProcess.objects(basename=basename)
translationProcess.update(set__translate_uploaded=True)
log.info('uploadTranslateDocx: ended at ' + str(getcurrenttime()) +
'total time elapsed : ' + str(getcurrenttime() - start_time))
return res.getres()
def save_ocr_data():
body = request.get_json()
if body['ocr_data'] is None:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
ocr_data = Ocrdata(created_on=str(int(time.time())),
data=body['ocr_data']['response'])
ocr_data.save()
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def update_sentences_status():
body = request.get_json()
if (body['sentences'] is None or not isinstance(body['sentences'], list)):
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
for sentence in body['sentences']:
corpus = Sentence.objects(_id=sentence['_id']['$oid'])
corpus.update(set__status=sentence['status'])
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def update_password_admin():
log.info('update_password : started')
body = request.get_json()
user_id = body['user_id']
high_court_code = body['high_court_code']
new_password = body['new_password']
log.info("high_court_code == " + high_court_code)
if high_court_code is not None:
userHighCourt = Userhighcourt.objects(user_id=user_id)
if userHighCourt is not None and len(userHighCourt) > 0:
log.info('high court with user exist ' + str(len(userHighCourt)))
userHighCourt.update(set__high_court_code=high_court_code)
else:
log.info('saving high court with user')
user_high_court = Userhighcourt(high_court_code=high_court_code,
user_id=user_id)
user_high_court.save()
profile = requests.get(PROFILE_REQ_URL + user_id).content
profile = json.loads(profile)
roles_ = get_user_roles_basic_auth(user_id)
data = {"status": "false"}
req = GATEWAY_SERVER_URL + 'credentials/basic-auth/' + user_id + '/status'
response = requests.put(req, json=data)
res = response.json()
status = res['status']
log.info("status == " + status)
if not status == 'Deactivated':
res = CustomResponse(Status.ERROR_GATEWAY.value, None)
return res.getres()
data = {
"credential": {
"password": new_password,
"scopes": roles_
},
"consumerId": user_id,
"type": "basic-auth"
}
if new_password is not None or new_password.__len__() == 0:
if new_password.__len__() < 6:
log.info(
'update_password : password is too weak, at least provide 6 characters'
)
res = CustomResponse(Status.ERROR_WEAK_PASSWORD.value, None)
return res.getres()
else:
req = GATEWAY_SERVER_URL + 'credentials'
response = requests.post(req, json=data)
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def create_user_oauth():
log.info('create_user_oauth : started')
body = request.get_json()
user_name = body['username']
try:
response = shell.create_oauth(user_name)
res = CustomResponse(Status.SUCCESS.value, response)
return res.getres()
except Exception as e:
log.info('create_user_oauth : error ' + str(e))
res = CustomResponse(Status.ERROR_GATEWAY.value, None)
return res.getres()
def check_ocr_data():
body = request.get_json()
if body['student_id'] is None or body['exam_id'] is None or body[
'ocr_data'] is None:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
student_fromdb = Student.objects(student_id=body['student_id'])
exam_fromdb = Exams.objects(exam_id=body['exam_id'])
if not (student_fromdb is not None and len(student_fromdb) > 0):
res = CustomResponse(Status.WRONG_STUDENT_CODE.value, None)
return res.getres()
elif not (exam_fromdb is not None and len(exam_fromdb) > 0):
res = CustomResponse(Status.WRONG_CODE.value, None)
return res.getres()
else:
try:
exam_obj = json.loads(exam_fromdb.to_json())
student_obj = json.loads(student_fromdb.to_json())
map_obj = conver_list_to_map(exam_obj[0]['data'])
ocr_data = body['ocr_data']
table_data = ocr_data['response'][1]['data']
students_data = ocr_data['response'][0]['data']
student_obj[0]['exam_id'] = body['exam_id']
ocr_data['metadata'] = student_obj[0]
if len(ocr_data['response'][0]['data']) > len(
ocr_data['response'][1]['data']):
table_data = ocr_data['response'][0]['data']
students_data = ocr_data['response'][1]['data']
for marks_data in table_data:
key = str(marks_data['col']) + str(marks_data['row'])
if key in map_obj:
marks_data['text'] = map_obj[key]
elif str(marks_data['col']) == '0' or str(
marks_data['col']) == '1' or str(
marks_data['col']) == '2':
marks_data['text'] = ''
if 'exam_date' in exam_obj[0]:
for student_data in students_data:
if student_data['col'] == 1 and student_data['row'] == 3:
student_data['text'] = exam_obj[0]['exam_date']
res = CustomResponse(Status.SUCCESS.value, ocr_data)
return res.getres()
except Exception as e:
res = CustomResponse(Status.ERR_GLOBAL_SYSTEM.value, ocr_data)
return res.getres(
), Status.ERR_GLOBAL_SYSTEM.value['http']['status']
def fetch_sentences():
global LANGUAGES
basename = request.args.get('basename')
totalcount = 0
(sentencesobj, totalcount) = Sentence.limit(request.args.get('pagesize'), basename, request.args.get('status'),
request.args.get('pageno'))
corpus_obj = Corpus.objects(basename=basename)
corpus_dict = json.loads(corpus_obj.to_json())
sentences_list = []
sources = []
if sentencesobj is not None:
for sent in sentencesobj:
sent_dict = json.loads(sent.to_json())
corpus = Sentence.objects(_id=sent_dict['_id']['$oid'])
if sent_dict['status'] == STATUS_PENDING:
corpus.update(set__status=STATUS_PROCESSING)
sources.append(sent_dict['source'])
target_lang = 'en'
if 'target_lang' in corpus_dict[0] and corpus_dict[0]['target_lang'] is not None:
target_lang = LANGUAGES[corpus_dict[0]['target_lang']]
translation_list = translatesinglesentence(sources, target_lang)
index = 0
for sent in sentencesobj:
sent_dict = json.loads(sent.to_json())
sent_dict['translation'] = translation_list[index]
sentences_list.append(sent_dict)
index += 1
# print()
# for sentence in sentencesobj:
# # sentence.update(set__status=STATUS_PROCESSING, set__locked=True, set__locked_time=datetime.now())
# sentence.update(set__status=STATUS_PROCESSING)
res = CustomResponse(Status.SUCCESS.value, sentences_list, totalcount)
return res.getres()
def roles():
body = request.get_json()
res = None
if body['operation'] is not None and body['role-type'] is not None:
if body['operation'] == "create":
if not body['role-type'] == '':
try:
response = requests.post(GATEWAY_SERVER_URL + 'scopes')
res = CustomResponse(Status.SUCCESS.value,
json.loads(response))
except:
res = CustomResponse(Status.FAILURE.value, None)
else:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
' role-type not provided ')
else:
res = CustomResponse(Status.OPERATION_NOT_PERMITTED.value,
'supported opertion type are : [create] ')
else:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
' please provide operation and role-type ')
return res.getres()
def save_student_masterdata():
body = request.get_json()
if body['student'] is None or body['student']['student_id'] is None:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
student_fromdb = Student.objects(student_id=body['student']['student_id'])
if student_fromdb is not None and len(student_fromdb) > 0:
res = CustomResponse(Status.USER_ALREADY_EXISTS.value, None)
return res.getres(), Status.USER_ALREADY_EXISTS.value['http']['status']
else:
student = Student(student_id=body['student']['student_id'],
student_name=body['student']['student_name'])
student.save()
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def post(self):
body = request.get_json()
userid = request.headers.get('userid')
if 'pages' not in body or 'process_identifier' not in body or userid is None:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value,
None)
return res.getresjson(), 400
results = body['pages']
process_identifier = body['process_identifier']
obj_to_be_saved = []
for result in results:
page_data = {}
page_data['page_no'] = result['page_no']
page_data['page_width'] = result['page_width']
page_data['page_height'] = result['page_height']
for block_type in BLOCK_TYPES:
if result[block_type['key']] is not None:
for data in result[block_type['key']]:
obj_to_be_saved = self.make_obj(
process_identifier, page_data, data,
block_type['key'], obj_to_be_saved, userid)
file_content_instances = [
FileContent(**data) for data in obj_to_be_saved
]
FileContent.objects.insert(file_content_instances)
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def create_user_basic_auth():
log.info('create_user_basic_auth : started')
body = request.get_json()
user_name = body['username']
firstname = body['firstname']
lastname = body['lastname']
password = body['password']
scope = body['roles']
high_court_code = body['high_court_code']
try:
profile = requests.get(PROFILE_REQ_URL + user_name)
try:
profile = profile.json()
if profile['isActive']:
# _id = profile['']
log.info('create_user_oauth : profile is = : ' + str(profile))
res = CustomResponse(Status.USER_ALREADY_EXISTS.value, None)
return res.getres()
except:
pass
log.info('here')
create_response = shell.create_user(user_name, firstname, lastname)
log.info('user created')
shell_response = shell.create_basic_auth_credentials(
user_name, password)
log.info('basic auth created')
response = shell.create_oauth(user_name)
log.info('oauth created')
user = shell.get_user_info(user_name)
log.info(str(user))
scope_response = shell.scope_add(user['id'], scope)
time.sleep(3)
log.info('scope added')
if high_court_code is not None:
user_high_court = Userhighcourt(high_court_code=high_court_code,
user_id=user['id'])
user_high_court.save()
res = CustomResponse(Status.SUCCESS.value, response)
return res.getres()
except Exception as e:
log.info(' create_user : error ' + str(e))
res = CustomResponse(Status.ERROR_GATEWAY.value, None)
return res.getres()
def fetch_corpus():
if request.headers.get('ad-userid') is not None:
log.info('fetch_corpus: initiated by ' + request.headers.get('ad-userid'))
else:
log.info('fetch_corpus: initiated by anonymous user')
corpus = Corpus.objects.to_json()
res = CustomResponse(Status.SUCCESS.value, json.loads(corpus))
return res.getres()
def save_exam_masterdata():
body = request.get_json()
if body['exam'] is None or body['exam']['exam_id'] is None or body['exam'][
'data'] is None:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
exam_fromdb = Exams.objects(exam_id=body['exam']['exam_id'])
if exam_fromdb is not None and len(exam_fromdb) > 0:
exam_fromdb.update(set__data=body['exam']['data'])
else:
exam = Exams(exam_id=body['exam']['exam_id'],
data=body['exam']['data'],
exam_date=body['exam']['exam_date'])
exam.save()
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def translate_source():
sources = []
source = request.args.get('source')
basename = request.args.get('basename')
if source is None or basename is None:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
sources.append(source)
corpus_obj = Corpus.objects(basename=basename)
corpus_dict = json.loads(corpus_obj.to_json())
target_lang = 'en'
if 'target_lang' in corpus_dict[0] and corpus_dict[0]['target_lang'] is not None:
target_lang = LANGUAGES[corpus_dict[0]['target_lang']]
translation_list = translatesinglesentence(sources, target_lang)
res = CustomResponse(Status.SUCCESS.value, translation_list)
return res.getres()
def create_parallel_corpus():
global STATUS_ACTIVE
body = request.get_json()
if body['source_corpus'] is None or len(body['source_corpus']) == 0:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
basename = str(uuid.uuid4())
target_corpus_id = str(uuid.uuid4())
source = body['source_corpus']
name = body['name']
domain = body['domain']
target_lang = body['target_lang']
source_lang = body['source_lang']
corpus = Singlecorpus(status=STATUS_ACTIVE,
created_on=datetime.now(),
name=name,
corpusid=target_corpus_id,
domain=domain,
lang=target_lang,
created_by=request.headers.get('ad-userid'))
corpus.tags = [BASE_CORPUS, target_lang]
corpus.save()
source_corpus = Singlecorpus.objects(corpusid=source)
if source_corpus is None or len(source_corpus) == 0:
res = CustomResponse(Status.DATA_NOT_FOUND.value, None)
return res.getres(), Status.DATA_NOT_FOUND.value['http']['status']
parallel_corpus = Parallelcorpus(source_lang=source_lang,
target_lang=target_lang,
name=name,
domain=domain,
basename=basename,
source_id=source,
target_id=target_corpus_id,
status=STATUS_ACTIVE)
parallel_corpus.save()
source_sentences = Corpussentence.objects(
Q(tags=source) & Q(original=True))
for source_sentence in source_sentences:
source_sentence_dict = json.loads(source_sentence.to_json())
source_sentence_tags = source_sentence_dict['parallelcorpusid']
source_sentence_tags.append(basename)
source_sentence.parallelcorpusid = source_sentence_tags
source_sentence.save()
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def update_password():
log.info('update_password : started')
body = request.get_json()
user_id = body['user_id']
user_name = body['user_name']
old_password = body['old_password']
new_password = body['new_password']
password_checked = check_password(user_name, old_password)
if not password_checked:
res = CustomResponse(Status.ERROR_WRONG_PASSWORD.value, None)
return res.getres()
if new_password is None or new_password.__len__() < 6:
log.info(
'update_password : password is too weak, at least provide 6 characters'
)
res = CustomResponse(Status.ERROR_WEAK_PASSWORD.value, None)
return res.getres()
profile = requests.get(PROFILE_REQ_URL + user_id).content
profile = json.loads(profile)
roles_ = get_user_roles_basic_auth(user_id)
data = {"status": "false"}
req = GATEWAY_SERVER_URL + 'credentials/basic-auth/' + user_id + '/status'
response = requests.put(req, json=data)
res = response.json()
status = res['status']
log.info("status == " + status)
if not status == 'Deactivated':
res = CustomResponse(Status.ERROR_GATEWAY.value, None)
return res.getres()
data = {
"credential": {
"password": new_password,
"scopes": roles_
},
"consumerId": user_id,
"type": "basic-auth"
}
req = GATEWAY_SERVER_URL + 'credentials'
response = requests.post(req, json=data)
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def tokenize_hin_sentence():
body = request.get_json()
if body['paragraphs'] is None or not isinstance(body['paragraphs'], list):
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
response = []
for paragraph in body['paragraphs']:
tokenizer = AnuvaadHinTokenizer()
if 'text' in paragraph:
tokenized_data = {}
tokenized_data['text'] = tokenizer.tokenize(paragraph['text'])
if 'page_no' in paragraph:
tokenized_data['page_no'] = paragraph['page_no']
response.append(tokenized_data)
else:
response.append(tokenizer.tokenize(paragraph))
res = CustomResponse(Status.SUCCESS.value, response)
return res.getres()
def upload_file():
pool = mp.Pool(mp.cpu_count())
basename = str(int(time.time()))
try:
name = request.form.getlist('name')
domain = request.form.getlist('domain')
source_lang = request.form.getlist('source_lang')
target_lang = request.form.getlist('target_lang')
comment = request.form.getlist('comment')
if comment is None or len(comment) == 0:
comment = ['']
if name is None or len(name) == 0 or len(name[0]) == 0 or domain is None or len(domain) == 0 or len(
domain[0]) == 0 or request.files is None or request.files['hindi'] is None or request.files[
'english'] is None:
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
else:
current_time = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
corpus = Corpus(status=STATUS_PROCESSING, name=name[0], domain=domain[0], created_on=current_time,source_lang=source_lang[0],target_lang=target_lang[0],
last_modified=current_time, author='', comment=comment[0], no_of_sentences=0,
basename=basename)
corpus.save()
f = request.files['hindi']
f_eng = request.files['english']
filepath = os.path.join(
app.config['UPLOAD_FOLDER'], basename + '_hin.pdf')
filepath_eng = os.path.join(
app.config['UPLOAD_FOLDER'], basename + '_eng.pdf')
f.save(filepath)
f_eng.save(filepath_eng)
pool.apply_async(converttoimage, args=(
filepath, app.config['UPLOAD_FOLDER'], basename, '_hin'), callback=capturetext)
pool.apply_async(converttoimage, args=(
filepath_eng, app.config['UPLOAD_FOLDER'], basename, '_eng'), callback=capturetext)
pool.close()
pool.join()
return process_files(basename)
except Exception as e:
print(e)
res = CustomResponse(Status.ERR_GLOBAL_SYSTEM.value, None)
return res.getres(), Status.ERR_GLOBAL_SYSTEM.value['http']['status']
def process_files_law(basename, name):
filtertext(app.config['UPLOAD_FOLDER'] + '/' + basename + '_hin.txt',
app.config['UPLOAD_FOLDER'] + '/' + basename + '_hin_filtered.txt')
filtertext(app.config['UPLOAD_FOLDER'] + '/' + basename + '_eng.txt',
app.config['UPLOAD_FOLDER'] + '/' + basename + '_eng_filtered.txt')
processhindi(app.config['UPLOAD_FOLDER'] +
'/' + basename + '_hin_filtered.txt')
processenglish(app.config['UPLOAD_FOLDER'] +
'/' + basename + '_eng_filtered.txt')
translatewithgoogle(app.config['UPLOAD_FOLDER'] +
'/' + basename + '_hin_filtered.txt', app.config['UPLOAD_FOLDER'] +
'/' + basename + '_eng_tran.txt')
os.system(
'./helpers/bleualign.py -s ' + os.getcwd() + '/upload/' + basename + '_hin_filtered' + '.txt' + ' -t ' + os.getcwd() + '/upload/' + basename +
'_eng_filtered' + '.txt' + ' --srctotarget ' + os.getcwd() + '/upload/' + basename + '_eng_tran' + '.txt' + ' -o ' + os.getcwd() + '/upload/' + basename + '_output')
english_res = []
hindi_res = []
english_points = []
english_points_words = []
hindi_points = []
hindi_points_words = []
f_eng = open(app.config['UPLOAD_FOLDER'] +
'/' + basename + '_output-t', 'r')
for f in f_eng:
english_res.append(f)
point = fetchwordsfromsentence(f, basename)
english_points.append(point['avg'])
english_points_words.append(point['values'])
f_eng.close()
f_hin = open(app.config['UPLOAD_FOLDER'] +
'/' + basename + '_output-s', 'r')
for f in f_hin:
hindi_res.append(f)
point = fetchwordsfromsentence(f, basename)
hindi_points.append(point['avg'])
hindi_points_words.append(point['values'])
f_hin.close()
data = {'hindi': hindi_res, 'english': english_res,
'english_scores': english_points, 'hindi_scores': hindi_points}
sentences = []
for i in range(0, len(hindi_res)):
sentence = Sentence(status=STATUS_PENDING, alignment_accuracy=english_res[i].split(':::::')[1], basename=name,
source=hindi_res[i], target=english_res[i].split(':::::')[0],
source_ocr_words=hindi_points_words[i], source_ocr=str(hindi_points[i]),
target_ocr_words=english_points_words[i], target_ocr=str(english_points[i]))
sentences.append(sentence)
# sentence.save()
Sentence.objects.insert(sentences)
for f in glob.glob(app.config['UPLOAD_FOLDER'] + '/' + basename + '*'):
os.remove(f)
res = CustomResponse(Status.SUCCESS.value, data)
# corpus = Corpus.objects(basename=basename)
# corpus.update(set__status=STATUS_PROCESSED,
# set__no_of_sentences=len(hindi_res))
return res.getres()
def remove_junk():
basename = str(int(time.time()))
f = request.files['file']
filepath_eng = os.path.join(
app.config['UPLOAD_FOLDER'], basename + '_junk.txt')
f.save(filepath_eng)
f_eng = open(app.config['UPLOAD_FOLDER'] + '/' + basename + '_junk.txt', 'r')
for t in f_eng:
Sentence.objects(source=t).delete()
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def update_corpus():
start_time = int(round(time.time() * 1000))
log.info('update_corpus: started at ' + str(start_time))
current_time = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
req_data = request.get_json()
validate_upload_corpus_request(req_data)
res = CustomResponse(Status.SUCCESS.value, 'no. of sentences are ')
end_time = int(round(time.time() * 1000))
log.info('upload_corpus: ended at ' + str(end_time) +
'total time elapsed = ' + str(end_time - start_time))
return res.getres()
def fetch_translation_process():
log.info('fetch_translation_process : started at ' + str(getcurrenttime()))
try:
transalationProcess = TranslationProcess.objects(created_by=request.headers.get('ad-userid')).order_by(
'-basename').to_json()
res = CustomResponse(Status.SUCCESS.value, json.loads(transalationProcess))
except:
log.info('fetch-translation-process : ERROR occured')
pass
log.info('fetch_translation_process : ended at ' + str(getcurrenttime()))
return res.getres()
def delete_process():
log.info('delete_process: started at ' + str(getcurrenttime()))
try:
basename = request.form.getlist('processname')[0]
log.info('delte_process : requested basename is : ' + basename)
translationProcess = TranslationProcess.objects(basename=basename).delete()
log.info('delete_process: ended at ' + str(getcurrenttime()))
res = CustomResponse(Status.SUCCESS.value, basename)
except:
log.info('delte_process : ERROR while processing basename : ' + basename)
res = CustomResponse(Status.FAILURE.value, basename)
return res.getres()
def update_sentences():
body = request.get_json()
if (body['sentences'] is None or not isinstance(body['sentences'], list)):
res = CustomResponse(
Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
for sentence in body['sentences']:
corpus = Sentence.objects(_id=sentence['_id']['$oid'])
corpus_dict = json.loads(corpus.to_json())
sentence_log = Sentencelog(source_words=corpus_dict[0]['source'].split(" "),
target_words=corpus_dict[0]['target'].split(" "),
source_edited_words=sentence['source'].split(" "),
updated_on=datetime.now(), edited_by=request.headers.get('ad-userid'),
parent_id=sentence['_id']['$oid'], target_edited_words=sentence['target'].split(" "),
basename=corpus_dict[0]['basename'], source=corpus_dict[0]['source'],
target=corpus_dict[0]['target'], source_edited=sentence['source'],
target_edited=sentence['target'])
sentence_log.save()
corpus.update(set__source=sentence['source'], set__target=sentence['target'], set__status=STATUS_EDITED)
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def upload_corpus():
global BASE_CORPUS, STATUS_ACTIVE
basename = str(uuid.uuid4())
f_corpus = request.files['corpus']
name = request.form.getlist('name')
domain = request.form.getlist('domain')
lang = request.form.getlist('lang')
if lang is None or len(lang) == 0 or name is None or len(name) == 0 or len(
name[0]) == 0 or domain is None or len(domain) == 0 or len(
domain[0]
) == 0 or request.files is None or request.files['corpus'] is None:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
description = ''
if request.form.getlist('description') is not None and len(
request.form.getlist('description')) > 0 and len(
request.form.getlist('description')[0]) > 0:
description = request.form.getlist('description')[0]
corpus = Singlecorpus(status=STATUS_ACTIVE,
created_on=datetime.now(),
name=name[0],
corpusid=basename,
domain=domain[0],
lang=lang[0],
description=description,
created_by=request.headers.get('ad-userid'))
corpus.tags = [BASE_CORPUS, lang[0]]
corpus.save()
index = 0
for f in f_corpus:
sentence = Corpussentence(sentence=str(f), index=index)
sentence.tags = [basename, lang[0]]
sentence.original = True
sentence.parallelcorpusid = []
sentence.created_by = request.headers.get('ad-userid')
sentence.save()
index += 1
res = CustomResponse(Status.SUCCESS.value, None)
return res.getres()
def ner_sentences():
data = request.get_json()
if 'sentences' not in data or data['sentences'] is None or not isinstance(
data['sentences'], list):
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
else:
output_ner = list()
for text in data['sentences']:
mix_model_dir = os.getcwd() + '/upload/models/exp_1_mix/'
model_dir_order = os.getcwd() + '/upload/models/exp_1_order/'
model_dir_judgment = os.getcwd(
) + '/upload/models/exp_1_judgement/'
result_ner = SC_ner_annotation(model_dir_judgment, model_dir_order,
mix_model_dir, text).main()
if result_ner is None or mix_model_dir is None:
return "something went wrong"
else:
output_ner.append(result_ner)
res = CustomResponse(Status.SUCCESS.value, output_ner)
return res.getres()
def get(self):
parse = reqparse.RequestParser()
parse.add_argument('filename', type=str, location='args',help='Filename is required', required=True)
args = parse.parse_args()
filename = args['filename']
filepath = os.path.join(config.download_folder, filename)
if(os.path.exists(filepath)):
with open(filepath) as json_file:
data = json.load(json_file)
res = CustomResponse(Status.SUCCESS.value, data)
return res.getres()
else:
res = CustomResponse(Status.ERROR_NOTFOUND_FILE.value, None)
return res.getresjson(), 400
def convert_to_pdf():
body = request.get_json()
upload_id = str(uuid4())
filename = body['filename']
filepath = os.path.join(NGINX_FOLDER, filename)
try:
result = convert_to(os.path.join(NGINX_FOLDER, 'pdf', upload_id), filepath, timeout=15)
copyfile(result, os.path.join(NGINX_FOLDER, upload_id+'.pdf'))
except LibreOfficeError:
raise InternalServerErrorError({'message': 'Error when converting file to PDF'})
except TimeoutExpired:
raise InternalServerErrorError({'message': 'Timeout when converting file to PDF'})
res = CustomResponse(Status.SUCCESS.value, upload_id+'.pdf')
return res.getres()
def get_parallel_corpus_sentences_list():
basename = request.args.get('basename')
if basename is None or len(basename) == 0:
res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None)
return res.getres(
), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status']
parallel_corpus = Parallelcorpus.objects(basename=basename)
if parallel_corpus is None or len(parallel_corpus) == 0:
res = CustomResponse(Status.DATA_NOT_FOUND.value, None)
return res.getres(), Status.DATA_NOT_FOUND.value['http']['status']
parallel_corpus_dict = json.loads(parallel_corpus.to_json())
source_sentences = Corpussentence.objects.filter(
Q(tags=parallel_corpus_dict[0]['source_id'])
& Q(parallelcorpusid=basename)).order_by('index')
target_sentences = Corpussentence.objects.filter(
Q(tags=parallel_corpus_dict[0]['target_id'])
& Q(parallelcorpusid=basename)).order_by('index')
data = {
'source': json.loads(source_sentences.to_json()),
'target': json.loads(target_sentences.to_json())
}
res = CustomResponse(Status.SUCCESS.value, data)
return res.getres()