def post(self): body = request.get_json() upload_id = str(uuid4()) filename = body['filename'] filepath = os.path.join(config.download_folder, filename) if filename.endswith('.pdf'): res = CustomResponse(Status.SUCCESS.value, filename) return res.getres() try: result = convert_to(os.path.join(config.download_folder, 'pdf', upload_id), filepath, timeout=60) copyfile(result, os.path.join(config.download_folder, upload_id + '.pdf')) userfile = UserFiles(created_by=request.headers.get('ad-userid'), filename=upload_id + '.pdf', created_on=datetime.now()) userfile.save() except LibreOfficeError as e: raise InternalServerErrorError( {'message': 'Error when converting file to PDF'}) except TimeoutExpired: raise InternalServerErrorError( {'message': 'Timeout when converting file to PDF'}) res = CustomResponse(Status.SUCCESS.value, upload_id + '.pdf') return res.getres()
def saveTranslateDocx(): start_time = int(round(time.time() * 1000)) log.info('uploadTranslateDocx: started at ' + str(start_time)) if (request.form.getlist('basename') is None or not isinstance(request.form.getlist('basename'), list)): res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] basename = request.form.getlist('basename')[0] current_time = datetime.now().strftime("%m/%d/%Y, %H:%M:%S") f = request.files['file'] filepath = os.path.join(app.config['UPLOAD_FOLDER'], basename + '_u.docx') index = 0 while (os.path.exists(filepath)): filepath = os.path.join(app.config['UPLOAD_FOLDER'], basename + '_' + str(index) + '_u.docx') index = index + 1 f.save(filepath) res = CustomResponse(Status.SUCCESS.value, basename + '_' + str(index) + '_u' + '.docx') translationProcess = TranslationProcess.objects(basename=basename) translationProcess.update(set__translate_uploaded=True) log.info('uploadTranslateDocx: ended at ' + str(getcurrenttime()) + 'total time elapsed : ' + str(getcurrenttime() - start_time)) return res.getres()
def save_ocr_data(): body = request.get_json() if body['ocr_data'] is None: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] ocr_data = Ocrdata(created_on=str(int(time.time())), data=body['ocr_data']['response']) ocr_data.save() res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def update_sentences_status(): body = request.get_json() if (body['sentences'] is None or not isinstance(body['sentences'], list)): res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres(), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] for sentence in body['sentences']: corpus = Sentence.objects(_id=sentence['_id']['$oid']) corpus.update(set__status=sentence['status']) res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def update_password_admin(): log.info('update_password : started') body = request.get_json() user_id = body['user_id'] high_court_code = body['high_court_code'] new_password = body['new_password'] log.info("high_court_code == " + high_court_code) if high_court_code is not None: userHighCourt = Userhighcourt.objects(user_id=user_id) if userHighCourt is not None and len(userHighCourt) > 0: log.info('high court with user exist ' + str(len(userHighCourt))) userHighCourt.update(set__high_court_code=high_court_code) else: log.info('saving high court with user') user_high_court = Userhighcourt(high_court_code=high_court_code, user_id=user_id) user_high_court.save() profile = requests.get(PROFILE_REQ_URL + user_id).content profile = json.loads(profile) roles_ = get_user_roles_basic_auth(user_id) data = {"status": "false"} req = GATEWAY_SERVER_URL + 'credentials/basic-auth/' + user_id + '/status' response = requests.put(req, json=data) res = response.json() status = res['status'] log.info("status == " + status) if not status == 'Deactivated': res = CustomResponse(Status.ERROR_GATEWAY.value, None) return res.getres() data = { "credential": { "password": new_password, "scopes": roles_ }, "consumerId": user_id, "type": "basic-auth" } if new_password is not None or new_password.__len__() == 0: if new_password.__len__() < 6: log.info( 'update_password : password is too weak, at least provide 6 characters' ) res = CustomResponse(Status.ERROR_WEAK_PASSWORD.value, None) return res.getres() else: req = GATEWAY_SERVER_URL + 'credentials' response = requests.post(req, json=data) res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def create_user_oauth(): log.info('create_user_oauth : started') body = request.get_json() user_name = body['username'] try: response = shell.create_oauth(user_name) res = CustomResponse(Status.SUCCESS.value, response) return res.getres() except Exception as e: log.info('create_user_oauth : error ' + str(e)) res = CustomResponse(Status.ERROR_GATEWAY.value, None) return res.getres()
def check_ocr_data(): body = request.get_json() if body['student_id'] is None or body['exam_id'] is None or body[ 'ocr_data'] is None: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] student_fromdb = Student.objects(student_id=body['student_id']) exam_fromdb = Exams.objects(exam_id=body['exam_id']) if not (student_fromdb is not None and len(student_fromdb) > 0): res = CustomResponse(Status.WRONG_STUDENT_CODE.value, None) return res.getres() elif not (exam_fromdb is not None and len(exam_fromdb) > 0): res = CustomResponse(Status.WRONG_CODE.value, None) return res.getres() else: try: exam_obj = json.loads(exam_fromdb.to_json()) student_obj = json.loads(student_fromdb.to_json()) map_obj = conver_list_to_map(exam_obj[0]['data']) ocr_data = body['ocr_data'] table_data = ocr_data['response'][1]['data'] students_data = ocr_data['response'][0]['data'] student_obj[0]['exam_id'] = body['exam_id'] ocr_data['metadata'] = student_obj[0] if len(ocr_data['response'][0]['data']) > len( ocr_data['response'][1]['data']): table_data = ocr_data['response'][0]['data'] students_data = ocr_data['response'][1]['data'] for marks_data in table_data: key = str(marks_data['col']) + str(marks_data['row']) if key in map_obj: marks_data['text'] = map_obj[key] elif str(marks_data['col']) == '0' or str( marks_data['col']) == '1' or str( marks_data['col']) == '2': marks_data['text'] = '' if 'exam_date' in exam_obj[0]: for student_data in students_data: if student_data['col'] == 1 and student_data['row'] == 3: student_data['text'] = exam_obj[0]['exam_date'] res = CustomResponse(Status.SUCCESS.value, ocr_data) return res.getres() except Exception as e: res = CustomResponse(Status.ERR_GLOBAL_SYSTEM.value, ocr_data) return res.getres( ), Status.ERR_GLOBAL_SYSTEM.value['http']['status']
def fetch_sentences(): global LANGUAGES basename = request.args.get('basename') totalcount = 0 (sentencesobj, totalcount) = Sentence.limit(request.args.get('pagesize'), basename, request.args.get('status'), request.args.get('pageno')) corpus_obj = Corpus.objects(basename=basename) corpus_dict = json.loads(corpus_obj.to_json()) sentences_list = [] sources = [] if sentencesobj is not None: for sent in sentencesobj: sent_dict = json.loads(sent.to_json()) corpus = Sentence.objects(_id=sent_dict['_id']['$oid']) if sent_dict['status'] == STATUS_PENDING: corpus.update(set__status=STATUS_PROCESSING) sources.append(sent_dict['source']) target_lang = 'en' if 'target_lang' in corpus_dict[0] and corpus_dict[0]['target_lang'] is not None: target_lang = LANGUAGES[corpus_dict[0]['target_lang']] translation_list = translatesinglesentence(sources, target_lang) index = 0 for sent in sentencesobj: sent_dict = json.loads(sent.to_json()) sent_dict['translation'] = translation_list[index] sentences_list.append(sent_dict) index += 1 # print() # for sentence in sentencesobj: # # sentence.update(set__status=STATUS_PROCESSING, set__locked=True, set__locked_time=datetime.now()) # sentence.update(set__status=STATUS_PROCESSING) res = CustomResponse(Status.SUCCESS.value, sentences_list, totalcount) return res.getres()
def roles(): body = request.get_json() res = None if body['operation'] is not None and body['role-type'] is not None: if body['operation'] == "create": if not body['role-type'] == '': try: response = requests.post(GATEWAY_SERVER_URL + 'scopes') res = CustomResponse(Status.SUCCESS.value, json.loads(response)) except: res = CustomResponse(Status.FAILURE.value, None) else: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, ' role-type not provided ') else: res = CustomResponse(Status.OPERATION_NOT_PERMITTED.value, 'supported opertion type are : [create] ') else: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, ' please provide operation and role-type ') return res.getres()
def save_student_masterdata(): body = request.get_json() if body['student'] is None or body['student']['student_id'] is None: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] student_fromdb = Student.objects(student_id=body['student']['student_id']) if student_fromdb is not None and len(student_fromdb) > 0: res = CustomResponse(Status.USER_ALREADY_EXISTS.value, None) return res.getres(), Status.USER_ALREADY_EXISTS.value['http']['status'] else: student = Student(student_id=body['student']['student_id'], student_name=body['student']['student_name']) student.save() res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def post(self): body = request.get_json() userid = request.headers.get('userid') if 'pages' not in body or 'process_identifier' not in body or userid is None: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getresjson(), 400 results = body['pages'] process_identifier = body['process_identifier'] obj_to_be_saved = [] for result in results: page_data = {} page_data['page_no'] = result['page_no'] page_data['page_width'] = result['page_width'] page_data['page_height'] = result['page_height'] for block_type in BLOCK_TYPES: if result[block_type['key']] is not None: for data in result[block_type['key']]: obj_to_be_saved = self.make_obj( process_identifier, page_data, data, block_type['key'], obj_to_be_saved, userid) file_content_instances = [ FileContent(**data) for data in obj_to_be_saved ] FileContent.objects.insert(file_content_instances) res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def create_user_basic_auth(): log.info('create_user_basic_auth : started') body = request.get_json() user_name = body['username'] firstname = body['firstname'] lastname = body['lastname'] password = body['password'] scope = body['roles'] high_court_code = body['high_court_code'] try: profile = requests.get(PROFILE_REQ_URL + user_name) try: profile = profile.json() if profile['isActive']: # _id = profile[''] log.info('create_user_oauth : profile is = : ' + str(profile)) res = CustomResponse(Status.USER_ALREADY_EXISTS.value, None) return res.getres() except: pass log.info('here') create_response = shell.create_user(user_name, firstname, lastname) log.info('user created') shell_response = shell.create_basic_auth_credentials( user_name, password) log.info('basic auth created') response = shell.create_oauth(user_name) log.info('oauth created') user = shell.get_user_info(user_name) log.info(str(user)) scope_response = shell.scope_add(user['id'], scope) time.sleep(3) log.info('scope added') if high_court_code is not None: user_high_court = Userhighcourt(high_court_code=high_court_code, user_id=user['id']) user_high_court.save() res = CustomResponse(Status.SUCCESS.value, response) return res.getres() except Exception as e: log.info(' create_user : error ' + str(e)) res = CustomResponse(Status.ERROR_GATEWAY.value, None) return res.getres()
def fetch_corpus(): if request.headers.get('ad-userid') is not None: log.info('fetch_corpus: initiated by ' + request.headers.get('ad-userid')) else: log.info('fetch_corpus: initiated by anonymous user') corpus = Corpus.objects.to_json() res = CustomResponse(Status.SUCCESS.value, json.loads(corpus)) return res.getres()
def save_exam_masterdata(): body = request.get_json() if body['exam'] is None or body['exam']['exam_id'] is None or body['exam'][ 'data'] is None: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] exam_fromdb = Exams.objects(exam_id=body['exam']['exam_id']) if exam_fromdb is not None and len(exam_fromdb) > 0: exam_fromdb.update(set__data=body['exam']['data']) else: exam = Exams(exam_id=body['exam']['exam_id'], data=body['exam']['data'], exam_date=body['exam']['exam_date']) exam.save() res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def translate_source(): sources = [] source = request.args.get('source') basename = request.args.get('basename') if source is None or basename is None: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres(), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] sources.append(source) corpus_obj = Corpus.objects(basename=basename) corpus_dict = json.loads(corpus_obj.to_json()) target_lang = 'en' if 'target_lang' in corpus_dict[0] and corpus_dict[0]['target_lang'] is not None: target_lang = LANGUAGES[corpus_dict[0]['target_lang']] translation_list = translatesinglesentence(sources, target_lang) res = CustomResponse(Status.SUCCESS.value, translation_list) return res.getres()
def create_parallel_corpus(): global STATUS_ACTIVE body = request.get_json() if body['source_corpus'] is None or len(body['source_corpus']) == 0: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] basename = str(uuid.uuid4()) target_corpus_id = str(uuid.uuid4()) source = body['source_corpus'] name = body['name'] domain = body['domain'] target_lang = body['target_lang'] source_lang = body['source_lang'] corpus = Singlecorpus(status=STATUS_ACTIVE, created_on=datetime.now(), name=name, corpusid=target_corpus_id, domain=domain, lang=target_lang, created_by=request.headers.get('ad-userid')) corpus.tags = [BASE_CORPUS, target_lang] corpus.save() source_corpus = Singlecorpus.objects(corpusid=source) if source_corpus is None or len(source_corpus) == 0: res = CustomResponse(Status.DATA_NOT_FOUND.value, None) return res.getres(), Status.DATA_NOT_FOUND.value['http']['status'] parallel_corpus = Parallelcorpus(source_lang=source_lang, target_lang=target_lang, name=name, domain=domain, basename=basename, source_id=source, target_id=target_corpus_id, status=STATUS_ACTIVE) parallel_corpus.save() source_sentences = Corpussentence.objects( Q(tags=source) & Q(original=True)) for source_sentence in source_sentences: source_sentence_dict = json.loads(source_sentence.to_json()) source_sentence_tags = source_sentence_dict['parallelcorpusid'] source_sentence_tags.append(basename) source_sentence.parallelcorpusid = source_sentence_tags source_sentence.save() res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def update_password(): log.info('update_password : started') body = request.get_json() user_id = body['user_id'] user_name = body['user_name'] old_password = body['old_password'] new_password = body['new_password'] password_checked = check_password(user_name, old_password) if not password_checked: res = CustomResponse(Status.ERROR_WRONG_PASSWORD.value, None) return res.getres() if new_password is None or new_password.__len__() < 6: log.info( 'update_password : password is too weak, at least provide 6 characters' ) res = CustomResponse(Status.ERROR_WEAK_PASSWORD.value, None) return res.getres() profile = requests.get(PROFILE_REQ_URL + user_id).content profile = json.loads(profile) roles_ = get_user_roles_basic_auth(user_id) data = {"status": "false"} req = GATEWAY_SERVER_URL + 'credentials/basic-auth/' + user_id + '/status' response = requests.put(req, json=data) res = response.json() status = res['status'] log.info("status == " + status) if not status == 'Deactivated': res = CustomResponse(Status.ERROR_GATEWAY.value, None) return res.getres() data = { "credential": { "password": new_password, "scopes": roles_ }, "consumerId": user_id, "type": "basic-auth" } req = GATEWAY_SERVER_URL + 'credentials' response = requests.post(req, json=data) res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def tokenize_hin_sentence(): body = request.get_json() if body['paragraphs'] is None or not isinstance(body['paragraphs'], list): res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] response = [] for paragraph in body['paragraphs']: tokenizer = AnuvaadHinTokenizer() if 'text' in paragraph: tokenized_data = {} tokenized_data['text'] = tokenizer.tokenize(paragraph['text']) if 'page_no' in paragraph: tokenized_data['page_no'] = paragraph['page_no'] response.append(tokenized_data) else: response.append(tokenizer.tokenize(paragraph)) res = CustomResponse(Status.SUCCESS.value, response) return res.getres()
def upload_file(): pool = mp.Pool(mp.cpu_count()) basename = str(int(time.time())) try: name = request.form.getlist('name') domain = request.form.getlist('domain') source_lang = request.form.getlist('source_lang') target_lang = request.form.getlist('target_lang') comment = request.form.getlist('comment') if comment is None or len(comment) == 0: comment = [''] if name is None or len(name) == 0 or len(name[0]) == 0 or domain is None or len(domain) == 0 or len( domain[0]) == 0 or request.files is None or request.files['hindi'] is None or request.files[ 'english'] is None: res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres(), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] else: current_time = datetime.now().strftime("%m/%d/%Y, %H:%M:%S") corpus = Corpus(status=STATUS_PROCESSING, name=name[0], domain=domain[0], created_on=current_time,source_lang=source_lang[0],target_lang=target_lang[0], last_modified=current_time, author='', comment=comment[0], no_of_sentences=0, basename=basename) corpus.save() f = request.files['hindi'] f_eng = request.files['english'] filepath = os.path.join( app.config['UPLOAD_FOLDER'], basename + '_hin.pdf') filepath_eng = os.path.join( app.config['UPLOAD_FOLDER'], basename + '_eng.pdf') f.save(filepath) f_eng.save(filepath_eng) pool.apply_async(converttoimage, args=( filepath, app.config['UPLOAD_FOLDER'], basename, '_hin'), callback=capturetext) pool.apply_async(converttoimage, args=( filepath_eng, app.config['UPLOAD_FOLDER'], basename, '_eng'), callback=capturetext) pool.close() pool.join() return process_files(basename) except Exception as e: print(e) res = CustomResponse(Status.ERR_GLOBAL_SYSTEM.value, None) return res.getres(), Status.ERR_GLOBAL_SYSTEM.value['http']['status']
def process_files_law(basename, name): filtertext(app.config['UPLOAD_FOLDER'] + '/' + basename + '_hin.txt', app.config['UPLOAD_FOLDER'] + '/' + basename + '_hin_filtered.txt') filtertext(app.config['UPLOAD_FOLDER'] + '/' + basename + '_eng.txt', app.config['UPLOAD_FOLDER'] + '/' + basename + '_eng_filtered.txt') processhindi(app.config['UPLOAD_FOLDER'] + '/' + basename + '_hin_filtered.txt') processenglish(app.config['UPLOAD_FOLDER'] + '/' + basename + '_eng_filtered.txt') translatewithgoogle(app.config['UPLOAD_FOLDER'] + '/' + basename + '_hin_filtered.txt', app.config['UPLOAD_FOLDER'] + '/' + basename + '_eng_tran.txt') os.system( './helpers/bleualign.py -s ' + os.getcwd() + '/upload/' + basename + '_hin_filtered' + '.txt' + ' -t ' + os.getcwd() + '/upload/' + basename + '_eng_filtered' + '.txt' + ' --srctotarget ' + os.getcwd() + '/upload/' + basename + '_eng_tran' + '.txt' + ' -o ' + os.getcwd() + '/upload/' + basename + '_output') english_res = [] hindi_res = [] english_points = [] english_points_words = [] hindi_points = [] hindi_points_words = [] f_eng = open(app.config['UPLOAD_FOLDER'] + '/' + basename + '_output-t', 'r') for f in f_eng: english_res.append(f) point = fetchwordsfromsentence(f, basename) english_points.append(point['avg']) english_points_words.append(point['values']) f_eng.close() f_hin = open(app.config['UPLOAD_FOLDER'] + '/' + basename + '_output-s', 'r') for f in f_hin: hindi_res.append(f) point = fetchwordsfromsentence(f, basename) hindi_points.append(point['avg']) hindi_points_words.append(point['values']) f_hin.close() data = {'hindi': hindi_res, 'english': english_res, 'english_scores': english_points, 'hindi_scores': hindi_points} sentences = [] for i in range(0, len(hindi_res)): sentence = Sentence(status=STATUS_PENDING, alignment_accuracy=english_res[i].split(':::::')[1], basename=name, source=hindi_res[i], target=english_res[i].split(':::::')[0], source_ocr_words=hindi_points_words[i], source_ocr=str(hindi_points[i]), target_ocr_words=english_points_words[i], target_ocr=str(english_points[i])) sentences.append(sentence) # sentence.save() Sentence.objects.insert(sentences) for f in glob.glob(app.config['UPLOAD_FOLDER'] + '/' + basename + '*'): os.remove(f) res = CustomResponse(Status.SUCCESS.value, data) # corpus = Corpus.objects(basename=basename) # corpus.update(set__status=STATUS_PROCESSED, # set__no_of_sentences=len(hindi_res)) return res.getres()
def remove_junk(): basename = str(int(time.time())) f = request.files['file'] filepath_eng = os.path.join( app.config['UPLOAD_FOLDER'], basename + '_junk.txt') f.save(filepath_eng) f_eng = open(app.config['UPLOAD_FOLDER'] + '/' + basename + '_junk.txt', 'r') for t in f_eng: Sentence.objects(source=t).delete() res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def update_corpus(): start_time = int(round(time.time() * 1000)) log.info('update_corpus: started at ' + str(start_time)) current_time = datetime.now().strftime("%m/%d/%Y, %H:%M:%S") req_data = request.get_json() validate_upload_corpus_request(req_data) res = CustomResponse(Status.SUCCESS.value, 'no. of sentences are ') end_time = int(round(time.time() * 1000)) log.info('upload_corpus: ended at ' + str(end_time) + 'total time elapsed = ' + str(end_time - start_time)) return res.getres()
def fetch_translation_process(): log.info('fetch_translation_process : started at ' + str(getcurrenttime())) try: transalationProcess = TranslationProcess.objects(created_by=request.headers.get('ad-userid')).order_by( '-basename').to_json() res = CustomResponse(Status.SUCCESS.value, json.loads(transalationProcess)) except: log.info('fetch-translation-process : ERROR occured') pass log.info('fetch_translation_process : ended at ' + str(getcurrenttime())) return res.getres()
def delete_process(): log.info('delete_process: started at ' + str(getcurrenttime())) try: basename = request.form.getlist('processname')[0] log.info('delte_process : requested basename is : ' + basename) translationProcess = TranslationProcess.objects(basename=basename).delete() log.info('delete_process: ended at ' + str(getcurrenttime())) res = CustomResponse(Status.SUCCESS.value, basename) except: log.info('delte_process : ERROR while processing basename : ' + basename) res = CustomResponse(Status.FAILURE.value, basename) return res.getres()
def update_sentences(): body = request.get_json() if (body['sentences'] is None or not isinstance(body['sentences'], list)): res = CustomResponse( Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres(), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] for sentence in body['sentences']: corpus = Sentence.objects(_id=sentence['_id']['$oid']) corpus_dict = json.loads(corpus.to_json()) sentence_log = Sentencelog(source_words=corpus_dict[0]['source'].split(" "), target_words=corpus_dict[0]['target'].split(" "), source_edited_words=sentence['source'].split(" "), updated_on=datetime.now(), edited_by=request.headers.get('ad-userid'), parent_id=sentence['_id']['$oid'], target_edited_words=sentence['target'].split(" "), basename=corpus_dict[0]['basename'], source=corpus_dict[0]['source'], target=corpus_dict[0]['target'], source_edited=sentence['source'], target_edited=sentence['target']) sentence_log.save() corpus.update(set__source=sentence['source'], set__target=sentence['target'], set__status=STATUS_EDITED) res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def upload_corpus(): global BASE_CORPUS, STATUS_ACTIVE basename = str(uuid.uuid4()) f_corpus = request.files['corpus'] name = request.form.getlist('name') domain = request.form.getlist('domain') lang = request.form.getlist('lang') if lang is None or len(lang) == 0 or name is None or len(name) == 0 or len( name[0]) == 0 or domain is None or len(domain) == 0 or len( domain[0] ) == 0 or request.files is None or request.files['corpus'] is None: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] description = '' if request.form.getlist('description') is not None and len( request.form.getlist('description')) > 0 and len( request.form.getlist('description')[0]) > 0: description = request.form.getlist('description')[0] corpus = Singlecorpus(status=STATUS_ACTIVE, created_on=datetime.now(), name=name[0], corpusid=basename, domain=domain[0], lang=lang[0], description=description, created_by=request.headers.get('ad-userid')) corpus.tags = [BASE_CORPUS, lang[0]] corpus.save() index = 0 for f in f_corpus: sentence = Corpussentence(sentence=str(f), index=index) sentence.tags = [basename, lang[0]] sentence.original = True sentence.parallelcorpusid = [] sentence.created_by = request.headers.get('ad-userid') sentence.save() index += 1 res = CustomResponse(Status.SUCCESS.value, None) return res.getres()
def ner_sentences(): data = request.get_json() if 'sentences' not in data or data['sentences'] is None or not isinstance( data['sentences'], list): res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] else: output_ner = list() for text in data['sentences']: mix_model_dir = os.getcwd() + '/upload/models/exp_1_mix/' model_dir_order = os.getcwd() + '/upload/models/exp_1_order/' model_dir_judgment = os.getcwd( ) + '/upload/models/exp_1_judgement/' result_ner = SC_ner_annotation(model_dir_judgment, model_dir_order, mix_model_dir, text).main() if result_ner is None or mix_model_dir is None: return "something went wrong" else: output_ner.append(result_ner) res = CustomResponse(Status.SUCCESS.value, output_ner) return res.getres()
def get(self): parse = reqparse.RequestParser() parse.add_argument('filename', type=str, location='args',help='Filename is required', required=True) args = parse.parse_args() filename = args['filename'] filepath = os.path.join(config.download_folder, filename) if(os.path.exists(filepath)): with open(filepath) as json_file: data = json.load(json_file) res = CustomResponse(Status.SUCCESS.value, data) return res.getres() else: res = CustomResponse(Status.ERROR_NOTFOUND_FILE.value, None) return res.getresjson(), 400
def convert_to_pdf(): body = request.get_json() upload_id = str(uuid4()) filename = body['filename'] filepath = os.path.join(NGINX_FOLDER, filename) try: result = convert_to(os.path.join(NGINX_FOLDER, 'pdf', upload_id), filepath, timeout=15) copyfile(result, os.path.join(NGINX_FOLDER, upload_id+'.pdf')) except LibreOfficeError: raise InternalServerErrorError({'message': 'Error when converting file to PDF'}) except TimeoutExpired: raise InternalServerErrorError({'message': 'Timeout when converting file to PDF'}) res = CustomResponse(Status.SUCCESS.value, upload_id+'.pdf') return res.getres()
def get_parallel_corpus_sentences_list(): basename = request.args.get('basename') if basename is None or len(basename) == 0: res = CustomResponse(Status.ERR_GLOBAL_MISSING_PARAMETERS.value, None) return res.getres( ), Status.ERR_GLOBAL_MISSING_PARAMETERS.value['http']['status'] parallel_corpus = Parallelcorpus.objects(basename=basename) if parallel_corpus is None or len(parallel_corpus) == 0: res = CustomResponse(Status.DATA_NOT_FOUND.value, None) return res.getres(), Status.DATA_NOT_FOUND.value['http']['status'] parallel_corpus_dict = json.loads(parallel_corpus.to_json()) source_sentences = Corpussentence.objects.filter( Q(tags=parallel_corpus_dict[0]['source_id']) & Q(parallelcorpusid=basename)).order_by('index') target_sentences = Corpussentence.objects.filter( Q(tags=parallel_corpus_dict[0]['target_id']) & Q(parallelcorpusid=basename)).order_by('index') data = { 'source': json.loads(source_sentences.to_json()), 'target': json.loads(target_sentences.to_json()) } res = CustomResponse(Status.SUCCESS.value, data) return res.getres()