def get_review_data(solution_id, file_id, status, doc_type, direction, query_string):
query = get_flow_id(file_id=file_id, status=status, doc_type=doc_type, direction=direction,
query_string=query_string)
# query = {"flow_file_id": file_id}
rows = MongoDbConn.find_one(DASHBOARD_CONFIG, query={"table": "DocumentReviewDetails"})
record = dict()
record.update(query)
del query["disable_prev"]
del query["disable_next"]
query["solution_id"] = solution_id
atts = list()
for field in rows['config']:
if field['level'] == 0:
record = add_url_recid(field, query, record)
else:
if atts == list():
atts = set_attributes(field, query)
else:
atts = update_attributes(atts, set_attributes(field, query))
if field['key'] == "Document Confidence" and record[field['key']] is not None:
record[field['key']] = str(round(get_score(field, record[field['key']]), 2)) + "%"
if field['key'] == "Processed" and record[field['key']] is not None:
record[field['key']] = str(conv_timestamp(record[field['key']]))
record['attributes'] = add_intent(add_nlp_flag_to_atts(atts, query), query)
return record
def save_template_element(solution_id, data):
try:
query = dict(solution_id=solution_id, template_id=data.pop("template_id"), is_deleted=False)
template = MongoDbConn.find_one(TEMPLATE_COLLECTION, query)
template = json.loads(template["template"]) if template else {}
# new element object
new = data
new["id"], is_update = (str(uuid4()), False) if "id" not in new else (new["id"], True)
new["section_id"] = new["id"] if "section_id" not in new else new["section_id"]
new["is_deleted"] = False
if new["type"] == "table":
template["domain_object_mapping"] = update_table_mapping(new, template["domain_object_mapping"])
else:
if "domain_mapping" in new and new["domain_mapping"] != "":
template["domain_object_mapping"] = update_template_domain_mapping(new["domain_mapping"],
new["id"],
template["domain_object_mapping"])
template["document_variables"].pop(new["id"], None)
if "doc_var" in new and new["doc_var"] != {}:
template["document_variables"] = update_template_document_variables(new["doc_var"],
new["id"],
template["document_variables"])
template["domain_object_mapping"].pop(new["id"], None)
elements_reformatted = update_template_elements(new, _obj=template["elements"], is_update=is_update)
template["elements"] = [elements_reformatted]
resp = post_save_template(solution_id, dict(document=template))
resp.update(dict(section_id=new['section_id'], id=new['id']))
return resp
except Exception:
return dict(success=False, msg="Failed to save element", error=traceback.format_exc(), status="failure")
def update_solutions(req_data):
solns = None
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
query = {
'is_deleted': False,
'solution_id': req_data["solution_id"]
}
projection = {'_id': 0}
solns = MongoDbConn.find_one(SOLUTION_COLLECTION,
query,
projection=projection)
# solns = Solution.objects.get(deleted=False,solution_id=req_data["solution_id"])
if solns:
solns['hocr_type'] = req_data["hocr_type"]
# Solution.objects.update(solns)
# solns.save()
MongoDbConn.update(SOLUTION_COLLECTION, query, solns)
return {'status': 'success', 'msg': 'updated Solutions list'}
else:
return {'status': 'success', 'msg': 'No solutions exists'}
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {'status': 'failure', 'msg': 'exception occerd : ' + str(e)}
finally:
context.end_span()
def get_solution_id(request):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
solution_id = common.get_solution_from_session(request)
solution_name = ""
query = {'solution_id': solution_id}
projection = {'_id': 0}
soln_exists = MongoDbConn.find_one(SOLUTION_COLLECTION,
query,
projection=projection)
if soln_exists:
solution_name = soln_exists['solution_name']
return {
"solution_id": solution_id,
'solution_name': solution_name,
'case_management_url': CASE_MANAGEMENT_SERVICE_URL,
"status": "success"
}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {"status": "failure", "msg": str(e)}
finally:
context.end_span()
def mapping_list_of_thresholds(mapping_query, map_list=[]):
mapping = MongoDbConn.find_one(MAPPING_COLLECTION, mapping_query)
context = tracer.get_context(request_id=str(uuid4()), log_level="ERROR")
context.start_span(component=__name__)
try:
if mapping:
section_data = mapping["sections"]
for section_id, data in section_data.items():
if "map_to" in data and not data["is_deleted"]:
map_list.append(data['map_to'][0]) if type(data['map_to']) == list and len(data['map_to']) == 1 \
else map_list.append(data['map_to'])
if "elements" in data:
element_data = data["elements"]
for element_id, element_data in element_data.items():
if "map_to" in element_data and not element_data[
"is_deleted"]:
for each in element_data["map_to"]:
map_list.append(each['map_to'])
return map_list
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {
"status": "failed",
"msg": "error in getting objects",
"error": str(e)
}
finally:
context.end_span()
def create_email_template(solution_id,payload):
job_id = None
temp_result = {"status" :"failure"}
template = MongoDbConn.find_one(TEMPLATE_COLLECTION,
{"solution_id": solution_id, "template_name": "email", "is_deleted": False})
if template is None:
template_data = format_template_data(solution_id)
response = post_job(TEMPLATE_CONFIG["SAVE"],template_data)
if 'job_id' in response:
job_id = response["job_id"]
if not is_request_timeout(response):
status, result = get_response(response)
if status:
template_id = get_nested_value(response, "result.result.metadata.template_id")
if template_id:
section_result = create_new_section(template_id,solution_id,DEFAULT_SECTION)
if section_result["status"] != "success":
return temp_result.update({'msg': 'Failed to create sections',
'error': section_result, 'job_id':job_id})
else:
return temp_result.update({'msg': 'Failed to create template',
'error': result, 'job_id':job_id})
else:
return temp_result.update({'msg': 'Request timed out',
'error': response, 'job_id':job_id})
else:
template_id = template["template_id"]
element_result = update_elements(template_id,solution_id,payload)
if element_result["status"] == "success":
return {'status':'success'}
else:
return temp_result.update({'msg': 'Failed to create elements', 'error': element_result})
def process_workflow_files(request):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
if request.method == "GET":
result = MongoDbConn.find(RESOURCES_COLLECTION,dict(type="camunda_workflow"))
workflow_files = list()
for file in result:
file.pop("_id")
workflow_files.append(file)
return {"data":workflow_files,"status":"success"}
elif request.method == "POST":
payload = json.loads(request.body.decode())
if "file_path" in payload:
with open(payload["file_path"]) as fp:
xml_string = fp.read()
return {"status": "success","xml_string" : xml_string}
if "resource_id" in payload:
file = MongoDbConn.find_one(RESOURCES_COLLECTION,dict(type="camunda_workflow",resource_id=payload["resource_id"]))
if file is not None:
with open(file["file_path"], 'r+') as f:
f.read()
f.seek(0)
f.write(payload["xml_string"])
f.truncate()
return {"status":"success","msg":"Workflow updated successfully"}
return {"status":"failure","msg":"Workflow update failed"}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {"status": "failure", "msg": str(e)}
finally:
context.end_span()
def get_source(self, solution_id, source_id):
"""
This function will fetch the source record
and return the dictionary as response
:param solution_id: Session Solution Id
:param source_id: Id of the source
:return: source record
"""
try:
if not source_id:
return {
'status': 'failure',
'status_code': STATUS_CODES['PRECONDITION_FAILED'],
'msg': 'Source Id is not available in the request.'
}
query = {
'solution_id': solution_id,
'source_type': self.source_type,
'source_id': source_id,
'is_deleted': False
}
projection = {'_id': 0}
source_rec = MongoDbConn.find_one(SOURCES_COLLECTION,
query,
projection=projection)
status_code = STATUS_CODES['OK']
return source_rec, query, status_code
except Exception as e:
self.context.log(message=str(e),
obj={"tb": traceback.format_exc()})
return None, None, None
def get_s3_bucket():
result = MongoDbConn.find_one(CONFIG_COLLECTION, {})
resp = dict()
if result is not None:
resp['default'] = result['s3_claims_bucket']
# resp['data'] = [e.bucket for e in S3Bucket.objects.all()]
resp['data'] = []
return resp
def get_template_type(template_id, solution_id):
if template_id and template_id != "unknown":
template = MongoDbConn.find_one(TEMPLATE_COLLECTION,
{"solution_id": solution_id, "template_id": template_id})
if template:
template_data = json.loads(template['template'])
if "template_type" in template_data and template_data["template_type"] not in UNKNOWN_TYPES:
return "known"
return "unknown"
def add_url_recid(field, query, record):
result = MongoDbConn.find_one(field['collection'], query)
if result is not None:
if field['type'] == "string":
record[field['key']] = str(get_value(field['value'], result))
elif field['type'] == "url":
record[field['key']] = str(get_value(field['value'], result))
return record
def add_nlp_flag_to_atts(atts, query):
record = MongoDbConn.find_one(CLAIMS_COLLECTION, query)
if record is not None and "doc_type" in record.keys():
form_type = record["doc_type"]
else:
form_type = None
result = MongoDbConn.find_one(FORMS_COLLECTION, {"form_type": form_type})
if form_type is not None and result is not None and "configuration" in result.keys():
config = result["configuration"]
for itm in atts:
itm["is_nlp_enabled"] = False
for key in config.keys():
upd_key = " ".join([i.strip() for i in str(key).split("_") if i != "_"]).strip().title()
if "key" in itm.keys() and itm["key"] == upd_key:
if "is_extract_intent" in config[key].keys() and config[key]["is_extract_intent"]:
itm["is_nlp_enabled"] = True
return atts
def save_unknown_template(solution_id, payload):
if "template_id" in payload:
query = dict(solution_id=solution_id, template_id=payload["template_id"], is_deleted=False)
template = MongoDbConn.find_one(TEMPLATE_COLLECTION, query)
template = json.loads(template["template"]) if template else {}
template["elements"] = payload["elements"]
return post_save_template(solution_id, dict(document=template))
else:
payload["request_type"] = "ingest_template"
return post_save_template(solution_id, data=payload, endpoint=TEMPLATE_INGEST_URL)
def get_email_details(solution_id):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
result = MongoDbConn.find_one(SOURCE_COLLECTION, dict(solution_id=solution_id,source_type="email"))
if result is not None:
template = MongoDbConn.find_one(TEMPLATE_COLLECTION,
{"solution_id": solution_id, "template_name" : "email", "is_deleted": False})
if result is not None:
email_mapping = template["fields"][0]["domain_mapping"]
else:
email_mapping = ""
result["configuration"]["email_body"] = email_mapping
return result["configuration"]
else:
return dict(email={})
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
finally:
context.end_span()
def add_email_info(data):
attachments = []
for doc_id in data["child_documents"]:
filter_query = {"doc_id": doc_id}
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, filter_query)
if document is not None:
attach_data = dict()
attach_data["doc_id"] = doc_id
attach_data["mime_type"] = document["metadata"]["extn"]
attach_data["file_name"] = document["metadata"]["file_name"]
attachments.append(attach_data)
return attachments
def change_doc_state(request):
solution_id = get_solution_from_session(request)
try:
payload = json.loads(request.body.decode())
except:
payload = request.POST
if payload["doc_state"] != "processed":
doc_id = payload["doc_id"]
query = dict(doc_id=doc_id, solution_id=solution_id)
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, query)
if "entity_feedback" in document and document["entity_feedback"]:
MongoDbConn.update(DOCUMENTS_COLLECTION, query, {"entity_feedback": None})
return post_doc_state_change(payload,solution_id, reset_cycle=True)
def update_template(solution_id, payload):
try:
# Finding template
filter_query = dict(solution_id=solution_id, template_id=payload.pop("template_id"), is_deleted=False)
template_data = MongoDbConn.find_one(TEMPLATE_COLLECTION, filter_query)
template = json.loads(template_data["template"]) if template_data else {}
# Updating template
endpoint = TEMPLATE_PUBLISH_URL if "is_draft" in payload else TEMPLATE_SAVE_URL
template.update(payload)
return post_save_template(solution_id, dict(document=template), endpoint=endpoint)
except Exception:
return dict(success=False, msg="Failed to publish template", error=traceback.format_exc(), status="failure")
def invoke_files_download(doc_id):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
result = MongoDbConn.find_one(TRAINING_SET_COLLECTION, {"_id": ObjectId(doc_id)})
if result is not None:
path = result['s3_key']
return download_file(AMAZON_AWS_BUCKET, path)
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
finally:
context.end_span()
def get_review_value(field, rec, query):
if "review_value" in field.keys() and get_value(field['review_value'], rec) is not None:
if field['type'] == 'bool':
return literal_eval(get_value(field['review_value'], rec))
else:
return get_value(field['review_value'], rec)
elif field['type'] == 'string':
return get_value(field['value'], rec)
elif field['type'] == 'bool':
return field['value']
elif field['type'] == "list" and "list_collection" in field.keys():
coord_list = MongoDbConn.find_one(field['list_collection'], query)[field['key']]
return get_coordinates(rec[field['value']], coord_list)
def document_data(doc_id, solution_id, entity_reqd=True, rules_reqd=True):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
filter_query = {"doc_id": doc_id}
projection = {"metadata": 1, "doc_id": 1, "confidence_score": 1, "elements": 1, "doc_state": 1, "root_id": 1,
"entity": 1, "_id": 0}
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, filter_query, projection=projection)
doc_type = get_doc_type(document["metadata"]["properties"]['extension'])
overall_doc_score = get_doc_confidence_score(document, doc_type)
document['document_confidence_score'] = overall_doc_score
template_id = document["metadata"]["template_info"]["id"]
template_type = get_template_type(template_id, solution_id)
document["doc_type"] = doc_type
if doc_type == "email":
document["attachments"] = add_email_info(document)
# elif doc_type == "excel":
# for page in document["pages"]:
# page["doc_html"] = get_html_data(page)
document["template_type"] = template_type
if "metadata" in document:
if "searchable_pdf" in document["metadata"].keys():
document["searchable_pdf"] = document["metadata"]["searchable_pdf"]
if "entity" in document and entity_reqd:
entity_data_orgnl = json.loads(document["entity"])
enrich_data = list(get_enrichments(entity_data_orgnl, "enrichments"))
filter_query["is_deleted"] = False
elements = get_all_elements(document["elements"], [])
review_data = dict(attributes_extracted=0, review_required=0, confidence=0)
review_data["entity_feedback"] = document["entity_feedback"] if "entity_feedback" in document else []
document["entity"] = format_entity_data(document["entity"], elements,
review_data, enrich_data, rules_reqd)
document['document_confidence_score'] = review_data["confidence"]
document["attributes_extracted"] = review_data["attributes_extracted"]
if rules_reqd:
document["review_required"] = 0
else:
document["review_required"] = review_data["review_required"]
document = remove_items(document, ["entity_feedback"])
document["elements"] = elements
else:
document.pop("entity", None)
review_state = get_review_state(entity_reqd, rules_reqd, doc_type, template_type)
data = {"data": document, "volume": MOUNT_PATH, "review_state": review_state}
return {"status": "success", "msg": "document data", "data": data}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {"status": "failure", "msg": str(e), "data": {}}
finally:
context.end_span()
def delete_template_element(solution_id, data):
try:
query = dict(solution_id=solution_id, template_id=data.pop("template_id"), is_deleted=False)
template = MongoDbConn.find_one(TEMPLATE_COLLECTION, query)
template = json.loads(template["template"]) if template else {}
id = data["id"]
template["elements"] = remove_element(template["elements"], id)
template["domain_object_mapping"].pop(id, None)
template["document_variables"].pop(id, None)
return post_save_template(solution_id, dict(document=template))
except Exception:
return dict(success=False, msg="failed to delete template", error=traceback.format_exc(), status="failure")
def page_group_review(request, doc_id):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
solution_id = common.get_solution_from_session(request)
if request.method == "GET":
query = {"doc_id": doc_id}
projection = {"doc_id": 1, "solution_id": 1, "pages": 1, "page_groups": 1, "metadata.properties": 1,
"_id": 0}
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, query, projection=projection)
if document is not None:
document["volume"] = MOUNT_PATH
return {"status": "success", "data": document}
else:
return {"status": "failure", "msg": "Failed to return document data"}
elif request.method == "POST":
payload = json.loads(request.body.decode())
query = {"doc_id": doc_id, "solution_id": solution_id}
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, query)
doc_groups = document["page_groups"]
feedback_list = get_groups_feedback(payload, doc_groups)
if feedback_list:
feedback_status = post_groups_feedback(feedback_list, doc_id, solution_id, document["root_id"])
else:
feedback_status = True
if feedback_status:
return process_complete_review(request, doc_id)
else:
return {'status': 'failure', 'msg': 'Error posting feedback'}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {'status': 'failure', 'msg': 'Internal error while submitting review', 'error': str(e)}
finally:
context.end_span()
def process_jobs():
try:
job = MongoDbConn.find_one(JOB_COLLECTION, {"is_complete": False})
print("Running " + job["name"] + " job.")
response = post_job(job["endpoint"], job["data"])
result = get_nested_value(response, job["key"])
if result and len(result) > 0:
for entity in result:
query = {"solution_id": job["solution_id"]}
query.update({a: entity[a] for a in job["unique_keys"]})
MongoDbConn.update(job['collection'], query, entity)
MongoDbConn.update(JOB_COLLECTION, {"_id": job["_id"]},
{"is_complete": True})
except Exception as e:
print(str(e))
def update_queue_extracted_feedback(document, doc_id, state):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
if not document:
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, {"doc_id": doc_id})
if "life_cycle" in document:
curr_state = check_current_status(document, state)
if curr_state and curr_state != "In Progress":
update_queue_status(document, state, "In Progress", update_reqd=True)
return {"status": "success", "msg": "Feedback submitted"}
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {"status": "failed", "msg": "Error updating queue status", "error": str(e)}
finally:
context.end_span()
def delete_solution(soln):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
solution_id = soln["solution_id"]
soln_exists = None
try:
query = {'solution_id': solution_id}
projection = {'_id': 0}
soln_exists = MongoDbConn.find_one(SOLUTION_COLLECTION,
query,
projection=projection)
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {
'status': 'failure',
'msg': 'Error occurred while deleting solution'
}
if soln_exists:
solution_name = soln_exists['solution_name']
remove_nifi_pipeline_config(solution_name)
SolutionService.delete_workflows_bpmn_queues(
solution_id, context)
soln_exists['is_deleted'] = True
soln_exists['updated_ts'] = datetime.utcnow().isoformat()
MongoDbConn.update(SOLUTION_COLLECTION, query, soln_exists)
status = {
'status': 'success',
'msg': solution_name + ' - solution has been deleted'
}
return status
else:
status = {
'status': 'failure',
'msg': 'Solution does not exists'
}
return status
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {
'status': 'failure',
'msg': 'Error occurred while deleting solution'
}
finally:
context.end_span()
def get_user_groups(self, queue_id, solution_id):
"""
This function will fetch the user_groups Ids present in the
specific queue and return the details of user_group ids as response
:param queue_id: Specific case queue id
:param solution_id: Session solution id
:return: details of user_group ids
"""
try:
query = {'id': queue_id, 'solution_id': solution_id}
projection = {'user_groups': 1, '_id': 0}
return MongoDbConn.find_one(WORKFLOW_QUEUE_COLLECTION,
query,
projection=projection)
except Exception as e:
self.context.log(message=str(e),
obj={'tb': traceback.format_exc()})
return dict()
def update_training_data(data):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
training_set = MongoDbConn.find_one(TRAINING_SET_COLLECTION,
{'_id': ObjectId(data['_id'])})
if training_set is not None:
MongoDbConn.update(TRAINING_SET_COLLECTION,
{'_id': ObjectId(data['_id'])}, {'description': data['description'],
'updated_ts': datetime.now()})
return True
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return False
finally:
context.end_span()
def insert_threshold(request, solution_id, workflow_id, task_id, threshold_id):
try:
if request.method == 'POST':
payload = json.loads(request.body.decode())
query = {
"solution_id": "developer_7f3970b4-8589-41a8-9030-d0065e407056"
}
wf_data_dict = MongoDbConn.find_one(WORKFLOW_COLLECTION, query)
for value, soln_id in zip(payload, wf_data_dict["case_object"]):
if (value == soln_id['variable_id']):
soln_id.update({"thresholds": payload[value]})
post_response = MongoDbConn.insert('thresholds_data',
payload)
return ({"response": "success"})
else:
return ({"response": "method not working out"})
except:
pass
def process_complete_review(request, doc_id):
context = tracer.get_context(request_id=str(uuid4()), log_level="ERROR")
context.start_span(component=__name__)
try:
solution_id = common.get_solution_from_session(request)
path = request.get_full_path()
if "text/" in path or "entity/" in path:
payload = json.loads(request.body.decode())
doc_id = payload["doc_id"]
if payload["feedback"]:
if "text/" in path:
feedback_status = process_text_feedback(request)
else:
feedback_status = process_entity_feedback(request)
if feedback_status["status"] != "success":
return {"status": "failure", "msg": "Failed to submit feedback"}
query = {"doc_id": doc_id, "solution_id": solution_id}
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, query)
data = dict(doc_id=doc_id,pipeline_name="manual_review",root_id=document["root_id"])
if 'completeReview/review/' in path:
data.update({"object_type": ["document", "domain", "recommendation"],"complete_review":True})
post_status = post(API_GATEWAY_POST_JOB_URI + PIPELINE["MANUAL_TRIGGER"],
{"solution_id": solution_id, "data": data})
if post_status["status"] != "success":
return {"status": "failure", "msg": "Error while posting review"}
state = ""
if "text/" in path:
state = "extracted"
elif "grouping/" in path:
state = "classified"
elif "entity/" in path:
state = "processed"
elif 'review/' in path:
state = 'reviewed'
update_queue_status(document, state, "Closed", update_reqd=True)
# context.end_span()
return {"status": "success", "msg": "Review completion Posted successfully"}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {"status": "failure", "msg": "Internal Error occured while posting review", "error": str(e)}
finally:
context.end_span()
def get_document_details(request, doc_id, page_no):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
solution_id = common.get_solution_from_session(request)
query = {"doc_id": doc_id, "page_no": int(page_no)}
projection = {"solution_id": 0, "updated_ts": 0, "created_ts": 0, "_id": 0, "doc_id": 0}
elements = MongoDbConn.find(DOC_ELEMENTS_COLLECTION, query, projection=projection)
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, {"doc_id": doc_id}, projection={"doc_id": 1, "entity": 1})
processed_rules = {}
if "entity" in document:
processed_rules = get_all_rules_processed(document["entity"])
mapping_data = get_doc_mapping_from_template(doc_id, solution_id)
element_list = []
for element in elements:
domain_mapping = get_domain_mapping(mapping_data, element_id=element["element_id"],
section_id=element["section_id"])
if element["type"] == "table":
table = dict()
if "headings" and "columns" in element:
table["table"], table["headings"] = construct_table_data(element["headings"], element["columns"],
domain_mapping)
element["tables"] = table
element = remove_items(element, ["headings", "columns"])
else:
element["domain_mapping"] = ""
if domain_mapping and isinstance(domain_mapping, dict) and "domain_mapping" in domain_mapping:
element["domain_mapping"] = domain_mapping["domain_mapping"]
if processed_rules:
element["rules"] = get_rules_info(element["domain_mapping"], processed_rules, solution_id,
element["text"])
if "score" not in element:
element["score"] = 0
element_list.append(element)
data = {"elements": element_list, "entity": {}}
return {"status": "success", "data": data}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {"status": "failure", "msg": "Error occured while processing", "error": str(e)}
finally:
context.end_span()