def ingest_file(request, collection, aws_bucket, aws_path): response = {"status":"failure"} if request.method == 'POST'and len(request.FILES) != 0: files_list = read_multiple_files(request) for file_key, file_value in files_list.items(): # Saving File to media folder. solution_id = get_solution_from_session(request) file_data = save_to_folder(solution_id, file_value[0],MOUNT_PATH,"documents","uploads",flag=True) if file_data["status"] == "success": # file_name = file_data["data"]["filename"] # filename = " ".join(file_name.split()).replace(" ","_") # uploaded_file_url = file_data['data']["file_path"] # # posting to Amazon S3 # resp = post_s3(str(filename), ROOT + uploaded_file_url, aws_bucket, aws_path) # # Formatting data for insert # data = create_data(None, file_data) # if resp['status'] == 'success': # result_id = MongoDbConn.insert(collection, data) # resp["document_id"] = str(result_id) # else: # response['msg'] = "Error while ingesting the files into S3" # return response #pipeline_name = PIPELINE_VARIABLES["INGEST_DOCUMENT"] #if solution_id == 'testcm_7ba1bb84-1362-434d-b596-0f01273c172e': pipeline_name = PIPELINE_VARIABLES["FILE_SOURCE"] payload = {"data": {"file_path" : file_data["data"]["relative_path"], "pipeline_name": pipeline_name, "request_type": SERVICE_REQUEST_TYPES["INGEST_DOCUMENT"]}, "solution_id":solution_id} resp = post(API_GATEWAY_POST_JOB_URI+PIPELINE["TRIGGER_PIPELINE"],payload) if resp['status'] != 'success': response["msg"] = resp["msg"] return response else: response['msg'] = "Error while saving the file" return response elif request.method == "POST": solution_id = get_solution_from_session(request) payload = json.loads(request.body.decode()) if "files" in payload and payload["files"]: for file in payload["files"]: sftp_data = get_mountpath_fromsftp(solution_id,file) payload = {"data": {"file_path" : sftp_data["relative_path"]}, "solution_id":solution_id} resp = post(API_GATEWAY_POST_JOB_URI+DOCUMENT_ENDPOINT["ingest_flow"],payload) if resp['status'] != 'success': response["msg"] = resp["msg"] return response else: response["msg"] = "No files attached to the request" return response response['status'] = "success" response['msg'] = "File(s) uploaded Successfully" return response
def upload_training_set(request): if request.method == 'POST' and request.FILES is not None and len(request.FILES)>0: files = request.FILES uploaded_file = files['file'] unique_folder = "/media/" + str(uuid4()) # Saving File to media folder. fs = FileSystemStorage(location=ROOT + unique_folder) filename = fs.save(uploaded_file.name, uploaded_file) uploaded_file_url = str(unique_folder + "/" + filename).replace("%20", " ") extn = str(str(filename).rsplit(".", 1)[1]) if extn == 'owl' or extn == "rdf": store_entity_definitions(str(os.getcwd()) + uploaded_file_url, get_solution_from_session(request)) # Adding file to bucket resp = post_s3(str(filename), ROOT + uploaded_file_url, AMAZON_AWS_BUCKET, AMAZON_AWS_KEY_PATH) # Sending file info to platform resp_api = post_to_ms(str(filename), resp['url'], extn, get_solution_from_session(request)) # Formatting data for insert data = create_data(dict(request.POST), filename, resp['url'], resp['key'], extn, get_solution_from_session(request)) data['created_ts'] = datetime.now() data['is_published'] = False if resp_api['status'] == 'success' and resp_api['status_code'] == 200: data['is_published'] = True services = get_file_contents('platform_services.json') if services is not None: platform_services = dict() for key in services.keys(): platform_services[key] = {'enabled': False} data['services'] = platform_services MongoDbConn.insert(TRAINING_SET_COLLECTION, data) # create_training_data_services(data) return {'status': 'success', 'msg': 'Resource uploaded to library'} elif request.method == 'POST': payload = json.loads(request.body.decode()) data = payload['data'] status = False if '_id' in data.keys(): status = update_training_data(data) if status: return {'status': 'success', 'status_code': 200, 'msg': 'Updated training set'} else: return {'status': 'failure', 'status_code': 500, 'msg': 'Failed in updated training set'} elif request.method == 'GET': delete_files() return training_set_get(TRAINING_SET_COLLECTION, dict(), get_solution_from_session(request)) elif request.method == 'DELETE': payload = json.loads(request.body.decode()) return training_set_delete(get_solution_from_session(request), payload)
def delete_s3_file(request): context = tracer.get_context(request_id=str(uuid4()), log_level="INFO") context.start_span(component=__name__) try: if request.method == "POST": solution_id = get_solution_from_session(request) payload = json.loads(request.body.decode()) file_path = payload["file_path"] files = StorageHandler.delete(AMAZON_AWS_BUCKET, file_path) return {"status": "success", "solution_id": solution_id, "status_code": STATUS_CODES["OK"]} else: return {"status": "failure", "error": "Request type not supported", "status_code": STATUS_CODES['METHOD_NOT_ALLOWED']} except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) tb = traceback.format_exc() return {"status": "failure", "error": e, "status_code": STATUS_CODES["INTERNAL_SERVER_ERROR"]} finally: context.end_span() # if __name__ == '__main__': # solution_id = "qafix_f04b26a1-241b-4aad-a91d-0be28375df73" # SFTPManager.init_solution(solution_id) # sftpm = SFTPManager(solution_id) # username = sftpm._get_username(solution_id) # print(username) # file_list = sftpm.upload_file(upload_file_path="/home/thiru/Downloads/cms_csf.pdf",server_file_path="/{0}/cms_csf.pdf".format(solution_id))
def process_tags(request): context = tracer.get_context(request_id=str(uuid4()), log_level="INFO") context.start_span(component=__name__) try: solution_id = common.get_solution_from_session(request) result = dict(status="failure") if request.method == "GET": data = dict(solution_id=solution_id) response = rm_rules.process(RULES_ENDPOINT["get_tags"],data) if response["status"]["success"]: tags_list = response["metadata"]["tags"] result.update({"status":"success","msg":"Tags retrieved successfully","data":tags_list}) else: result["msg"] = response["status"]["msg"] if request.method == "DELETE": payload = json.loads(request.body.decode()) data = payload data["solution_id"] = solution_id response = rm_rules.process(RULES_ENDPOINT["del_tag"], data) if response["status"]["success"]: result.update({"status": "success", "msg": "Tags deleted successfully"}) else: result["msg"] = response["status"]["msg"] return result # TODO raise specific exception except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return {"status":"failure","msg":"Internal Error occured","error":str(e)} finally: context.end_span()
def process_hierarchy(request): ''' :param request:wsgi request :return:class hierarchy details ''' context = tracer.get_context(request_id=str(uuid4()), log_level="INFO") context.start_span(component=__name__) try: solution_id = get_solution_from_session(request) result = dict(status="failure") if request.method == "POST": payload = json.loads(request.body.decode()) data = {"solution_id": solution_id, "tag": payload['tag']} response = rm_rules.process(RULES_ENDPOINT["get_hierarchy"], data) if response["status"]["success"]: tag_info = response["metadata"] formatted_data = rule_info(tag_info) result.update({"status": "success", "msg": "data formatted successfully", "data": formatted_data}) else: result["msg"] = response["status"]["msg"] return result # TODO raise specific exception except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return {"status": "failure", "msg": "Internal Error occurred", "error": str(e)} finally: context.end_span()
def tables_services(request): solution_id = get_solution_from_session(request) if request.method == "POST": payload = json.loads(request.body.decode()) return tables_save(TABLES_CONFIG, payload, solution_id) elif request.method == "GET": return tables_get(TABLES_CONFIG, solution_id)
def ner_service_label(self, request): """ This function will fetch the list of all the ner labels and return the dictionary format response :param request: Http Request :return: Dict response """ try: solution_id = get_solution_from_session(request) path = request.get_full_path() if request.method == 'GET': if 'nerservice/' in path: return self.get_ner_labels(solution_id) if request.method == 'POST': try: payload = json.loads(request.body.decode()) except: payload = request.POST if 'nerservice/' in path: return self.create_ner_label(solution_id, payload) except Exception as e: self.context.log(message=str(e), obj={"tb": traceback.format_exc()}) return {'status': 'failure', 'msg': 'Internal error occurred', 'status_code': STATUS_CODES['INTERNAL_SERVER_ERROR'], 'error': str(e)} finally: self.context.end_span()
def process_rules(request): context = tracer.get_context(request_id=str(uuid4()), log_level="INFO") context.start_span(component=__name__) try: solution_id = common.get_solution_from_session(request) if request.method == "GET": return get_rule(solution_id) elif request.method == "POST": data = json.loads(request.body.decode()) if "rule_id" in data: msg = "Rule Updated Successfully" else: msg = "Rule Added Successfully" response = update_rule(solution_id,data) if response["status"] == "success": return {"status": "success", "msg":msg, "rule_id" : response["rule_id"]} else: return response elif request.method == "DELETE": data = json.loads(request.body.decode()) response = delete_rule(solution_id,data["rule_id"]) if response["status"] == "success": return {"status": "success", "msg":"Rule Deleted Successfully"} else: return response # TODO raise specific exception except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return {"status": "failure", "msg": str(e)} finally: context.end_span()
def process_learning_datasets(request, path=None): context = tracer.get_context(request_id=str(uuid4()), log_level="ERROR") context.start_span(component=__name__) try: solution_id = get_solution_from_session(request) if request.method == "GET": if path is None: return get_all_datasets(solution_id, None) else: if path.startswith("/"): path = path.split('/', 1)[1] file_path = os.path.join(MOUNT_PATH, path) return download_file_from_efs(file_path, solution_id) elif request.method == "POST" and len(request.FILES) != 0: payload = request.POST return upload_dataset(request.FILES['file'], solution_id, payload) elif request.method == "POST": payload = json.loads(request.body.decode()) path = request.get_full_path() if 'dataset/list/' in path: return get_all_datasets(solution_id, payload=payload) elif "files" in payload: return upload_dataset(None, solution_id, payload) else: return update_dataset(solution_id, payload) except Exception as e: return { "status": "failure", "msg": "Internal Error occurred", "error": str(e) }
def get_solution_id(request): context = tracer.get_context(request_id=str(uuid4()), log_level="INFO") context.start_span(component=__name__) try: solution_id = common.get_solution_from_session(request) solution_name = "" query = {'solution_id': solution_id} projection = {'_id': 0} soln_exists = MongoDbConn.find_one(SOLUTION_COLLECTION, query, projection=projection) if soln_exists: solution_name = soln_exists['solution_name'] return { "solution_id": solution_id, 'solution_name': solution_name, 'case_management_url': CASE_MANAGEMENT_SERVICE_URL, "status": "success" } # TODO raise specific exception except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return {"status": "failure", "msg": str(e)} finally: context.end_span()
def process_custom_rules(request,type): context = tracer.get_context(request_id=str(uuid4()), log_level="INFO") context.start_span(component=__name__) try: solution_id = common.get_solution_from_session(request) if request.method == "GET": return(get_custom_rules(solution_id,type)) elif request.method == "POST": data = json.loads(request.body.decode()) response = update_custom_rule(solution_id,data) if response["status"] == "success": return {"status": "success", "msg":"Custom rule added Successfully"} else: return response elif request.method == "DELETE": data = json.loads(request.body.decode()) response = delete_custom_rule(solution_id, data) if response["status"] == "success": return {"status": "success", "msg":"Custom rule deleted Successfully"} else: return response except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return {"status": "failure", "msg": str(e)} finally: context.end_span()
def pipeline_ingest(request): solution_id = get_solution_from_session(request) return JsonResponse( ingest_file(request, UPLOAD_COLLECTION, aws_path=solution_id + "/ingest/", aws_bucket=AMAZON_AWS_BUCKET))
def email_service(request): solution_id = get_solution_from_session(request) if request.method == "GET": return get_email_details(solution_id) elif request.method == 'POST': payload = json.loads(request.body.decode()) return update_email_details(solution_id,payload)
def fetch_sections_data(request, doc_id): """ :param request: :param doc_id: for which we need to fetch sections :return: Response in Json format which consists the all sections data for the given doc_id """ context = tracer.get_context(request_id=str(uuid4()), log_level="INFO") context.start_span(component=__name__) try: if doc_id != "": solution_id = common.get_solution_from_session(request) document_info = document_data(doc_id, solution_id, False) data = document_info["data"]["data"] review_state = document_info["data"]["review_state"] counts = {"extracted": 0, "reviewed": 0} data["elements"] = process_elements(data["elements"], counts) data['need_review_count'] = counts["reviewed"] data['attributes_extracted'] = counts["extracted"] return {"status": "success", "data": data, "volume": MOUNT_PATH, "review_state": review_state, "msg": "successfully returned document sections data"} else: return {"status": "failure", "msg": "Failed to return document sections data"} # TODO raise specific exception except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return {'status': 'failure', 'msg': 'Internal error while submitting section data', 'error': str(e)} finally: context.end_span()
def get_previous_run_details(request): """ :param request: Http request :return: result Json """ context = tracer.get_context(request_id=str(uuid4()), log_level="ERROR") context.start_span(component=__name__) try: result = {"status": "failure"} if request.method == "POST": try: payload = json.loads(request.body.decode()) except: payload = request.POST solution_id = get_solution_from_session(request) data = { "solution_id": solution_id, "data": { 'model': { 'model_id': payload['model_id'], 'version_id': payload['version_id'] } } } runs_result = post_job(LEARNING_CONFIG['get_prev_run'], data) if 'job_id' in runs_result: result["job_id"] = runs_result["job_id"] if not is_request_timeout(runs_result): status, msg = get_response(runs_result) if status: result["status"] = "success" runs_result = get_nested_value( runs_result, "result.result.metadata.run_data") runs_result.sort(key=lambda f: f['update_ts'], reverse=True) result['data'] = runs_result result['total_runs'] = len(runs_result) else: if 'message' in msg and 'error_message' in msg['message']: result["error"] = msg['message']['error_message'] else: result["error"] = 'Error' result[ "msg"] = "Error in retrieving the previous runs information" else: result["msg"] = "Request timed out" return result else: result["msg"] = 'POST api is expected' return result except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return { "status": "failure", "msg": "Binaries list not available.", "error": str(e) } finally: context.end_span()
def get_insights(request, doc_id=""): solution_id = get_solution_from_session(request) if request.method == "GET": return JsonResponse(document_data(doc_id, solution_id)) elif request.method == "POST": payload = json.loads(request.body.decode()) filter_obj = payload[ 'filter_obj'] if payload is not None and 'filter_obj' in payload else None return JsonResponse(documents_data(solution_id, filter_obj))
def download_training_data(request, doc_id=""): solution_id = get_solution_from_session(request) path = request.get_full_path() if "download/efs/" in path: asset_path = path.rsplit("download/efs/", 1)[1] # payload = json.loads(request.body.decode()) return download_file_from_efs(asset_path, solution_id) else: return invoke_files_download(doc_id)
def get_review(request, status, doc_type, file_flow_id, direction, query=""): if request.method == "GET": return JsonResponse(get_review_data(get_solution_from_session(request), file_id=file_flow_id, status=status, doc_type=doc_type, direction=direction, query_string=query), safe=False)
def insight_request(request, type): solution_id = get_solution_from_session(request) if request.method == 'GET': result = InsightService(type, solution_id).process(request.method, None) return JsonResponse(result, safe=False) else: payload = json.loads(request.body.decode()) result = InsightService(type, solution_id).process(request.method, payload) return JsonResponse(result, safe=False)
def threshold_data(request): """ :param request: :return: response in json format """ solution_id = get_solution_from_session(request) if request.method == "POST": payload = json.loads(request.body.decode()) return JsonResponse(save_threshold_data(solution_id, payload))
def custom_functions(self, request, function_name=None): """ This function will fetch the list of all the custom functions and return the dictionary format response :param request: Http Request :param function_name: Specific function name :return: Dict response """ try: solution_id = get_solution_from_session(request) path = request.get_full_path() if request.method == 'GET': if 'customfunctions/' in path: return self.get_custom_functions_detail( solution_id, function_name) if request.method == 'POST': try: payload = json.loads(request.body.decode()) except: payload = request.POST if 'enable/' in path: return self.enable_custom_function(solution_id, payload) elif 'open/' in path: return self.open_custom_function(solution_id, payload) elif 'save/' in path: return self.save_custom_function(solution_id, payload) elif 'create/' in path: return self.create_custom_function(solution_id, payload) elif 'publish/' in path: return self.publish_custom_function(solution_id, payload) elif 'test/' in path: return self.test_custom_function(solution_id, payload) elif 'logs/' in path: return self.custom_function_logs(solution_id, payload) if 'enable_version/' in path: return self.enable_version_custom_function( solution_id, payload) return self.get_custom_functions_list(solution_id, payload) if request.method == 'DELETE': try: payload = json.loads(request.body.decode()) except: payload = request.POST if 'customfunctions/' in path: return self.delete_custom_function(solution_id, payload) except Exception as e: self.context.log(message=str(e), obj={"tb": traceback.format_exc()}) return { 'status': 'failure', 'msg': 'Internal error occurred', 'status_code': STATUS_CODES['INTERNAL_SERVER_ERROR'], 'error': str(e) } finally: self.context.end_span()
def get_binaries(request): """ :param request: API request :return: json response """ context = tracer.get_context(request_id=str(uuid4()), log_level="ERROR") context.start_span(component=__name__) try: result = {"status": "failure"} if request.method == "POST": try: payload = json.loads(request.body.decode()) except: payload = request.POST filter_obj = None if 'filter_obj' in payload: filter_obj = payload['filter_obj'] solution_id = get_solution_from_session(request) data = {"solution_id": solution_id, "data": {}, 'metadata': {}} bin_result = post_job(LEARNING_CONFIG['get_binaries'], data) if 'job_id' in bin_result: result["job_id"] = bin_result["job_id"] if not is_request_timeout(bin_result): status, msg = get_response(bin_result) if status: result["status"] = "success" binaries = get_nested_value( bin_result, "result.result.metadata.binaries") # for ele in binaries: # ele['file_path'] = MOUNT_PATH + ele['file_path'] if filter_obj: result["data"], total_binaries = implement_pagination( binaries, filter_obj, 'created_ts') else: result['data'] = binaries total_binaries = len(binaries) result['total_binaries'] = total_binaries else: result["error"] = msg result[ "msg"] = "Error in retrieving the binaries information" else: result["msg"] = "Request timed out" return result else: result["msg"] = 'POST api is expected' return result except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return { "status": "failure", "msg": "Binaries list not available.", "error": str(e) } finally: context.end_span()
def review_list(request, status, doc_type, query=""): try: if request.method == "GET": return JsonResponse(generate_review_list( get_solution_from_session(request), status, doc_type, query), safe=False) except Exception as e: print(str(e)) import traceback traceback.print_exc()
def feedback_service(request): if request.method == "POST": solution_id = get_solution_from_session(request) payload = json.loads(request.body.decode()) response = feedback(payload, solution_id) if response["status"] == "success": return JsonResponse( update_queue_extracted_feedback(None, payload["doc_id"])) else: return JsonResponse(response)
def process_resources(request): payload = json.loads(request.body.decode()) solution_id = common.get_solution_from_session(request) if request.method in ["POST","DELETE"]: payload["solution_id"] = solution_id if request.method == "POST": type = "ADD" else: type = "DELETE" return save_data_changes(payload,type)
def process_file_sources(request, solution_id=None, source_id=None): ss_obj = SourcesServices() if not solution_id: solution_id = get_solution_from_session(request) source_type = 'manualupload' return JsonResponse(ss_obj.process_source_request(request, source_type, solution_id=solution_id, source_id=source_id), safe=False)
def get_attributes_thresholds(request, template_id): ''' :param request: get request from the ui end with solution_id :param template_id: template_id :return:list of all mapping thresholds ''' solution_id = get_solution_from_session(request) context = tracer.get_context(request_id=str(uuid4()), log_level="ERROR") context.start_span(component=__name__) solution_id = get_solution_from_session(request) try: if template_id: query = {"solution_id": solution_id, "template_id": template_id} template = MongoDbConn.find_one(TEMPLATE_COLLECTION, query) if template["template_type"] not in UNKNOWN_TYPES: map_list = mapping_list_of_thresholds(query) elif template["template_type"] in UNKNOWN_TYPES: map_list = mapping_list_of_thresholds(query) try: config = ENTITY_CONFIG["default"] all_domain_mappings = entity_get(solution_id, config, "domainmapping") map_list = threshold_mapping_list(all_domain_mappings, map_list) except Exception as e: return str(e) return { "status": "success", "msg": "objects formed Successfully", "data": map_list } # TODO raise specific exception except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) return { "status": "failed", "msg": "error in getting objects", "error": str(e) } finally: context.end_span()
def template_train_trigger(request): job_id = None context = tracer.get_context(request_id=str(uuid4()), log_level="INFO") context.start_span(component=__name__) try: solution_id = get_solution_from_session(request) payload = json.loads(request.body.decode()) # assertions for essentials assert "template_id" in payload assert "documents" in payload assert len(payload["documents"]) > 0 data = {"template_id": payload["template_id"], "documents": payload["documents"]} # payload to post payload = {"solution_id": solution_id, "data": data} response = post_job(TEMPLATE_TRAIN_TRIGGER_ENDPOINT, payload) if 'job_id' in response: job_id = response["job_id"] if not is_request_timeout(response): status, result = get_response(response) if status: return {"status": "success", "msg": "triggered training successfully", 'job_id': job_id} else: return {"status": "failure", "msg": "failed to trigger training", 'error': result, 'job_id':job_id} else: return {"status": "failure", "msg": "request timeout", 'error': response, 'job_id':job_id} # TODO raise specific exception except AssertionError as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) tb = traceback.format_exc() if job_id: return {"status": "failure", "msg": "Assertion failed, " + str(e), "traceback": str(tb), 'job_id':job_id} else: return {"status": "failure", "msg": "Assertion failed, " + str(e), "traceback": str(tb)} # TODO raise specific exception except Exception as e: context.log(message=str(e), obj={"tb": traceback.format_exc()}) tb = traceback.format_exc() if job_id: return {"status": "failure", "msg": "unknown error, " + str(e), "traceback": str(tb), 'job_id':job_id} else: return {"status": "failure", "msg": "unknown error, " + str(e), "traceback": str(tb)} finally: context.end_span()
def change_doc_state(request): solution_id = get_solution_from_session(request) try: payload = json.loads(request.body.decode()) except: payload = request.POST if payload["doc_state"] != "processed": doc_id = payload["doc_id"] query = dict(doc_id=doc_id, solution_id=solution_id) document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, query) if "entity_feedback" in document and document["entity_feedback"]: MongoDbConn.update(DOCUMENTS_COLLECTION, query, {"entity_feedback": None}) return post_doc_state_change(payload,solution_id, reset_cycle=True)
def get_template_request(request, template_id=""): if request.method in ["GET", "POST"]: solution_id = get_solution_from_session(request) if template_id in ["known", "unknown", "allpublished"]: payload = json.loads( request.body.decode()) if request.method == "POST" else {} return JsonResponse( get_template(solution_id, template_type=template_id, payload=payload)) else: return JsonResponse( get_template(solution_id, template_id=template_id))