def delete_source(self, solution_id, source_id):
"""
This function will delete the particular source
and return the dictionary as response
:param solution_id: Session Solution Id
:param source_id: Sources Id
:return: Dictionary as response
"""
try:
source_rec, query, status_code = self.get_source(
solution_id, source_id)
if not source_rec or \
status_code == STATUS_CODES['PRECONDITION_FAILED']:
return {
'status': 'failure',
'status_code': STATUS_CODES['NO_CONTENT'],
'msg': 'Source/s not available to delete.'
}
source_rec['is_deleted'] = True
source_rec['updated_ts'] = datetime.utcnow().isoformat()
MongoDbConn.update(SOURCES_COLLECTION, query, source_rec)
return {
'status': 'success',
'status_code': STATUS_CODES['OK'],
'msg': 'Source/s has been deleted successfully.'
}
except Exception as e:
self.context.log(message=str(e),
obj={"tb": traceback.format_exc()})
return {
'status': 'failure',
'status_code': STATUS_CODES['INTERNAL_SERVER_ERROR'],
'msg': 'Failed to delete the source/s.'
}
def update_email_details(solution_id,payload):
job_id = None
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
data = dict(solution_id = solution_id, data=dict(source_type="email",service_name="source",solution_id=solution_id,
configuration=payload))
response = post_job(CONFIGURE_EMAIL,data)
if 'job_id' in response:
job_id = response["job_id"]
if not is_request_timeout(response):
status, result = get_response(response)
if status:
MongoDbConn.update(SOURCE_COLLECTION, where_clause=dict(solution_id=solution_id,source_type="email"), query=data["data"])
temp_result = create_email_template(solution_id,payload)
if temp_result["status"] == "success":
return {"status": "success", "msg": "Updated email details.",
'job_id': job_id}
else:
return temp_result
else:
return {'status': 'failure', 'msg': 'Error in updating emails',
'error': result, 'job_id':job_id}
else:
return {'status': 'failure', 'msg': 'Request timeout',
"error": response, 'job_id':job_id}
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
if job_id:
return {"status": "failure", "msg": "Failed to update details.",
'job_id':job_id}
else:
return {"status": "failure", "msg": "Failed to update details."}
finally:
context.end_span()
def update_solutions(req_data):
solns = None
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
query = {
'is_deleted': False,
'solution_id': req_data["solution_id"]
}
projection = {'_id': 0}
solns = MongoDbConn.find_one(SOLUTION_COLLECTION,
query,
projection=projection)
# solns = Solution.objects.get(deleted=False,solution_id=req_data["solution_id"])
if solns:
solns['hocr_type'] = req_data["hocr_type"]
# Solution.objects.update(solns)
# solns.save()
MongoDbConn.update(SOLUTION_COLLECTION, query, solns)
return {'status': 'success', 'msg': 'updated Solutions list'}
else:
return {'status': 'success', 'msg': 'No solutions exists'}
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {'status': 'failure', 'msg': 'exception occerd : ' + str(e)}
finally:
context.end_span()
def store_entity_definitions(file_path, solution_id):
resp = dict()
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
data = get_entity_definitions_from_ontology(file_path)
for itm in data:
itm['solution_id'] = solution_id
if "entity_name" in itm.keys():
MongoDbConn.update(ENTITY_DEFN_COLL,
where_clause={
"entity_name": itm["entity_name"],
"solution_id": itm["solution_id"]
},
query=itm)
resp['status'] = 'success'
resp['msg'] = 'Entity definitions successfully updated/created'
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
resp['status'] = 'failure'
resp['msg'] = "Failed to update Entity Definitions " + str(e)
context.end_span()
return resp
def test_get_template():
query = dict(solution_id=test_solution,
is_deleted=False,
template_id=test_template_obj["doc_id"])
MongoDbConn.update(TEMPLATE_COLLECTION, query,
{"template": json.dumps(test_template_obj)})
response = get_template(test_solution)
assert len(response["data"]) == 1
def update_queue_status(data, state, status, update_reqd=False):
if "life_cycle" in data and state in data["life_cycle"]:
update_data = dict()
data["life_cycle"][state]["status"] = status
if status == "Closed" and state not in ["processed", "reviewed"]:
data["life_cycle"][state]["closed_ts"] = datetime.now()
update_data["doc_state"] = "processing"
update_data["life_cycle"] = data["life_cycle"]
if update_reqd:
MongoDbConn.update(DOCUMENTS_COLLECTION, {"doc_id": data["doc_id"]}, update_data)
return data
def update_s3_bucket(request):
if request.method == 'POST':
payload = json.loads(request.body.decode())
try:
MongoDbConn.update(CONFIG_COLLECTION, where_clause={},
query={"s3_claims_bucket": payload["s3_claims_bucket"]})
resp = {"status": "success", "msg": "updated bucket."}
except:
resp = {"status": "failure", "msg": "failed to update bucket."}
return resp
elif request.method == "GET":
return get_s3_bucket()
def change_doc_state(request):
solution_id = get_solution_from_session(request)
try:
payload = json.loads(request.body.decode())
except:
payload = request.POST
if payload["doc_state"] != "processed":
doc_id = payload["doc_id"]
query = dict(doc_id=doc_id, solution_id=solution_id)
document = MongoDbConn.find_one(DOCUMENTS_COLLECTION, query)
if "entity_feedback" in document and document["entity_feedback"]:
MongoDbConn.update(DOCUMENTS_COLLECTION, query, {"entity_feedback": None})
return post_doc_state_change(payload,solution_id, reset_cycle=True)
def process_action_delete(result_id):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
query = {'is_deleted': True, 'deleted_ts': datetime.now()}
MongoDbConn.update(TRAINING_SET_MODELS_COLLECTION,
where_clause={"_id": ObjectId(result_id)},
query=query)
return {'status': 'success', 'msg': 'Model has been deleted'}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {"status": "failure", "msg": "failed to delete the model"}
finally:
context.end_span()
def update_intent_review(request):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
payload = json.loads(request.body.decode())
doc_id = payload["doc_id"]
MongoDbConn.update(DOCUMENTS_COLLECTION, {"doc_id": doc_id},
{"elements_updated": payload["elements"]})
return {"status": "success", "msg": "Intent updated successfully"}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {"status": "failure", "msg": str(e)}
finally:
context.end_span()
def process_jobs():
try:
job = MongoDbConn.find_one(JOB_COLLECTION, {"is_complete": False})
print("Running " + job["name"] + " job.")
response = post_job(job["endpoint"], job["data"])
result = get_nested_value(response, job["key"])
if result and len(result) > 0:
for entity in result:
query = {"solution_id": job["solution_id"]}
query.update({a: entity[a] for a in job["unique_keys"]})
MongoDbConn.update(job['collection'], query, entity)
MongoDbConn.update(JOB_COLLECTION, {"_id": job["_id"]},
{"is_complete": True})
except Exception as e:
print(str(e))
def update_sources(self, payload, solution_id, source_id):
"""
This function will save the sources in the MongoDB
and return the dictionary as response
:param payload: Request payload
:param solution_id: session Solution Id
:param source_id: Id of the Source which needs to be updated
:return: dictionary as response
"""
try:
source_rec, query, status_code = self.get_source(
solution_id, source_id)
if not source_rec or \
status_code == STATUS_CODES['PRECONDITION_FAILED']:
return {
'status': 'failure',
'status_code': STATUS_CODES['NO_CONTENT'],
'msg': 'Source/s not available to update.'
}
payload['updated_ts'] = datetime.utcnow().isoformat()
if 'triggers' in payload:
triggers = payload['triggers']
for trigger in triggers:
trigger['updated_ts'] = datetime.utcnow().isoformat()
payload['triggers'] = triggers
if 'schedules' in payload:
schedules = payload['schedules']
for schedule in schedules:
schedule['updated_ts'] = datetime.utcnow().isoformat()
payload['schedules'] = schedules
MongoDbConn.update(SOURCES_COLLECTION, query, payload)
return {
'status':
'success',
'status_code':
STATUS_CODES['OK'],
'msg':
'Source ' + payload['name'] +
' updated successfully in this solution.'
}
except Exception as e:
self.context.log(message=str(e),
obj={"tb": traceback.format_exc()})
return {
'status': 'failure',
'status_code': STATUS_CODES['INTERNAL_SERVER_ERROR'],
'msg': 'Failed to save the source/s information.'
}
def process_action_retrain(solution_id, payload, model, result_id):
file_name = model['name'] + '_' + str(len(model['model_ref']) +
1) + '.json'
with open(file_name, 'w') as outfile:
json.dump(payload['data'], outfile)
s3_resp = post_s3(str(file_name), ROOT + '/' + str(file_name),
AMAZON_AWS_BUCKET, AMAZON_AWS_KEY_PATH)
if s3_resp['status'] == 'success':
os.remove(file_name)
uri = TRAINING_SET_TRAIN_MODEL_URI
if model['type'] == 'action_classifier':
uri = TRAINING_SET_ACTION_CLASSIFIER_URI
response = post_train_model_job(solution_id, model, s3_resp['key'],
uri)
if not is_request_timeout(response):
if is_message_published(response):
model['model_ref'].append({
'bucket_name':
AMAZON_AWS_BUCKET,
'key_name':
AMAZON_AWS_KEY_PATH + file_name
})
query = {
'is_enabled': True,
'updated_ts': datetime.now(),
'model_ref': model['model_ref']
}
MongoDbConn.update(TRAINING_SET_MODELS_COLLECTION,
where_clause={"_id": ObjectId(result_id)},
query=query)
return {'status': 'success', 'msg': 'Retrain model completed'}
else:
return {
'status': 'failure',
'msg': 'Error in service while publishing retrained model'
}
else:
return {
'status': 'failure',
'msg': 'Timeout Error while processing retrained model'
}
else:
return {
'status': 'failure',
'msg': 'Error in uploading retrained model to s3'
}
def delete_solution(soln):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
solution_id = soln["solution_id"]
soln_exists = None
try:
query = {'solution_id': solution_id}
projection = {'_id': 0}
soln_exists = MongoDbConn.find_one(SOLUTION_COLLECTION,
query,
projection=projection)
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {
'status': 'failure',
'msg': 'Error occurred while deleting solution'
}
if soln_exists:
solution_name = soln_exists['solution_name']
remove_nifi_pipeline_config(solution_name)
SolutionService.delete_workflows_bpmn_queues(
solution_id, context)
soln_exists['is_deleted'] = True
soln_exists['updated_ts'] = datetime.utcnow().isoformat()
MongoDbConn.update(SOLUTION_COLLECTION, query, soln_exists)
status = {
'status': 'success',
'msg': solution_name + ' - solution has been deleted'
}
return status
else:
status = {
'status': 'failure',
'msg': 'Solution does not exists'
}
return status
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {
'status': 'failure',
'msg': 'Error occurred while deleting solution'
}
finally:
context.end_span()
def update_training_data(data):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
training_set = MongoDbConn.find_one(TRAINING_SET_COLLECTION,
{'_id': ObjectId(data['_id'])})
if training_set is not None:
MongoDbConn.update(TRAINING_SET_COLLECTION,
{'_id': ObjectId(data['_id'])}, {'description': data['description'],
'updated_ts': datetime.now()})
return True
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return False
finally:
context.end_span()
def process_action_enable(solution_id, result_id, model):
job_data = dict()
data = dict()
job_data['solution_id'] = solution_id
data['model_name'] = model['name']
data['model_type'] = model['type']
job_data['data'] = data
response = post_job(TRAINING_SET_TRAIN_MODEL_STATUS_URI, job_data)
is_timeout = True if response is not None and 'msg' in response.keys(
) and 'Timeout' == response['msg'] else False
if not is_timeout:
result = response['result'] if response is not None and 'result' in response.keys() \
else None
if result is not None and 'status' in result.keys(
) and result['status']['success']:
query = {'is_enabled': True, 'updated_ts': datetime.now()}
disable_query = {'is_enabled': False, 'updated_ts': datetime.now()}
MongoDbConn.update(TRAINING_SET_MODELS_COLLECTION,
where_clause={
"solution_id": solution_id,
'type': model['type']
},
query=disable_query)
MongoDbConn.update(TRAINING_SET_MODELS_COLLECTION,
where_clause={"_id": ObjectId(result_id)},
query=query)
return {
'status': 'success',
'msg': 'Updated model status to enabled'
}
else:
return {
'status': 'failure',
'msg': 'Error from service while updating the model status'
}
else:
return {
'status': 'failure',
'msg': 'Timeout error while updating the model status'
}
def update_rule(solution_id,rule):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
rules = RuleManager()
rule["solution_id"] = solution_id
result = rules.process("saveRule",rule)
status = result["status"]
if status["success"]:
rule_id = get_nested_value(result,"metadata.rule_id")
where_clause = dict(solution_id=solution_id, rule_id=rule_id)
MongoDbConn.update(RULES_COLLECTION,where_clause,rule)
response = {"status":"success","rule_id":rule_id}
else:
response = {"status":"failure","msg":status["msg"]}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
response = {"status": "failure","msg":str(e)}
context.end_span()
return response
def post_review_data(payload, flow_file_id):
resp = dict()
resp['status'] = "failure"
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
# Inserting payload in feedback
for record in payload:
if "rec_id" in record.keys():
MongoDbConn.update(CLAIMS_COLLECTION, where_clause={"_id": ObjectId(str(record['rec_id']))},
query={"status": "reviewed"})
if "is_update" in record.keys() or "is_accept" in record.keys():
if record['is_update'] or record['is_accept']:
record_id = record['id']
record.pop("id")
MongoDbConn.update(CLAIMS_FIELDS_COLLECTION, where_clause={"_id": ObjectId(record_id)},
query={"reviewed": record})
if record['is_update']:
record['flow_file_id'] = flow_file_id
record['is_processed'] = False
record_ins = deepcopy(record)
[record_ins.pop(key) for key in ["is_update", "is_accept", "is_nlp_enabled"]]
MongoDbConn.update(FEEDBACK_COLLECTION, where_clause={"_id": ObjectId(record_id)},
query=record_ins)
if "is_nlp_enabled" in record.keys() and record["is_nlp_enabled"]:
data = {"solution_id": SOLUTION_ID, "data": {"text": record["value"]}}
result = http.post_job(NLP_CONFIG['EXTRACT_INTENT'], data)
if 'job_id' in result:
resp['status'] = result["job_id"]
if result is not None:
result_data = http.get_nested_value(result, "result.result.metadata")
MongoDbConn.update(CLAIMS_FIELDS_COLLECTION, {"_id": ObjectId(record_id)},
{"intent": result_data})
resp['status'] = "success"
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
resp['status'] = "failure"
context.end_span()
return resp
def process_action_disable(result_id):
query = {'is_enabled': False, 'updated_ts': datetime.now()}
MongoDbConn.update(TRAINING_SET_MODELS_COLLECTION,
where_clause={"_id": ObjectId(result_id)},
query=query)
return {'status': 'success', 'msg': 'Updated model status to disabled'}
def update_insight_templates(self, insight_template):
filter_query = {'solution_id': self.solution_id, 'template_key': insight_template['template_key']}
update_query = {'template_value': insight_template['template_value'], 'updated_ts': datetime.now()}
MongoDbConn.update(self.collection_name, filter_query, update_query)
def update_case_queue(var):
MongoDbConn.update(CASE_QUEUE_COLLECTION, var, {"is_deleted": True})
return True
def custom_threshold(request,
solution_id=None,
workflow_id=None,
task_id=None):
"""this function will handle all the GET, POST and PUT methods
from the API call to Update Threshold values"""
context = tracer.get_context(request_id=str(uuid4()), log_level="ERROR")
context.start_span(component=__name__)
try:
if request.method == 'GET':
query = {
'task_id': task_id,
'solution_id': solution_id,
'workflow_id': workflow_id,
'is_deleted': False
}
projection = {'_id': 0}
thresholds = MongoDbConn.find_one(THRESHOLD_COLLECTION,
query,
projection=projection)
if not thresholds:
thresholds = {}
return {
"status": "success",
"data": thresholds,
'status_code': STATUS_CODES['OK'],
'msg': 'Threshold fetched successfully.'
}
elif request.method == 'POST':
try:
payload = json.loads(request.body.decode())
except:
payload = request.POST
query = {
'task_id': task_id,
'solution_id': solution_id,
'workflow_id': workflow_id,
'is_deleted': False
}
projection = {'_id': 0}
thresholds = MongoDbConn.find_one(THRESHOLD_COLLECTION,
query,
projection=projection)
if thresholds:
return {
'status': 'failure',
'status_code': STATUS_CODES['FOUND'],
'msg':
'threshold already available for this configuration.'
}
data_dict = dict()
data_dict.update({
'created_ts': datetime.utcnow().isoformat(),
'updated_ts': datetime.utcnow().isoformat(),
'is_deleted': False,
'solution_id': solution_id,
'workflow_id': workflow_id,
'task_id': task_id,
'thresholds': payload['threshold'],
'applicable': payload['applicable']
})
try:
MongoDbConn.insert(THRESHOLD_COLLECTION, data_dict)
return {
"status": "success",
'status_code': STATUS_CODES['OK'],
'msg': 'Threshold saved successfully.'
}
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {
"status": "Failure",
"msg": "not able to save threshold",
'status_code': STATUS_CODES['FORBIDDEN']
}
elif request.method == 'PUT':
try:
payload = json.loads(request.body.decode())
except:
payload = request.POST
payload.update({'updated_ts': datetime.utcnow().isoformat()})
query = {
'task_id': task_id,
'solution_id': solution_id,
'workflow_id': workflow_id,
'is_deleted': False
}
projection = {'_id': 0}
thresholds = MongoDbConn.find_one(THRESHOLD_COLLECTION,
query,
projection=projection)
if not thresholds:
return {
'status': 'failure',
'status_code': STATUS_CODES['NOT_FOUND'],
'msg':
'threshold record not available for this configuration.'
}
thresholds['thresholds'] = payload['threshold']
thresholds['applicable'] = payload['applicable']
thresholds.update({'updated_ts': datetime.utcnow().isoformat()})
MongoDbConn.update(THRESHOLD_COLLECTION, query, thresholds)
return {
"status": "success",
'status_code': STATUS_CODES['OK'],
'msg': 'Threshold updated successfully.'
}
elif request.method == 'DELETE':
query = {
'task_id': task_id,
'solution_id': solution_id,
'workflow_id': workflow_id
}
projection = {'_id': 0}
th_recs = MongoDbConn.find_one(THRESHOLD_COLLECTION,
query,
projection=projection)
if th_recs:
th_recs['is_deleted'] = True
th_recs['updated_ts'] = datetime.utcnow().isoformat()
MongoDbConn.update(THRESHOLD_COLLECTION, query, th_recs)
return {
"status": "success",
'status_code': STATUS_CODES['OK'],
'msg': 'Threshold deleted successfully.'
}
else:
return {
"status": "failure",
'status_code': STATUS_CODES['OK'],
'msg': 'Threshold not available.'
}
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {
'status': 'failure',
'msg': 'Internal error occurred while fetching '
'the custom functions list.',
'status_code': STATUS_CODES['INTERNAL_SERVER_ERROR'],
'error': str(e)
}