def test_insight_template(self, data):
job_id = None
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
# construct insight configure message
req_data = dict()
payload = data['template_value']
payload['request_type'] = data['template_key']
req_data['entity_id'] = DEFAULT_ENTITY_ID
req_data['solution_id'] = self.solution_id
req_data['data'] = payload
response = post_job(INSIGHT_CONFIG['get_insight'], req_data)
if 'job_id' in response:
job_id = response["job_id"]
if not is_request_timeout(response):
if is_message_published(response):
processed_request = False
req_data['data'] = {'insight_id': self.get_insight_id_from_response(response),
'request_type': 'default'}
initiated_dt = datetime.now()
while not processed_request:
response = post_job(INSIGHT_CONFIG['get_insight'], req_data)
if 'job_id' in response:
job_id = response["job_id"]
if not is_request_timeout(response) and is_message_published(response):
data = self.get_insight_from_response(response)
curr_date = datetime.now()
if ('insights' in data.keys() and len(data['insights']) > 0) or \
(curr_date - initiated_dt).total_seconds() > 60:
return {'status': 'success', 'msg': 'Get insight response',
'data': data, 'job_id': job_id}
else:
return {'status': 'failure', 'msg': 'Request timed out',
'job_id': job_id}
else:
return {'status': 'failure', 'msg': 'Failed in configure insight template',
'error': '', 'job_id': job_id}
return {'status': 'failure', 'msg': 'Request timed out', 'job_id': job_id}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
traceback.print_exc()
if job_id:
return {'status': 'failure', 'msg': 'Error in configure insight template',
'error': traceback.format_exc(), 'job_id': job_id}
else:
return {'status': 'failure', 'msg': 'Error in configure insight template',
'error': traceback.format_exc()}
finally:
context.end_span()
def tables_save(config, data, solution_id):
response = post_job(config['SAVE'], {
"solution_id": solution_id,
"data": data
})
if not is_request_timeout(response):
if is_message_published(response):
return {'status': 'success', 'msg': 'Tables config saved'}
else:
return {'status': 'failure', 'msg': 'Failed to save config'}
else:
return {'status': 'failure', 'msg': 'Request timeout'}
def process_action_retrain(solution_id, payload, model, result_id):
file_name = model['name'] + '_' + str(len(model['model_ref']) +
1) + '.json'
with open(file_name, 'w') as outfile:
json.dump(payload['data'], outfile)
s3_resp = post_s3(str(file_name), ROOT + '/' + str(file_name),
AMAZON_AWS_BUCKET, AMAZON_AWS_KEY_PATH)
if s3_resp['status'] == 'success':
os.remove(file_name)
uri = TRAINING_SET_TRAIN_MODEL_URI
if model['type'] == 'action_classifier':
uri = TRAINING_SET_ACTION_CLASSIFIER_URI
response = post_train_model_job(solution_id, model, s3_resp['key'],
uri)
if not is_request_timeout(response):
if is_message_published(response):
model['model_ref'].append({
'bucket_name':
AMAZON_AWS_BUCKET,
'key_name':
AMAZON_AWS_KEY_PATH + file_name
})
query = {
'is_enabled': True,
'updated_ts': datetime.now(),
'model_ref': model['model_ref']
}
MongoDbConn.update(TRAINING_SET_MODELS_COLLECTION,
where_clause={"_id": ObjectId(result_id)},
query=query)
return {'status': 'success', 'msg': 'Retrain model completed'}
else:
return {
'status': 'failure',
'msg': 'Error in service while publishing retrained model'
}
else:
return {
'status': 'failure',
'msg': 'Timeout Error while processing retrained model'
}
else:
return {
'status': 'failure',
'msg': 'Error in uploading retrained model to s3'
}
def initialize_learning_service(self):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
default_config = get_file_contents("learning_default_config.json")
# construct insight configure message
req_data = dict()
req_data['configuration'] = default_config
req_data['service_name'] = 'learning-microservice'
req = {"solution_id": self.solution_id, "data": req_data, "metadata":{}}
response = post_job(LEARNING_CONFIG['initialize_defaults_api'], req)
if not is_request_timeout(response):
if is_message_published(response):
return True
# TODO raise specific exception
except Exception as e :
traceback.print_exc()
context.log(message=str(e), obj={"tb": traceback.format_exc()})
context.end_span()
return False
def process_action_test(solution_id, payload, model):
data = dict()
data['solution_id'] = solution_id
data['data'] = payload['data']
data['data']['model_name'] = model['name']
data['data']['model_type'] = model['type']
response = post_job(TRAINING_SET_TRAIN_MODEL_TEST_URI, data)
if not is_request_timeout(response):
if is_message_published(response):
return response
else:
return {
'status': 'failure',
'msg': 'Error in service while publishing model to test'
}
else:
return {
'status': 'failure',
'msg': 'Timeout Error while processing model to test'
}
def configure_insight_template(self, data):
job_id = None
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
# construct insight configure message
req_data = dict()
#req_data['configuration'] = {"defaults": {"realtime_requests": {data['template_key']: data['template_value']}}}
req_data['configuration'] = {"defaults": {"realtime_requests": data}}
req_data['service_name'] = 'insight-microservice'
req = {"solution_id": self.solution_id, "data": req_data}
response = post_job(INSIGHT_CONFIG['configure_template_api'], req)
if 'job_id' in response:
job_id = response["job_id"]
if not is_request_timeout(response):
if is_message_published(response):
#self.update_insight_templates(data)
return {'status': 'success',
'msg': 'Configured insight template',
'job_id': job_id}
else:
return {'status': 'failure',
'msg': 'Failed in configure insight template',
'error': '', 'job_id':job_id}
return {'status': 'failure', 'msg': 'Request timed out',
'job_id': job_id}
# TODO raise specific exception
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
traceback.print_exc()
if job_id:
return {'status': 'failure', 'msg': 'Error in configure insight template',
'error': traceback.format_exc(), 'job_id': job_id}
else:
return {'status': 'failure', 'msg': 'Error in configure insight template',
'error': traceback.format_exc()}
finally:
context.end_span()
def solution_trigger(solution_id):
context = tracer.get_context(request_id=str(uuid4()), log_level="INFO")
context.start_span(component=__name__)
try:
job_id = None
payload = {
"solution_id": solution_id,
"entity_id": DEFAULT_ENTITY_ID,
"data": {}
}
response = post_job(NEW_SOLN_TRIGGER, payload)
if 'job_id' in response:
job_id = response["job_id"]
if not is_request_timeout(response):
if is_message_published(response):
return {
'status': 'success',
'msg': 'Solution initialized',
'job_id': job_id
}
else:
return {
'status': 'failure',
'msg': 'Failed to initialise',
'job_id': job_id
}
else:
return {
'status': 'failure',
'msg': 'Request timeout',
'job_id': job_id
}
except Exception as e:
context.log(message=str(e), obj={"tb": traceback.format_exc()})
return {'status': 'failure', 'msg': 'exception occerd : ' + str(e)}
finally:
context.end_span()
def process_action_default(solution_id, payload, request):
training_set_id = payload['training_set_id']
training_set = MongoDbConn.find_one(TRAINING_SET_COLLECTION,
{'_id': ObjectId(training_set_id)})
if training_set is not None:
# post job
uri = TRAINING_SET_TRAIN_MODEL_URI
if payload['type'] == 'action_classifier':
uri = TRAINING_SET_ACTION_CLASSIFIER_URI
file_name = AMAZON_AWS_KEY_PATH + training_set['file_name']
if 's3_key' in training_set.keys():
file_name = training_set['s3_key']
response = post_train_model_job(solution_id, payload, file_name, uri)
is_published = False
if not is_request_timeout(response):
if is_message_published(response):
data = dict()
data['name'] = payload['name']
data['model_id'] = response['result']['result']['metadata'][
'model_id']
data['description'] = payload[
'description'] if 'description' in payload.keys() else ''
data['type'] = payload['type']
data['solution_id'] = get_solution_from_session(request)
data['model_ref'] = []
data['model_ref'].append({
'bucket_name':
AMAZON_AWS_BUCKET,
'key_name':
AMAZON_AWS_KEY_PATH + training_set['file_name']
})
data['is_published'] = is_published
data['service'] = payload['service']
data['created_ts'] = datetime.now()
data['updated_ts'] = datetime.now()
data['is_enabled'] = False
data['is_deleted'] = False
MongoDbConn.insert(TRAINING_SET_MODELS_COLLECTION, query=data)
return {
'status': 'success',
'msg': payload['name'] + ' model created'
}
else:
status = response['result']['status']
return {
'status': 'failure',
'msg': 'Error from service while creating model',
'error': status['message']
}
else:
return {
'status': 'failure',
'msg': 'Service is not running or taking more time to process',
'error': response
}
else:
return {
'status': 'failure',
'msg': 'Selected training set is not available'
}
