def produce_task(websocket_id, model_id, spec):
model = load_model_helper(websocket_id, model_id)
if not model:
return
if DEBUG_MODE:
produce_data = model.produce(spec)
else:
try:
produce_data = model.produce(spec)
except Exception:
logger.info("caught traceback when running produce:")
logger.info(traceback.format_exc())
ws_msg = WebsocketMessage.get_fail_message_with_data( \
RECEIVE_PRODUCE_MSG,
'produce failed due to exception',
data={
'model_id': model.model_id,
'search_id': model.search_id,
'system': model.system
})
ws_msg.send_message(websocket_id)
return
ws_msg = WebsocketMessage.get_success_message(
RECEIVE_PRODUCE_MSG,
"produce successfully completed",
data=produce_data)
ws_msg.send_message(websocket_id)
def describe_task(websocket_id, model_id):
model = load_model_helper(websocket_id, model_id)
if not model:
return
try:
result = model.describe()
except Exception:
logger.info("caught traceback when running describe:")
logger.info(traceback.format_exc())
ws_msg = WebsocketMessage.get_fail_message_with_data( \
RECEIVE_DESCRIBE_MSG,
'describe failed due to exception',
data={
'model_id': model.model_id,
'search_id': model.search_id,
'system': model.system
})
ws_msg.send_message(websocket_id)
return
ws_msg = WebsocketMessage.get_success_message(
RECEIVE_DESCRIBE_MSG, "describe successfully completed", data=result)
ws_msg.send_message(websocket_id)
def make_produce_solution_call(pipeline_id, websocket_id, user_id, produce_params, **kwargs):
"""Celery task to make TA2 calls for:
ProduceSolution and GetProduceSolutionResults"""
assert pipeline_id, "pipeline_id must be set"
assert websocket_id, "websocket_id must be set"
assert user_id, "user_id must be set"
assert produce_params, "produce_params must be set"
produce_helper = ProduceSolutionHelper(\
pipeline_id, websocket_id,
user_id, produce_params, **kwargs)
print('produce 1')
if produce_helper.has_error():
user_msg = ('ProduceSolution failure for pipeline (%s): %s') % \
(pipeline_id, produce_helper.get_error_message())
ws_msg = WebsocketMessage.get_fail_message(\
ta2_static.PRODUCE_SOLUTION, user_msg)
ws_msg.send_message(websocket_id)
LOGGER.info('ProduceSolutionHelper: %s', user_msg)
return
LOGGER.info('ProduceSolutionHelper: OK!')
print('produce 2')
produce_helper.run_process()
def send_websocket_err_msg(self, grpc_call, user_msg=''):
"""Send an error messsage over websockets"""
assert grpc_call, 'grpc_call is required'
user_msg = '%s error; pipeline %s; dataset %s: %s' % \
(grpc_call,
self.pipeline_id,
self.produce_dataset_name,
user_msg)
# ----------------------------------
# Send Websocket message
# ----------------------------------
ws_msg = WebsocketMessage.get_fail_message(grpc_call, user_msg)
ws_msg.send_message(self.websocket_id)
# ----------------------------------
# Log it
# ----------------------------------
LOGGER.info('ProduceSolutionHelper: %s', user_msg)
# ----------------------------------
# Add error message to class
# ----------------------------------
self.add_err_msg(user_msg)
def send_websocket_err_msg(self, grpc_call, user_msg='', pipeline_id=None):
"""Send an error messsage over websockets"""
assert grpc_call, 'grpc_call is required'
if pipeline_id:
user_msg = '%s error; pipeline %s: %s' % \
(grpc_call,
pipeline_id,
user_msg)
else:
user_msg = '%s error: %s' % \
(grpc_call,
user_msg)
# Send Websocket message
#
ws_msg = WebsocketMessage.get_fail_message(grpc_call, user_msg)
ws_msg.send_message(self.websocket_id)
# Log it
#
#LOGGER.info('SearchSolutionsHelper: %s', user_msg)
# Add error to class
#
self.add_err_msg(user_msg)
def add_err_msg(self, user_msg):
"""Add to base base "add_err_msg", also send a websocket message"""
# call the base "add_err_msg"
#
super().add_err_msg(user_msg)
if not self.websocket_id:
return
user_msg = '%s (dataset search)' % \
(user_msg,)
# ----------------------------------
# Send Websocket message
# ----------------------------------
data = {}
if self.datamart_name:
data['datamart_name'] = self.datamart_name
ws_msg = WebsocketMessage.get_fail_message_with_data(\
dm_static.DATAMART_SEARCH_BY_DATASET,
user_msg,
data=data)
print('send to websocket id: %s' % self.websocket_id)
ws_msg.send_message(self.websocket_id)
# ----------------------------------
# Log it
# ----------------------------------
LOGGER.info('WebsocketMessage: %s', user_msg)
def send_websocket_success_message(self):
"""Send a websocket message with the materialize_result data"""
LOGGER.info('(5) send_websocket_success_message')
if self.has_error():
return
if not self.websocket_id:
LOGGER.info('(5a) no websocket_id')
return
LOGGER.info('(5b) send the message!')
ws_string_info = json_dumps(self.new_workspace.to_dict())
if not ws_string_info.success:
user_msg = ('Sorry! An error occurred. (Created workspace but'
' failed JSON conversion.)')
self.add_err_msg(user_msg)
return
ws_msg = WebsocketMessage.get_success_message(\
dm_static.DATAMART_AUGMENT_PROCESS,
('The dataset has been augmented '
'and a new workspace created'),
msg_cnt=99,
data={
'workspace_json_string': ws_string_info.result_obj,
'augment_params': self.augment_params
})
ws_msg.send_message(self.websocket_id)
LOGGER.info('(5c) sent!')
def solve_task(websocket_id,
system_id,
specification,
system_params=None,
search_id=None):
system_params = system_params or {}
solver = Solve(system=system_id,
specification=specification,
callback_found='solve',
callback_arguments={
'specification': specification,
'websocket_id': websocket_id
},
system_params=system_params,
search_id=search_id)
start_time = time.time()
if DEBUG_MODE:
result = solver.run()
else:
try:
result = solver.run()
except Exception:
logger.info("caught traceback when running solver:")
logger.info(traceback.format_exc())
ws_msg = WebsocketMessage.get_fail_message_with_data( \
RECEIVE_SOLVE_MSG,
'solve failed due to exception',
data={
'search_id': solver.search.search_id,
'system': solver.system
})
ws_msg.send_message(websocket_id)
return
stop_time = time.time()
if KEY_DATA not in result:
result[KEY_DATA] = {}
result[KEY_DATA]['elapsed_time'] = stop_time - start_time
ws_msg = WebsocketMessage.get_success_message(RECEIVE_SOLVE_MSG,
result.get('message'),
data=result.get('data'))
ws_msg.send_message(websocket_id)
def load_model_helper(websocket_id, model_id):
""" utility for loading a model with websocket error handling """
try:
return Model.load(model_id)
except Exception:
logger.info("caught traceback when running describe:")
logger.info(traceback.format_exc())
ws_msg = WebsocketMessage.get_fail_message_with_data( \
RECEIVE_DESCRIBE_MSG,
f'describe failed loading model: could not find model {model_id}',
data={
'model_id': model_id
})
ws_msg.send_message(websocket_id)
def send_websocket_success_message(self):
"""Send a websocket message with the materialize_result data"""
LOGGER.info('(5) send_websocket_success_message')
if self.has_error():
return
if not self.websocket_id:
LOGGER.info('(5a) not websocket_id')
return
LOGGER.info('(5b) send the message!')
ws_msg = WebsocketMessage.get_success_message(\
DATAMART_MATERIALIZE_PROCESS,
'The dataset has been materialized',
#data=dict(dog=1))
data=self.materialize_result)
ws_msg.send_message(self.websocket_id)
LOGGER.info('(5c) sent!')
def run_construct_dataset(self):
"""Go through the steps...."""
LOGGER.info('>>> run_construct_dataset')
try:
self.user = User.objects.get(pk=self.user_id)
except User.DoesNotExist:
user_msg = 'No user found for id: %s' % self.user_id
self.send_websocket_err_msg(user_msg)
return
for src_file in self.orig_source_files:
if not isfile(src_file):
user_msg = f'File does not exists: {src_file}'
self.send_websocket_err_msg(user_msg)
return
if not isdir(self.writable_output_dir):
user_msg = f'Directory does not exists: {self.writable_output_dir}'
self.send_websocket_err_msg(user_msg)
return
LOGGER.info('(1) construct_folders')
if not self.construct_folders():
return
LOGGER.info('(2) create dataset doc + move file to .csv')
if not self.create_files():
return
LOGGER.info('(3) create_new_config')
self.create_new_config()
# self.send_websocket_err_msg(':( - the augment did not work')
if not self.has_error() and self.websocket_id:
ws_msg = WebsocketMessage.get_success_message(\
ADD_USER_DATASET_PROCESS,
'New user workspace created: %s' % self.new_workspace,
msg_cnt=1)
ws_msg.send_message(self.websocket_id)
def make_score_solutions_call(pipeline_id, websocket_id, user_id,
score_params, **kwargs):
print('make_score_solutions_call 1')
assert pipeline_id, "pipeline_id must be set"
assert websocket_id, "websocket_id must be set"
assert user_id, "user_id must be set"
assert score_params, "score_params must be set"
score_helper = ScoreSolutionHelper(pipeline_id, websocket_id, user_id,
score_params, **kwargs)
if score_helper.has_error():
user_msg = ('ScoreSolution failure for pipeline (%s): %s') % \
(pipeline_id, score_helper.get_error_message())
ws_msg = WebsocketMessage.get_fail_message(\
ta2_static.SCORE_SOLUTION, user_msg)
ws_msg.send_message(websocket_id)
LOGGER.error(user_msg)
return
score_helper.run_process()
def send_websocket_success_message(self):
"""Send a websocket message with the materialize_result data"""
LOGGER.info('(5) send_websocket_success_message')
if self.has_error():
return
if not self.websocket_id:
LOGGER.info('(5a) not websocket_id')
return
LOGGER.info('(5b) send the message!')
resp_info = dict(datamart_name=self.datamart_name,
search_results=self.search_results)
ws_msg = WebsocketMessage.get_success_message(\
dm_static.DATAMART_SEARCH_BY_DATASET,
'The dataset search is complete',
data=resp_info)
ws_msg.send_message(self.websocket_id)
LOGGER.info('(5c) sent!')
def run_construct_dataset(self):
"""Go through the steps...."""
LOGGER.info('>>> run_construct_dataset')
print(self.orig_source_file)
if not isfile(self.orig_source_file):
user_msg = 'File does not exists: %s' % self.orig_source_file
self.send_websocket_err_msg(user_msg)
return
LOGGER.info('(1) retrieve workspace')
if not self.retrieve_workspace():
return
LOGGER.info('(2) construct_folders')
if not self.construct_folders():
return
LOGGER.info('(3) move_source_files')
if not self.move_source_files():
return
#LOGGER.info('(4) create problem docs (and dataset doc, if needed)')
#if not self.create_problem_data_docs():
# return
LOGGER.info('(4) create_new_config')
self.create_new_config()
# self.send_websocket_err_msg(':( - the augment did not work')
if not self.has_error() and self.websocket_id:
ws_msg = WebsocketMessage.get_success_message(\
DATAMART_AUGMENT_PROCESS,
'New user workspace created: %s' % self.new_workspace,
msg_cnt=1)
ws_msg.send_message(self.websocket_id)
def send_websocket_err_msg(self, user_msg):
"""Send an error messsage over websockets"""
# ----------------------------------
# Add error message to class
# ----------------------------------
self.add_err_msg(user_msg)
if not self.websocket_id:
return
user_msg = '%s (datamart augment)' % \
(user_msg,)
# ----------------------------------
# Send Websocket message
# ----------------------------------
ws_msg = WebsocketMessage.get_fail_message(DATAMART_AUGMENT_PROCESS,
user_msg)
ws_msg.send_message(self.websocket_id)
# ----------------------------------
# Log it
# ----------------------------------
LOGGER.info('WebsocketMessage: %s', user_msg)
def add_err_msg(self, user_msg):
"""Add to base base "add_err_msg", also send a websocket message"""
# call the base "add_err_msg"
#
super().add_err_msg(user_msg)
if not self.websocket_id:
return
user_msg = '%s (datamart augment)' % \
(user_msg,)
# ----------------------------------
# Send Websocket message
# ----------------------------------
ws_msg = WebsocketMessage.get_fail_message(
dm_static.DATAMART_AUGMENT_PROCESS, user_msg)
print('send to websocket id: %s' % self.websocket_id)
ws_msg.send_message(self.websocket_id)
# ----------------------------------
# Log it
# ----------------------------------
LOGGER.info('WebsocketMessage: %s', user_msg)
def stream_and_store_results(raven_json_str, stored_request_id,
grpc_req_obj_name, grpc_call_name, **kwargs):
"""Make the grpc call which has a streaming response
grpc_req_obj_name: "core_pb2.GetSearchSolutionsResultsRequest", etc
grpc_call_name: "GetSearchSolutionsResults", etc
"""
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
StoredRequestUtil.set_error_status(stored_request_id, err_msg)
return
# optional: used to stream messages back to client via channels
#
websocket_id = kwargs.get('websocket_id', None)
#
grpc_req_obj = eval(grpc_req_obj_name)
grpc_rpc_call_function = eval('core_stub.%s' % grpc_call_name)
# --------------------------------
# convert the JSON string to a gRPC request
# Yes: done for the 2nd time
# --------------------------------
try:
req = Parse(raven_json_str, grpc_req_obj())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
StoredRequestUtil.set_error_status(stored_request_id, err_msg)
return
# --------------------------------
# Send the gRPC request
# --------------------------------
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# -----------------------------------------
for reply in grpc_rpc_call_function(\
req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
stored_resp = None # to hold a StoredResponse object
# -----------------------------------------
# parse the response
# -----------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
# -----------------------------------------
# does it look ok?
# -----------------------------------------
if not msg_json_info.success:
print('PROBLEM HERE TO LOG!')
user_msg = 'failed to store response: %s' % \
msg_json_info.err_msg
ws_msg = WebsocketMessage.get_fail_message(\
grpc_call_name, user_msg, msg_cnt=msg_cnt)
ws_msg.send_message(websocket_id)
continue
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_response(\
stored_request_id,
response=msg_json_info.result_obj)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
user_msg = 'failed to store response: %s' % \
msg_json_info.err_msg
ws_msg = WebsocketMessage.get_fail_message(\
grpc_call_name, user_msg, msg_cnt=msg_cnt)
ws_msg.send_message(websocket_id)
# Wait for the next response...
continue
# -----------------------------------------------
# send responses back to any open WebSockets
# ---------------------------------------------
if websocket_id:
stored_resp = stored_resp_info.result_obj
ws_msg = WebsocketMessage.get_success_message(\
grpc_call_name,
'it worked',
msg_cnt=msg_cnt,
data=stored_resp.as_dict())
print('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(websocket_id)
StoredResponse.mark_as_read(stored_resp)
# -----------------------------------------------
print('msg received #%d' % msg_cnt)
except grpc.RpcError as err_obj:
StoredRequestUtil.set_error_status(\
stored_request_id,
str(err_obj))
return
#except Exception as err_obj:
# StoredRequestUtil.set_error_status(\
# stored_request_id,
# str(err_obj))
# return
StoredRequestUtil.set_finished_ok_status(stored_request_id)
def run_describe_solution(self, pipeline_id, solution_id, msg_cnt=-1):
"""sync: Run a DescribeSolution call for each solution_id"""
print(f'run_describe_solution 1. pipeline_id: {pipeline_id}')
# ----------------------------------
# Create the input
# ----------------------------------
req_params = {ta2_static.KEY_SOLUTION_ID: solution_id}
json_str_info = json_dumps(req_params)
if not json_str_info.success:
self.add_err_msg(json_str_info.err_msg)
return
json_str_input = json_str_info.result_obj
# --------------------------------
# (2) Save request
# --------------------------------
stored_request = StoredRequest(\
user=self.user_object,
search_id=self.search_id,
pipeline_id=pipeline_id,
workspace='(not specified)',
request_type=ta2_static.DESCRIBE_SOLUTION,
is_finished=False,
request=req_params)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.DESCRIBE_SOLUTION,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SUMMARIZATION,
other=req_params)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
print(
f'run_describe_solution 2. stored_request.pipeline_id: {stored_request.pipeline_id}'
)
# ----------------------------------
# Run Describe Solution
# ----------------------------------
describe_info = describe_solution(json_str_input)
if not describe_info.success:
self.add_err_msg(describe_info.err_msg)
StoredResponse.add_err_response(\
stored_request,
describe_info.err_msg)
return
# ----------------------------------
# Parse the DescribeSolutionResponse
# ----------------------------------
describe_data_info = json_loads(describe_info.result_obj)
if not describe_data_info.success:
self.add_err_msg(describe_data_info.err_msg)
StoredResponse.add_err_response(\
stored_request,
describe_data_info.err_msg)
return
# -----------------------------------------------
# Add the pipline id to the result
# -----------------------------------------------
describe_data = describe_data_info.result_obj
describe_data[ta2_static.KEY_PIPELINE_ID] = pipeline_id
describe_data[ta2_static.KEY_SEARCH_ID] = self.search_id
describe_data[ta2_static.KEY_SOLUTION_ID] = solution_id
describe_data.move_to_end(ta2_static.KEY_PIPELINE_ID, last=False)
# params = dict()
# if not stored_request.pipeline_id:
# params['pipeline_id'] = describe_data[KEY_PIPELINE_ID]
stored_info = StoredResponse.add_success_response(\
stored_request,
describe_data,
pipeline_id=pipeline_id)
if not stored_info.success:
print('stored info fail!', stored_info.err_msg)
print(
f'run_describe_solution 3. stored_info.result_obj.pipeline_id: {stored_info.result_obj.pipeline_id}'
)
print(
f'run_describe_solution 4. stored_request.pipeline_id: {stored_request.pipeline_id}'
)
# -----------------------------------------
# Tracking this in the behavioral log,
# e.g. checking time lapse between creation
# of solution and if user investigates this model,
# later, if at all
# -----------------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.DESCRIBE_SOLUTION_RESPONSE,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=describe_data)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# -----------------------------------------------
# send responses back to WebSocket
# ---------------------------------------------
ws_msg = WebsocketMessage.get_success_message(\
'DescribeSolution',
'it worked',
msg_cnt=msg_cnt,
data=describe_data)
print('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(self.websocket_id)
def run_get_search_solution_results(self):
"""Run SearchSolutions against a TA2"""
# -----------------------------------
# (1) make GRPC request object
# -----------------------------------
params_dict = dict(searchId=self.search_id)
params_info = json_dumps(params_dict)
if not params_info.success:
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
params_info.err_msg)
return
try:
grpc_req = Parse(params_info.result_obj,
core_pb2.GetSearchSolutionsResultsRequest())
except ParseError as err_obj:
err_msg = ('GetSearchSolutionsResultsRequest: Failed to'
' convert JSON to gRPC: %s') % (err_obj)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
params_info.err_msg)
return
# --------------------------------
# (2) Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=self.user_object,
search_id=self.search_id,
workspace='(not specified)',
request_type=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
is_finished=False,
request=params_dict)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=params_dict)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# --------------------------------
# (3) Make the gRPC request
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# Note: The StoredResponse.id becomes the pipeline id
# -----------------------------------------
for reply in core_stub.GetSearchSolutionsResults(\
grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
# -----------------------------------------------
# Parse the response into JSON + store response
# -----------------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
if not msg_json_info.success:
user_msg = 'Failed to convert response to JSON: %s' % \
msg_json_info.err_msg
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
# Wait for next response....
continue
result_json = msg_json_info.result_obj
# TA2s (specifically NYU) responds once when trying a new pipeline, with a message missing a solutionId
# the same process responds again once the solution contains a solutionId
print('results json from TA2')
print(result_json)
if not result_json.get('solutionId'):
continue
if ta2_static.KEY_SOLUTION_ID not in result_json:
user_msg = '"%s" not found in response to JSON: %s' % \
(ta2_static.KEY_SOLUTION_ID, result_json)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
# Wait for next response....
continue
# Solution id used for DescribeSolution...
#
solution_id = result_json[ta2_static.KEY_SOLUTION_ID]
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_stream_success_response(\
stored_request, result_json)
# -----------------------------------------
# Tracking this in the behavioral log,
# e.g. checking time lapse between creation
# of solution and if user investigates this model,
# later, if at all
# -----------------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.
GET_SEARCH_SOLUTIONS_RESULTS_RESPONSE,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=result_json)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
user_msg = 'Failed to store response from %s: %s' % \
(ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
msg_json_info.err_msg)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
# Wait for the next response...
continue
# ---------------------------------------------
# Looks good! Get the StoredResponse
# - This id will be used as the pipeline id
# ---------------------------------------------
stored_response = stored_resp_info.result_obj
stored_response.use_id_as_pipeline_id()
StoredResponse.add_stream_success_response(\
stored_response, stored_response)
# -----------------------------------------------
# send responses back to WebSocket
# ---------------------------------------------
ws_msg = WebsocketMessage.get_success_message(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
'it worked',
msg_cnt=msg_cnt,
data=stored_response.as_dict())
print('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(self.websocket_id)
stored_response.mark_as_sent_to_user()
print('msg received #%d' % msg_cnt)
# -----------------------------------------------
# continue the process describe/score/etc
# -----------------------------------------------
# DescribeSolution - run sync
#
self.run_describe_solution(stored_response.pipeline_id,
solution_id, msg_cnt)
# FitSolution - run async
#
print('PRE run_fit_solution')
self.run_fit_solution(stored_response.pipeline_id, solution_id)
print('POST run_fit_solution')
print('PRE run_score_solution')
self.run_score_solution(stored_response.pipeline_id,
solution_id)
print('POST run_score_solution')
# -----------------------------------------------
# All results arrived, send message to UI
# -----------------------------------------------
ws_msg = WebsocketMessage.get_success_message( \
ta2_static.ENDGetSearchSolutionsResults,
{'searchId': self.search_id, 'message': 'it worked'})
print('ws_msg: %s' % ws_msg)
ws_msg.send_message(self.websocket_id)
except grpc.RpcError as err_obj:
stored_request.set_error_status(str(err_obj))
return
except Exception as err_obj:
stored_request.set_error_status(str(err_obj))
return
StoredRequestUtil.set_finished_ok_status(stored_request.id)
def run_get_score_solution_responses(self, request_id):
"""(2) Run GetScoreSolutionResults"""
if self.has_error():
return
if not request_id:
self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS,
'request_id must be set')
return
# -----------------------------------
# (1) make GRPC request object
# -----------------------------------
params_dict = {ta2_static.KEY_REQUEST_ID: request_id}
params_info = json_dumps(params_dict)
try:
grpc_req = Parse(params_info.result_obj,
core_pb2.GetScoreSolutionResultsRequest())
except ParseError as err_obj:
err_msg = ('Failed to convert JSON to gRPC: %s') % (err_obj)
self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS,
err_msg)
return
# --------------------------------
# (2) Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=self.user_object,
request_type=ta2_static.GET_SCORE_SOLUTION_RESULTS,
search_id=self.search_id,
pipeline_id=self.pipeline_id,
is_finished=False,
request=params_dict)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.GET_SCORE_SOLUTION_RESULTS,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SUMMARIZATION,
other=params_dict)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# --------------------------------
# (3) Make the gRPC request
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# Note: The StoredResponse.id becomes the pipeline id
# -----------------------------------------
for reply in core_stub.GetScoreSolutionResults(\
grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
stored_response = None # to hold a StoredResponse object
# -----------------------------------------------
# Parse the response into JSON + store response
# -----------------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
if not msg_json_info.success:
err_msg = ('Failed to convert JSON to gRPC: %s') % \
(err_obj,)
StoredResponse.add_stream_err_response(
stored_request, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
err_msg)
# Wait for next response....
continue
result_json = msg_json_info.result_obj
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_stream_success_response(\
stored_request, result_json)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
self.send_websocket_err_msg(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
stored_resp_info.err_msg)
#
StoredResponse.add_stream_err_response(\
stored_request, stored_resp_info.err_msg)
#
continue
# ---------------------------------------------
# Looks good! Get the StoredResponse
# - send responses back to WebSocket
# ---------------------------------------------
stored_response = stored_resp_info.result_obj
stored_response.set_pipeline_id(self.pipeline_id)
# ---------------------------------------------
# If progress is complete,
# send response back to WebSocket
# ---------------------------------------------
progress_val = get_dict_value(\
result_json,
[ta2_static.KEY_PROGRESS,
ta2_static.KEY_PROGRESS_STATE])
if (not progress_val.success) or \
(progress_val.result_obj != ta2_static.KEY_PROGRESS_COMPLETED):
user_msg = 'GetScoreSolutionResultsResponse is not yet complete'
LOGGER.info(user_msg)
# wait for next message...
continue
ws_msg = WebsocketMessage.get_success_message(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
'it worked',
msg_cnt=msg_cnt,
data=stored_response.as_dict())
LOGGER.info('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(self.websocket_id)
# stored_response.mark_as_sent_to_user()
except grpc.RpcError as err_obj:
stored_request.set_error_status(str(err_obj))
return
except Exception as err_obj:
stored_request.set_error_status(str(err_obj))
return
StoredRequestUtil.set_finished_ok_status(stored_request.id)