def start_session(raven_json_str=None):
"""Start session command
This command sends a UserAgent and the protocol version
to the TA2 service
"""
if raven_json_str is None:
# Default if the user_agent is not from the UI
raven_dict = dict(user_agent=settings.TA2_GPRC_USER_AGENT)
else:
# The UI has sent JSON in string format that contains the user_agent
try:
raven_dict = json.loads(raven_json_str)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_sess_response(err_msg)
# The protocol version always comes from the latest
# version we have in the repo (just copied in for now)
#
raven_dict['version'] = TA2Connection.get_protocol_version()
# --------------------------------
# Convert back to string for TA2 call
# --------------------------------
content = json.dumps(raven_dict)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(content, core_pb2.SessionRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_sess_response(err_msg)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_sess_response(err_msg)
#return dict(status=core_pb2.FAILED_PRECONDITION,
# details=err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.StartSession(req)
except Exception as ex:
return get_failed_precondition_sess_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return MessageToJson(reply)
def get_data_flow_results(info_str=None):
"""Ask a TA2 to GetDataflowResults via gRPC"""
if info_str is None:
err_msg = 'UI Str for PipelineReference is None'
return get_failed_precondition_response(err_msg)
# --------------------------------
# Is this valid JSON?
# --------------------------------
try:
raven_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str, dataflow_ext_pb2.PipelineReference())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
info_dict = dict(pipelineId=raven_dict.get('pipelineId'))
return get_grpc_test_json(
'test_responses/get_dataflow_results_ok.json', info_dict)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
dataflow_stub, err_msg = TA2Connection.get_grpc_dataflow_stub()
if err_msg:
return get_failed_precondition_sess_response(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = dataflow_stub.GetDataflowResults(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
if reply and str(reply) == VAL_GRPC_STATE_CODE_NONE:
err_msg = ('Unkown gRPC state.'
' (Was an GetDataflowResults request sent?)')
return get_failed_precondition_response(err_msg)
# --------------------------------
# Convert the reply to JSON and send it on
# --------------------------------
results = map(MessageToJson, reply)
result_str = '[' + ', '.join(results) + ']'
return result_str
def set_problem_doc(info_str=None):
"""
SetProblemDocRequest={"ReplaceProblemDocField":{"metric":"ROC_AUC"}}
Accept UI input as JSON *string* similar to
{"context": {"session_id": "session_0"}, "ReplaceProblemDocField": {"metric": "ACCURACY", "taskType": "CLASSIFICATION"}}
"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for SetProblemDoc is None'
return get_failed_precondition_response(err_msg)
# --------------------------------
# Convert info string to dict
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
#content = json.dumps(info_dict)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str, core_pb2.SetProblemDocRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
return get_grpc_test_json('test_responses/set_problem_doc_ok.json',
dict())
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.SetProblemDoc(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it on
# --------------------------------
return MessageToJson(reply)
def end_session(raven_json_str):
"""end session command
This command needs a session id from the start_session cmd
e.g. string: '{"session_id" : "123556"}'
"""
# The UI has sent JSON in string format that contains the session_id
try:
raven_dict = json.loads(raven_json_str)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON for end_session: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# The protocol version always comes from the latest
# version we have in the repo (just copied in for now)
#
if not 'session_id' in raven_dict:
err_msg = 'No session_id found: %s' % (raven_json_str)
return get_failed_precondition_response(err_msg)
# --------------------------------
# Convert back to string for TA2 call
# --------------------------------
content = json.dumps(raven_dict)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(content, core_pb2.SessionContext())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
#return dict(status=core_pb2.FAILED_PRECONDITION,
# details=err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.EndSession(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return MessageToJson(reply)
def fit_solution(raven_json_str=None):
"""
Send a FitSolutionRequest to the FitSolution command
"""
if raven_json_str is None:
err_msg = 'No data found for the FitSolutionRequest'
return err_resp(err_msg)
# --------------------------------
# Make sure it's valid JSON
# --------------------------------
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.FitSolutionRequest())
except ParseError as err_obj:
err_msg = ('Failed to convert JSON to gRPC: %s'
' (req_search_solutions)'
'\nraven_json_str: %s') % \
(err_obj, raven_json_str)
print('-' * 40)
print(err_msg)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp = core_pb2.FitSolutionResponse(\
request_id='requestId_%s' % get_alphanumeric_string(6))
return ok_resp(message_to_json(resp))
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.FitSolution(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return ok_resp(message_to_json(reply))
def describe_data_flow(raven_json_str=None):
"""
Send a PipelineReference to the DescribeDataflow command
"""
if raven_json_str is None:
err_msg = 'No data found for the PipelineReference'
return get_failed_precondition_sess_response(err_msg)
# --------------------------------
# The UI has sent JSON in string format that contains the PipelineReference
# Make sure it's valid JSON
# --------------------------------
try:
raven_dict = json.loads(raven_json_str)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_sess_response(err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, dataflow_ext_pb2.PipelineReference())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_sess_response(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
info_dict = dict(pipelineId=raven_dict.get('pipelineId'))
return get_grpc_test_json('test_responses/describe_data_flow_ok.json',
info_dict)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
dataflow_stub, err_msg = TA2Connection.get_grpc_dataflow_stub()
if err_msg:
return get_failed_precondition_sess_response(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = dataflow_stub.DataflowDescription(req)
except Exception as ex:
return get_failed_precondition_sess_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return MessageToJson(reply)
def ta2_hello():
"""Hello. This is a "heartbeat" request for the TA2"""
# --------------------------------
# convert the JSON string to a gRPC request
# for this call,this step is un-needed, just keeping it
# in case things change...
# --------------------------------
try:
req = Parse("{}", core_pb2.HelloRequest())
#req = core_pb2.HelloRequest()
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
content = MessageToJson(req, including_default_value_fields=True)
# print('content as JSON:\n', content)
# In test mode, check if the incoming JSON is legit (in line above)
# -- then return canned response
#
if settings.TA2_STATIC_TEST_MODE:
info = dict(TA3TA2_API_VERSION=TA3TA2Util.get_api_version(),
TA3_GRPC_USER_AGENT=settings.TA3_GRPC_USER_AGENT)
resp_str = get_grpc_test_json(\
'test_responses/Hello_ok.json',
info)
return ok_resp(resp_str)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.Hello(req,
timeout=settings.TA2_GRPC_FAST_TIMEOUT)
except Exception as ex:
return err_resp(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
user_msg_json = MessageToJson(reply, including_default_value_fields=True)
return ok_resp(user_msg_json)
def delete_pipelines(info_str):
"""Ask a TA2 to DeletePipelines via gRPC"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for DeletePipelines is None'
return get_failed_precondition_response(err_msg)
# --------------------------------
# Is this valid JSON?
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str, core_pb2.PipelineDeleteRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
return get_grpc_test_json('test_responses/list_pipelines_ok.json',
dict())
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request - returns a stream
# --------------------------------
try:
reply = core_stub.DeletePipelines(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return MessageToJson(reply)
def describe_solution(raven_json_str=None):
"""
Send a DescribeSolutionRequest to the DescribeSolution command
"""
if raven_json_str is None:
err_msg = 'No data found for the DescribeSolutionRequest'
return err_resp(err_msg)
# --------------------------------
# Make sure it's valid JSON
# --------------------------------
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.DescribeSolutionRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp_str = get_grpc_test_json(\
'test_responses/DescribeSolutionResponse_ok.json',
dict())
return ok_resp(resp_str)
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.DescribeSolution(\
req,
timeout=settings.TA2_GRPC_FAST_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return ok_resp(message_to_json(reply))
def list_primitives():
"""
Send a ListPrimitivesRequest to the ListPrimitives command
"""
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse("{}", core_pb2.ListPrimitivesRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp_str = get_grpc_test_json(\
'test_responses/ListPrimitivesResponse_ok.json',
dict())
return ok_resp(resp_str)
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.ListPrimitives(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return ok_resp(message_to_json(reply))
def execute_pipeline(info_str=None):
"""Ask a TA2 to ListPipelines via gRPC
This call is a bit different b/c it writes part of the data to a file
and places that file uri into the original request
Success: (updated request str, grpc json response)
Failure: (None, error message)
"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for PipelineListResult is None'
return None, get_failed_precondition_response(err_msg)
if info_str.find(VAL_DATA_URI) == -1:
err_msg = ('Expected to see place holder for file uri.'
' Placeholder is "%s"') % VAL_DATA_URI
return None, get_failed_precondition_response(err_msg)
d3m_config = get_latest_d3m_config()
if not d3m_config:
err_msg = ('The D3M configuration is not available.'
' Therefore, there is no "temp_storage_root" directory to'
' write the data.')
return None, get_failed_precondition_response(err_msg)
# --------------------------------
# Is this valid JSON?
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return None, get_failed_precondition_response(err_msg)
if not KEY_DATA in info_dict:
err_msg = ('The JSON request did not contain a "%s" key.') % KEY_DATA
return None, get_failed_precondition_response(err_msg)
file_uri, err_msg = write_data_for_execute_pipeline(
d3m_config, info_dict[KEY_DATA])
if err_msg is not None:
return None, get_failed_precondition_response(err_msg)
# Reformat the original content
#
# (1) remove the data key
if KEY_DATA in info_dict:
del info_dict[KEY_DATA]
# (2) convert it back to a JSON string
info_str = json.dumps(info_dict)
# (3) replace the VAL_DATA_URI with the file_uri
info_str_formatted = info_str.replace(VAL_DATA_URI, file_uri)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str_formatted, core_pb2.PipelineExecuteRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return None, get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
#return info_str_formatted,\
# get_grpc_test_json('test_responses/execute_results_1pipe_ok.json',
# dict())
#---
template_info = get_predict_file_info_dict()
template_str = get_grpc_test_json(
'test_responses/execute_results_1pipe_ok.json', template_info)
# These next lines embed file uri content into the JSON
embed_util = FileEmbedUtil(template_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
test_note = ('Test. An actual result would be the test JSON with'
' the "data" section removed and DATA_URI replaced'
' with a file path to where the "data" section was'
' written.')
return json.dumps(dict(note=test_note)), embed_util.get_final_results()
#---
#return info_str_formatted,\
# get_grpc_test_json('test_responses/execute_results_1pipe_ok.json',
# dict())
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return None, get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request - returns a stream
# --------------------------------
try:
reply = core_stub.ExecutePipeline(req)
except Exception as ex:
return None, get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it on
# --------------------------------
results = map(MessageToJson, reply)
result_str = '[' + ', '.join(results) + ']'
embed_util = FileEmbedUtil(result_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
return info_str_formatted, embed_util.get_final_results()
def end_session(raven_json_str):
"""end session command
This command needs a session id from the start_session cmd
e.g. string: '{"session_id" : "123556"}'
"""
# The UI has sent JSON in string format that contains the session_id
try:
raven_dict = json.loads(raven_json_str)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON for end_session: %s' % (
err_obj)
return get_failed_precondition_response(err_msg)
# The protocol version always comes from the latest
# version we have in the repo (just copied in for now)
#
if not KEY_SESSION_ID_FROM_UI in raven_dict:
return get_failed_precondition_response(ERR_NO_SESSION_ID)
# --------------------------------
# Convert back to string for TA2 call
# --------------------------------
content = json.dumps(raven_dict)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(content, core_pb2.SessionContext())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# In test mode, check if the incoming JSON is legit (in line above)
# -- then return canned response below
#
if settings.TA2_STATIC_TEST_MODE:
rnd_session_id = random_info.get_alphanumeric_string(7)
tinfo = dict(session_id=rnd_session_id)
#if random.randint(1, 3) == 3:
# return get_grpc_test_json('test_responses/endsession_badassertion.json')
return get_grpc_test_json('test_responses/endsession_ok.json', tinfo)
if settings.TA2_STATIC_TEST_MODE:
rnd_session_id = random_info.get_alphanumeric_string(7)
tinfo = dict(session_id=rnd_session_id)
if random.randint(1, 3) == 3:
return get_grpc_test_json(
request, 'test_responses/endsession_badassertion.json')
return get_grpc_test_json(request, 'test_responses/endsession_ok.json',
tinfo)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.EndSession(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return MessageToJson(reply)
def stream_and_store_results(raven_json_str, stored_request_id,
grpc_req_obj_name, grpc_call_name, **kwargs):
"""Make the grpc call which has a streaming response
grpc_req_obj_name: "core_pb2.GetSearchSolutionsResultsRequest", etc
grpc_call_name: "GetSearchSolutionsResults", etc
"""
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
StoredRequestUtil.set_error_status(stored_request_id, err_msg)
return
# optional: used to stream messages back to client via channels
#
websocket_id = kwargs.get('websocket_id', None)
#
grpc_req_obj = eval(grpc_req_obj_name)
grpc_rpc_call_function = eval('core_stub.%s' % grpc_call_name)
# --------------------------------
# convert the JSON string to a gRPC request
# Yes: done for the 2nd time
# --------------------------------
try:
req = Parse(raven_json_str, grpc_req_obj())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
StoredRequestUtil.set_error_status(stored_request_id, err_msg)
return
# --------------------------------
# Send the gRPC request
# --------------------------------
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# -----------------------------------------
for reply in grpc_rpc_call_function(\
req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
stored_resp = None # to hold a StoredResponse object
# -----------------------------------------
# parse the response
# -----------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
# -----------------------------------------
# does it look ok?
# -----------------------------------------
if not msg_json_info.success:
print('PROBLEM HERE TO LOG!')
user_msg = 'failed to store response: %s' % \
msg_json_info.err_msg
ws_msg = WebsocketMessage.get_fail_message(\
grpc_call_name, user_msg, msg_cnt=msg_cnt)
ws_msg.send_message(websocket_id)
continue
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_response(\
stored_request_id,
response=msg_json_info.result_obj)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
user_msg = 'failed to store response: %s' % \
msg_json_info.err_msg
ws_msg = WebsocketMessage.get_fail_message(\
grpc_call_name, user_msg, msg_cnt=msg_cnt)
ws_msg.send_message(websocket_id)
# Wait for the next response...
continue
# -----------------------------------------------
# send responses back to any open WebSockets
# ---------------------------------------------
if websocket_id:
stored_resp = stored_resp_info.result_obj
ws_msg = WebsocketMessage.get_success_message(\
grpc_call_name,
'it worked',
msg_cnt=msg_cnt,
data=stored_resp.as_dict())
print('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(websocket_id)
StoredResponse.mark_as_read(stored_resp)
# -----------------------------------------------
print('msg received #%d' % msg_cnt)
except grpc.RpcError as err_obj:
StoredRequestUtil.set_error_status(\
stored_request_id,
str(err_obj))
return
#except Exception as err_obj:
# StoredRequestUtil.set_error_status(\
# stored_request_id,
# str(err_obj))
# return
StoredRequestUtil.set_finished_ok_status(stored_request_id)
def run_get_search_solution_results(self):
"""Run SearchSolutions against a TA2"""
# -----------------------------------
# (1) make GRPC request object
# -----------------------------------
params_dict = dict(searchId=self.search_id)
params_info = json_dumps(params_dict)
if not params_info.success:
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
params_info.err_msg)
return
try:
grpc_req = Parse(params_info.result_obj,
core_pb2.GetSearchSolutionsResultsRequest())
except ParseError as err_obj:
err_msg = ('GetSearchSolutionsResultsRequest: Failed to'
' convert JSON to gRPC: %s') % (err_obj)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
params_info.err_msg)
return
# --------------------------------
# (2) Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=self.user_object,
search_id=self.search_id,
workspace='(not specified)',
request_type=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
is_finished=False,
request=params_dict)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=params_dict)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# --------------------------------
# (3) Make the gRPC request
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# Note: The StoredResponse.id becomes the pipeline id
# -----------------------------------------
for reply in core_stub.GetSearchSolutionsResults(\
grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
# -----------------------------------------------
# Parse the response into JSON + store response
# -----------------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
if not msg_json_info.success:
user_msg = 'Failed to convert response to JSON: %s' % \
msg_json_info.err_msg
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
# Wait for next response....
continue
result_json = msg_json_info.result_obj
# TA2s (specifically NYU) responds once when trying a new pipeline, with a message missing a solutionId
# the same process responds again once the solution contains a solutionId
print('results json from TA2')
print(result_json)
if not result_json.get('solutionId'):
continue
if ta2_static.KEY_SOLUTION_ID not in result_json:
user_msg = '"%s" not found in response to JSON: %s' % \
(ta2_static.KEY_SOLUTION_ID, result_json)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
# Wait for next response....
continue
# Solution id used for DescribeSolution...
#
solution_id = result_json[ta2_static.KEY_SOLUTION_ID]
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_stream_success_response(\
stored_request, result_json)
# -----------------------------------------
# Tracking this in the behavioral log,
# e.g. checking time lapse between creation
# of solution and if user investigates this model,
# later, if at all
# -----------------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.
GET_SEARCH_SOLUTIONS_RESULTS_RESPONSE,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=result_json)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
user_msg = 'Failed to store response from %s: %s' % \
(ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
msg_json_info.err_msg)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
# Wait for the next response...
continue
# ---------------------------------------------
# Looks good! Get the StoredResponse
# - This id will be used as the pipeline id
# ---------------------------------------------
stored_response = stored_resp_info.result_obj
stored_response.use_id_as_pipeline_id()
StoredResponse.add_stream_success_response(\
stored_response, stored_response)
# -----------------------------------------------
# send responses back to WebSocket
# ---------------------------------------------
ws_msg = WebsocketMessage.get_success_message(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
'it worked',
msg_cnt=msg_cnt,
data=stored_response.as_dict())
print('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(self.websocket_id)
stored_response.mark_as_sent_to_user()
print('msg received #%d' % msg_cnt)
# -----------------------------------------------
# continue the process describe/score/etc
# -----------------------------------------------
# DescribeSolution - run sync
#
self.run_describe_solution(stored_response.pipeline_id,
solution_id, msg_cnt)
# FitSolution - run async
#
print('PRE run_fit_solution')
self.run_fit_solution(stored_response.pipeline_id, solution_id)
print('POST run_fit_solution')
print('PRE run_score_solution')
self.run_score_solution(stored_response.pipeline_id,
solution_id)
print('POST run_score_solution')
# -----------------------------------------------
# All results arrived, send message to UI
# -----------------------------------------------
ws_msg = WebsocketMessage.get_success_message( \
ta2_static.ENDGetSearchSolutionsResults,
{'searchId': self.search_id, 'message': 'it worked'})
print('ws_msg: %s' % ws_msg)
ws_msg.send_message(self.websocket_id)
except grpc.RpcError as err_obj:
stored_request.set_error_status(str(err_obj))
return
except Exception as err_obj:
stored_request.set_error_status(str(err_obj))
return
StoredRequestUtil.set_finished_ok_status(stored_request.id)
def run_get_score_solution_responses(self, request_id):
"""(2) Run GetScoreSolutionResults"""
if self.has_error():
return
if not request_id:
self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS,
'request_id must be set')
return
# -----------------------------------
# (1) make GRPC request object
# -----------------------------------
params_dict = {ta2_static.KEY_REQUEST_ID: request_id}
params_info = json_dumps(params_dict)
try:
grpc_req = Parse(params_info.result_obj,
core_pb2.GetScoreSolutionResultsRequest())
except ParseError as err_obj:
err_msg = ('Failed to convert JSON to gRPC: %s') % (err_obj)
self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS,
err_msg)
return
# --------------------------------
# (2) Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=self.user_object,
request_type=ta2_static.GET_SCORE_SOLUTION_RESULTS,
search_id=self.search_id,
pipeline_id=self.pipeline_id,
is_finished=False,
request=params_dict)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.GET_SCORE_SOLUTION_RESULTS,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SUMMARIZATION,
other=params_dict)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# --------------------------------
# (3) Make the gRPC request
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# Note: The StoredResponse.id becomes the pipeline id
# -----------------------------------------
for reply in core_stub.GetScoreSolutionResults(\
grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
stored_response = None # to hold a StoredResponse object
# -----------------------------------------------
# Parse the response into JSON + store response
# -----------------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
if not msg_json_info.success:
err_msg = ('Failed to convert JSON to gRPC: %s') % \
(err_obj,)
StoredResponse.add_stream_err_response(
stored_request, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
err_msg)
# Wait for next response....
continue
result_json = msg_json_info.result_obj
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_stream_success_response(\
stored_request, result_json)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
self.send_websocket_err_msg(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
stored_resp_info.err_msg)
#
StoredResponse.add_stream_err_response(\
stored_request, stored_resp_info.err_msg)
#
continue
# ---------------------------------------------
# Looks good! Get the StoredResponse
# - send responses back to WebSocket
# ---------------------------------------------
stored_response = stored_resp_info.result_obj
stored_response.set_pipeline_id(self.pipeline_id)
# ---------------------------------------------
# If progress is complete,
# send response back to WebSocket
# ---------------------------------------------
progress_val = get_dict_value(\
result_json,
[ta2_static.KEY_PROGRESS,
ta2_static.KEY_PROGRESS_STATE])
if (not progress_val.success) or \
(progress_val.result_obj != ta2_static.KEY_PROGRESS_COMPLETED):
user_msg = 'GetScoreSolutionResultsResponse is not yet complete'
LOGGER.info(user_msg)
# wait for next message...
continue
ws_msg = WebsocketMessage.get_success_message(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
'it worked',
msg_cnt=msg_cnt,
data=stored_response.as_dict())
LOGGER.info('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(self.websocket_id)
# stored_response.mark_as_sent_to_user()
except grpc.RpcError as err_obj:
stored_request.set_error_status(str(err_obj))
return
except Exception as err_obj:
stored_request.set_error_status(str(err_obj))
return
StoredRequestUtil.set_finished_ok_status(stored_request.id)
def solution_export3(user, raven_json, **kwargs):
"""
Send a SolutionExportRequest to the SolutionExport command
"""
if not isinstance(user, User):
err_msg = '"user" must be a User object'
return err_resp(err_msg)
if not isinstance(raven_json, dict):
err_msg = 'raven_dict must be a python dict'
return err_resp(err_msg)
if not ta2_static.KEY_SEARCH_ID in raven_json:
err_msg = (f'Key: "{ta2_static.KEY_SEARCH_ID}" not found in the'
f' "raven_json" dict. (solution_export3)')
return err_resp(err_msg)
search_id = raven_json.pop(
ta2_static.KEY_SEARCH_ID) # not needed for GRPC call
session_key = kwargs.get(SESSION_KEY, '')
# --------------------------------
# Convert dict to string
# --------------------------------
raven_json_info = json_dumps(raven_json)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
raven_json_str = raven_json_info.result_obj
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.SolutionExportRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp = core_pb2.SolutionExportResponse()
return ok_resp(message_to_json(resp))
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=user,
search_id=search_id,
workspace='(not specified)',
request_type=ta2_static.SOLUTION_EXPORT,
is_finished=False,
request=raven_json)
stored_request.save()
# --------------------------------
# Behavioral logging
# --------------------------------
log_data = dict(session_key=session_key,
feature_id=ta2_static.SOLUTION_EXPORT,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_EXPORT,
other=raven_json)
LogEntryMaker.create_ta2ta3_entry(user, log_data)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.SolutionExport(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
user_msg = f'Error: {err_obj}'
StoredResponse.add_err_response(stored_request, user_msg)
return err_resp(user_msg)
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
resp_json_str = message_to_json(reply)
resp_json_dict_info = json_loads(resp_json_str)
if not resp_json_dict_info.success:
user_msg = (f'Failed to convert GRPC response to JSON:'
f' {resp_json_dict_info.err_msg}')
StoredResponse.add_err_response(stored_request, user_msg)
return err_resp(user_msg)
StoredResponse.add_success_response(stored_request,
resp_json_dict_info.result_obj)
return ok_resp(resp_json_str)
def update_problem_schema(info_str=None):
"""
Accept UI input as JSON *string* similar to
{"taskType" : "REGRESSION",
"taskSubtype" : "TASK_SUBTYPE_UNDEFINED",
"outputType" : "REAL",
"metric" : "ROOT_MEAN_SQUARED_ERROR"}
"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for UpdateProblemSchema is None'
return get_failed_precondition_response(err_msg)
# --------------------------------
# Convert info string to dict
# --------------------------------
try:
info_dict = json.loads(info_str)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# --------------------------------
# create UpdateProblemSchemaRequest compatible JSON
# --------------------------------
updates_list = []
for key, val in info_dict.items():
updates_list.append({key : val})
final_dict = dict(updates=updates_list)
content = json.dumps(final_dict)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(content, core_pb2.UpdateProblemSchemaRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.UpdateProblemSchema(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it on
# --------------------------------
return MessageToJson(reply)
def search_solutions(raven_json_str=None):
"""
Send a SearchSolutionsRequest to the SearchSolutions command
"""
print('raven_json_str', raven_json_str)
if raven_json_str is None:
err_msg = 'No data found for the SearchSolutionsRequest'
return err_resp(err_msg)
# This is a dict or OrderedDict, make it a json string
#
if isinstance(raven_json_str, dict):
json_str_info = json_dumps(raven_json_str)
if not json_str_info.success:
return json_str_info
raven_json_str = json_str_info.result_obj
else:
# Make sure the string is valid JSON
#
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
print('SearchSolutionsRequest (string)', raven_json_str)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.SearchSolutionsRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
print('req', req)
print('-' * 40)
print('raven_json_str', raven_json_str)
print('-' * 40)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
search_id = 'search_id_%s' % get_alphanumeric_string(6)
resp = core_pb2.SearchSolutionsResponse(search_id=search_id)
# print('message_to_json(req)', message_to_json(resp))
return ok_resp(message_to_json(resp))
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.SearchSolutions(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return ok_resp(message_to_json(reply))
def end_search_solutions(raven_json_str=None, **kwargs):
"""
Send a EndSearchSolutionsRequest to the EndSearchSolutions command
optional kwargs:
user = User object for logging
"""
if raven_json_str is None:
err_msg = 'No data found for the EndSearchSolutionsRequest'
return err_resp(err_msg)
# --------------------------------
# Make sure it's valid JSON
# --------------------------------
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
if not ta2_static.KEY_SEARCH_ID in raven_json_info.result_obj:
err_msg = (f'The send solutions request did not include'
f' a "{ta2_static.KEY_SEARCH_ID}" key')
return err_resp(err_msg)
search_id = raven_json_info.result_obj[ta2_static.KEY_SEARCH_ID]
# --------------------------------
# optional logging
# --------------------------------
user = kwargs.get('user')
stored_request = None
# Begin to log D3M call
#
if user:
stored_request = StoredRequest(\
user=user,
request_type=ta2_static.END_SEARCH_SOLUTIONS,
search_id=search_id,
is_finished=False,
request=raven_json_info.result_obj)
stored_request.save()
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.EndSearchSolutionsRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp = core_pb2.EndSearchSolutionsResponse()
# print('message_to_json(req)', message_to_json(resp))
return ok_resp(message_to_json(resp))
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.EndSearchSolutions(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
# This returns a JSON string
reply_json_str = message_to_json(reply)
# Double-check, make sure it converts back to a python dict
#
json_format_info = json_loads(reply_json_str)
if not json_format_info.success:
if user:
StoredResponse.add_err_response(stored_request,
json_format_info.err_msg)
return err_resp(json_format_info.err_msg)
# Looks good, save response and return value
#
if user:
StoredResponse.add_success_response(stored_request,
json_format_info.result_obj)
return ok_resp(json_format_info.result_obj)
def get_create_pipeline_results(info_str=None):
"""Send the pipeline create request via gRPC"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for %s is None' % PIPELINE_CREATE_RESULTS_REQUEST
return get_failed_precondition_response(err_msg)
# --------------------------------
# Convert info string to dict
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if KEY_CONTEXT_FROM_UI not in info_dict:
return get_failed_precondition_response(ERR_NO_CONTEXT)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str, core_pb2.PipelineCreateResultsRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
template_info = get_predict_file_info_dict(info_dict.get('task'))
template_str = get_grpc_test_json(
'test_responses/createpipeline_ok.json', template_info)
# These next lines embed file uri content into the JSON
embed_util = FileEmbedUtil(template_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
return embed_util.get_final_results()
#return get_grpc_test_json('test_responses/createpipeline_ok.json',
# template_info)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.GetCreatePipelineResults(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
try:
print(MessageToJson(reply))
except:
print('failed unary convert to JSON')
# --------------------------------
# Convert the reply to JSON and send it on
# --------------------------------
results = map(MessageToJson, reply)
result_str = '[' + ', '.join(results) + ']'
embed_util = FileEmbedUtil(result_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
return embed_util.get_final_results()
def export_pipeline(info_str=None, call_entry=None):
"""Ask a TA2 to ExportPipeline via gRPC"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for ExportPipeline is None'
return get_failed_precondition_response(err_msg)
if info_str.find(VAL_EXECUTABLE_URI) == -1:
err_msg = ('Expected to see place holder for executable uri.'
' Placeholder is "%s"') % VAL_EXECUTABLE_URI
return None, get_failed_precondition_response(err_msg)
d3m_config = get_latest_d3m_config()
if not d3m_config:
err_msg = ('The D3M configuration is not available.'
' Therefore, there is no "executables_root" directory to'
' write the data.')
return None, get_failed_precondition_response(err_msg)
# --------------------------------
# Is this valid JSON?
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# --------------------------------
# Construct and set a write directory for the executable
# --------------------------------
# get the pipeline id
pipeline_id, err_msg = get_pipeline_id(info_dict)
if err_msg:
return get_failed_precondition_response(err_msg)
# dir = d3m_config.executables_root + pipeline_id
executable_write_dir = join('file://%s' % d3m_config.executables_root,
pipeline_id)
# update the dict + info_str
info_dict[KEY_PIPELINE_EXEC_URI] = executable_write_dir
if KEY_PIPELINE_EXEC_URI_FROM_UI in info_dict:
del info_dict[KEY_PIPELINE_EXEC_URI_FROM_UI]
try:
info_str = json.dumps(info_dict)
except TypeError as ex_obj:
err_msg = 'Failed to PipelineExportRequest info to JSON: %s' % ex_obj
return get_failed_precondition_response(err_msg)
#print('info_str', info_str)
if call_entry:
call_entry.request_msg = info_str
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str, core_pb2.PipelineExportRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
return get_grpc_test_json('test_responses/export_pipeline_ok.json',
dict())
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request - returns a stream
# --------------------------------
try:
reply = core_stub.ExportPipeline(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return MessageToJson(reply)
def start_session(raven_json_str=None):
"""Start session command
This command sends a UserAgent and the protocol version
to the TA2 service
"""
if raven_json_str is None:
err_msg = 'No data found. Please send a "user_agent"'
return get_failed_precondition_sess_response(err_msg)
# Default if the user_agent is not from the UI
#raven_dict = dict(user_agent=settings.TA2_GPRC_USER_AGENT)
# The UI has sent JSON in string format that contains the user_agent
try:
raven_dict = json.loads(raven_json_str)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_sess_response(err_msg)
# check for a user_agent
#
if not KEY_USER_AGENT_FROM_UI in raven_dict:
return get_failed_precondition_sess_response(ERR_MSG_NO_USER_AGENT)
# The protocol version always comes from the latest
# version we have in the repo (just copied in for now)
#
raven_dict['version'] = TA2Connection.get_protocol_version()
# --------------------------------
# Convert back to string for TA2 call
# --------------------------------
content = json.dumps(raven_dict)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(content, core_pb2.SessionRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_sess_response(err_msg)
# In test mode, check if the incoming JSON is legit (in line above)
# -- then return canned response
#
if settings.TA2_STATIC_TEST_MODE:
rnd_session_id = random_info.get_alphanumeric_string(7)
info_dict = dict(session_id=rnd_session_id,
api_version=TA3TA2Util.get_api_version())
return get_grpc_test_json('test_responses/startsession_ok.json',
info_dict)
#if random.randint(1,10) == 3:
# return get_grpc_test_json('test_responses/startsession_badassertion.json')
#else:
# return get_grpc_test_json('test_responses/startsession_ok.json', d)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_sess_response(err_msg)
#return dict(status=core_pb2.FAILED_PRECONDITION,
# details=err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.StartSession(req)
except Exception as ex:
return get_failed_precondition_sess_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return MessageToJson(reply)
def get_execute_pipeline_results(info_str=None):
"""Ask a TA2 to GetExecutePipelineResults via gRPC"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for PipelineExecuteResultsRequest is None'
return get_failed_precondition_response(err_msg)
# --------------------------------
# Is this valid JSON?
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str, core_pb2.PipelineExecuteResultsRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
template_info = get_predict_file_info_dict()
template_str = get_grpc_test_json(
'test_responses/execute_results_ok.json', template_info)
embed_util = FileEmbedUtil(template_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
return embed_util.get_final_results()
#return get_grpc_test_json('test_responses/execute_results_ok.json',
# dict())
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
#print('req: %s' % req)
# --------------------------------
# Send the gRPC request - returns a stream
# --------------------------------
try:
reply = core_stub.GetExecutePipelineResults(req)
except grpc.RpcError as ex:
return get_failed_precondition_response(str(ex))
except Exception as ex:
return get_failed_precondition_response(str(ex))
#print('reply', reply)
"""
if reply and str(reply) == VAL_GRPC_STATE_CODE_NONE:
err_msg = ('Unknown gRPC state.'
' (Was an ExecutePipeline request sent?)')
return get_failed_precondition_response(err_msg)
"""
try:
print(MessageToJson(reply))
except:
print('failed unary convert to JSON')
#print('reply: %s' % reply)
# --------------------------------
# Convert the reply to JSON and send it on
# --------------------------------
results = map(MessageToJson, reply)
result_str = '[' + ', '.join(results) + ']'
embed_util = FileEmbedUtil(result_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
return embed_util.get_final_results()
def pipeline_create(info_str=None):
"""Send the pipeline create request via gRPC"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for %s is None' % PIPELINE_CREATE_REQUEST
return get_failed_precondition_response(err_msg)
# --------------------------------
# Convert info string to dict
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if KEY_CONTEXT_FROM_UI not in info_dict:
return get_failed_precondition_response(ERR_NO_CONTEXT)
if KEY_SESSION_ID_FROM_UI not in info_dict[KEY_CONTEXT_FROM_UI]:
return get_failed_precondition_response(ERR_NO_SESSION_ID)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str, core_pb2.PipelineCreateRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
template_info = get_predict_file_info_dict(info_dict.get('task'))
template_str = get_grpc_test_json('test_responses/createpipeline_ok.json',
template_info)
# These next lines embed file uri content into the JSON
embed_util = FileEmbedUtil(template_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
return embed_util.get_final_results()
#return get_grpc_test_json('test_responses/createpipeline_ok.json',
# template_info)
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
messages = []
try:
for reply in core_stub.CreatePipelines(req):
user_msg = MessageToJson(reply)
print(user_msg)
messages.append(user_msg)
except Exception as ex:
return get_reply_exception_response(str(ex))
print('end of queue. make message list')
result_str = '['+', '.join(messages)+']'
print('embed file contents')
embed_util = FileEmbedUtil(result_str)
if embed_util.has_error:
print('file embed error')
return get_failed_precondition_response(embed_util.error_message)
print('return results')
return embed_util.get_final_results()