def clean_query(self):
"""Load this as JSON, make sure that it contains at least 1 search param"""
query_info_json = json_loads(self.cleaned_data.get('query'))
if not query_info_json.success:
user_msg = ('The search is not formatted correctly.'
' Please try again.'
' (Expected a JSON string)')
raise forms.ValidationError(user_msg)
query_dict = query_info_json.result_obj
clear_dict(query_dict)
if not query_dict:
user_msg = ('The search must contain at least 1 keyword or'
' variable. Please try again. (id:1)')
raise forms.ValidationError(user_msg)
if not 'keywords' in query_dict:
if not 'variables' in query_dict:
user_msg = ('The search must contain at least 1 keyword or'
' variable. Please try again. (id:2)')
raise forms.ValidationError(user_msg)
return query_dict
def get_config_file_contents(d3m_config, config_key, as_dict=True):
"""Get contents of a file specified in the config"""
if not isinstance(d3m_config, D3MConfiguration):
return err_resp('d3m_config must be a D3MConfiguration object')
if not config_key in D3M_FILE_ATTRIBUTES:
return err_resp('config_key not found!')
filepath_info = get_d3m_filepath(d3m_config, config_key)
if not filepath_info.success:
return err_resp(filepath_info.err_msg)
fpath = filepath_info.result_obj
try:
with open(fpath, "r") as fh:
contents = fh.read()
except IOError as err_obj:
user_msg = 'Failed to read file: %s\n%s' % \
(fpath, err_obj)
return err_resp(user_msg)
if not as_dict:
return ok_resp(contents)
doc_info = json_loads(contents)
if not doc_info.success:
return err_resp(doc_info.err_msg)
return ok_resp(doc_info.result_obj)
def read_problem_doc_if_exists(self):
"""Verify the problem path
example: "/input/TRAIN/problem_TRAIN/problemDoc.json"
Note: As of 7/17/2019, it's ok if there's no problem doc
"""
if self.has_error():
return False
if not isfile(self.env_config.D3MPROBLEMPATH):
user_msg = ('D3MPROBLEMPATH file non-existent or'
' can\'t be reached: %s') % \
self.env_config.D3MPROBLEMPATH
print('Note: ', user_msg)
# self.add_err_msg(user_msg)
return True
json_content = open(self.env_config.D3MPROBLEMPATH, 'r').read()
pdoc_info = json_loads(json_content)
if not pdoc_info.success:
user_msg = ('D3MPROBLEMPATH file not JSON. %s\nError: %s') % \
(self.env_config.D3MPROBLEMPATH, pdoc_info.err_msg)
print('Note: ', user_msg)
# self.add_err_msg(user_msg)
return True
self.problem_doc = pdoc_info.result_obj
return True
def clean_search_result(self):
json_info = json_loads(self.cleaned_data.get('search_result'))
if not json_info.success:
raise forms.ValidatonError(\
("The 'search_result' is not valid JSON."
" %s") % json_info.err_msg)
return json_info.result_obj
def clean_search_result(self):
"""Convert search_result to a python dict"""
json_info = json_loads(self.cleaned_data.get('search_result'))
if not json_info.success:
user_msg = (f"The 'search_result' is not valid JSON."
f' {json_info.err_msg}')
raise forms.ValidatonError(user_msg)
return json_info.result_obj
def fit_solution(raven_json_str=None):
"""
Send a FitSolutionRequest to the FitSolution command
"""
if raven_json_str is None:
err_msg = 'No data found for the FitSolutionRequest'
return err_resp(err_msg)
# --------------------------------
# Make sure it's valid JSON
# --------------------------------
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.FitSolutionRequest())
except ParseError as err_obj:
err_msg = ('Failed to convert JSON to gRPC: %s'
' (req_search_solutions)'
'\nraven_json_str: %s') % \
(err_obj, raven_json_str)
print('-' * 40)
print(err_msg)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp = core_pb2.FitSolutionResponse(\
request_id='requestId_%s' % get_alphanumeric_string(6))
return ok_resp(message_to_json(resp))
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.FitSolution(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return ok_resp(message_to_json(reply))
def view_hello(request):
"""gRPC: Call from UI as a hearbeat"""
user_info = get_authenticated_user(request)
if not user_info.success:
return JsonResponse(get_json_error(user_info.err_msg))
# --------------------------------
# Behavioral logging
# --------------------------------
log_data = dict(session_key=get_session_key(request),
feature_id=ta2_static.HELLO,
activity_l1=bl_static.L1_SYSTEM_ACTIVITY,
activity_l2=bl_static.L2_APP_LAUNCH)
LogEntryMaker.create_ta2ta3_entry(user_info.result_obj, log_data)
# note: this is just a heartbeat, so no params are sent
#
# Begin to log D3M call
#
call_entry = None
if ServiceCallEntry.record_d3m_call():
call_entry = ServiceCallEntry.get_dm3_entry(\
request_obj=request,
call_type='Hello',
request_msg=('no params for this call'))
# Let's call the TA2!
#
resp_info = ta2_hello()
if not resp_info.success:
return JsonResponse(get_json_error(resp_info.err_msg))
json_str = resp_info.result_obj
# Convert JSON str to python dict - err catch here
# - let it blow up for now--should always return JSON
json_format_info = json_loads(json_str)
if not json_format_info.success:
return JsonResponse(get_json_error(json_format_info.err_msg))
# Save D3M log
#
if call_entry:
call_entry.save_d3m_response(json_format_info.result_obj)
json_info = get_json_success('success!',
data=json_format_info.result_obj)
return JsonResponse(json_info, safe=False)
def view_list_primitives(request):
"""gRPC: Call from UI with a ListPrimitivesRequest"""
user_info = get_authenticated_user(request)
if not user_info.success:
return JsonResponse(get_json_error(user_info.err_msg))
# --------------------------------
# (2) Begin to log D3M call
# --------------------------------
call_entry = None
if ServiceCallEntry.record_d3m_call():
call_entry = ServiceCallEntry.get_dm3_entry(\
request_obj=request,
call_type='ListPrimitives',
request_msg='no params for this call')
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=get_session_key(request),
feature_id=ta2_static.LIST_PRIMITIVES,
activity_l1=bl_static.L1_SYSTEM_ACTIVITY,
activity_l2=bl_static.L2_ACTIVITY_BLANK)
LogEntryMaker.create_ta2ta3_entry(user_info.result_obj, log_data)
# Let's call the TA2!
#
search_info = list_primitives()
#print('search_info', search_info)
if not search_info.success:
return JsonResponse(get_json_error(search_info.err_msg))
# Convert JSON str to python dict - err catch here
#
json_format_info = json_loads(search_info.result_obj)
if not json_format_info.success:
return JsonResponse(get_json_error(json_format_info.err_msg))
# Save D3M log
#
if call_entry:
call_entry.save_d3m_response(json_format_info.result_obj)
json_info = get_json_success('success!', data=json_format_info.result_obj)
return JsonResponse(json_info, safe=False)
def augment_isi_file(self):
"""Augment the file via the ISI API"""
# self.add_err_msg(('ISI Augment is disabled!!!!'
# ' (augment_util.augment_isi_file)'))
# return False
if self.has_error():
return False
# user_workspace, data_path, search_result, left_columns,
# right_columns, exact_match=False, **kwargs
search_result_info = json_loads(self.augment_params['search_result'])
if not search_result_info.success:
err_msg = (f"Failed to load augment_params['search_result']"
f" as JSON: {search_result_info.err_msg}")
self.add_err_msg(err_msg)
return
search_result_json = search_result_info.result_obj
extra_params = dict() # none for now...
augment_info = self.datamart_util.datamart_augment(\
self.user_workspace,
self.augment_params[dm_static.KEY_DATA_PATH],
search_result_json,
exact_match=self.augment_params.get('exact_match'),
**extra_params)
if not augment_info.success:
self.add_err_msg(augment_info.err_msg)
return False
augment_dict = augment_info.result_obj
keys_to_check = [
dm_static.KEY_DATA_PATH, dm_static.KEY_DATASET_DOC_PATH
]
for key in keys_to_check:
if key not in augment_dict:
user_msg = (f'Key "{key}" not found in the NYU augment_dict.'
f' Keys: {augment_dict.keys()}')
self.add_err_msg(user_msg)
return False
self.augment_new_filepath = augment_dict[dm_static.KEY_DATA_PATH]
self.augment_new_datasetdoc = augment_dict[
dm_static.KEY_DATASET_DOC_PATH]
return True
def describe_solution(raven_json_str=None):
"""
Send a DescribeSolutionRequest to the DescribeSolution command
"""
if raven_json_str is None:
err_msg = 'No data found for the DescribeSolutionRequest'
return err_resp(err_msg)
# --------------------------------
# Make sure it's valid JSON
# --------------------------------
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.DescribeSolutionRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp_str = get_grpc_test_json(\
'test_responses/DescribeSolutionResponse_ok.json',
dict())
return ok_resp(resp_str)
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.DescribeSolution(\
req,
timeout=settings.TA2_GRPC_FAST_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return ok_resp(message_to_json(reply))
def read_file_contents(fpath, as_dict=True):
"""Given a valid filepath, read the file and return it.
Used for smaller files"""
if not isfile(fpath):
return err_resp(f'File doesn\'t exist: {fpath}')
try:
with open(fpath, "r") as fh:
contents = fh.read()
except IOError as err_obj:
user_msg = 'Failed to read file: %s\n%s' % \
(fpath, err_obj)
return err_resp(user_msg)
if not as_dict:
return ok_resp(contents)
return json_loads(contents)
def load_and_return_json_file(self, fpath):
"""Load a JSON file; assumes fpath exists and has undergone prelim checks"""
assert isfile(
fpath), "fpath must exist; check before using this method"
json_info = None
with open(fpath) as f:
fcontent = f.read()
json_info = json_loads(fcontent)
if not json_info.success:
return self.format_embed_err(ERR_CODE_FILE_INVALID_JSON,
json_info.err_msg)
embed_snippet = OrderedDict()
embed_snippet[KEY_SUCCESS] = True
embed_snippet[KEY_DATA] = json_info.result_obj
return embed_snippet
def view_get_config_file(request, config_key, d3m_config_id=None):
"""Get contents of a file specified in the config"""
if not config_key in D3M_FILE_ATTRIBUTES:
user_msg = (f'Config key "{config_key}" not found!'
f' (view_get_config_file)')
return JsonResponse(get_json_error(user_msg))
if d3m_config_id is None:
d3m_config = get_latest_d3m_config()
else:
d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first()
if d3m_config is None:
user_msg = 'Config not found!'
return JsonResponse(get_json_error(user_msg))
# Make sure the config has a value.
# For example the D3MPROBLEMPATH may be blank
#
if not getattr(d3m_config, config_key):
user_msg = f'Sorry! The config does not have a "{config_key}" value!'
return JsonResponse(get_json_error(user_msg))
filepath_info = get_d3m_filepath(d3m_config, config_key)
if not filepath_info.success:
user_msg = f'{filepath_info.err_msg} (view_get_config_file)'
return JsonResponse(get_json_error(user_msg))
# Relatively small files...
# response = FileResponse(open(filepath_info.result_obj, 'rb'))
# return response
fcontent = open(filepath_info.result_obj, 'r').read()
json_info = json_loads(fcontent)
if not json_info.success:
user_msg = f'{json_info.err_msg} (view_get_config_file)'
return JsonResponse(get_json_error(user_msg))
return JsonResponse(get_json_success(\
'Success!',
data=json_info.result_obj))
def view_hello_heartbeat(request):
"""Hello to TA2 with no logging. Used for testing"""
# Let's call the TA2!
#
resp_info = ta2_hello()
if not resp_info.success:
return JsonResponse(get_json_error(resp_info.err_msg))
json_str = resp_info.result_obj
# Convert JSON str to python dict - err catch here
# - let it blow up for now--should always return JSON
json_format_info = json_loads(json_str)
if not json_format_info.success:
return JsonResponse(get_json_error(json_format_info.err_msg))
json_info = get_json_success('success!',
data=json_format_info.result_obj)
return JsonResponse(json_info)
def get_num_pipeline_steps(self):
"""Return the number of pipeline steps in the DescribeSolutionResponse"""
if not self.has_steps():
return -1
#return err_resp('(no steps found--error)')
steps_info = json_loads(self.solution_response)
if not steps_info.success:
return -1
#return err_resp(steps_info.err_msg)
steps_dict = steps_info.result_obj
if not KEY_PIPELINE in steps_dict:
return -1
if not KEY_STEPS in steps_dict[KEY_PIPELINE]:
return -1
#return err_resp('Key "%s" not found in response' % KEY_STEPS)
num_steps = len(steps_dict[KEY_PIPELINE][KEY_STEPS])
#return ok_resp(num_steps)
return num_steps
def view_search_solutions(request):
"""gRPC: Call from UI with a SearchSolutionsRequest"""
req_body_info = get_request_body(request)
if not req_body_info.success:
return JsonResponse(get_json_error(req_body_info.err_msg))
# Begin to log D3M call
#
call_entry = None
if ServiceCallEntry.record_d3m_call():
call_entry = ServiceCallEntry.get_dm3_entry(\
request_obj=request,
call_type=ta2_static.SEARCH_SOLUTIONS,
request_msg=req_body_info.result_obj)
# Let's call the TA2!
#
search_info = search_solutions(req_body_info.result_obj)
#print('search_info', search_info)
if not search_info.success:
return JsonResponse(get_json_error(search_info.err_msg))
# Convert JSON str to python dict - err catch here
# - let it blow up for now--should always return JSON
#json_dict = json.loads(search_info.result_obj, object_pairs_hook=OrderedDict)
json_format_info = json_loads(search_info.result_obj)
if not json_format_info.success:
return JsonResponse(get_json_error(json_format_info.err_msg))
# Save D3M log
#
if call_entry:
call_entry.save_d3m_response(json_format_info.result_obj)
json_info = get_json_success('success!', data=json_format_info.result_obj)
return JsonResponse(json_info, safe=False)
def view_produce_solution(request):
"""gRPC: Call from UI with a ProduceSolutionRequest"""
req_body_info = get_request_body(request)
if not req_body_info.success:
return JsonResponse(get_json_error(req_body_info.err_msg))
# Begin to log D3M call
#
call_entry = None
if ServiceCallEntry.record_d3m_call():
call_entry = ServiceCallEntry.get_dm3_entry(\
request_obj=request,
call_type='ProduceSolution',
request_msg=req_body_info.result_obj)
# Let's call the TA2!
#
search_info = produce_solution(req_body_info.result_obj)
#print('search_info', search_info)
if not search_info.success:
return JsonResponse(get_json_error(search_info.err_msg))
# Convert JSON str to python dict - err catch here
#
json_format_info = json_loads(search_info.result_obj)
if not json_format_info.success:
return JsonResponse(get_json_error(json_format_info.err_msg))
# Save D3M log
#
if call_entry:
call_entry.save_d3m_response(json_format_info.result_obj)
json_info = get_json_success('success!', data=json_format_info.result_obj)
return JsonResponse(json_info, safe=False)
def view_solution_export3(request):
"""gRPC: Call from UI with a SolutionExportRequest"""
# Retrieve the User
#
user_info = get_authenticated_user(request)
if not user_info.success:
return JsonResponse(get_json_error(user_info.err_msg))
user = user_info.result_obj
req_body_info = get_request_body_as_json(request)
if not req_body_info.success:
return JsonResponse(get_json_error(req_body_info.err_msg))
session_key = get_session_key(request)
# Let's call the TA2!
#
search_info = solution_export3(user,
req_body_info.result_obj,
session_key=session_key)
# print('search_info', search_info)
if not search_info.success:
return JsonResponse(get_json_error(search_info.err_msg))
# Convert JSON str to python dict - err catch here
#
json_format_info = json_loads(search_info.result_obj)
if not json_format_info.success:
return JsonResponse(get_json_error(json_format_info.err_msg))
json_info = get_json_success('success!', data=json_format_info.result_obj)
return JsonResponse(json_info, safe=False)
def get_score_solutions_results(raven_json_str, user_obj, websocket_id=None):
"""
Send a GetScoreSolutionResultsRequest to the GetScoreSolutionResults command
Expected input from raven_json_str:
{requestId: scoreId,
pipelineId: response1.id}
"""
if user_obj is None:
return err_resp("The user_obj cannot be None")
if not raven_json_str:
err_msg = 'No data found for the GetScoreSolutionResultsRequest'
return err_resp(err_msg)
# --------------------------------
# Make sure it's valid JSON
# --------------------------------
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
# --------------------------------
# convert the JSON string to a gRPC request
# Done for error checking; call repeated in celery task
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.GetScoreSolutionResultsRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
# --------------------------------
# Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=user_obj,
workspace='(not specified)',
request_type='GetScoreSolutionResults',
is_finished=False,
request=raven_json_info.result_obj)
stored_request.save()
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp_str = get_grpc_test_json(\
'test_responses/GetScoreSolutionResultsResponse_ok.json',
dict())
resp_info = json_loads(resp_str)
if not resp_info.success:
return err_resp(resp_info.err_msg)
# Save the stored response
#
StoredResponse.add_response(\
stored_request.id,
response=resp_info.result_obj)
StoredRequestUtil.set_finished_ok_status(stored_request.id)
# Return the stored **request** (not response)
#
return ok_resp(stored_request.as_dict())
stream_and_store_results.delay(raven_json_str,
stored_request.id,
'core_pb2.GetScoreSolutionResultsRequest',
'GetScoreSolutionResults',
websocket_id=websocket_id)
return ok_resp(stored_request.as_dict())
def stream_and_store_results(raven_json_str, stored_request_id,
grpc_req_obj_name, grpc_call_name, **kwargs):
"""Make the grpc call which has a streaming response
grpc_req_obj_name: "core_pb2.GetSearchSolutionsResultsRequest", etc
grpc_call_name: "GetSearchSolutionsResults", etc
"""
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
StoredRequestUtil.set_error_status(stored_request_id, err_msg)
return
# optional: used to stream messages back to client via channels
#
websocket_id = kwargs.get('websocket_id', None)
#
grpc_req_obj = eval(grpc_req_obj_name)
grpc_rpc_call_function = eval('core_stub.%s' % grpc_call_name)
# --------------------------------
# convert the JSON string to a gRPC request
# Yes: done for the 2nd time
# --------------------------------
try:
req = Parse(raven_json_str, grpc_req_obj())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
StoredRequestUtil.set_error_status(stored_request_id, err_msg)
return
# --------------------------------
# Send the gRPC request
# --------------------------------
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# -----------------------------------------
for reply in grpc_rpc_call_function(\
req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
stored_resp = None # to hold a StoredResponse object
# -----------------------------------------
# parse the response
# -----------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
# -----------------------------------------
# does it look ok?
# -----------------------------------------
if not msg_json_info.success:
print('PROBLEM HERE TO LOG!')
user_msg = 'failed to store response: %s' % \
msg_json_info.err_msg
ws_msg = WebsocketMessage.get_fail_message(\
grpc_call_name, user_msg, msg_cnt=msg_cnt)
ws_msg.send_message(websocket_id)
continue
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_response(\
stored_request_id,
response=msg_json_info.result_obj)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
user_msg = 'failed to store response: %s' % \
msg_json_info.err_msg
ws_msg = WebsocketMessage.get_fail_message(\
grpc_call_name, user_msg, msg_cnt=msg_cnt)
ws_msg.send_message(websocket_id)
# Wait for the next response...
continue
# -----------------------------------------------
# send responses back to any open WebSockets
# ---------------------------------------------
if websocket_id:
stored_resp = stored_resp_info.result_obj
ws_msg = WebsocketMessage.get_success_message(\
grpc_call_name,
'it worked',
msg_cnt=msg_cnt,
data=stored_resp.as_dict())
print('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(websocket_id)
StoredResponse.mark_as_read(stored_resp)
# -----------------------------------------------
print('msg received #%d' % msg_cnt)
except grpc.RpcError as err_obj:
StoredRequestUtil.set_error_status(\
stored_request_id,
str(err_obj))
return
#except Exception as err_obj:
# StoredRequestUtil.set_error_status(\
# stored_request_id,
# str(err_obj))
# return
StoredRequestUtil.set_finished_ok_status(stored_request_id)
def run_describe_solution(self, pipeline_id, solution_id, msg_cnt=-1):
"""sync: Run a DescribeSolution call for each solution_id"""
print(f'run_describe_solution 1. pipeline_id: {pipeline_id}')
# ----------------------------------
# Create the input
# ----------------------------------
req_params = {ta2_static.KEY_SOLUTION_ID: solution_id}
json_str_info = json_dumps(req_params)
if not json_str_info.success:
self.add_err_msg(json_str_info.err_msg)
return
json_str_input = json_str_info.result_obj
# --------------------------------
# (2) Save request
# --------------------------------
stored_request = StoredRequest(\
user=self.user_object,
search_id=self.search_id,
pipeline_id=pipeline_id,
workspace='(not specified)',
request_type=ta2_static.DESCRIBE_SOLUTION,
is_finished=False,
request=req_params)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.DESCRIBE_SOLUTION,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SUMMARIZATION,
other=req_params)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
print(
f'run_describe_solution 2. stored_request.pipeline_id: {stored_request.pipeline_id}'
)
# ----------------------------------
# Run Describe Solution
# ----------------------------------
describe_info = describe_solution(json_str_input)
if not describe_info.success:
self.add_err_msg(describe_info.err_msg)
StoredResponse.add_err_response(\
stored_request,
describe_info.err_msg)
return
# ----------------------------------
# Parse the DescribeSolutionResponse
# ----------------------------------
describe_data_info = json_loads(describe_info.result_obj)
if not describe_data_info.success:
self.add_err_msg(describe_data_info.err_msg)
StoredResponse.add_err_response(\
stored_request,
describe_data_info.err_msg)
return
# -----------------------------------------------
# Add the pipline id to the result
# -----------------------------------------------
describe_data = describe_data_info.result_obj
describe_data[ta2_static.KEY_PIPELINE_ID] = pipeline_id
describe_data[ta2_static.KEY_SEARCH_ID] = self.search_id
describe_data[ta2_static.KEY_SOLUTION_ID] = solution_id
describe_data.move_to_end(ta2_static.KEY_PIPELINE_ID, last=False)
# params = dict()
# if not stored_request.pipeline_id:
# params['pipeline_id'] = describe_data[KEY_PIPELINE_ID]
stored_info = StoredResponse.add_success_response(\
stored_request,
describe_data,
pipeline_id=pipeline_id)
if not stored_info.success:
print('stored info fail!', stored_info.err_msg)
print(
f'run_describe_solution 3. stored_info.result_obj.pipeline_id: {stored_info.result_obj.pipeline_id}'
)
print(
f'run_describe_solution 4. stored_request.pipeline_id: {stored_request.pipeline_id}'
)
# -----------------------------------------
# Tracking this in the behavioral log,
# e.g. checking time lapse between creation
# of solution and if user investigates this model,
# later, if at all
# -----------------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.DESCRIBE_SOLUTION_RESPONSE,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=describe_data)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# -----------------------------------------------
# send responses back to WebSocket
# ---------------------------------------------
ws_msg = WebsocketMessage.get_success_message(\
'DescribeSolution',
'it worked',
msg_cnt=msg_cnt,
data=describe_data)
print('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(self.websocket_id)
def run_get_search_solution_results(self):
"""Run SearchSolutions against a TA2"""
# -----------------------------------
# (1) make GRPC request object
# -----------------------------------
params_dict = dict(searchId=self.search_id)
params_info = json_dumps(params_dict)
if not params_info.success:
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
params_info.err_msg)
return
try:
grpc_req = Parse(params_info.result_obj,
core_pb2.GetSearchSolutionsResultsRequest())
except ParseError as err_obj:
err_msg = ('GetSearchSolutionsResultsRequest: Failed to'
' convert JSON to gRPC: %s') % (err_obj)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
params_info.err_msg)
return
# --------------------------------
# (2) Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=self.user_object,
search_id=self.search_id,
workspace='(not specified)',
request_type=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
is_finished=False,
request=params_dict)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=params_dict)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# --------------------------------
# (3) Make the gRPC request
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# Note: The StoredResponse.id becomes the pipeline id
# -----------------------------------------
for reply in core_stub.GetSearchSolutionsResults(\
grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
# -----------------------------------------------
# Parse the response into JSON + store response
# -----------------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
if not msg_json_info.success:
user_msg = 'Failed to convert response to JSON: %s' % \
msg_json_info.err_msg
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
# Wait for next response....
continue
result_json = msg_json_info.result_obj
# TA2s (specifically NYU) responds once when trying a new pipeline, with a message missing a solutionId
# the same process responds again once the solution contains a solutionId
print('results json from TA2')
print(result_json)
if not result_json.get('solutionId'):
continue
if ta2_static.KEY_SOLUTION_ID not in result_json:
user_msg = '"%s" not found in response to JSON: %s' % \
(ta2_static.KEY_SOLUTION_ID, result_json)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
# Wait for next response....
continue
# Solution id used for DescribeSolution...
#
solution_id = result_json[ta2_static.KEY_SOLUTION_ID]
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_stream_success_response(\
stored_request, result_json)
# -----------------------------------------
# Tracking this in the behavioral log,
# e.g. checking time lapse between creation
# of solution and if user investigates this model,
# later, if at all
# -----------------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.
GET_SEARCH_SOLUTIONS_RESULTS_RESPONSE,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=result_json)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
user_msg = 'Failed to store response from %s: %s' % \
(ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
msg_json_info.err_msg)
StoredResponse.add_stream_err_response(\
stored_response, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
user_msg)
# Wait for the next response...
continue
# ---------------------------------------------
# Looks good! Get the StoredResponse
# - This id will be used as the pipeline id
# ---------------------------------------------
stored_response = stored_resp_info.result_obj
stored_response.use_id_as_pipeline_id()
StoredResponse.add_stream_success_response(\
stored_response, stored_response)
# -----------------------------------------------
# send responses back to WebSocket
# ---------------------------------------------
ws_msg = WebsocketMessage.get_success_message(\
ta2_static.GET_SEARCH_SOLUTIONS_RESULTS,
'it worked',
msg_cnt=msg_cnt,
data=stored_response.as_dict())
print('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(self.websocket_id)
stored_response.mark_as_sent_to_user()
print('msg received #%d' % msg_cnt)
# -----------------------------------------------
# continue the process describe/score/etc
# -----------------------------------------------
# DescribeSolution - run sync
#
self.run_describe_solution(stored_response.pipeline_id,
solution_id, msg_cnt)
# FitSolution - run async
#
print('PRE run_fit_solution')
self.run_fit_solution(stored_response.pipeline_id, solution_id)
print('POST run_fit_solution')
print('PRE run_score_solution')
self.run_score_solution(stored_response.pipeline_id,
solution_id)
print('POST run_score_solution')
# -----------------------------------------------
# All results arrived, send message to UI
# -----------------------------------------------
ws_msg = WebsocketMessage.get_success_message( \
ta2_static.ENDGetSearchSolutionsResults,
{'searchId': self.search_id, 'message': 'it worked'})
print('ws_msg: %s' % ws_msg)
ws_msg.send_message(self.websocket_id)
except grpc.RpcError as err_obj:
stored_request.set_error_status(str(err_obj))
return
except Exception as err_obj:
stored_request.set_error_status(str(err_obj))
return
StoredRequestUtil.set_finished_ok_status(stored_request.id)
def make_search_solutions_call(all_params, websocket_id, user_id,
**kwargs):
"""Return the result of a SearchSolutions call.
If successful, an async process is kicked off"""
if not websocket_id:
return err_resp('websocket_id must be set')
print('make_search_solutions_call 1')
param_check = SearchSolutionsHelper.check_params(all_params)
if not param_check.success:
return param_check
print('make_search_solutions_call 2')
try:
user_obj = User.objects.get(pk=user_id)
except User.DoesNotExist:
user_msg = 'No user found for id: %s' % user_id
return err_resp(user_msg)
search_solution_params = all_params[
ta2_static.KEY_SEARCH_SOLUTION_PARAMS]
# --------------------------------
# (2) Logging
# --------------------------------
stored_request = StoredRequest(\
user=user_obj,
# search_id=self.search_id,
workspace='(not specified)',
request_type=ta2_static.SEARCH_SOLUTIONS,
is_finished=False,
request=search_solution_params)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
session_key = kwargs.get(SESSION_KEY, None)
log_data = dict(session_key=session_key,
feature_id=ta2_static.SEARCH_SOLUTIONS,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SEARCH,
other=search_solution_params)
LogEntryMaker.create_ta2ta3_entry(user_obj, log_data)
# 11/6/2019 - late night hack, these variables shouldn't be here
# - introduced somewhere in the .js when setting a problem
#
search_solution_params.pop('id', None)
search_solution_params.pop('session_key', None)
# Run SearchSolutions against the TA2
#
search_info = search_solutions(search_solution_params)
if not search_info.success:
StoredResponse.add_err_response(stored_request,
search_info.err_msg)
return search_info
print('make_search_solutions_call 2')
search_info_json = json_loads(search_info.result_obj)
if not search_info_json.success:
StoredResponse.add_err_response(stored_request,
search_info_json.err_msg)
return search_info_json
search_info_data = search_info_json.result_obj
print('search_info_data', json_dumps(search_info_data)[1])
print('make_search_solutions_call 3')
if not ta2_static.KEY_SEARCH_ID in search_info_data:
user_msg = 'searchId not found in the SearchSolutionsResponse'
StoredResponse.add_err_response(stored_request, user_msg)
return err_resp(user_msg)
search_id = search_info_data['searchId']
StoredResponse.add_success_response(stored_request,
search_info_data,
search_id=search_id)
# Async task to run GetSearchSolutionsResults
#
extra_params = {SESSION_KEY: session_key}
SearchSolutionsHelper.kick_off_solution_results.delay(\
search_id, websocket_id, user_id,
all_search_params=all_params,
**extra_params)
# Back to the UI, looking good
#
return ok_resp(search_info_data)
def run_get_score_solution_responses(self, request_id):
"""(2) Run GetScoreSolutionResults"""
if self.has_error():
return
if not request_id:
self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS,
'request_id must be set')
return
# -----------------------------------
# (1) make GRPC request object
# -----------------------------------
params_dict = {ta2_static.KEY_REQUEST_ID: request_id}
params_info = json_dumps(params_dict)
try:
grpc_req = Parse(params_info.result_obj,
core_pb2.GetScoreSolutionResultsRequest())
except ParseError as err_obj:
err_msg = ('Failed to convert JSON to gRPC: %s') % (err_obj)
self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS,
err_msg)
return
# --------------------------------
# (2) Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=self.user_object,
request_type=ta2_static.GET_SCORE_SOLUTION_RESULTS,
search_id=self.search_id,
pipeline_id=self.pipeline_id,
is_finished=False,
request=params_dict)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.GET_SCORE_SOLUTION_RESULTS,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SUMMARIZATION,
other=params_dict)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# --------------------------------
# (3) Make the gRPC request
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
msg_cnt = 0
try:
# -----------------------------------------
# Iterate through the streaming responses
# Note: The StoredResponse.id becomes the pipeline id
# -----------------------------------------
for reply in core_stub.GetScoreSolutionResults(\
grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):
msg_cnt += 1
stored_response = None # to hold a StoredResponse object
# -----------------------------------------------
# Parse the response into JSON + store response
# -----------------------------------------------
msg_json_str = message_to_json(reply)
msg_json_info = json_loads(msg_json_str)
if not msg_json_info.success:
err_msg = ('Failed to convert JSON to gRPC: %s') % \
(err_obj,)
StoredResponse.add_stream_err_response(
stored_request, user_msg)
self.send_websocket_err_msg(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
err_msg)
# Wait for next response....
continue
result_json = msg_json_info.result_obj
# -----------------------------------------
# Looks good, save the response
# -----------------------------------------
stored_resp_info = StoredResponse.add_stream_success_response(\
stored_request, result_json)
# -----------------------------------------
# Make sure the response was saved (probably won't happen)
# -----------------------------------------
if not stored_resp_info.success:
# Not good but probably won't happen
# send a message to the user...
#
self.send_websocket_err_msg(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
stored_resp_info.err_msg)
#
StoredResponse.add_stream_err_response(\
stored_request, stored_resp_info.err_msg)
#
continue
# ---------------------------------------------
# Looks good! Get the StoredResponse
# - send responses back to WebSocket
# ---------------------------------------------
stored_response = stored_resp_info.result_obj
stored_response.set_pipeline_id(self.pipeline_id)
# ---------------------------------------------
# If progress is complete,
# send response back to WebSocket
# ---------------------------------------------
progress_val = get_dict_value(\
result_json,
[ta2_static.KEY_PROGRESS,
ta2_static.KEY_PROGRESS_STATE])
if (not progress_val.success) or \
(progress_val.result_obj != ta2_static.KEY_PROGRESS_COMPLETED):
user_msg = 'GetScoreSolutionResultsResponse is not yet complete'
LOGGER.info(user_msg)
# wait for next message...
continue
ws_msg = WebsocketMessage.get_success_message(\
ta2_static.GET_SCORE_SOLUTION_RESULTS,
'it worked',
msg_cnt=msg_cnt,
data=stored_response.as_dict())
LOGGER.info('ws_msg: %s' % ws_msg)
#print('ws_msg', ws_msg.as_dict())
ws_msg.send_message(self.websocket_id)
# stored_response.mark_as_sent_to_user()
except grpc.RpcError as err_obj:
stored_request.set_error_status(str(err_obj))
return
except Exception as err_obj:
stored_request.set_error_status(str(err_obj))
return
StoredRequestUtil.set_finished_ok_status(stored_request.id)
def run_process(self):
"""(1) Run ScoreSolution"""
if self.has_error():
return
# ----------------------------------
# Create the input
# ----------------------------------
LOGGER.info('ScoreSolutionHelper.run_process 2')
json_str_info = json_dumps(self.score_params)
if not json_str_info.success:
self.add_err_msg(json_str_info.err_msg)
return
json_str_input = json_str_info.result_obj
# ----------------------------------
# (2) Save the request
# ----------------------------------
stored_request = StoredRequest(\
user=self.user_object,
search_id=self.search_id,
pipeline_id=self.pipeline_id,
workspace='(not specified)',
request_type=ta2_static.SCORE_SOLUTION,
is_finished=False,
request=self.score_params)
stored_request.save()
# --------------------------------
# (2a) Behavioral logging
# --------------------------------
log_data = dict(session_key=self.session_key,
feature_id=ta2_static.SCORE_SOLUTION,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_SUMMARIZATION,
other=self.score_params)
LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data)
# ----------------------------------
# Run ScoreSolution
# ----------------------------------
LOGGER.info('run ScoreSolution: %s', json_str_input)
fit_info = score_solution(json_str_input)
if not fit_info.success:
print('ScoreSolution err_msg: ', fit_info.err_msg)
StoredResponse.add_err_response(stored_request, fit_info.err_msg)
self.send_websocket_err_msg(ta2_static.SCORE_SOLUTION,
fit_info.err_msg)
return
# ----------------------------------
# Parse the ScoreSolutionResponse
# ----------------------------------
response_info = json_loads(fit_info.result_obj)
if not response_info.success:
print('ScoreSolution grpc err_msg: ', response_info.err_msg)
StoredResponse.add_err_response(stored_request,
response_info.err_msg)
self.send_websocket_err_msg(ta2_static.SCORE_SOLUTION,
response_info.err_msg)
return
result_json = response_info.result_obj
# ----------------------------------
# Get the requestId
# ----------------------------------
if not ta2_static.KEY_REQUEST_ID in result_json:
user_msg = (' "%s" not found in response to JSON: %s') % \
(ta2_static.KEY_REQUEST_ID, result_json)
StoredResponse.add_err_response(stored_request, user_msg)
self.send_websocket_err_msg(ta2_static.SCORE_SOLUTION, user_msg)
return
StoredResponse.add_success_response(stored_request, result_json)
self.run_get_score_solution_responses(
result_json[ta2_static.KEY_REQUEST_ID])
def solution_export3(user, raven_json, **kwargs):
"""
Send a SolutionExportRequest to the SolutionExport command
"""
if not isinstance(user, User):
err_msg = '"user" must be a User object'
return err_resp(err_msg)
if not isinstance(raven_json, dict):
err_msg = 'raven_dict must be a python dict'
return err_resp(err_msg)
if not ta2_static.KEY_SEARCH_ID in raven_json:
err_msg = (f'Key: "{ta2_static.KEY_SEARCH_ID}" not found in the'
f' "raven_json" dict. (solution_export3)')
return err_resp(err_msg)
search_id = raven_json.pop(
ta2_static.KEY_SEARCH_ID) # not needed for GRPC call
session_key = kwargs.get(SESSION_KEY, '')
# --------------------------------
# Convert dict to string
# --------------------------------
raven_json_info = json_dumps(raven_json)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
raven_json_str = raven_json_info.result_obj
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.SolutionExportRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp = core_pb2.SolutionExportResponse()
return ok_resp(message_to_json(resp))
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Save the request to the db
# --------------------------------
stored_request = StoredRequest(\
user=user,
search_id=search_id,
workspace='(not specified)',
request_type=ta2_static.SOLUTION_EXPORT,
is_finished=False,
request=raven_json)
stored_request.save()
# --------------------------------
# Behavioral logging
# --------------------------------
log_data = dict(session_key=session_key,
feature_id=ta2_static.SOLUTION_EXPORT,
activity_l1=bl_static.L1_MODEL_SELECTION,
activity_l2=bl_static.L2_MODEL_EXPORT,
other=raven_json)
LogEntryMaker.create_ta2ta3_entry(user, log_data)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.SolutionExport(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
user_msg = f'Error: {err_obj}'
StoredResponse.add_err_response(stored_request, user_msg)
return err_resp(user_msg)
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
resp_json_str = message_to_json(reply)
resp_json_dict_info = json_loads(resp_json_str)
if not resp_json_dict_info.success:
user_msg = (f'Failed to convert GRPC response to JSON:'
f' {resp_json_dict_info.err_msg}')
StoredResponse.add_err_response(stored_request, user_msg)
return err_resp(user_msg)
StoredResponse.add_success_response(stored_request,
resp_json_dict_info.result_obj)
return ok_resp(resp_json_str)
def search_solutions(raven_json_str=None):
"""
Send a SearchSolutionsRequest to the SearchSolutions command
"""
print('raven_json_str', raven_json_str)
if raven_json_str is None:
err_msg = 'No data found for the SearchSolutionsRequest'
return err_resp(err_msg)
# This is a dict or OrderedDict, make it a json string
#
if isinstance(raven_json_str, dict):
json_str_info = json_dumps(raven_json_str)
if not json_str_info.success:
return json_str_info
raven_json_str = json_str_info.result_obj
else:
# Make sure the string is valid JSON
#
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
print('SearchSolutionsRequest (string)', raven_json_str)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.SearchSolutionsRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
print('req', req)
print('-' * 40)
print('raven_json_str', raven_json_str)
print('-' * 40)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
search_id = 'search_id_%s' % get_alphanumeric_string(6)
resp = core_pb2.SearchSolutionsResponse(search_id=search_id)
# print('message_to_json(req)', message_to_json(resp))
return ok_resp(message_to_json(resp))
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.SearchSolutions(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return ok_resp(message_to_json(reply))
def end_search_solutions(raven_json_str=None, **kwargs):
"""
Send a EndSearchSolutionsRequest to the EndSearchSolutions command
optional kwargs:
user = User object for logging
"""
if raven_json_str is None:
err_msg = 'No data found for the EndSearchSolutionsRequest'
return err_resp(err_msg)
# --------------------------------
# Make sure it's valid JSON
# --------------------------------
raven_json_info = json_loads(raven_json_str)
if not raven_json_info.success:
return err_resp(raven_json_info.err_msg)
if not ta2_static.KEY_SEARCH_ID in raven_json_info.result_obj:
err_msg = (f'The send solutions request did not include'
f' a "{ta2_static.KEY_SEARCH_ID}" key')
return err_resp(err_msg)
search_id = raven_json_info.result_obj[ta2_static.KEY_SEARCH_ID]
# --------------------------------
# optional logging
# --------------------------------
user = kwargs.get('user')
stored_request = None
# Begin to log D3M call
#
if user:
stored_request = StoredRequest(\
user=user,
request_type=ta2_static.END_SEARCH_SOLUTIONS,
search_id=search_id,
is_finished=False,
request=raven_json_info.result_obj)
stored_request.save()
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(raven_json_str, core_pb2.EndSearchSolutionsRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return err_resp(err_msg)
# In test mode, return canned response
#
if settings.TA2_STATIC_TEST_MODE:
resp = core_pb2.EndSearchSolutionsResponse()
# print('message_to_json(req)', message_to_json(resp))
return ok_resp(message_to_json(resp))
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return err_resp(err_msg)
# --------------------------------
# Send the gRPC request
# --------------------------------
try:
reply = core_stub.EndSearchSolutions(\
req,
timeout=settings.TA2_GRPC_SHORT_TIMEOUT)
except Exception as err_obj:
return err_resp(str(err_obj))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
# This returns a JSON string
reply_json_str = message_to_json(reply)
# Double-check, make sure it converts back to a python dict
#
json_format_info = json_loads(reply_json_str)
if not json_format_info.success:
if user:
StoredResponse.add_err_response(stored_request,
json_format_info.err_msg)
return err_resp(json_format_info.err_msg)
# Looks good, save response and return value
#
if user:
StoredResponse.add_success_response(stored_request,
json_format_info.result_obj)
return ok_resp(json_format_info.result_obj)
def view_upload_dataset(request):
"""Upload dataset and metadata"""
print('FILE_UPLOAD_MAX_MEMORY_SIZE:', settings.FILE_UPLOAD_MAX_MEMORY_SIZE)
user_workspace_info = get_latest_user_workspace(request)
if not user_workspace_info.success:
return JsonResponse(get_json_error(user_workspace_info.err_msg))
user_workspace = user_workspace_info.result_obj
# Destination directory for learningData.csv, learningData#.csv, etc.
# and about.json
#
dest_dir_info = create_directory_add_timestamp(\
join(settings.TWORAVENS_USER_DATASETS_DIR,
f'uploads_{user_workspace.user.id}',
get_alpha_string(6)))
if not dest_dir_info.success:
return JsonResponse(get_json_error(dest_dir_info.err_msg))
dest_directory = dest_dir_info.result_obj
print('view_upload_dataset. dest_directory', dest_directory)
# Save the about.json
#
json_info = json_loads(request.POST.get('metadata'))
if not json_info.success:
return JsonResponse(get_json_error(json_info.err_msg))
# save json data
dataset_name = None
if dp_static.DATASET_NAME_FROM_UI in json_info.result_obj:
dataset_name = json_info.result_obj[dp_static.DATASET_NAME_FROM_UI]
#with open(os.path.join(dest_directory, 'about.json'), 'w') as metadata_file:
# json.dump(json_info.result_obj, metadata_file)
# Save data files. They don't have to be .csv, that's handled latter,
# e.g. convert from .tab, .tsv, xls, etc.
#
for idx, file in enumerate(request.FILES.getlist('files')):
print(file.name)
_fname, fext = splitext(file.name)
if not fext.lower() in dp_static.VALID_EXTENSIONS:
# no extension found, won't be able to open it
user_msg = (
f'The extension for this file was not recognized: "{file.name}".'
f' Valid extensions: {", ".join(dp_static.VALID_EXTENSIONS)}.')
return JsonResponse(get_json_error(user_msg))
new_filename = join(
dest_directory,
f'learningData{idx + 1 if idx else ""}{fext.lower()}')
with open(new_filename, 'wb+') as outfile:
for chunk in file.chunks():
outfile.write(chunk)
print('dest_directory', dest_directory)
# Create new dataset folders/etc
#
additional_inputs_dir = user_workspace.d3m_config.additional_inputs
created = create_directory(additional_inputs_dir)
if not created.success:
return JsonResponse(get_json_error(created.err_msg))
new_dataset_info = UserDatasetUtil.make_new_dataset(\
user_workspace.user.id,
dest_directory,
settings.TWORAVENS_USER_DATASETS_DIR,
**{dp_static.DATASET_NAME: dataset_name})
if not new_dataset_info.success:
return JsonResponse(get_json_error(new_dataset_info.err_msg))
#udu = UserDatasetUtil(1, input_files, output_dir)
return JsonResponse(get_json_success('file upload completed successfully'))
def augment_nyu_file(self):
"""Augment the file via NYU API"""
if self.has_error():
return False
# -----------------------------
# TEMPORARILY DISABLED
# -----------------------------
#err_msg = ('NYU augment functionality is currently disabled')
#self.add_err_msg(err_msg)
#return False
#print('augment_params', self.augment_params)
# Check for required keys and convert them python dicts
#
req_keys = ['search_result', 'left_columns', 'right_columns']
keys_not_found = []
jsonified_data = {}
for rk in req_keys:
if not rk in self.augment_params:
keys_not_found.append(rk)
else:
json_info = json_loads(self.augment_params[rk])
if not json_info.success:
user_msg = (f'Sorry! Augment failed. (The data for'
f' "{rk}" was not JSON)')
self.add_err_msg(user_msg)
return False
jsonified_data[rk] = json_info.result_obj
if keys_not_found:
user_msg = (f'Sorry! Augment failed. (These required fields'
f' weren\'t found: {req_keys})')
self.add_err_msg(user_msg)
return
# Format the task for augment submission
#
task_data = jsonified_data['search_result']
task_data['augmentation'] = {\
'type': 'join',
'left_columns': jsonified_data['left_columns'], # game id user's dataset
'right_columns': jsonified_data['right_columns'] # game id in datamart dataset
}
extra_params = dict() # none for now...
# Different params than ISI
#
augment_info = self.datamart_util.datamart_augment(\
self.user_workspace,
self.augment_params[dm_static.KEY_DATA_PATH],
task_data,
**extra_params)
if not augment_info.success:
self.add_err_msg(augment_info.err_msg)
return False
# ----------------------------------------
# Looks like the augment worked.
# It should have returned:
# - a data file path
# - a dataset doc path
# ----------------------------------------
augment_dict = augment_info.result_obj
keys_to_check = [
dm_static.KEY_DATA_PATH, dm_static.KEY_DATASET_DOC_PATH
]
for key in keys_to_check:
if key not in augment_dict:
user_msg = (f'Key "{key}" not found in the NYU augment_dict.'
f' Keys: {augment_dict.keys()}')
self.add_err_msg(user_msg)
return False
self.augment_new_filepath = augment_dict[dm_static.KEY_DATA_PATH]
self.augment_new_datasetdoc = augment_dict[
dm_static.KEY_DATASET_DOC_PATH]
return True
