def clean_query(self): """Load this as JSON, make sure that it contains at least 1 search param""" query_info_json = json_loads(self.cleaned_data.get('query')) if not query_info_json.success: user_msg = ('The search is not formatted correctly.' ' Please try again.' ' (Expected a JSON string)') raise forms.ValidationError(user_msg) query_dict = query_info_json.result_obj clear_dict(query_dict) if not query_dict: user_msg = ('The search must contain at least 1 keyword or' ' variable. Please try again. (id:1)') raise forms.ValidationError(user_msg) if not 'keywords' in query_dict: if not 'variables' in query_dict: user_msg = ('The search must contain at least 1 keyword or' ' variable. Please try again. (id:2)') raise forms.ValidationError(user_msg) return query_dict
def get_config_file_contents(d3m_config, config_key, as_dict=True): """Get contents of a file specified in the config""" if not isinstance(d3m_config, D3MConfiguration): return err_resp('d3m_config must be a D3MConfiguration object') if not config_key in D3M_FILE_ATTRIBUTES: return err_resp('config_key not found!') filepath_info = get_d3m_filepath(d3m_config, config_key) if not filepath_info.success: return err_resp(filepath_info.err_msg) fpath = filepath_info.result_obj try: with open(fpath, "r") as fh: contents = fh.read() except IOError as err_obj: user_msg = 'Failed to read file: %s\n%s' % \ (fpath, err_obj) return err_resp(user_msg) if not as_dict: return ok_resp(contents) doc_info = json_loads(contents) if not doc_info.success: return err_resp(doc_info.err_msg) return ok_resp(doc_info.result_obj)
def read_problem_doc_if_exists(self): """Verify the problem path example: "/input/TRAIN/problem_TRAIN/problemDoc.json" Note: As of 7/17/2019, it's ok if there's no problem doc """ if self.has_error(): return False if not isfile(self.env_config.D3MPROBLEMPATH): user_msg = ('D3MPROBLEMPATH file non-existent or' ' can\'t be reached: %s') % \ self.env_config.D3MPROBLEMPATH print('Note: ', user_msg) # self.add_err_msg(user_msg) return True json_content = open(self.env_config.D3MPROBLEMPATH, 'r').read() pdoc_info = json_loads(json_content) if not pdoc_info.success: user_msg = ('D3MPROBLEMPATH file not JSON. %s\nError: %s') % \ (self.env_config.D3MPROBLEMPATH, pdoc_info.err_msg) print('Note: ', user_msg) # self.add_err_msg(user_msg) return True self.problem_doc = pdoc_info.result_obj return True
def clean_search_result(self): json_info = json_loads(self.cleaned_data.get('search_result')) if not json_info.success: raise forms.ValidatonError(\ ("The 'search_result' is not valid JSON." " %s") % json_info.err_msg) return json_info.result_obj
def clean_search_result(self): """Convert search_result to a python dict""" json_info = json_loads(self.cleaned_data.get('search_result')) if not json_info.success: user_msg = (f"The 'search_result' is not valid JSON." f' {json_info.err_msg}') raise forms.ValidatonError(user_msg) return json_info.result_obj
def fit_solution(raven_json_str=None): """ Send a FitSolutionRequest to the FitSolution command """ if raven_json_str is None: err_msg = 'No data found for the FitSolutionRequest' return err_resp(err_msg) # -------------------------------- # Make sure it's valid JSON # -------------------------------- raven_json_info = json_loads(raven_json_str) if not raven_json_info.success: return err_resp(raven_json_info.err_msg) # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(raven_json_str, core_pb2.FitSolutionRequest()) except ParseError as err_obj: err_msg = ('Failed to convert JSON to gRPC: %s' ' (req_search_solutions)' '\nraven_json_str: %s') % \ (err_obj, raven_json_str) print('-' * 40) print(err_msg) return err_resp(err_msg) # In test mode, return canned response # if settings.TA2_STATIC_TEST_MODE: resp = core_pb2.FitSolutionResponse(\ request_id='requestId_%s' % get_alphanumeric_string(6)) return ok_resp(message_to_json(resp)) core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return err_resp(err_msg) # -------------------------------- # Send the gRPC request # -------------------------------- try: reply = core_stub.FitSolution(\ req, timeout=settings.TA2_GRPC_SHORT_TIMEOUT) except Exception as err_obj: return err_resp(str(err_obj)) # -------------------------------- # Convert the reply to JSON and send it back # -------------------------------- return ok_resp(message_to_json(reply))
def view_hello(request): """gRPC: Call from UI as a hearbeat""" user_info = get_authenticated_user(request) if not user_info.success: return JsonResponse(get_json_error(user_info.err_msg)) # -------------------------------- # Behavioral logging # -------------------------------- log_data = dict(session_key=get_session_key(request), feature_id=ta2_static.HELLO, activity_l1=bl_static.L1_SYSTEM_ACTIVITY, activity_l2=bl_static.L2_APP_LAUNCH) LogEntryMaker.create_ta2ta3_entry(user_info.result_obj, log_data) # note: this is just a heartbeat, so no params are sent # # Begin to log D3M call # call_entry = None if ServiceCallEntry.record_d3m_call(): call_entry = ServiceCallEntry.get_dm3_entry(\ request_obj=request, call_type='Hello', request_msg=('no params for this call')) # Let's call the TA2! # resp_info = ta2_hello() if not resp_info.success: return JsonResponse(get_json_error(resp_info.err_msg)) json_str = resp_info.result_obj # Convert JSON str to python dict - err catch here # - let it blow up for now--should always return JSON json_format_info = json_loads(json_str) if not json_format_info.success: return JsonResponse(get_json_error(json_format_info.err_msg)) # Save D3M log # if call_entry: call_entry.save_d3m_response(json_format_info.result_obj) json_info = get_json_success('success!', data=json_format_info.result_obj) return JsonResponse(json_info, safe=False)
def view_list_primitives(request): """gRPC: Call from UI with a ListPrimitivesRequest""" user_info = get_authenticated_user(request) if not user_info.success: return JsonResponse(get_json_error(user_info.err_msg)) # -------------------------------- # (2) Begin to log D3M call # -------------------------------- call_entry = None if ServiceCallEntry.record_d3m_call(): call_entry = ServiceCallEntry.get_dm3_entry(\ request_obj=request, call_type='ListPrimitives', request_msg='no params for this call') # -------------------------------- # (2a) Behavioral logging # -------------------------------- log_data = dict(session_key=get_session_key(request), feature_id=ta2_static.LIST_PRIMITIVES, activity_l1=bl_static.L1_SYSTEM_ACTIVITY, activity_l2=bl_static.L2_ACTIVITY_BLANK) LogEntryMaker.create_ta2ta3_entry(user_info.result_obj, log_data) # Let's call the TA2! # search_info = list_primitives() #print('search_info', search_info) if not search_info.success: return JsonResponse(get_json_error(search_info.err_msg)) # Convert JSON str to python dict - err catch here # json_format_info = json_loads(search_info.result_obj) if not json_format_info.success: return JsonResponse(get_json_error(json_format_info.err_msg)) # Save D3M log # if call_entry: call_entry.save_d3m_response(json_format_info.result_obj) json_info = get_json_success('success!', data=json_format_info.result_obj) return JsonResponse(json_info, safe=False)
def augment_isi_file(self): """Augment the file via the ISI API""" # self.add_err_msg(('ISI Augment is disabled!!!!' # ' (augment_util.augment_isi_file)')) # return False if self.has_error(): return False # user_workspace, data_path, search_result, left_columns, # right_columns, exact_match=False, **kwargs search_result_info = json_loads(self.augment_params['search_result']) if not search_result_info.success: err_msg = (f"Failed to load augment_params['search_result']" f" as JSON: {search_result_info.err_msg}") self.add_err_msg(err_msg) return search_result_json = search_result_info.result_obj extra_params = dict() # none for now... augment_info = self.datamart_util.datamart_augment(\ self.user_workspace, self.augment_params[dm_static.KEY_DATA_PATH], search_result_json, exact_match=self.augment_params.get('exact_match'), **extra_params) if not augment_info.success: self.add_err_msg(augment_info.err_msg) return False augment_dict = augment_info.result_obj keys_to_check = [ dm_static.KEY_DATA_PATH, dm_static.KEY_DATASET_DOC_PATH ] for key in keys_to_check: if key not in augment_dict: user_msg = (f'Key "{key}" not found in the NYU augment_dict.' f' Keys: {augment_dict.keys()}') self.add_err_msg(user_msg) return False self.augment_new_filepath = augment_dict[dm_static.KEY_DATA_PATH] self.augment_new_datasetdoc = augment_dict[ dm_static.KEY_DATASET_DOC_PATH] return True
def describe_solution(raven_json_str=None): """ Send a DescribeSolutionRequest to the DescribeSolution command """ if raven_json_str is None: err_msg = 'No data found for the DescribeSolutionRequest' return err_resp(err_msg) # -------------------------------- # Make sure it's valid JSON # -------------------------------- raven_json_info = json_loads(raven_json_str) if not raven_json_info.success: return err_resp(raven_json_info.err_msg) # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(raven_json_str, core_pb2.DescribeSolutionRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return err_resp(err_msg) # In test mode, return canned response # if settings.TA2_STATIC_TEST_MODE: resp_str = get_grpc_test_json(\ 'test_responses/DescribeSolutionResponse_ok.json', dict()) return ok_resp(resp_str) core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return err_resp(err_msg) # -------------------------------- # Send the gRPC request # -------------------------------- try: reply = core_stub.DescribeSolution(\ req, timeout=settings.TA2_GRPC_FAST_TIMEOUT) except Exception as err_obj: return err_resp(str(err_obj)) # -------------------------------- # Convert the reply to JSON and send it back # -------------------------------- return ok_resp(message_to_json(reply))
def read_file_contents(fpath, as_dict=True): """Given a valid filepath, read the file and return it. Used for smaller files""" if not isfile(fpath): return err_resp(f'File doesn\'t exist: {fpath}') try: with open(fpath, "r") as fh: contents = fh.read() except IOError as err_obj: user_msg = 'Failed to read file: %s\n%s' % \ (fpath, err_obj) return err_resp(user_msg) if not as_dict: return ok_resp(contents) return json_loads(contents)
def load_and_return_json_file(self, fpath): """Load a JSON file; assumes fpath exists and has undergone prelim checks""" assert isfile( fpath), "fpath must exist; check before using this method" json_info = None with open(fpath) as f: fcontent = f.read() json_info = json_loads(fcontent) if not json_info.success: return self.format_embed_err(ERR_CODE_FILE_INVALID_JSON, json_info.err_msg) embed_snippet = OrderedDict() embed_snippet[KEY_SUCCESS] = True embed_snippet[KEY_DATA] = json_info.result_obj return embed_snippet
def view_get_config_file(request, config_key, d3m_config_id=None): """Get contents of a file specified in the config""" if not config_key in D3M_FILE_ATTRIBUTES: user_msg = (f'Config key "{config_key}" not found!' f' (view_get_config_file)') return JsonResponse(get_json_error(user_msg)) if d3m_config_id is None: d3m_config = get_latest_d3m_config() else: d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first() if d3m_config is None: user_msg = 'Config not found!' return JsonResponse(get_json_error(user_msg)) # Make sure the config has a value. # For example the D3MPROBLEMPATH may be blank # if not getattr(d3m_config, config_key): user_msg = f'Sorry! The config does not have a "{config_key}" value!' return JsonResponse(get_json_error(user_msg)) filepath_info = get_d3m_filepath(d3m_config, config_key) if not filepath_info.success: user_msg = f'{filepath_info.err_msg} (view_get_config_file)' return JsonResponse(get_json_error(user_msg)) # Relatively small files... # response = FileResponse(open(filepath_info.result_obj, 'rb')) # return response fcontent = open(filepath_info.result_obj, 'r').read() json_info = json_loads(fcontent) if not json_info.success: user_msg = f'{json_info.err_msg} (view_get_config_file)' return JsonResponse(get_json_error(user_msg)) return JsonResponse(get_json_success(\ 'Success!', data=json_info.result_obj))
def view_hello_heartbeat(request): """Hello to TA2 with no logging. Used for testing""" # Let's call the TA2! # resp_info = ta2_hello() if not resp_info.success: return JsonResponse(get_json_error(resp_info.err_msg)) json_str = resp_info.result_obj # Convert JSON str to python dict - err catch here # - let it blow up for now--should always return JSON json_format_info = json_loads(json_str) if not json_format_info.success: return JsonResponse(get_json_error(json_format_info.err_msg)) json_info = get_json_success('success!', data=json_format_info.result_obj) return JsonResponse(json_info)
def get_num_pipeline_steps(self): """Return the number of pipeline steps in the DescribeSolutionResponse""" if not self.has_steps(): return -1 #return err_resp('(no steps found--error)') steps_info = json_loads(self.solution_response) if not steps_info.success: return -1 #return err_resp(steps_info.err_msg) steps_dict = steps_info.result_obj if not KEY_PIPELINE in steps_dict: return -1 if not KEY_STEPS in steps_dict[KEY_PIPELINE]: return -1 #return err_resp('Key "%s" not found in response' % KEY_STEPS) num_steps = len(steps_dict[KEY_PIPELINE][KEY_STEPS]) #return ok_resp(num_steps) return num_steps
def view_search_solutions(request): """gRPC: Call from UI with a SearchSolutionsRequest""" req_body_info = get_request_body(request) if not req_body_info.success: return JsonResponse(get_json_error(req_body_info.err_msg)) # Begin to log D3M call # call_entry = None if ServiceCallEntry.record_d3m_call(): call_entry = ServiceCallEntry.get_dm3_entry(\ request_obj=request, call_type=ta2_static.SEARCH_SOLUTIONS, request_msg=req_body_info.result_obj) # Let's call the TA2! # search_info = search_solutions(req_body_info.result_obj) #print('search_info', search_info) if not search_info.success: return JsonResponse(get_json_error(search_info.err_msg)) # Convert JSON str to python dict - err catch here # - let it blow up for now--should always return JSON #json_dict = json.loads(search_info.result_obj, object_pairs_hook=OrderedDict) json_format_info = json_loads(search_info.result_obj) if not json_format_info.success: return JsonResponse(get_json_error(json_format_info.err_msg)) # Save D3M log # if call_entry: call_entry.save_d3m_response(json_format_info.result_obj) json_info = get_json_success('success!', data=json_format_info.result_obj) return JsonResponse(json_info, safe=False)
def view_produce_solution(request): """gRPC: Call from UI with a ProduceSolutionRequest""" req_body_info = get_request_body(request) if not req_body_info.success: return JsonResponse(get_json_error(req_body_info.err_msg)) # Begin to log D3M call # call_entry = None if ServiceCallEntry.record_d3m_call(): call_entry = ServiceCallEntry.get_dm3_entry(\ request_obj=request, call_type='ProduceSolution', request_msg=req_body_info.result_obj) # Let's call the TA2! # search_info = produce_solution(req_body_info.result_obj) #print('search_info', search_info) if not search_info.success: return JsonResponse(get_json_error(search_info.err_msg)) # Convert JSON str to python dict - err catch here # json_format_info = json_loads(search_info.result_obj) if not json_format_info.success: return JsonResponse(get_json_error(json_format_info.err_msg)) # Save D3M log # if call_entry: call_entry.save_d3m_response(json_format_info.result_obj) json_info = get_json_success('success!', data=json_format_info.result_obj) return JsonResponse(json_info, safe=False)
def view_solution_export3(request): """gRPC: Call from UI with a SolutionExportRequest""" # Retrieve the User # user_info = get_authenticated_user(request) if not user_info.success: return JsonResponse(get_json_error(user_info.err_msg)) user = user_info.result_obj req_body_info = get_request_body_as_json(request) if not req_body_info.success: return JsonResponse(get_json_error(req_body_info.err_msg)) session_key = get_session_key(request) # Let's call the TA2! # search_info = solution_export3(user, req_body_info.result_obj, session_key=session_key) # print('search_info', search_info) if not search_info.success: return JsonResponse(get_json_error(search_info.err_msg)) # Convert JSON str to python dict - err catch here # json_format_info = json_loads(search_info.result_obj) if not json_format_info.success: return JsonResponse(get_json_error(json_format_info.err_msg)) json_info = get_json_success('success!', data=json_format_info.result_obj) return JsonResponse(json_info, safe=False)
def get_score_solutions_results(raven_json_str, user_obj, websocket_id=None): """ Send a GetScoreSolutionResultsRequest to the GetScoreSolutionResults command Expected input from raven_json_str: {requestId: scoreId, pipelineId: response1.id} """ if user_obj is None: return err_resp("The user_obj cannot be None") if not raven_json_str: err_msg = 'No data found for the GetScoreSolutionResultsRequest' return err_resp(err_msg) # -------------------------------- # Make sure it's valid JSON # -------------------------------- raven_json_info = json_loads(raven_json_str) if not raven_json_info.success: return err_resp(raven_json_info.err_msg) # -------------------------------- # convert the JSON string to a gRPC request # Done for error checking; call repeated in celery task # -------------------------------- try: req = Parse(raven_json_str, core_pb2.GetScoreSolutionResultsRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return err_resp(err_msg) # -------------------------------- # Save the request to the db # -------------------------------- stored_request = StoredRequest(\ user=user_obj, workspace='(not specified)', request_type='GetScoreSolutionResults', is_finished=False, request=raven_json_info.result_obj) stored_request.save() # In test mode, return canned response # if settings.TA2_STATIC_TEST_MODE: resp_str = get_grpc_test_json(\ 'test_responses/GetScoreSolutionResultsResponse_ok.json', dict()) resp_info = json_loads(resp_str) if not resp_info.success: return err_resp(resp_info.err_msg) # Save the stored response # StoredResponse.add_response(\ stored_request.id, response=resp_info.result_obj) StoredRequestUtil.set_finished_ok_status(stored_request.id) # Return the stored **request** (not response) # return ok_resp(stored_request.as_dict()) stream_and_store_results.delay(raven_json_str, stored_request.id, 'core_pb2.GetScoreSolutionResultsRequest', 'GetScoreSolutionResults', websocket_id=websocket_id) return ok_resp(stored_request.as_dict())
def stream_and_store_results(raven_json_str, stored_request_id, grpc_req_obj_name, grpc_call_name, **kwargs): """Make the grpc call which has a streaming response grpc_req_obj_name: "core_pb2.GetSearchSolutionsResultsRequest", etc grpc_call_name: "GetSearchSolutionsResults", etc """ core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: StoredRequestUtil.set_error_status(stored_request_id, err_msg) return # optional: used to stream messages back to client via channels # websocket_id = kwargs.get('websocket_id', None) # grpc_req_obj = eval(grpc_req_obj_name) grpc_rpc_call_function = eval('core_stub.%s' % grpc_call_name) # -------------------------------- # convert the JSON string to a gRPC request # Yes: done for the 2nd time # -------------------------------- try: req = Parse(raven_json_str, grpc_req_obj()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) StoredRequestUtil.set_error_status(stored_request_id, err_msg) return # -------------------------------- # Send the gRPC request # -------------------------------- msg_cnt = 0 try: # ----------------------------------------- # Iterate through the streaming responses # ----------------------------------------- for reply in grpc_rpc_call_function(\ req, timeout=settings.TA2_GRPC_LONG_TIMEOUT): msg_cnt += 1 stored_resp = None # to hold a StoredResponse object # ----------------------------------------- # parse the response # ----------------------------------------- msg_json_str = message_to_json(reply) msg_json_info = json_loads(msg_json_str) # ----------------------------------------- # does it look ok? # ----------------------------------------- if not msg_json_info.success: print('PROBLEM HERE TO LOG!') user_msg = 'failed to store response: %s' % \ msg_json_info.err_msg ws_msg = WebsocketMessage.get_fail_message(\ grpc_call_name, user_msg, msg_cnt=msg_cnt) ws_msg.send_message(websocket_id) continue # ----------------------------------------- # Looks good, save the response # ----------------------------------------- stored_resp_info = StoredResponse.add_response(\ stored_request_id, response=msg_json_info.result_obj) # ----------------------------------------- # Make sure the response was saved (probably won't happen) # ----------------------------------------- if not stored_resp_info.success: # Not good but probably won't happen # send a message to the user... # user_msg = 'failed to store response: %s' % \ msg_json_info.err_msg ws_msg = WebsocketMessage.get_fail_message(\ grpc_call_name, user_msg, msg_cnt=msg_cnt) ws_msg.send_message(websocket_id) # Wait for the next response... continue # ----------------------------------------------- # send responses back to any open WebSockets # --------------------------------------------- if websocket_id: stored_resp = stored_resp_info.result_obj ws_msg = WebsocketMessage.get_success_message(\ grpc_call_name, 'it worked', msg_cnt=msg_cnt, data=stored_resp.as_dict()) print('ws_msg: %s' % ws_msg) #print('ws_msg', ws_msg.as_dict()) ws_msg.send_message(websocket_id) StoredResponse.mark_as_read(stored_resp) # ----------------------------------------------- print('msg received #%d' % msg_cnt) except grpc.RpcError as err_obj: StoredRequestUtil.set_error_status(\ stored_request_id, str(err_obj)) return #except Exception as err_obj: # StoredRequestUtil.set_error_status(\ # stored_request_id, # str(err_obj)) # return StoredRequestUtil.set_finished_ok_status(stored_request_id)
def run_describe_solution(self, pipeline_id, solution_id, msg_cnt=-1): """sync: Run a DescribeSolution call for each solution_id""" print(f'run_describe_solution 1. pipeline_id: {pipeline_id}') # ---------------------------------- # Create the input # ---------------------------------- req_params = {ta2_static.KEY_SOLUTION_ID: solution_id} json_str_info = json_dumps(req_params) if not json_str_info.success: self.add_err_msg(json_str_info.err_msg) return json_str_input = json_str_info.result_obj # -------------------------------- # (2) Save request # -------------------------------- stored_request = StoredRequest(\ user=self.user_object, search_id=self.search_id, pipeline_id=pipeline_id, workspace='(not specified)', request_type=ta2_static.DESCRIBE_SOLUTION, is_finished=False, request=req_params) stored_request.save() # -------------------------------- # (2a) Behavioral logging # -------------------------------- log_data = dict(session_key=self.session_key, feature_id=ta2_static.DESCRIBE_SOLUTION, activity_l1=bl_static.L1_MODEL_SELECTION, activity_l2=bl_static.L2_MODEL_SUMMARIZATION, other=req_params) LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data) print( f'run_describe_solution 2. stored_request.pipeline_id: {stored_request.pipeline_id}' ) # ---------------------------------- # Run Describe Solution # ---------------------------------- describe_info = describe_solution(json_str_input) if not describe_info.success: self.add_err_msg(describe_info.err_msg) StoredResponse.add_err_response(\ stored_request, describe_info.err_msg) return # ---------------------------------- # Parse the DescribeSolutionResponse # ---------------------------------- describe_data_info = json_loads(describe_info.result_obj) if not describe_data_info.success: self.add_err_msg(describe_data_info.err_msg) StoredResponse.add_err_response(\ stored_request, describe_data_info.err_msg) return # ----------------------------------------------- # Add the pipline id to the result # ----------------------------------------------- describe_data = describe_data_info.result_obj describe_data[ta2_static.KEY_PIPELINE_ID] = pipeline_id describe_data[ta2_static.KEY_SEARCH_ID] = self.search_id describe_data[ta2_static.KEY_SOLUTION_ID] = solution_id describe_data.move_to_end(ta2_static.KEY_PIPELINE_ID, last=False) # params = dict() # if not stored_request.pipeline_id: # params['pipeline_id'] = describe_data[KEY_PIPELINE_ID] stored_info = StoredResponse.add_success_response(\ stored_request, describe_data, pipeline_id=pipeline_id) if not stored_info.success: print('stored info fail!', stored_info.err_msg) print( f'run_describe_solution 3. stored_info.result_obj.pipeline_id: {stored_info.result_obj.pipeline_id}' ) print( f'run_describe_solution 4. stored_request.pipeline_id: {stored_request.pipeline_id}' ) # ----------------------------------------- # Tracking this in the behavioral log, # e.g. checking time lapse between creation # of solution and if user investigates this model, # later, if at all # ----------------------------------------- log_data = dict(session_key=self.session_key, feature_id=ta2_static.DESCRIBE_SOLUTION_RESPONSE, activity_l1=bl_static.L1_MODEL_SELECTION, activity_l2=bl_static.L2_MODEL_SEARCH, other=describe_data) LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data) # ----------------------------------------------- # send responses back to WebSocket # --------------------------------------------- ws_msg = WebsocketMessage.get_success_message(\ 'DescribeSolution', 'it worked', msg_cnt=msg_cnt, data=describe_data) print('ws_msg: %s' % ws_msg) #print('ws_msg', ws_msg.as_dict()) ws_msg.send_message(self.websocket_id)
def run_get_search_solution_results(self): """Run SearchSolutions against a TA2""" # ----------------------------------- # (1) make GRPC request object # ----------------------------------- params_dict = dict(searchId=self.search_id) params_info = json_dumps(params_dict) if not params_info.success: self.send_websocket_err_msg(\ ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, params_info.err_msg) return try: grpc_req = Parse(params_info.result_obj, core_pb2.GetSearchSolutionsResultsRequest()) except ParseError as err_obj: err_msg = ('GetSearchSolutionsResultsRequest: Failed to' ' convert JSON to gRPC: %s') % (err_obj) self.send_websocket_err_msg(\ ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, params_info.err_msg) return # -------------------------------- # (2) Save the request to the db # -------------------------------- stored_request = StoredRequest(\ user=self.user_object, search_id=self.search_id, workspace='(not specified)', request_type=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, is_finished=False, request=params_dict) stored_request.save() # -------------------------------- # (2a) Behavioral logging # -------------------------------- log_data = dict(session_key=self.session_key, feature_id=ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, activity_l1=bl_static.L1_MODEL_SELECTION, activity_l2=bl_static.L2_MODEL_SEARCH, other=params_dict) LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data) # -------------------------------- # (3) Make the gRPC request # -------------------------------- core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return err_resp(err_msg) msg_cnt = 0 try: # ----------------------------------------- # Iterate through the streaming responses # Note: The StoredResponse.id becomes the pipeline id # ----------------------------------------- for reply in core_stub.GetSearchSolutionsResults(\ grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT): msg_cnt += 1 # ----------------------------------------------- # Parse the response into JSON + store response # ----------------------------------------------- msg_json_str = message_to_json(reply) msg_json_info = json_loads(msg_json_str) if not msg_json_info.success: user_msg = 'Failed to convert response to JSON: %s' % \ msg_json_info.err_msg self.send_websocket_err_msg(\ ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, user_msg) StoredResponse.add_stream_err_response(\ stored_response, user_msg) # Wait for next response.... continue result_json = msg_json_info.result_obj # TA2s (specifically NYU) responds once when trying a new pipeline, with a message missing a solutionId # the same process responds again once the solution contains a solutionId print('results json from TA2') print(result_json) if not result_json.get('solutionId'): continue if ta2_static.KEY_SOLUTION_ID not in result_json: user_msg = '"%s" not found in response to JSON: %s' % \ (ta2_static.KEY_SOLUTION_ID, result_json) StoredResponse.add_stream_err_response(\ stored_response, user_msg) self.send_websocket_err_msg(\ ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, user_msg) # Wait for next response.... continue # Solution id used for DescribeSolution... # solution_id = result_json[ta2_static.KEY_SOLUTION_ID] # ----------------------------------------- # Looks good, save the response # ----------------------------------------- stored_resp_info = StoredResponse.add_stream_success_response(\ stored_request, result_json) # ----------------------------------------- # Tracking this in the behavioral log, # e.g. checking time lapse between creation # of solution and if user investigates this model, # later, if at all # ----------------------------------------- log_data = dict(session_key=self.session_key, feature_id=ta2_static. GET_SEARCH_SOLUTIONS_RESULTS_RESPONSE, activity_l1=bl_static.L1_MODEL_SELECTION, activity_l2=bl_static.L2_MODEL_SEARCH, other=result_json) LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data) # ----------------------------------------- # Make sure the response was saved (probably won't happen) # ----------------------------------------- if not stored_resp_info.success: # Not good but probably won't happen # send a message to the user... # user_msg = 'Failed to store response from %s: %s' % \ (ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, msg_json_info.err_msg) StoredResponse.add_stream_err_response(\ stored_response, user_msg) self.send_websocket_err_msg(\ ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, user_msg) # Wait for the next response... continue # --------------------------------------------- # Looks good! Get the StoredResponse # - This id will be used as the pipeline id # --------------------------------------------- stored_response = stored_resp_info.result_obj stored_response.use_id_as_pipeline_id() StoredResponse.add_stream_success_response(\ stored_response, stored_response) # ----------------------------------------------- # send responses back to WebSocket # --------------------------------------------- ws_msg = WebsocketMessage.get_success_message(\ ta2_static.GET_SEARCH_SOLUTIONS_RESULTS, 'it worked', msg_cnt=msg_cnt, data=stored_response.as_dict()) print('ws_msg: %s' % ws_msg) #print('ws_msg', ws_msg.as_dict()) ws_msg.send_message(self.websocket_id) stored_response.mark_as_sent_to_user() print('msg received #%d' % msg_cnt) # ----------------------------------------------- # continue the process describe/score/etc # ----------------------------------------------- # DescribeSolution - run sync # self.run_describe_solution(stored_response.pipeline_id, solution_id, msg_cnt) # FitSolution - run async # print('PRE run_fit_solution') self.run_fit_solution(stored_response.pipeline_id, solution_id) print('POST run_fit_solution') print('PRE run_score_solution') self.run_score_solution(stored_response.pipeline_id, solution_id) print('POST run_score_solution') # ----------------------------------------------- # All results arrived, send message to UI # ----------------------------------------------- ws_msg = WebsocketMessage.get_success_message( \ ta2_static.ENDGetSearchSolutionsResults, {'searchId': self.search_id, 'message': 'it worked'}) print('ws_msg: %s' % ws_msg) ws_msg.send_message(self.websocket_id) except grpc.RpcError as err_obj: stored_request.set_error_status(str(err_obj)) return except Exception as err_obj: stored_request.set_error_status(str(err_obj)) return StoredRequestUtil.set_finished_ok_status(stored_request.id)
def make_search_solutions_call(all_params, websocket_id, user_id, **kwargs): """Return the result of a SearchSolutions call. If successful, an async process is kicked off""" if not websocket_id: return err_resp('websocket_id must be set') print('make_search_solutions_call 1') param_check = SearchSolutionsHelper.check_params(all_params) if not param_check.success: return param_check print('make_search_solutions_call 2') try: user_obj = User.objects.get(pk=user_id) except User.DoesNotExist: user_msg = 'No user found for id: %s' % user_id return err_resp(user_msg) search_solution_params = all_params[ ta2_static.KEY_SEARCH_SOLUTION_PARAMS] # -------------------------------- # (2) Logging # -------------------------------- stored_request = StoredRequest(\ user=user_obj, # search_id=self.search_id, workspace='(not specified)', request_type=ta2_static.SEARCH_SOLUTIONS, is_finished=False, request=search_solution_params) stored_request.save() # -------------------------------- # (2a) Behavioral logging # -------------------------------- session_key = kwargs.get(SESSION_KEY, None) log_data = dict(session_key=session_key, feature_id=ta2_static.SEARCH_SOLUTIONS, activity_l1=bl_static.L1_MODEL_SELECTION, activity_l2=bl_static.L2_MODEL_SEARCH, other=search_solution_params) LogEntryMaker.create_ta2ta3_entry(user_obj, log_data) # 11/6/2019 - late night hack, these variables shouldn't be here # - introduced somewhere in the .js when setting a problem # search_solution_params.pop('id', None) search_solution_params.pop('session_key', None) # Run SearchSolutions against the TA2 # search_info = search_solutions(search_solution_params) if not search_info.success: StoredResponse.add_err_response(stored_request, search_info.err_msg) return search_info print('make_search_solutions_call 2') search_info_json = json_loads(search_info.result_obj) if not search_info_json.success: StoredResponse.add_err_response(stored_request, search_info_json.err_msg) return search_info_json search_info_data = search_info_json.result_obj print('search_info_data', json_dumps(search_info_data)[1]) print('make_search_solutions_call 3') if not ta2_static.KEY_SEARCH_ID in search_info_data: user_msg = 'searchId not found in the SearchSolutionsResponse' StoredResponse.add_err_response(stored_request, user_msg) return err_resp(user_msg) search_id = search_info_data['searchId'] StoredResponse.add_success_response(stored_request, search_info_data, search_id=search_id) # Async task to run GetSearchSolutionsResults # extra_params = {SESSION_KEY: session_key} SearchSolutionsHelper.kick_off_solution_results.delay(\ search_id, websocket_id, user_id, all_search_params=all_params, **extra_params) # Back to the UI, looking good # return ok_resp(search_info_data)
def run_get_score_solution_responses(self, request_id): """(2) Run GetScoreSolutionResults""" if self.has_error(): return if not request_id: self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS, 'request_id must be set') return # ----------------------------------- # (1) make GRPC request object # ----------------------------------- params_dict = {ta2_static.KEY_REQUEST_ID: request_id} params_info = json_dumps(params_dict) try: grpc_req = Parse(params_info.result_obj, core_pb2.GetScoreSolutionResultsRequest()) except ParseError as err_obj: err_msg = ('Failed to convert JSON to gRPC: %s') % (err_obj) self.send_websocket_err_msg(ta2_static.GET_SCORE_SOLUTION_RESULTS, err_msg) return # -------------------------------- # (2) Save the request to the db # -------------------------------- stored_request = StoredRequest(\ user=self.user_object, request_type=ta2_static.GET_SCORE_SOLUTION_RESULTS, search_id=self.search_id, pipeline_id=self.pipeline_id, is_finished=False, request=params_dict) stored_request.save() # -------------------------------- # (2a) Behavioral logging # -------------------------------- log_data = dict(session_key=self.session_key, feature_id=ta2_static.GET_SCORE_SOLUTION_RESULTS, activity_l1=bl_static.L1_MODEL_SELECTION, activity_l2=bl_static.L2_MODEL_SUMMARIZATION, other=params_dict) LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data) # -------------------------------- # (3) Make the gRPC request # -------------------------------- core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return err_resp(err_msg) msg_cnt = 0 try: # ----------------------------------------- # Iterate through the streaming responses # Note: The StoredResponse.id becomes the pipeline id # ----------------------------------------- for reply in core_stub.GetScoreSolutionResults(\ grpc_req, timeout=settings.TA2_GRPC_LONG_TIMEOUT): msg_cnt += 1 stored_response = None # to hold a StoredResponse object # ----------------------------------------------- # Parse the response into JSON + store response # ----------------------------------------------- msg_json_str = message_to_json(reply) msg_json_info = json_loads(msg_json_str) if not msg_json_info.success: err_msg = ('Failed to convert JSON to gRPC: %s') % \ (err_obj,) StoredResponse.add_stream_err_response( stored_request, user_msg) self.send_websocket_err_msg(\ ta2_static.GET_SCORE_SOLUTION_RESULTS, err_msg) # Wait for next response.... continue result_json = msg_json_info.result_obj # ----------------------------------------- # Looks good, save the response # ----------------------------------------- stored_resp_info = StoredResponse.add_stream_success_response(\ stored_request, result_json) # ----------------------------------------- # Make sure the response was saved (probably won't happen) # ----------------------------------------- if not stored_resp_info.success: # Not good but probably won't happen # send a message to the user... # self.send_websocket_err_msg(\ ta2_static.GET_SCORE_SOLUTION_RESULTS, stored_resp_info.err_msg) # StoredResponse.add_stream_err_response(\ stored_request, stored_resp_info.err_msg) # continue # --------------------------------------------- # Looks good! Get the StoredResponse # - send responses back to WebSocket # --------------------------------------------- stored_response = stored_resp_info.result_obj stored_response.set_pipeline_id(self.pipeline_id) # --------------------------------------------- # If progress is complete, # send response back to WebSocket # --------------------------------------------- progress_val = get_dict_value(\ result_json, [ta2_static.KEY_PROGRESS, ta2_static.KEY_PROGRESS_STATE]) if (not progress_val.success) or \ (progress_val.result_obj != ta2_static.KEY_PROGRESS_COMPLETED): user_msg = 'GetScoreSolutionResultsResponse is not yet complete' LOGGER.info(user_msg) # wait for next message... continue ws_msg = WebsocketMessage.get_success_message(\ ta2_static.GET_SCORE_SOLUTION_RESULTS, 'it worked', msg_cnt=msg_cnt, data=stored_response.as_dict()) LOGGER.info('ws_msg: %s' % ws_msg) #print('ws_msg', ws_msg.as_dict()) ws_msg.send_message(self.websocket_id) # stored_response.mark_as_sent_to_user() except grpc.RpcError as err_obj: stored_request.set_error_status(str(err_obj)) return except Exception as err_obj: stored_request.set_error_status(str(err_obj)) return StoredRequestUtil.set_finished_ok_status(stored_request.id)
def run_process(self): """(1) Run ScoreSolution""" if self.has_error(): return # ---------------------------------- # Create the input # ---------------------------------- LOGGER.info('ScoreSolutionHelper.run_process 2') json_str_info = json_dumps(self.score_params) if not json_str_info.success: self.add_err_msg(json_str_info.err_msg) return json_str_input = json_str_info.result_obj # ---------------------------------- # (2) Save the request # ---------------------------------- stored_request = StoredRequest(\ user=self.user_object, search_id=self.search_id, pipeline_id=self.pipeline_id, workspace='(not specified)', request_type=ta2_static.SCORE_SOLUTION, is_finished=False, request=self.score_params) stored_request.save() # -------------------------------- # (2a) Behavioral logging # -------------------------------- log_data = dict(session_key=self.session_key, feature_id=ta2_static.SCORE_SOLUTION, activity_l1=bl_static.L1_MODEL_SELECTION, activity_l2=bl_static.L2_MODEL_SUMMARIZATION, other=self.score_params) LogEntryMaker.create_ta2ta3_entry(self.user_object, log_data) # ---------------------------------- # Run ScoreSolution # ---------------------------------- LOGGER.info('run ScoreSolution: %s', json_str_input) fit_info = score_solution(json_str_input) if not fit_info.success: print('ScoreSolution err_msg: ', fit_info.err_msg) StoredResponse.add_err_response(stored_request, fit_info.err_msg) self.send_websocket_err_msg(ta2_static.SCORE_SOLUTION, fit_info.err_msg) return # ---------------------------------- # Parse the ScoreSolutionResponse # ---------------------------------- response_info = json_loads(fit_info.result_obj) if not response_info.success: print('ScoreSolution grpc err_msg: ', response_info.err_msg) StoredResponse.add_err_response(stored_request, response_info.err_msg) self.send_websocket_err_msg(ta2_static.SCORE_SOLUTION, response_info.err_msg) return result_json = response_info.result_obj # ---------------------------------- # Get the requestId # ---------------------------------- if not ta2_static.KEY_REQUEST_ID in result_json: user_msg = (' "%s" not found in response to JSON: %s') % \ (ta2_static.KEY_REQUEST_ID, result_json) StoredResponse.add_err_response(stored_request, user_msg) self.send_websocket_err_msg(ta2_static.SCORE_SOLUTION, user_msg) return StoredResponse.add_success_response(stored_request, result_json) self.run_get_score_solution_responses( result_json[ta2_static.KEY_REQUEST_ID])
def solution_export3(user, raven_json, **kwargs): """ Send a SolutionExportRequest to the SolutionExport command """ if not isinstance(user, User): err_msg = '"user" must be a User object' return err_resp(err_msg) if not isinstance(raven_json, dict): err_msg = 'raven_dict must be a python dict' return err_resp(err_msg) if not ta2_static.KEY_SEARCH_ID in raven_json: err_msg = (f'Key: "{ta2_static.KEY_SEARCH_ID}" not found in the' f' "raven_json" dict. (solution_export3)') return err_resp(err_msg) search_id = raven_json.pop( ta2_static.KEY_SEARCH_ID) # not needed for GRPC call session_key = kwargs.get(SESSION_KEY, '') # -------------------------------- # Convert dict to string # -------------------------------- raven_json_info = json_dumps(raven_json) if not raven_json_info.success: return err_resp(raven_json_info.err_msg) raven_json_str = raven_json_info.result_obj # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(raven_json_str, core_pb2.SolutionExportRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return err_resp(err_msg) # In test mode, return canned response # if settings.TA2_STATIC_TEST_MODE: resp = core_pb2.SolutionExportResponse() return ok_resp(message_to_json(resp)) core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return err_resp(err_msg) # -------------------------------- # Save the request to the db # -------------------------------- stored_request = StoredRequest(\ user=user, search_id=search_id, workspace='(not specified)', request_type=ta2_static.SOLUTION_EXPORT, is_finished=False, request=raven_json) stored_request.save() # -------------------------------- # Behavioral logging # -------------------------------- log_data = dict(session_key=session_key, feature_id=ta2_static.SOLUTION_EXPORT, activity_l1=bl_static.L1_MODEL_SELECTION, activity_l2=bl_static.L2_MODEL_EXPORT, other=raven_json) LogEntryMaker.create_ta2ta3_entry(user, log_data) # -------------------------------- # Send the gRPC request # -------------------------------- try: reply = core_stub.SolutionExport(\ req, timeout=settings.TA2_GRPC_SHORT_TIMEOUT) except Exception as err_obj: user_msg = f'Error: {err_obj}' StoredResponse.add_err_response(stored_request, user_msg) return err_resp(user_msg) # -------------------------------- # Convert the reply to JSON and send it back # -------------------------------- resp_json_str = message_to_json(reply) resp_json_dict_info = json_loads(resp_json_str) if not resp_json_dict_info.success: user_msg = (f'Failed to convert GRPC response to JSON:' f' {resp_json_dict_info.err_msg}') StoredResponse.add_err_response(stored_request, user_msg) return err_resp(user_msg) StoredResponse.add_success_response(stored_request, resp_json_dict_info.result_obj) return ok_resp(resp_json_str)
def search_solutions(raven_json_str=None): """ Send a SearchSolutionsRequest to the SearchSolutions command """ print('raven_json_str', raven_json_str) if raven_json_str is None: err_msg = 'No data found for the SearchSolutionsRequest' return err_resp(err_msg) # This is a dict or OrderedDict, make it a json string # if isinstance(raven_json_str, dict): json_str_info = json_dumps(raven_json_str) if not json_str_info.success: return json_str_info raven_json_str = json_str_info.result_obj else: # Make sure the string is valid JSON # raven_json_info = json_loads(raven_json_str) if not raven_json_info.success: return err_resp(raven_json_info.err_msg) print('SearchSolutionsRequest (string)', raven_json_str) # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(raven_json_str, core_pb2.SearchSolutionsRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return err_resp(err_msg) print('req', req) print('-' * 40) print('raven_json_str', raven_json_str) print('-' * 40) # In test mode, return canned response # if settings.TA2_STATIC_TEST_MODE: search_id = 'search_id_%s' % get_alphanumeric_string(6) resp = core_pb2.SearchSolutionsResponse(search_id=search_id) # print('message_to_json(req)', message_to_json(resp)) return ok_resp(message_to_json(resp)) core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return err_resp(err_msg) # -------------------------------- # Send the gRPC request # -------------------------------- try: reply = core_stub.SearchSolutions(\ req, timeout=settings.TA2_GRPC_SHORT_TIMEOUT) except Exception as err_obj: return err_resp(str(err_obj)) # -------------------------------- # Convert the reply to JSON and send it back # -------------------------------- return ok_resp(message_to_json(reply))
def end_search_solutions(raven_json_str=None, **kwargs): """ Send a EndSearchSolutionsRequest to the EndSearchSolutions command optional kwargs: user = User object for logging """ if raven_json_str is None: err_msg = 'No data found for the EndSearchSolutionsRequest' return err_resp(err_msg) # -------------------------------- # Make sure it's valid JSON # -------------------------------- raven_json_info = json_loads(raven_json_str) if not raven_json_info.success: return err_resp(raven_json_info.err_msg) if not ta2_static.KEY_SEARCH_ID in raven_json_info.result_obj: err_msg = (f'The send solutions request did not include' f' a "{ta2_static.KEY_SEARCH_ID}" key') return err_resp(err_msg) search_id = raven_json_info.result_obj[ta2_static.KEY_SEARCH_ID] # -------------------------------- # optional logging # -------------------------------- user = kwargs.get('user') stored_request = None # Begin to log D3M call # if user: stored_request = StoredRequest(\ user=user, request_type=ta2_static.END_SEARCH_SOLUTIONS, search_id=search_id, is_finished=False, request=raven_json_info.result_obj) stored_request.save() # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(raven_json_str, core_pb2.EndSearchSolutionsRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return err_resp(err_msg) # In test mode, return canned response # if settings.TA2_STATIC_TEST_MODE: resp = core_pb2.EndSearchSolutionsResponse() # print('message_to_json(req)', message_to_json(resp)) return ok_resp(message_to_json(resp)) core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return err_resp(err_msg) # -------------------------------- # Send the gRPC request # -------------------------------- try: reply = core_stub.EndSearchSolutions(\ req, timeout=settings.TA2_GRPC_SHORT_TIMEOUT) except Exception as err_obj: return err_resp(str(err_obj)) # -------------------------------- # Convert the reply to JSON and send it back # -------------------------------- # This returns a JSON string reply_json_str = message_to_json(reply) # Double-check, make sure it converts back to a python dict # json_format_info = json_loads(reply_json_str) if not json_format_info.success: if user: StoredResponse.add_err_response(stored_request, json_format_info.err_msg) return err_resp(json_format_info.err_msg) # Looks good, save response and return value # if user: StoredResponse.add_success_response(stored_request, json_format_info.result_obj) return ok_resp(json_format_info.result_obj)
def view_upload_dataset(request): """Upload dataset and metadata""" print('FILE_UPLOAD_MAX_MEMORY_SIZE:', settings.FILE_UPLOAD_MAX_MEMORY_SIZE) user_workspace_info = get_latest_user_workspace(request) if not user_workspace_info.success: return JsonResponse(get_json_error(user_workspace_info.err_msg)) user_workspace = user_workspace_info.result_obj # Destination directory for learningData.csv, learningData#.csv, etc. # and about.json # dest_dir_info = create_directory_add_timestamp(\ join(settings.TWORAVENS_USER_DATASETS_DIR, f'uploads_{user_workspace.user.id}', get_alpha_string(6))) if not dest_dir_info.success: return JsonResponse(get_json_error(dest_dir_info.err_msg)) dest_directory = dest_dir_info.result_obj print('view_upload_dataset. dest_directory', dest_directory) # Save the about.json # json_info = json_loads(request.POST.get('metadata')) if not json_info.success: return JsonResponse(get_json_error(json_info.err_msg)) # save json data dataset_name = None if dp_static.DATASET_NAME_FROM_UI in json_info.result_obj: dataset_name = json_info.result_obj[dp_static.DATASET_NAME_FROM_UI] #with open(os.path.join(dest_directory, 'about.json'), 'w') as metadata_file: # json.dump(json_info.result_obj, metadata_file) # Save data files. They don't have to be .csv, that's handled latter, # e.g. convert from .tab, .tsv, xls, etc. # for idx, file in enumerate(request.FILES.getlist('files')): print(file.name) _fname, fext = splitext(file.name) if not fext.lower() in dp_static.VALID_EXTENSIONS: # no extension found, won't be able to open it user_msg = ( f'The extension for this file was not recognized: "{file.name}".' f' Valid extensions: {", ".join(dp_static.VALID_EXTENSIONS)}.') return JsonResponse(get_json_error(user_msg)) new_filename = join( dest_directory, f'learningData{idx + 1 if idx else ""}{fext.lower()}') with open(new_filename, 'wb+') as outfile: for chunk in file.chunks(): outfile.write(chunk) print('dest_directory', dest_directory) # Create new dataset folders/etc # additional_inputs_dir = user_workspace.d3m_config.additional_inputs created = create_directory(additional_inputs_dir) if not created.success: return JsonResponse(get_json_error(created.err_msg)) new_dataset_info = UserDatasetUtil.make_new_dataset(\ user_workspace.user.id, dest_directory, settings.TWORAVENS_USER_DATASETS_DIR, **{dp_static.DATASET_NAME: dataset_name}) if not new_dataset_info.success: return JsonResponse(get_json_error(new_dataset_info.err_msg)) #udu = UserDatasetUtil(1, input_files, output_dir) return JsonResponse(get_json_success('file upload completed successfully'))
def augment_nyu_file(self): """Augment the file via NYU API""" if self.has_error(): return False # ----------------------------- # TEMPORARILY DISABLED # ----------------------------- #err_msg = ('NYU augment functionality is currently disabled') #self.add_err_msg(err_msg) #return False #print('augment_params', self.augment_params) # Check for required keys and convert them python dicts # req_keys = ['search_result', 'left_columns', 'right_columns'] keys_not_found = [] jsonified_data = {} for rk in req_keys: if not rk in self.augment_params: keys_not_found.append(rk) else: json_info = json_loads(self.augment_params[rk]) if not json_info.success: user_msg = (f'Sorry! Augment failed. (The data for' f' "{rk}" was not JSON)') self.add_err_msg(user_msg) return False jsonified_data[rk] = json_info.result_obj if keys_not_found: user_msg = (f'Sorry! Augment failed. (These required fields' f' weren\'t found: {req_keys})') self.add_err_msg(user_msg) return # Format the task for augment submission # task_data = jsonified_data['search_result'] task_data['augmentation'] = {\ 'type': 'join', 'left_columns': jsonified_data['left_columns'], # game id user's dataset 'right_columns': jsonified_data['right_columns'] # game id in datamart dataset } extra_params = dict() # none for now... # Different params than ISI # augment_info = self.datamart_util.datamart_augment(\ self.user_workspace, self.augment_params[dm_static.KEY_DATA_PATH], task_data, **extra_params) if not augment_info.success: self.add_err_msg(augment_info.err_msg) return False # ---------------------------------------- # Looks like the augment worked. # It should have returned: # - a data file path # - a dataset doc path # ---------------------------------------- augment_dict = augment_info.result_obj keys_to_check = [ dm_static.KEY_DATA_PATH, dm_static.KEY_DATASET_DOC_PATH ] for key in keys_to_check: if key not in augment_dict: user_msg = (f'Key "{key}" not found in the NYU augment_dict.' f' Keys: {augment_dict.keys()}') self.add_err_msg(user_msg) return False self.augment_new_filepath = augment_dict[dm_static.KEY_DATA_PATH] self.augment_new_datasetdoc = augment_dict[ dm_static.KEY_DATASET_DOC_PATH] return True