def retrieve_examine_workspace(self): """Was the workspace set. If not, see if it can be retrieved""" if self.has_error(): return # If the workspace is set, make sure it's the right type # if self.user_workspace: if not isinstance(self.user_workspace, UserWorkspace): self.add_err_msg(('user_workspace must be a UserWorkspace' ' object or None')) return # Looks like we have a user workspace! # If there's not a user workspace, see if we can get it # via the request_obj # if self.request_obj and not self.user_workspace: ws_info = ws_util.get_latest_user_workspace(self.request_obj) if ws_info.success: # Got one! self.user_workspace = ws_info.result_obj # Also use the d3m_config from it # This overrides a manually set d3m_config self.d3m_config = self.user_workspace.d3m_config # See if there's a default d3m_config # if not self.d3m_config: self.d3m_config = get_latest_d3m_config()
def view_pebbles_home(request): """Serve up the workspace, the current home page. Include global js settings""" app_config = AppConfiguration.get_config() if app_config is None: return HttpResponseRedirect(reverse('view_no_domain_config_error')) # Is this D3M Mode? If so, make sure: # (1) there is D3M config information # (2) user is logged in # if app_config.is_d3m_domain(): # (1) Is there a valid D3M config? d3m_config = get_latest_d3m_config() if not d3m_config: return HttpResponseRedirect(\ reverse('view_d3m_config_error')) # (2) Is the user authenticated? if not request.user.is_authenticated(): return HttpResponseRedirect(\ reverse('login')) session_key = get_session_key(request) dinfo = dict(title='TwoRavens', session_key=session_key, app_config=app_config.convert_to_dict()) return render(request, 'index.html', dinfo)
def view_get_problem_data_info(request, d3m_config_id=None): """Get info on train data and target files, if they exist""" if d3m_config_id is None: d3m_config = get_latest_d3m_config() else: d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first() if d3m_config is None: user_msg = 'Config not found! (view_get_problem_data_info)' return JsonResponse(get_json_error(user_msg)) is_pretty = request.GET.get('pretty', False) train_data_info = get_train_data_info(d3m_config) if not train_data_info.success: resp_dict = get_json_error(train_data_info.err_msg) else: resp_dict = get_json_success('It worked', data=train_data_info.result_obj) if is_pretty is not False: # return this as a formatted string? config_str = '<pre>%s<pre>' % \ (json.dumps(resp_dict, indent=4)) return HttpResponse(config_str) return JsonResponse(resp_dict)
def view_d3m_config_error(request): """Show this when the app is in D3M mode but there's no config info available""" # Only show this if: # (a) in D3M mode # app_config = AppConfiguration.get_config() if app_config is None: return HttpResponseRedirect(reverse('view_no_domain_config_error')) if not app_config.is_d3m_domain(): return HttpResponseRedirect(reverse('home')) # and (b) not D3M config info is in the db # d3m_config_info = get_latest_d3m_config() #get_latest_d3m_user_config_by_request(request) if d3m_config_info: return HttpResponseRedirect(reverse('home')) dinfo = dict(title='D3M configuration error') return render(request, 'content_pages/no_config_error.html', dinfo)
def view_get_problem_data_info(request, d3m_config_id=None): """Get info on train data and target files, if they exist""" if d3m_config_id is None: d3m_config = get_latest_d3m_config() else: d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first() if d3m_config is None: raise Http404('Config not found!') is_pretty = request.GET.get('pretty', False) info_dict, err_msg = get_train_data_info(d3m_config) if err_msg: resp_dict = dict(success=False, message=err_msg) else: resp_dict = dict(success=True, data=info_dict) if is_pretty is not False: # return this as a formatted string? config_str = '<pre>%s<pre>' % \ (json.dumps(resp_dict, indent=4)) return HttpResponse(config_str) return JsonResponse(resp_dict)
def delete_user_workspaces(user): """Used to reset UserWorkspace objects based on a user/problem""" if not isinstance(user, User): return err_resp('user must be a "User" object, not: "%s"' % user) d3m_config = get_latest_d3m_config() if not d3m_config: return err_resp('No default D3MConfiguration set.') params = dict(user=user) workspaces = UserWorkspace.objects.filter(**params) cnt = workspaces.count() workspaces.delete() return ok_resp('Workspaces cleared. %d deleted' % cnt)
def view_d3m_details_json_latest(request, as_eval_dict=False): """Return the "latest" D3m configuration as JSON. "latest" may be most recently added or a "default" of some kind""" is_pretty = request.GET.get('pretty', False) # Is there a default config? d3m_config = get_latest_d3m_config() if not d3m_config: raise Http404('no configs available') if is_pretty is not False: # return this as a formatted string? config_str = '<pre>%s<pre>' % \ (json.dumps(d3m_config.to_dict(as_eval_dict), indent=4)) return HttpResponse(config_str) # return as JSON! return JsonResponse(d3m_config.to_dict(as_eval_dict))
def view_get_config_file(request, config_key, d3m_config_id=None): """Get contents of a file specified in the config""" if not config_key in D3M_FILE_ATTRIBUTES: user_msg = (f'Config key "{config_key}" not found!' f' (view_get_config_file)') return JsonResponse(get_json_error(user_msg)) if d3m_config_id is None: d3m_config = get_latest_d3m_config() else: d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first() if d3m_config is None: user_msg = 'Config not found!' return JsonResponse(get_json_error(user_msg)) # Make sure the config has a value. # For example the D3MPROBLEMPATH may be blank # if not getattr(d3m_config, config_key): user_msg = f'Sorry! The config does not have a "{config_key}" value!' return JsonResponse(get_json_error(user_msg)) filepath_info = get_d3m_filepath(d3m_config, config_key) if not filepath_info.success: user_msg = f'{filepath_info.err_msg} (view_get_config_file)' return JsonResponse(get_json_error(user_msg)) # Relatively small files... # response = FileResponse(open(filepath_info.result_obj, 'rb')) # return response fcontent = open(filepath_info.result_obj, 'r').read() json_info = json_loads(fcontent) if not json_info.success: user_msg = f'{json_info.err_msg} (view_get_config_file)' return JsonResponse(get_json_error(user_msg)) return JsonResponse(get_json_success(\ 'Success!', data=json_info.result_obj))
def view_get_config_file(request, config_key, d3m_config_id=None): """Get contents of a file specified in the config""" if not config_key in D3M_FILE_ATTRIBUTES: raise Http404('config_key not found!') if d3m_config_id is None: d3m_config = get_latest_d3m_config() else: d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first() if d3m_config is None: raise Http404('Config not found!') filepath, err_msg_or_None = get_d3m_filepath(d3m_config, config_key) if err_msg_or_None is not None: return JsonResponse(dict(success=False, message=err_msg_or_None)) response = FileResponse(open(filepath, 'rb')) return response
def get_user_workspaces(user, create_if_not_found=True): """Get UserWorkspace list based on the active D3M config - This is trickier now. - An augmented workspace may have the active D3M config as the original_workspace """ if not isinstance(user, User): return err_resp('user must be a "User" object, not: "%s"' % user) d3m_config = get_latest_d3m_config() print('d3m_config', d3m_config) if not d3m_config: return err_resp('No default D3MConfiguration set.') params = dict(user=user) params = get_default_workspace_params(**params) params['d3m_config'] = d3m_config params_original_ws = dict(original_workspace__d3m_config=d3m_config) workspaces = UserWorkspace.objects.filter(\ Q(**params) | Q(**params_original_ws)) print('workspaces', workspaces) # Make sure the list has a current workspace # has_current_workspace = [ ws for ws in workspaces if ws.is_current_workspace ] if (not has_current_workspace) or (workspaces.count() == 0): if create_if_not_found: ws_info = create_new_user_workspace(user, d3m_config) if not ws_info.success: return err_resp('%s (get_user_workspaces)' %\ (ws_info.err_msg)) return ok_resp([ws_info.result_obj]) return err_resp('No workspaces found for the User and default config') return ok_resp(list(workspaces))
def get_latest_d3m_user_config(user, create_if_not_found=True, **kwargs): """Find the lastest UserWorkspace and return the attached d3m_config return_full_workspace = True : return UserWorkspace instead of D3MConfiguration """ if not isinstance(user, User): return err_resp('user must be a "User" object, not: "%s"' % user) return_full_workspace = kwargs.get('return_full_workspace', False) d3m_config = get_latest_d3m_config() if not d3m_config: return err_resp('No default D3MConfiguration set.') params = dict( user=user, is_current_workspace=True, ) params = get_default_workspace_params(**params) latest_workspace = UserWorkspace.objects.filter(**params).first() if latest_workspace: if return_full_workspace: return ok_resp(latest_workspace) return ok_resp(latest_workspace.d3m_config) if create_if_not_found: ws_info = create_new_user_workspace(user, d3m_config) if not ws_info.success: return err_resp('%s (get_latest_d3m_user_config)' %\ (ws_info.err_msg)) new_workspace = ws_info.result_obj if return_full_workspace: return ok_resp(new_workspace) return ok_resp(new_workspace.d3m_config) return err_resp('No workspace found for the User and default config')
def execute_pipeline(info_str=None): """Ask a TA2 to ListPipelines via gRPC This call is a bit different b/c it writes part of the data to a file and places that file uri into the original request Success: (updated request str, grpc json response) Failure: (None, error message) """ if info_str is None: info_str = get_test_info_str() if info_str is None: err_msg = 'UI Str for PipelineListResult is None' return None, get_failed_precondition_response(err_msg) if info_str.find(VAL_DATA_URI) == -1: err_msg = ('Expected to see place holder for file uri.' ' Placeholder is "%s"') % VAL_DATA_URI return None, get_failed_precondition_response(err_msg) d3m_config = get_latest_d3m_config() if not d3m_config: err_msg = ('The D3M configuration is not available.' ' Therefore, there is no "temp_storage_root" directory to' ' write the data.') return None, get_failed_precondition_response(err_msg) # -------------------------------- # Is this valid JSON? # -------------------------------- try: info_dict = json.loads(info_str, object_pairs_hook=OrderedDict) except json.decoder.JSONDecodeError as err_obj: err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj) return None, get_failed_precondition_response(err_msg) if not KEY_DATA in info_dict: err_msg = ('The JSON request did not contain a "%s" key.') % KEY_DATA return None, get_failed_precondition_response(err_msg) file_uri, err_msg = write_data_for_execute_pipeline( d3m_config, info_dict[KEY_DATA]) if err_msg is not None: return None, get_failed_precondition_response(err_msg) # Reformat the original content # # (1) remove the data key if KEY_DATA in info_dict: del info_dict[KEY_DATA] # (2) convert it back to a JSON string info_str = json.dumps(info_dict) # (3) replace the VAL_DATA_URI with the file_uri info_str_formatted = info_str.replace(VAL_DATA_URI, file_uri) # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(info_str_formatted, core_pb2.PipelineExecuteRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return None, get_failed_precondition_response(err_msg) if settings.TA2_STATIC_TEST_MODE: #return info_str_formatted,\ # get_grpc_test_json('test_responses/execute_results_1pipe_ok.json', # dict()) #--- template_info = get_predict_file_info_dict() template_str = get_grpc_test_json( 'test_responses/execute_results_1pipe_ok.json', template_info) # These next lines embed file uri content into the JSON embed_util = FileEmbedUtil(template_str) if embed_util.has_error: return get_failed_precondition_response(embed_util.error_message) test_note = ('Test. An actual result would be the test JSON with' ' the "data" section removed and DATA_URI replaced' ' with a file path to where the "data" section was' ' written.') return json.dumps(dict(note=test_note)), embed_util.get_final_results() #--- #return info_str_formatted,\ # get_grpc_test_json('test_responses/execute_results_1pipe_ok.json', # dict()) # -------------------------------- # Get the connection, return an error if there are channel issues # -------------------------------- core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return None, get_failed_precondition_response(err_msg) # -------------------------------- # Send the gRPC request - returns a stream # -------------------------------- try: reply = core_stub.ExecutePipeline(req) except Exception as ex: return None, get_failed_precondition_response(str(ex)) # -------------------------------- # Convert the reply to JSON and send it on # -------------------------------- results = map(MessageToJson, reply) result_str = '[' + ', '.join(results) + ']' embed_util = FileEmbedUtil(result_str) if embed_util.has_error: return get_failed_precondition_response(embed_util.error_message) return info_str_formatted, embed_util.get_final_results()
def export_pipeline(info_str=None, call_entry=None): """Ask a TA2 to ExportPipeline via gRPC""" if info_str is None: info_str = get_test_info_str() if info_str is None: err_msg = 'UI Str for ExportPipeline is None' return get_failed_precondition_response(err_msg) if info_str.find(VAL_EXECUTABLE_URI) == -1: err_msg = ('Expected to see place holder for executable uri.' ' Placeholder is "%s"') % VAL_EXECUTABLE_URI return None, get_failed_precondition_response(err_msg) d3m_config = get_latest_d3m_config() if not d3m_config: err_msg = ('The D3M configuration is not available.' ' Therefore, there is no "executables_root" directory to' ' write the data.') return None, get_failed_precondition_response(err_msg) # -------------------------------- # Is this valid JSON? # -------------------------------- try: info_dict = json.loads(info_str, object_pairs_hook=OrderedDict) except json.decoder.JSONDecodeError as err_obj: err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj) return get_failed_precondition_response(err_msg) # -------------------------------- # Construct and set a write directory for the executable # -------------------------------- # get the pipeline id pipeline_id, err_msg = get_pipeline_id(info_dict) if err_msg: return get_failed_precondition_response(err_msg) # dir = d3m_config.executables_root + pipeline_id executable_write_dir = join('file://%s' % d3m_config.executables_root, pipeline_id) # update the dict + info_str info_dict[KEY_PIPELINE_EXEC_URI] = executable_write_dir if KEY_PIPELINE_EXEC_URI_FROM_UI in info_dict: del info_dict[KEY_PIPELINE_EXEC_URI_FROM_UI] try: info_str = json.dumps(info_dict) except TypeError as ex_obj: err_msg = 'Failed to PipelineExportRequest info to JSON: %s' % ex_obj return get_failed_precondition_response(err_msg) #print('info_str', info_str) if call_entry: call_entry.request_msg = info_str # -------------------------------- # convert the JSON string to a gRPC request # -------------------------------- try: req = Parse(info_str, core_pb2.PipelineExportRequest()) except ParseError as err_obj: err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj) return get_failed_precondition_response(err_msg) if settings.TA2_STATIC_TEST_MODE: return get_grpc_test_json('test_responses/export_pipeline_ok.json', dict()) # -------------------------------- # Get the connection, return an error if there are channel issues # -------------------------------- core_stub, err_msg = TA2Connection.get_grpc_stub() if err_msg: return get_failed_precondition_response(err_msg) # -------------------------------- # Send the gRPC request - returns a stream # -------------------------------- try: reply = core_stub.ExportPipeline(req) except Exception as ex: return get_failed_precondition_response(str(ex)) # -------------------------------- # Convert the reply to JSON and send it back # -------------------------------- return MessageToJson(reply)
def get_ta2_run_command(self): """Return the Docker run command""" if self.has_error(): return self.error_message ta2_info = self.get_ta2_info() if not ta2_info: return # The new config has already been sent, so get env variables related to it d3m_config = get_latest_d3m_config() if not d3m_config: print('d3m_config not found! (get_ta2_run_command)') return env_str = d3m_config.get_docker_env_settings() if env_str is None: env_str = '' image_name = ta2_info[1] additional_options = ta2_info[2] print('INPUT', self.data_input_dir) print('OUTPUT', self.data_output_dir) # sudo is not necessary if just trying to run a local image? # If trying to download and run into an issue, then use sudo docker_cmd = ('docker run --rm' ' --name ta2_server' ' {4}' ' {2}' ' -v {0}:/input' ' -v {1}:/output' ' -v /ravens_volume:/ravens_volume' ' {3}' '').format(self.data_input_dir, self.data_output_dir, additional_options, image_name, env_str) print('docker_cmd', docker_cmd) xdocker_cmd = ('docker run --rm' ' --name ta2_server' ' -e D3MTIMEOUT=60' ' -e D3MINPUTDIR=/input' ' -e D3MOUTPUTDIR=/output' ' -e D3MRUN=ta2ta3' ' {2}' ' -v {0}:/input' ' -v {1}:/output' ' -v /ravens_volume:/ravens_volume' ' {3}' '').format(self.data_input_dir, self.data_output_dir, additional_options, image_name) return docker_cmd
def view_pebbles_home(request): """Serve up the workspace, the current home page. Include global js settings""" if not request.user.is_authenticated: return HttpResponseRedirect(reverse('login')) app_config = AppConfiguration.get_config() if app_config is None: return HttpResponseRedirect(reverse('view_no_domain_config_error')) user_info = get_authenticated_user(request) if not user_info.success: return JsonResponse(get_json_error(user_info.err_msg)) user = user_info.result_obj # Is this D3M Mode? If so, make sure: # (1) there is D3M config information # (2) user is logged in # if app_config.is_d3m_domain(): # (1) Is there a valid D3M config? d3m_config_info = get_latest_d3m_config() if not d3m_config_info: return HttpResponseRedirect(reverse('view_list_dataset_choices_html')) # return HttpResponseRedirect(reverse('view_d3m_config_error')) session_key = get_session_key(request) else: session_key = '(event-data-no-session-key)' dinfo = dict(title='TwoRavens', session_key=session_key, DEBUG=settings.DEBUG, ALLOW_SOCIAL_AUTH=settings.ALLOW_SOCIAL_AUTH, CSRF_COOKIE_NAME=settings.CSRF_COOKIE_NAME, app_config=app_config.convert_to_dict(), # TA2_STATIC_TEST_MODE=settings.TA2_STATIC_TEST_MODE, TA2_TEST_SERVER_URL=settings.TA2_TEST_SERVER_URL, # TA2_D3M_SOLVER_ENABLED=pybool_to_js(settings.TA2_D3M_SOLVER_ENABLED), TA2_WRAPPED_SOLVERS=settings.TA2_WRAPPED_SOLVERS, # TA3_GRPC_USER_AGENT=settings.TA3_GRPC_USER_AGENT, TA3TA2_API_VERSION=TA3TA2Util.get_api_version(), DISPLAY_DATAMART_UI=settings.DISPLAY_DATAMART_UI, WEBSOCKET_PREFIX=settings.WEBSOCKET_PREFIX) log_data = dict(session_key=session_key, feature_id=bl_static.FID_START_RAVENS_PEBBLES_PAGE, activity_l1=bl_static.L1_DATA_PREPARATION, activity_l2=bl_static.L2_DATA_OPEN) LogEntryMaker.create_system_entry(user, log_data) #print('-' * 40) #print(dinfo['app_config']) return render(request, 'index.html', dinfo)