def _run_dynamic_service_internal(self, app_id, params, tag, version, cell_id, run_id, **kwargs): # Intro tests: self.spec_manager.check_app(app_id, tag, raise_exception=True) if version is not None and tag != "release": raise ValueError( "App versions only apply to released app modules!") # Get the spec & params spec = self.spec_manager.get_spec(app_id, tag) if 'behavior' not in spec: raise ValueError( "This app appears invalid - it has no defined behavior") behavior = spec['behavior'] if 'script_module' in behavior or 'script_name' in behavior: # It's an old NJS script. These don't work anymore. raise ValueError( 'This app relies on a service that is now obsolete. Please contact the administrator.' ) # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'username': system_variable('user_id'), 'ws': system_variable('workspace') } kblogging.log_event(self._log, "run_dynamic_service", log_info) # Silly to keep this here, but we do not validate the incoming parameters. # If they are provided by the UI (we have cell_id), they are constructed # according to the spec, so are trusted; # Otherwise, if they are the product of direct code cell entry, this is a mode we do not # "support", so we can let it fail hard. # In the future when code cell interaction is supported for users, we will need to provide # robust validation and error reporting, but this may end up being (should be) provided by the # sdk execution infrastructure anyway input_vals = params function_name = spec['behavior']['kb_service_name'] + '.' + spec[ 'behavior']['kb_service_method'] try: result = clients.get("service").sync_call(function_name, input_vals, service_version=tag)[0] # if a ui call (a cell_id is defined) we send a result message, otherwise # just the raw result for display in a code cell. This is how we "support" # code cells for internal usage. if cell_id: self.send_cell_message('result', cell_id, run_id, {'result': result}) else: return result except: raise
def _run_widget_app_internal(self, app_id, tag, version, cell_id, run_id): self._send_comm_message('run_status', { 'event': 'validating_app', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id }) # Intro tests: self.spec_manager.check_app(app_id, tag, raise_exception=True) if version is not None and tag != "release": raise ValueError("App versions only apply to released app modules!") # Get the spec & params spec = self.spec_manager.get_spec(app_id, tag) if 'behavior' not in spec: raise ValueError("This app appears invalid - it has no defined behavior") behavior = spec['behavior'] if 'kb_service_input_mapping' in behavior: # it's a service! Should run this with run_app! raise ValueError('This app appears to be a long-running job! Please start it using the run_app function instead.') if 'script_module' in behavior or 'script_name' in behavior: # It's an old NJS script. These don't work anymore. raise ValueError('This app relies on a service that is now obsolete. Please contact the administrator.') # Here, we just deal with two behaviors: # 1. None of the above - it's a viewer. # 2. ***TODO*** python_class / python_function. Import and exec the python code. # for now, just map the inputs to outputs. # First, validate. # Preflight check the params - all required ones are present, all values are the right type, all numerical values are in given ranges #spec_params = self.spec_manager.app_params(spec) #(params, ws_refs) = self._validate_parameters(app_id, tag, spec_params, kwargs) log_info = { 'app_id': app_id, 'tag': tag, 'username': system_variable('user_id'), 'ws': system_variable('workspace') } kblogging.log_event(self._log, "run_widget_app", log_info) self._send_comm_message('run_status', { 'event': 'success', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id }) # now just map onto outputs. custom_widget = spec.get('widgets', {}).get('input', None) return WidgetManager().show_custom_widget(custom_widget, app_id, version, tag, spec, cell_id)
def _run_local_app_internal(self, app_id, params, widget_state, tag, version, cell_id, run_id): self._send_comm_message('run_status', { 'event': 'validating_app', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id }) spec = self._get_validated_app_spec(app_id, tag, False, version=version) # Here, we just deal with two behaviors: # 1. None of the above - it's a viewer. # 2. ***TODO*** python_class / python_function. # Import and exec the python code. # for now, just map the inputs to outputs. # First, validate. # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) (params, ws_refs) = validate_parameters(app_id, tag, spec_params, params) # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'username': system_variable('user_id'), 'ws': system_variable('workspace') } kblogging.log_event(self._log, "run_local_app", log_info) self._send_comm_message('run_status', { 'event': 'success', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id }) (output_widget, widget_params) = map_outputs_from_state([], params, spec) # All a local app does is route the inputs to outputs through the # spec's mapping, and then feed that into the specified output widget. wm = WidgetManager() if widget_state is not None: return wm.show_advanced_viewer_widget( output_widget, widget_params, widget_state, cell_id=cell_id, tag=tag ) else: return wm.show_output_widget( output_widget, widget_params, cell_id=cell_id, tag=tag )
def list_objects(obj_type=None, name=None, fuzzy_name=True): """ Returns a list of all objects in the current workspace with type=obj_type obj_type is a string. if None, return all visible objects (no reports, data palettes, etc.) name is a string. if None, then return everything. if not None, use that string to filter the search. if fuzzy_name is set to True, use that string as a search filter. e.g., "foo" would match "Foobar" and "Bazfoo" However, it doesn't go the other way. If name="Foobar" it will not match an object named "foo" If fuzzy_name is False, only exact (case-insensitive) matches are allowed. This has limited use, I know, but it's useful for fetching UPAs for objects you know, or names you're iterating over another way. This first prototype just returns a list of dictionaries, where each dict contains 'type', 'upa', and 'name' keys for each object. """ ws_name = system_variable('workspace') service = clients.get('service') service_params = {'ws_name': ws_name} if obj_type is not None: # matches: # foo.bar # foo.bar-1.0 # doesn't match: # foo # foo.bar- # foobar- # foo.bar-1.2.0 if not re.match(r"[A-Za-z]+\.[A-Za-z]+(-\d+\.\d+)?$", obj_type): raise ValueError( '{} is not a valid type. Valid types are of the format "Module.Type" or "Module.Type-Version"' .format(obj_type)) service_params['types'] = [obj_type] all_obj = service.sync_call('NarrativeService.list_objects_with_sets', [service_params])[0] obj_list = list() for obj in all_obj['data']: # filtration! # 1. ignore narratives if 'KBaseNarrative.Narrative' in obj['object_info'][2]: continue # 2. name filter if name is not None: name = str(name).lower() # if we're not strict, just search for the string if fuzzy_name is True and name not in obj['object_info'][1].lower( ): continue elif fuzzy_name is False and name != obj['object_info'][1].lower(): continue upa_prefix = '' # gavin's gonna wreck me. if 'dp_info' in obj: # seriously. upa_prefix = obj['dp_info'][ 'ref'] + ';' # not like I want to support this, either... info = obj['object_info'] obj_list.append({ "upa": "{}{}/{}/{}".format(upa_prefix, info[6], info[0], info[4]), "name": info[1], "type": info[2] }) return obj_list
def get_df(ws_ref, col_attributes=(), row_attributes=(), clustergrammer=False): """ Gets a dataframe from the WS object :param ws_ref: The Workspace reference of the 2DMatrix containing object :param col_attributes: Which column attributes should appear in the resulting DataFrame as a multiIndex. Defaults to all attributes, pass None to use a simple index of only ID. :param row_attributes: Which row attributes should appear in the resulting DataFrame as a multiIndex. Defaults to all attributes, pass None to use a simple index of only ID. :param clustergrammer: Returns a DataFrame with Clustergrammer compatible indices and columns. Defaults to False. :return: A Pandas DataFrame """ ws = clients.get('workspace') if "/" not in ws_ref: ws_ref = "{}/{}".format(system_variable('workspace'), ws_ref) generic_data = ws.get_objects2({'objects': [{'ref': ws_ref}]})['data'][0]['data'] if not _is_compatible_matrix(generic_data): raise ValueError("{} is not a compatible data type for this viewer. Data type must " "contain a 'data' key with a FloatMatrix2D type value".format(ws_ref)) cols = _get_categories(generic_data['data']['col_ids'], ws_ref, generic_data.get('col_attributemapping_ref'), generic_data.get('col_mapping'), col_attributes, clustergrammer) rows = _get_categories(generic_data['data']['row_ids'], ws_ref, generic_data.get('row_attributemapping_ref'), generic_data.get('row_mapping'), row_attributes, clustergrammer) return pd.DataFrame(data=generic_data['data']['values'], columns=cols, index=rows)
def _run_dynamic_service_internal(self, app_id, params, tag, version, cell_id, run_id): spec = self._get_validated_app_spec(app_id, tag, False, version=version) # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'username': system_variable('user_id'), 'ws': system_variable('workspace') } kblogging.log_event(self._log, "run_dynamic_service", log_info) # Silly to keep this here, but we do not validate the incoming parameters. # If they are provided by the UI (we have cell_id), they are constructed # according to the spec, so are trusted; # Otherwise, if they are the product of direct code cell entry, this is a mode we do not # "support", so we can let it fail hard. # In the future when code cell interaction is supported for users, we will need to provide # robust validation and error reporting, but this may end up being (should be) provided by the # sdk execution infrastructure anyway input_vals = params function_name = spec['behavior']['kb_service_name'] + '.' + spec[ 'behavior']['kb_service_method'] try: result = clients.get("service").sync_call(function_name, input_vals, service_version=tag)[0] # if a ui call (a cell_id is defined) we send a result message, otherwise # just the raw result for display in a code cell. This is how we "support" # code cells for internal usage. if cell_id: self.send_cell_message('result', cell_id, run_id, {'result': result}) else: return result except: raise
def _run_dynamic_service_internal(self, app_id, params, tag, version, cell_id, run_id): spec = self._get_validated_app_spec(app_id, tag, False, version=version) # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'username': system_variable('user_id'), 'ws': system_variable('workspace') } kblogging.log_event(self._log, "run_dynamic_service", log_info) # Silly to keep this here, but we do not validate the incoming parameters. # If they are provided by the UI (we have cell_id), they are constructed # according to the spec, so are trusted; # Otherwise, if they are the product of direct code cell entry, this is a mode we do not # "support", so we can let it fail hard. # In the future when code cell interaction is supported for users, we will need to provide # robust validation and error reporting, but this may end up being (should be) provided by the # sdk execution infrastructure anyway input_vals = params function_name = spec['behavior']['kb_service_name'] + '.' + spec['behavior']['kb_service_method'] try: result = clients.get("service").sync_call( function_name, input_vals, service_version=tag )[0] # if a ui call (a cell_id is defined) we send a result message, otherwise # just the raw result for display in a code cell. This is how we "support" # code cells for internal usage. if cell_id: self.send_cell_message('result', cell_id, run_id, { 'result': result }) else: return result except: raise
def initialize_jobs(self): """ Initializes this JobManager. This is expected to be run by a running Narrative, and naturally linked to a workspace. So it does the following steps. 1. app_util.system_variable('workspace_id') 2. get list of jobs with that ws id from UJS (also gets tag, cell_id, run_id) 3. initialize the Job objects by running NJS.get_job_params (also gets app_id) 4. start the status lookup loop. """ ws_id = system_variable("workspace_id") job_states = dict() kblogging.log_event(self._log, "JobManager.initialize_jobs", {"ws_id": ws_id}) try: job_states = clients.get("execution_engine2").check_workspace_jobs( { "workspace_id": ws_id, "return_list": 0 }) self._running_jobs = dict() except Exception as e: kblogging.log_event(self._log, "init_error", {"err": str(e)}) new_e = transform_job_exception(e) raise new_e for job_id, job_state in job_states.items(): job_input = job_state.get("job_input", {}) job_meta = job_input.get("narrative_cell_info", {}) status = job_state.get("status") job = Job.from_state( job_id, job_input, job_state.get("user"), app_id=job_input.get("app_id"), tag=job_meta.get("tag", "release"), cell_id=job_meta.get("cell_id", None), run_id=job_meta.get("run_id", None), token_id=job_meta.get("token_id", None), meta=job_meta, ) self._running_jobs[job_id] = { "refresh": 1 if status not in ["completed", "errored", "terminated"] else 0, "job": job, }
def initialize_jobs(self, cell_ids: List[str] = None) -> None: """ Initializes this JobManager. This is expected to be run by a running Narrative, and naturally linked to a workspace. So it does the following steps. 1. gets the current workspace ID from app_util.system_variable('workspace_id') 2. get list of jobs with that ws id from ee2 (also gets tag, cell_id, run_id) 3. initialize the Job objects and add them to the running jobs list 4. start the status lookup loop. """ ws_id = system_variable("workspace_id") job_states = dict() kblogging.log_event(self._log, "JobManager.initialize_jobs", {"ws_id": ws_id}) try: job_states = clients.get("execution_engine2").check_workspace_jobs( { "workspace_id": ws_id, "return_list": 0, # do not remove "exclude_fields": JOB_INIT_EXCLUDED_JOB_STATE_FIELDS, } ) except Exception as e: kblogging.log_event(self._log, "init_error", {"err": str(e)}) new_e = transform_job_exception(e, "Unable to initialize jobs") raise new_e self._running_jobs = dict() job_states = self._reorder_parents_children(job_states) for job_state in job_states.values(): child_jobs = None if job_state.get("batch_job"): child_jobs = [ self.get_job(child_id) for child_id in job_state.get("child_jobs", []) ] job = Job(job_state, children=child_jobs) # Set to refresh when job is not in terminal state # and when job is present in cells (if given) # and when it is not part of a batch refresh = not job.was_terminal() and not job.batch_id if cell_ids is not None: refresh = refresh and job.in_cells(cell_ids) self.register_new_job(job, refresh)
def initialize_jobs(self): """ Initializes this JobManager. This is expected to be run by a running Narrative, and naturally linked to a workspace. So it does the following steps. 1. app_util.system_variable('workspace_id') 2. get list of jobs with that ws id from UJS (also gets tag, cell_id, run_id) 3. initialize the Job objects by running NJS.get_job_params (also gets app_id) 4. start the status lookup loop. """ ws_id = system_variable("workspace_id") job_states = dict() kblogging.log_event(self._log, "JobManager.initialize_jobs", {'ws_id': ws_id}) try: job_states = clients.get('execution_engine2').check_workspace_jobs( { 'workspace_id': ws_id, 'return_list': 0 }) self._running_jobs = dict() except Exception as e: kblogging.log_event(self._log, 'init_error', {'err': str(e)}) new_e = transform_job_exception(e) raise new_e for job_id, job_state in job_states.items(): job_input = job_state.get('job_input', {}) job_meta = job_input.get('narrative_cell_info', {}) status = job_state.get('status') job = Job.from_state(job_id, job_input, job_state.get('user'), app_id=job_input.get('app_id'), tag=job_meta.get('tag', 'release'), cell_id=job_meta.get('cell_id', None), run_id=job_meta.get('run_id', None), token_id=job_meta.get('token_id', None), meta=job_meta) self._running_jobs[job_id] = { 'refresh': 1 if status not in ['completed', 'errored', 'terminated'] else 0, 'job': job }
def get_df(ws_ref, col_attributes=(), row_attributes=(), clustergrammer=False): """ Gets a dataframe from the WS object :param ws_ref: The Workspace reference of the 2DMatrix containing object :param col_attributes: Which column attributes should appear in the resulting DataFrame as a multiIndex. Defaults to all attributes, pass None to use a simple index of only ID. :param row_attributes: Which row attributes should appear in the resulting DataFrame as a multiIndex. Defaults to all attributes, pass None to use a simple index of only ID. :param clustergrammer: Returns a DataFrame with Clustergrammer compatible indices and columns. Defaults to False. :return: A Pandas DataFrame """ ws = clients.get("workspace") if "/" not in ws_ref: ws_ref = "{}/{}".format(system_variable("workspace"), ws_ref) generic_data = ws.get_objects2({"objects": [{"ref": ws_ref}]})["data"][0]["data"] if not _is_compatible_matrix(generic_data): raise ValueError( "{} is not a compatible data type for this viewer. Data type must " "contain a 'data' key with a FloatMatrix2D type value".format(ws_ref) ) cols = _get_categories( generic_data["data"]["col_ids"], ws_ref, generic_data.get("col_attributemapping_ref"), generic_data.get("col_mapping"), col_attributes, clustergrammer, ) rows = _get_categories( generic_data["data"]["row_ids"], ws_ref, generic_data.get("row_attributemapping_ref"), generic_data.get("row_mapping"), row_attributes, clustergrammer, ) return pd.DataFrame(data=generic_data["data"]["values"], columns=cols, index=rows)
def _transform_input(self, transform_type, value): """ Transforms an input according to the rules given in NarrativeMethodStore.ServiceMethodInputMapping Really, there are three types of transforms possible: 1. ref - turns the input string into a workspace ref. 2. int - tries to coerce the input string into an int. 3. list<type> - turns the given list into a list of the given type. (4.) none or None - doesn't transform. Returns a transformed (or not) value. """ if transform_type == "none" or transform_type is None: return value elif transform_type == "ref": # make a workspace ref if value is not None: value = system_variable('workspace') + '/' + value return value elif transform_type == "int": # make it an integer, OR 0. if value is None or len(str(value).strip()) == 0: return None return int(value) elif transform_type.startswith("list<") and transform_type.endswith(">"): # make it a list of transformed types. list_type = transform_type[5:-1] if isinstance(value, list): ret = [] for pos in range(0, len(value)): ret.append(self._transform_input(list_type, value[pos])) return ret else: return [self._transform_input(list_type, value)] else: raise ValueError("Unsupported Transformation type: " + transform_type)
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id, dry_run): """ Attemps to run the app, returns a Job with the running app info. Should *hopefully* also inject that app into the Narrative's metadata. Probably need some kind of JavaScript-foo to get that to work. Parameters: ----------- app_id - should be from the app spec, e.g. 'build_a_metabolic_model' or 'MegaHit/run_megahit'. params - a dictionary of parameters. tag - optional, one of [release|beta|dev] (default=release) version - optional, a semantic version string. Only released modules have versions, so if the tag is not 'release', and a version is given, a ValueError will be raised. **kwargs - these are the set of parameters to be used with the app. They can be found by using the app_usage function. If any non-optional apps are missing, a ValueError will be raised. """ ws_id = strict_system_variable('workspace_id') spec = self._get_validated_app_spec(app_id, tag, True, version=version) # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) spec_params_map = dict((spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params))) ws_input_refs = extract_ws_refs(app_id, tag, spec_params, params) input_vals = self._map_inputs( spec['behavior']['kb_service_input_mapping'], params, spec_params_map) service_method = spec['behavior']['kb_service_method'] service_name = spec['behavior']['kb_service_name'] service_ver = spec['behavior'].get('kb_service_version', None) # Let the given version override the spec's version. if version is not None: service_ver = version # This is what calls the function in the back end - Module.method # This isn't the same as the app spec id. function_name = service_name + '.' + service_method job_meta = {'tag': tag} if cell_id is not None: job_meta['cell_id'] = cell_id if run_id is not None: job_meta['run_id'] = run_id # This is the input set for NJSW.run_job. Now we need the workspace id # and whatever fits in the metadata. job_runner_inputs = { 'method': function_name, 'service_ver': service_ver, 'params': input_vals, 'app_id': app_id, 'wsid': ws_id, 'meta': job_meta } if len(ws_input_refs) > 0: job_runner_inputs['source_ws_objects'] = ws_input_refs if dry_run: return job_runner_inputs # We're now almost ready to run the job. Last, we need an agent token. try: token_name = 'KBApp_{}'.format(app_id) token_name = token_name[:self.__MAX_TOKEN_NAME_LEN] agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name) except Exception as e: raise job_runner_inputs['meta']['token_id'] = agent_token['id'] # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'version': service_ver, 'username': system_variable('user_id'), 'wsid': ws_id } kblogging.log_event(self._log, "run_app", log_info) try: job_id = clients.get("job_service", token=agent_token['token']).run_job(job_runner_inputs) except Exception as e: log_info.update({'err': str(e)}) kblogging.log_event(self._log, "run_app_error", log_info) raise transform_job_exception(e) new_job = Job(job_id, app_id, input_vals, system_variable('user_id'), tag=tag, app_version=service_ver, cell_id=cell_id, run_id=run_id, token_id=agent_token['id']) self._send_comm_message('run_status', { 'event': 'launched_job', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id, 'job_id': job_id }) JobManager().register_new_job(new_job) if cell_id is not None: return else: return new_job
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id, **kwargs): """ Attemps to run the app, returns a Job with the running app info. Should *hopefully* also inject that app into the Narrative's metadata. Probably need some kind of JavaScript-foo to get that to work. Parameters: ----------- app_id - should be from the app spec, e.g. 'build_a_metabolic_model' or 'MegaHit/run_megahit'. params - the dictionary of parameters. tag - optional, one of [release|beta|dev] (default=release) version - optional, a semantic version string. Only released modules have versions, so if the tag is not 'release', and a version is given, a ValueError will be raised. **kwargs - these are the set of parameters to be used with the app. They can be found by using the app_usage function. If any non-optional apps are missing, a ValueError will be raised. Example: -------- my_job = mm.run_app('MegaHit/run_megahit', version=">=1.0.0", read_library_name="My_PE_Library", output_contigset_name="My_Contig_Assembly") """ ### TODO: this needs restructuring so that we can send back validation failure ### messages. Perhaps a separate function and catch the errors, or return an ### error structure. # Intro tests: self.spec_manager.check_app(app_id, tag, raise_exception=True) if version is not None and tag != "release": if re.match(version, '\d+\.\d+\.\d+') is not None: raise ValueError("Semantic versions only apply to released app modules. You can use a Git commit hash instead to specify a version.") # Get the spec & params spec = self.spec_manager.get_spec(app_id, tag) # There's some branching to do here. # Cases: # app has behavior.kb_service_input_mapping -- is a valid long-running app. # app only has behavior.output_mapping - not kb_service_input_mapping or script_module - it's a viewer and should return immediately # app has other things besides kb_service_input_mapping -- not a valid app. if 'behavior' not in spec: raise Exception("This app appears invalid - it has no defined behavior") if 'kb_service_input_mapping' not in spec['behavior']: raise Exception("This app does not appear to be a long-running job! Please use 'run_local_app' to start this instead.") # Preflight check the params - all required ones are present, all values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) (params, ws_input_refs) = self._validate_parameters(app_id, tag, spec_params, params) ws_id = system_variable('workspace_id') if ws_id is None: raise ValueError('Unable to retrive current Narrative workspace information!') input_vals = self._map_inputs(spec['behavior']['kb_service_input_mapping'], params) service_method = spec['behavior']['kb_service_method'] service_name = spec['behavior']['kb_service_name'] service_ver = spec['behavior'].get('kb_service_version', None) service_url = spec['behavior']['kb_service_url'] # Let the given version override the spec's version. if version is not None: service_ver = version # This is what calls the function in the back end - Module.method # This isn't the same as the app spec id. function_name = service_name + '.' + service_method job_meta = {'tag': tag} if cell_id is not None: job_meta['cell_id'] = cell_id if run_id is not None: job_meta['run_id'] = run_id # This is the input set for NJSW.run_job. Now we need the worksapce id and whatever fits in the metadata. job_runner_inputs = { 'method': function_name, 'service_ver': service_ver, 'params': input_vals, 'app_id': app_id, 'wsid': ws_id, 'meta': job_meta } if len(ws_input_refs) > 0: job_runner_inputs['source_ws_objects'] = ws_input_refs # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'version': service_ver, 'username': system_variable('user_id'), 'wsid': ws_id } kblogging.log_event(self._log, "run_app", log_info) try: job_id = self.njs.run_job(job_runner_inputs) except Exception as e: log_info.update({'err': str(e)}) kblogging.log_event(self._log, "run_app_error", log_info) raise transform_job_exception(e) new_job = Job(job_id, app_id, [params], system_variable('user_id'), tag=tag, app_version=service_ver, cell_id=cell_id, run_id=run_id) self._send_comm_message('run_status', { 'event': 'launched_job', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id, 'job_id': job_id }) JobManager().register_new_job(new_job) if cell_id is not None: return else: return new_job
def test_sys_var_user_none(self): if 'KB_AUTH_TOKEN' in os.environ: del os.environ['KB_AUTH_TOKEN'] self.assertIsNone(system_variable('user_id'))
def test_sys_var_bad_token(self): if 'KB_AUTH_TOKEN' in os.environ: del os.environ['KB_AUTH_TOKEN'] self.assertIsNone(system_variable('token'))
def test_sys_var_workspace_id(self): os.environ['KB_WORKSPACE_ID'] = self.workspace self.assertEquals(system_variable('workspace_id'), 12345)
def load_widget_info(self, tag="release", verbose=False): """ Loads widget info and mapping. Eventually will fetch from kbase-ui, a kbase CDN, or the catalog service. For now, it gets known vis widgets from all method specs. This returns the a Dict where all keys are the name of a widget, and all values contain widget information in this structure: { "params": { "param_name": { "is_constant": boolean, "param_type": one of (string|boolean|dropdown), "allowed_values": list of strings (exists when param_type==dropdown), "allowed_types": list of data types (when param_type==string), "param_value": something, mainly when is_constant==True } } """ check_tag(tag, raise_exception=True) methods = self._sm.app_specs[tag].values() all_widgets = dict() # keys = widget names / namespaced require path / etc. # Individual widget values should be: # {params: { # name1: { # is_constant: boolean, # value: (***something*** | None) (something = any structure), # allowed: [ list of allowed values, optional ], # type: (string, int, float, boolean, etc. list? hash?) # allowed_types: [ list of allowed ws types, optional ] # }, # name2: { is_constant, value } # } for method in methods: if 'output' not in method['widgets']: widget_name = self._default_output_widget else: widget_name = method['widgets']['output'] if widget_name == 'null': if verbose: print("Ignoring a widget named 'null' in {} - {}".format(tag, method['info']['id'])) continue out_mapping = method['behavior'].get('kb_service_output_mapping', method['behavior'].get('output_mapping', None)) if out_mapping is not None: params = {} for p in out_mapping: param_name = p['target_property'] allowed_values = set() is_constant = False param_value = None param_type = 'string' allowed_types = set() if 'constant_value' in p: # add this val to a set of constant values for that param in that widget. # if more than one possible, this need to be optional is_constant = True allowed_values.add(p['constant_value']) if 'input_parameter' in p: # this is a user given input. look up what it expects from the # associated parameter of that name in_param = p['input_parameter'] for spec_param in method['parameters']: if spec_param['id'] == in_param: # want its: # field_type = text, float, number, ... in_type = spec_param['field_type'] if in_type == 'text': param_type = 'string' if spec_param.has_key('text_options'): validate_as = spec_param['text_options'].get('validate_as', None) if validate_as == 'int': param_type = 'int' elif validate_as == 'float': param_type = 'float' if spec_param['text_options'].has_key('valid_ws_types'): allowed_types.update(spec_param['text_options']['valid_ws_types']) elif param_type == 'textarea': param_type = 'string' elif param_type == 'checkbox': param_type = 'boolean' elif param_type == 'dropdown': param_type = 'dropdown' allowed_values.update([o['value'] for o in spec_param['dropdown_options']]) if 'narrative_system_variable' in p: # this is something like the ws name or token that needs to get fetched # by the system. Shouldn't be handled by the user. is_constant = True param_value = system_variable(p['narrative_system_variable']) if 'service_method_output_path' in p: param_type = 'from_service_output' param_info = { 'is_constant': is_constant, 'param_type': param_type, } if allowed_values: param_info['allowed_values'] = allowed_values if allowed_types: param_info['allowed_types'] = allowed_types if param_value: param_info['param_value'] = param_value params[param_name] = param_info if widget_name in all_widgets: # if it's already there, just update the allowed_types and allowed_values for some params that have them for p_name in params.keys(): if 'allowed_types' in params[p_name]: if p_name not in all_widgets[widget_name]['params']: all_widgets[widget_name]['params'][p_name] = params[p_name] else: widget_types = all_widgets[widget_name]['params'].get(p_name, {}).get('allowed_types', set()) widget_types.update(params[p_name]['allowed_types']) all_widgets[widget_name]['params'][p_name]['allowed_types'] = widget_types if 'allowed_values' in params[p_name]: if p_name not in all_widgets[widget_name]['params']: all_widgets[widget_name]['params'][p_name] = params[p_name] else: widget_vals = all_widgets[widget_name]['params'].get(p_name, {}).get('allowed_values', set()) widget_vals.update(params[p_name]['allowed_values']) all_widgets[widget_name]['params'][p_name]['allowed_values'] = widget_vals else: all_widgets[widget_name] = { 'params': params } # finally, turn all sets into lists for w in all_widgets: for p in all_widgets[w]["params"]: if "allowed_types" in all_widgets[w]["params"][p]: all_widgets[w]["params"][p]["allowed_types"] = list(all_widgets[w]["params"][p]["allowed_types"]) if "allowed_values" in all_widgets[w]['params'][p]: all_widgets[w]["params"][p]["allowed_values"] = list(all_widgets[w]["params"][p]["allowed_values"]) return all_widgets
def run_local_app( self, app_id, params, tag="release", version=None, cell_id=None, run_id=None, widget_state=None, ): """ Attempts to run a local app. These do not return a Job object, but just the result of the app. In most cases, this will be a Javascript display of the result, but could be anything. If the app_spec looks like it makes a service call, then this raises a ValueError. Otherwise, it validates each parameter in **kwargs against the app spec, executes it, and returns the result. Parameters: ----------- app_id - should be from the app spec, e.g. 'view_expression_profile' params - the dictionary of parameters for the app. Should be key-value pairs where they keys are strings. If any non-optional parameters are missing, an informative string will be printed. tag - optional, one of [release|beta|dev] (default=release) version - optional, a semantic version string. Only released modules have versions, so if the tag is not 'release', and a version is given, a ValueError will be raised. Example: run_local_app('NarrativeViewers/view_expression_profile', { "input_expression_matrix": "MyMatrix", "input_gene_ids": "1234" }, version='0.0.1', input_expression_matrix="MyMatrix") """ spec = self._get_validated_app_spec(app_id, tag, False, version=version) # Here, we just deal with two behaviors: # 1. None of the above - it's a viewer. # 2. ***TODO*** python_class / python_function. # Import and exec the python code. # for now, just map the inputs to outputs. # First, validate. # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) (params, ws_refs) = validate_parameters(app_id, tag, spec_params, params) # Log that we're trying to run a job... log_info = { "app_id": app_id, "tag": tag, "username": system_variable("user_id"), "ws": system_variable("workspace"), } kblogging.log_event(self._log, "run_local_app", log_info) self._send_comm_message( MESSAGE_TYPE["RUN_STATUS"], { "event": "success", "event_at": timestamp(), "cell_id": cell_id, "run_id": run_id, }, ) (output_widget, widget_params) = map_outputs_from_state([], params, spec) # All a local app does is route the inputs to outputs through the # spec's mapping, and then feed that into the specified output widget. wm = WidgetManager() if widget_state is not None: return wm.show_advanced_viewer_widget( output_widget, widget_params, widget_state, cell_id=cell_id, tag=tag ) else: return wm.show_output_widget( output_widget, widget_params, cell_id=cell_id, tag=tag )
def _run_local_app_internal(self, app_id, params, widget_state, tag, version, cell_id, run_id): self._send_comm_message( 'run_status', { 'event': 'validating_app', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id }) spec = self._get_validated_app_spec(app_id, tag, False, version=version) # Here, we just deal with two behaviors: # 1. None of the above - it's a viewer. # 2. ***TODO*** python_class / python_function. # Import and exec the python code. # for now, just map the inputs to outputs. # First, validate. # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) (params, ws_refs) = validate_parameters(app_id, tag, spec_params, params) # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'username': system_variable('user_id'), 'ws': system_variable('workspace') } kblogging.log_event(self._log, "run_local_app", log_info) self._send_comm_message( 'run_status', { 'event': 'success', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id }) (output_widget, widget_params) = map_outputs_from_state([], params, spec) # All a local app does is route the inputs to outputs through the # spec's mapping, and then feed that into the specified output widget. wm = WidgetManager() if widget_state is not None: return wm.show_advanced_viewer_widget(output_widget, widget_params, widget_state, cell_id=cell_id, tag=tag) else: return wm.show_output_widget(output_widget, widget_params, cell_id=cell_id, tag=tag)
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id, dry_run): """ Attemps to run the app, returns a Job with the running app info. Should *hopefully* also inject that app into the Narrative's metadata. Probably need some kind of JavaScript-foo to get that to work. Parameters: ----------- app_id - should be from the app spec, e.g. 'build_a_metabolic_model' or 'MegaHit/run_megahit'. params - a dictionary of parameters. tag - optional, one of [release|beta|dev] (default=release) version - optional, a semantic version string. Only released modules have versions, so if the tag is not 'release', and a version is given, a ValueError will be raised. **kwargs - these are the set of parameters to be used with the app. They can be found by using the app_usage function. If any non-optional apps are missing, a ValueError will be raised. """ ws_id = strict_system_variable('workspace_id') spec = self._get_validated_app_spec(app_id, tag, True, version=version) # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) spec_params_map = dict((spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params))) ws_input_refs = extract_ws_refs(app_id, tag, spec_params, params) input_vals = self._map_inputs( spec['behavior']['kb_service_input_mapping'], params, spec_params_map) service_method = spec['behavior']['kb_service_method'] service_name = spec['behavior']['kb_service_name'] service_ver = spec['behavior'].get('kb_service_version', None) # Let the given version override the spec's version. if version is not None: service_ver = version # This is what calls the function in the back end - Module.method # This isn't the same as the app spec id. function_name = service_name + '.' + service_method job_meta = {'tag': tag} if cell_id is not None: job_meta['cell_id'] = cell_id if run_id is not None: job_meta['run_id'] = run_id # This is the input set for NJSW.run_job. Now we need the workspace id # and whatever fits in the metadata. job_runner_inputs = { 'method': function_name, 'service_ver': service_ver, 'params': input_vals, 'app_id': app_id, 'wsid': ws_id, 'meta': job_meta } if len(ws_input_refs) > 0: job_runner_inputs['source_ws_objects'] = ws_input_refs if dry_run: return job_runner_inputs # We're now almost ready to run the job. Last, we need an agent token. try: token_name = 'KBApp_{}'.format(app_id) token_name = token_name[:self.__MAX_TOKEN_NAME_LEN] agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name) except Exception as e: raise job_runner_inputs['meta']['token_id'] = agent_token['id'] # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': tag, 'version': service_ver, 'username': system_variable('user_id'), 'wsid': ws_id } kblogging.log_event(self._log, "run_app", log_info) try: job_id = clients.get( "execution_engine2", token=agent_token['token']).run_job(job_runner_inputs) except Exception as e: log_info.update({'err': str(e)}) kblogging.log_event(self._log, "run_app_error", log_info) raise transform_job_exception(e) new_job = Job(job_id, app_id, input_vals, system_variable('user_id'), tag=tag, app_version=service_ver, cell_id=cell_id, run_id=run_id, token_id=agent_token['id']) self._send_comm_message( 'run_status', { 'event': 'launched_job', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id, 'job_id': job_id }) self.register_new_job(new_job) if cell_id is not None: return else: return new_job
def _run_app_batch_internal(self, app_id, params, tag, version, cell_id, run_id, dry_run): batch_method = "kb_BatchApp.run_batch" batch_app_id = "kb_BatchApp/run_batch" batch_method_ver = "dev" batch_method_tag = "dev" ws_id = strict_system_variable('workspace_id') spec = self._get_validated_app_spec(app_id, tag, True, version=version) # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) # A list of lists of UPAs, used for each subjob. batch_ws_upas = list() # The list of actual input values, post-mapping. batch_run_inputs = list() for param_set in params: spec_params_map = dict((spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params))) batch_ws_upas.append( extract_ws_refs(app_id, tag, spec_params, param_set)) batch_run_inputs.append( self._map_inputs(spec['behavior']['kb_service_input_mapping'], param_set, spec_params_map)) service_method = spec['behavior']['kb_service_method'] service_name = spec['behavior']['kb_service_name'] service_ver = spec['behavior'].get('kb_service_version', None) # Let the given version override the spec's version. if version is not None: service_ver = version # This is what calls the function in the back end - Module.method # This isn't the same as the app spec id. job_meta = { 'tag': batch_method_tag, 'batch_app': app_id, 'batch_tag': tag, 'batch_size': len(params), } if cell_id is not None: job_meta['cell_id'] = cell_id if run_id is not None: job_meta['run_id'] = run_id # Now put these all together in a way that can be sent to the batch processing app. batch_params = [{ "module_name": service_name, "method_name": service_method, "service_ver": service_ver, "wsid": ws_id, "meta": job_meta, "batch_params": [{ "params": batch_run_inputs[i], "source_ws_objects": batch_ws_upas[i] } for i in range(len(batch_run_inputs))], }] # We're now almost ready to run the job. Last, we need an agent token. try: token_name = 'KBApp_{}'.format(app_id) token_name = token_name[:self.__MAX_TOKEN_NAME_LEN] agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name) except Exception as e: raise job_meta['token_id'] = agent_token['id'] # This is the input set for NJSW.run_job. Now we need the workspace id # and whatever fits in the metadata. job_runner_inputs = { 'method': batch_method, 'service_ver': batch_method_ver, 'params': batch_params, 'app_id': batch_app_id, 'wsid': ws_id, 'meta': job_meta } # if len(ws_input_refs) > 0: # job_runner_inputs['source_ws_objects'] = ws_input_refs # if we're doing a dry run, just return the inputs that we made. if dry_run: return job_runner_inputs # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': batch_method_tag, 'version': service_ver, 'username': system_variable('user_id'), 'wsid': ws_id } kblogging.log_event(self._log, "run_batch_app", log_info) try: job_id = clients.get( "execution_engine2", token=agent_token['token']).run_job(job_runner_inputs) except Exception as e: log_info.update({'err': str(e)}) kblogging.log_event(self._log, "run_batch_app_error", log_info) raise transform_job_exception(e) new_job = Job(job_id, batch_app_id, batch_params, system_variable('user_id'), tag=batch_method_tag, app_version=batch_method_ver, cell_id=cell_id, run_id=run_id, token_id=agent_token['id'], meta=job_meta) self._send_comm_message( 'run_status', { 'event': 'launched_job', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id, 'job_id': job_id }) self.register_new_job(new_job) if cell_id is not None: return else: return new_job
def initialize_jobs(self, start_lookup_thread=True): """ Initializes this JobManager. This is expected to be run by a running Narrative, and naturally linked to a workspace. So it does the following steps. 1. app_util.system_variable('workspace_id') 2. get list of jobs with that ws id from UJS (also gets tag, cell_id, run_id) 3. initialize the Job objects by running NJS.get_job_params (also gets app_id) 4. start the status lookup loop. """ the_time = int(round(time.time() * 1000)) self._send_comm_message('start', {'time': the_time}) ws_id = system_variable('workspace_id') try: nar_jobs = clients.get('user_and_job_state').list_jobs2({ 'authstrat': 'kbaseworkspace', 'authparams': [str(ws_id)] }) except Exception as e: kblogging.log_event(self._log, 'init_error', {'err': str(e)}) new_e = transform_job_exception(e) error = { 'error': 'Unable to get initial jobs list', 'message': getattr(new_e, 'message', 'Unknown reason'), 'code': getattr(new_e, 'code', -1), 'source': getattr(new_e, 'source', 'jobmanager'), 'name': getattr(new_e, 'name', type(e).__name__), 'service': 'user_and_job_state' } self._send_comm_message('job_init_err', error) raise new_e job_ids = [j[0] for j in nar_jobs] job_states = clients.get('job_service').check_jobs({ 'job_ids': job_ids, 'with_job_params': 1 }) job_param_info = job_states.get('job_params', {}) job_check_error = job_states.get('check_error', {}) error_jobs = dict() for info in nar_jobs: job_id = info[0] user_info = info[1] job_meta = info[10] try: if job_id in job_param_info: job_info = job_param_info[job_id] job = Job.from_state(job_id, job_info, user_info[0], app_id=job_info.get('app_id'), tag=job_meta.get('tag', 'release'), cell_id=job_meta.get('cell_id', None), run_id=job_meta.get('run_id', None), token_id=job_meta.get('token_id', None), meta=job_meta) # Note that when jobs for this narrative are initially loaded, # they are set to not be refreshed. Rather, if a client requests # updates via the start_job_update message, the refresh flag will # be set to True. self._running_jobs[job_id] = { 'refresh': 0, 'job': job } elif job_id in job_check_error: job_err_state = { 'job_state': 'error', 'error': { 'error': 'KBase execution engine returned an error while looking up this job.', 'message': job_check_error[job_id].get('message', 'No error message available'), 'name': 'Job Error', 'code': job_check_error[job_id].get('code', -999), 'exception': { 'error_message': 'Job lookup in execution engine failed', 'error_type': job_check_error[job_id].get('name', 'unknown'), 'error_stacktrace': job_check_error[job_id].get('error', '') } }, 'cell_id': job_meta.get('cell_id', None), 'run_id': job_meta.get('run_id', None), } error_jobs[job_id] = job_err_state except Exception as e: kblogging.log_event(self._log, 'init_error', {'err': str(e)}) new_e = transform_job_exception(e) error = { 'error': 'Unable to get job info on initial lookup', 'job_id': job_id, 'message': getattr(new_e, 'message', 'Unknown reason'), 'code': getattr(new_e, 'code', -1), 'source': getattr(new_e, 'source', 'jobmanager'), 'name': getattr(new_e, 'name', type(e).__name__), 'service': 'job_service' } self._send_comm_message('job_init_lookup_err', error) raise new_e # should crash and burn on any of these. if len(job_check_error): err_str = 'Unable to find info for some jobs on initial lookup' err_type = 'job_init_partial_err' if len(job_check_error) == len(nar_jobs): err_str = 'Unable to get info for any job on initial lookup' err_type = 'job_init_lookup_err' error = { 'error': err_str, 'job_errors': error_jobs, 'message': 'Job information was unavailable from the server', 'code': -2, 'source': 'jobmanager', 'name': 'jobmanager', 'service': 'job_service', } self._send_comm_message(err_type, error) if not self._running_lookup_loop and start_lookup_thread: # only keep one loop at a time in cause this gets called again! if self._lookup_timer is not None: self._lookup_timer.cancel() self._running_lookup_loop = True self._lookup_job_status_loop() else: self._lookup_all_job_status()
def test_sys_var_no_ws_id(self): if 'KB_WORKSPACE_ID' in os.environ: del os.environ['KB_WORKSPACE_ID'] self.assertIsNone(system_variable('workspace_id'))
def test_sys_var_workspace(self): self.assertEquals(system_variable('workspace'), self.workspace)
def test_sys_var_workspace_id_except(self): os.environ['KB_WORKSPACE_ID'] = 'invalid_workspace' self.assertIsNone(system_variable('workspace_id'))
def test_sys_var_token(self): os.environ['KB_AUTH_TOKEN'] = self.good_fake_token self.assertEquals(system_variable('token'), self.good_fake_token)
def test_sys_var_user_bad(self): biokbase.auth.set_environ_token(self.bad_fake_token) self.assertIsNone(system_variable('user_id'))
def initialize_jobs(self): """ Initializes this JobManager. This is expected to be run by a running Narrative, and naturally linked to a workspace. So it does the following steps. 1. app_util.system_variable('workspace_id') 2. get list of jobs with that ws id from UJS (also gets tag, cell_id, run_id) 3. initialize the Job objects by running NJS.get_job_params on each of those (also gets app_id) 4. start the status lookup loop. """ ws_id = system_variable('workspace_id') try: nar_jobs = clients.get('user_and_job_state').list_jobs2({ 'authstrat': 'kbaseworkspace', 'authparams': [str(ws_id)] }) except Exception as e: kblogging.log_event(self._log, 'init_error', {'err': str(e)}) new_e = transform_job_exception(e) error = { 'error': 'Unable to get initial jobs list', 'message': getattr(new_e, 'message', 'Unknown reason'), 'code': getattr(new_e, 'code', -1), 'source': getattr(new_e, 'source', 'jobmanager'), 'name': getattr(new_e, 'name', type(e).__name__), 'service': 'user_and_job_state' } self._send_comm_message('job_init_err', error) raise new_e for info in nar_jobs: job_id = info[0] user_info = info[1] job_meta = info[10] try: job_info = clients.get('job_service').get_job_params(job_id)[0] self._running_jobs[job_id] = { 'refresh': True, 'job': Job.from_state(job_id, job_info, user_info[0], app_id=job_info.get('app_id'), tag=job_meta.get('tag', 'release'), cell_id=job_meta.get('cell_id', None), run_id=job_meta.get('run_id', None)) } except Exception as e: kblogging.log_event(self._log, 'init_error', {'err': str(e)}) new_e = transform_job_exception(e) error = { 'error': 'Unable to get job info on initial lookup', 'job_id': job_id, 'message': getattr(new_e, 'message', 'Unknown reason'), 'code': getattr(new_e, 'code', -1), 'source': getattr(new_e, 'source', 'jobmanager'), 'name': getattr(new_e, 'name', type(e).__name__), 'service': 'job_service' } self._send_comm_message('job_init_lookup_err', error) raise new_e # should crash and burn on any of these. if not self._running_lookup_loop: # only keep one loop at a time in cause this gets called again! if self._lookup_timer is not None: self._lookup_timer.cancel() self._running_lookup_loop = True self._lookup_job_status_loop() else: self._lookup_all_job_status()
def test_sys_var_bad(self): self.assertIsNone(system_variable(self.bad_tag))
def _validate_parameters(self, app_id, tag, spec_params, params): """ Validates the dict of params against the spec_params. If all is good, it updates a few parameters that need it - checkboxes go from True/False to 1/0, and sets default values where necessary. Then it returns a tuple like this: (dict_of_params, list_of_ws_refs) where list_of_ws_refs is the list of workspace references for objects being passed into the app. If it fails, this will raise a ValueError with a description of the problem and a (hopefully useful!) hint for the user as to what went wrong. """ spec_param_ids = [p['id'] for p in spec_params] # First, test for presence. missing_params = list() for p in spec_params: if not p['optional'] and not p['default'] and not params.get(p['id'], None): missing_params.append(p['id']) if len(missing_params): raise ValueError('Missing required parameters {} - try executing app_usage("{}", tag="{}") for more information'.format(json.dumps(missing_params), app_id, tag)) # Next, test for extra params that don't make sense extra_params = list() for p in params.keys(): if p not in spec_param_ids: extra_params.append(p) if len(extra_params): raise ValueError('Unknown parameters {} - maybe something was misspelled?\nexecute app_usage("{}", tag="{}") for more information'.format(json.dumps(extra_params), app_id, tag)) # Now, validate parameter values. # Should also check if input (NOT OUTPUT) object variables are present in the current workspace workspace = system_variable('workspace') ws_id = system_variable('workspace_id') if workspace is None or ws_id is None: raise ValueError('Unable to retrive current Narrative workspace information! workspace={}, workspace_id={}'.format(workspace, ws_id)) param_errors = list() # If they're workspace objects, track their refs in a list we'll pass to run_job as # a separate param to track provenance. ws_input_refs = list() for p in spec_params: if p['id'] in params: (wsref, err) = self._check_parameter(p, params[p['id']], workspace) if err is not None: param_errors.append("{} - {}".format(p['id'], err)) if wsref is not None: if isinstance(wsref, list): for ref in wsref: if ref is not None: ws_input_refs.append(ref) else: ws_input_refs.append(wsref) if len(param_errors): raise ValueError('Parameter value errors found!\n{}'.format("\n".join(param_errors))) # Hooray, parameters are validated. Set them up for transfer. for p in spec_params: # If any param is a checkbox, need to map from boolean to actual expected value in p['checkbox_map'] # note that True = 0th elem, False = 1st if p['type'] == 'checkbox': if p['id'] in params: checkbox_idx = 0 if params[p['id']] else 1 params[p['id']] = p['checkbox_map'][checkbox_idx] # While we're at it, set the default values for any unset parameters that have them if p['default'] and p['id'] not in params: params[p['id']] = p['default'] return (params, ws_input_refs)
def test_sys_var_time_ms(self): cur_t = int(time.time() * 1000) ts = system_variable('timestamp_epoch_ms') self.assertTrue(cur_t <= ts) self.assertTrue(ts - cur_t < 1000)
def _map_inputs(self, input_mapping, params): """ Maps the dictionary of parameters and inputs based on rules provided in the input_mapping. This iterates over the list of input_mappings, and uses them as a filter to apply to each parameter. Returns a list of inputs that can be passed directly to NJSW.run_job input_mapping is a list of dicts, as defined by NarrativeMethodStore.ServiceMethodInputMapping. params is a dict of key-value-pairs, each key is the input_parameter field of some parameter. """ inputs_dict = dict() for p in input_mapping: # 2 steps - figure out the proper value, then figure out the proper position. # value first! p_value = None if 'input_parameter' in p: p_value = params.get(p['input_parameter'], None) # turn empty strings into None if isinstance(p_value, basestring) and len(p_value) == 0: p_value = None elif 'narrative_system_variable' in p: p_value = system_variable(p['narrative_system_variable']) if 'constant_value' in p and p_value is None: p_value = p['constant_value'] if 'generated_value' in p and p_value is None: p_value = self._generate_input(generated_value) if 'target_type_transform' in p: p_value = self._transform_input(p['target_type_transform'], p_value) # get position! arg_position = p.get('target_argument_position', 0) target_prop = p.get('target_property', None) if target_prop is not None: final_input = inputs_dict.get(arg_position, dict()) if '/' in target_prop: ## This is case when slashes in target_prop separeate elements in nested maps. ## We ignore escaped slashes (separate backslashes should be escaped as well). bck_slash = u"\u244A" fwd_slash = u"\u20EB" temp_string = target_prop.replace("\\\\", bck_slash).replace("\\/", fwd_slash) temp_path = [] for part in temp_string.split("/"): part = part.replace(bck_slash, "\\").replace(fwd_slash, "/") temp_path.append(part.encode('ascii','ignore')) temp_map = final_input temp_key = None ## We're going along the path and creating intermediate dictionaries. for temp_path_item in temp_path: if temp_key: if temp_key not in temp_map: temp_map[temp_key] = {} temp_map = temp_map[temp_key] temp_key = temp_path_item ## temp_map points to deepest nested map now, temp_key is last item in path temp_map[temp_key] = p_value else: final_input[target_prop] = p_value inputs_dict[arg_position] = final_input else: inputs_dict[arg_position] = p_value inputs_list = list() keys = sorted(inputs_dict.keys()) for k in keys: inputs_list.append(inputs_dict[k]) return inputs_list
def test_sys_var_time_sec(self): cur_t = int(time.time()) ts = system_variable('timestamp_epoch_sec') self.assertTrue(cur_t <= ts) self.assertTrue(ts - cur_t < 1)
def _run_app_batch_internal(self, app_id, params, tag, version, cell_id, run_id, dry_run): batch_method = "kb_BatchApp.run_batch" batch_app_id = "kb_BatchApp/run_batch" batch_method_ver = "dev" batch_method_tag = "dev" ws_id = strict_system_variable('workspace_id') spec = self._get_validated_app_spec(app_id, tag, True, version=version) # Preflight check the params - all required ones are present, all # values are the right type, all numerical values are in given ranges spec_params = self.spec_manager.app_params(spec) # A list of lists of UPAs, used for each subjob. batch_ws_upas = list() # The list of actual input values, post-mapping. batch_run_inputs = list() for param_set in params: spec_params_map = dict((spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params))) batch_ws_upas.append(extract_ws_refs(app_id, tag, spec_params, param_set)) batch_run_inputs.append(self._map_inputs( spec['behavior']['kb_service_input_mapping'], param_set, spec_params_map)) service_method = spec['behavior']['kb_service_method'] service_name = spec['behavior']['kb_service_name'] service_ver = spec['behavior'].get('kb_service_version', None) # Let the given version override the spec's version. if version is not None: service_ver = version # This is what calls the function in the back end - Module.method # This isn't the same as the app spec id. job_meta = { 'tag': batch_method_tag, 'batch_app': app_id, 'batch_tag': tag, 'batch_size': len(params), } if cell_id is not None: job_meta['cell_id'] = cell_id if run_id is not None: job_meta['run_id'] = run_id # Now put these all together in a way that can be sent to the batch processing app. batch_params = [{ "module_name": service_name, "method_name": service_method, "service_ver": service_ver, "wsid": ws_id, "meta": job_meta, "batch_params": [{ "params": batch_run_inputs[i], "source_ws_objects": batch_ws_upas[i] } for i in range(len(batch_run_inputs))], }] # We're now almost ready to run the job. Last, we need an agent token. try: token_name = 'KBApp_{}'.format(app_id) token_name = token_name[:self.__MAX_TOKEN_NAME_LEN] agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name) except Exception as e: raise job_meta['token_id'] = agent_token['id'] # This is the input set for NJSW.run_job. Now we need the workspace id # and whatever fits in the metadata. job_runner_inputs = { 'method': batch_method, 'service_ver': batch_method_ver, 'params': batch_params, 'app_id': batch_app_id, 'wsid': ws_id, 'meta': job_meta } # if len(ws_input_refs) > 0: # job_runner_inputs['source_ws_objects'] = ws_input_refs # if we're doing a dry run, just return the inputs that we made. if dry_run: return job_runner_inputs # Log that we're trying to run a job... log_info = { 'app_id': app_id, 'tag': batch_method_tag, 'version': service_ver, 'username': system_variable('user_id'), 'wsid': ws_id } kblogging.log_event(self._log, "run_batch_app", log_info) try: job_id = clients.get("job_service", token=agent_token['token']).run_job(job_runner_inputs) except Exception as e: log_info.update({'err': str(e)}) kblogging.log_event(self._log, "run_batch_app_error", log_info) raise transform_job_exception(e) new_job = Job(job_id, batch_app_id, batch_params, system_variable('user_id'), tag=batch_method_tag, app_version=batch_method_ver, cell_id=cell_id, run_id=run_id, token_id=agent_token['id'], meta=job_meta) self._send_comm_message('run_status', { 'event': 'launched_job', 'event_at': datetime.datetime.utcnow().isoformat() + 'Z', 'cell_id': cell_id, 'run_id': run_id, 'job_id': job_id }) JobManager().register_new_job(new_job) if cell_id is not None: return else: return new_job
def test_sys_var_token(self): if (self.user_token): biokbase.auth.set_environ_token(self.user_token) self.assertEquals(system_variable('token'), self.user_token)
def test_sys_var_user_bad(self): os.environ['KB_AUTH_TOKEN'] = self.bad_fake_token self.assertIsNone(system_variable('user_id'))
def _map_inputs(self, input_mapping, params, spec_params): """ Maps the dictionary of parameters and inputs based on rules provided in the input_mapping. This iterates over the list of input_mappings, and uses them as a filter to apply to each parameter. Returns a list of inputs that can be passed directly to NJSW.run_job input_mapping is a list of dicts, as defined by NarrativeMethodStore.ServiceMethodInputMapping. params is a dict of key-value-pairs, each key is the input_parameter field of some parameter. """ inputs_dict = {} for p in input_mapping: # 2 steps - figure out the proper value, then figure out the # proper position. value first! p_value = None input_param_id = None if "input_parameter" in p: input_param_id = p["input_parameter"] p_value = params.get(input_param_id, None) if spec_params[input_param_id].get("type", "") == "group": p_value = self._map_group_inputs( p_value, spec_params[input_param_id], spec_params ) # turn empty strings into None if isinstance(p_value, str) and len(p_value) == 0: p_value = None elif "narrative_system_variable" in p: p_value = system_variable(p["narrative_system_variable"]) if "constant_value" in p and p_value is None: p_value = p["constant_value"] if "generated_value" in p and p_value is None: p_value = self._generate_input(p["generated_value"]) spec_param = None if input_param_id: spec_param = spec_params[input_param_id] p_value = transform_param_value( p.get("target_type_transform"), p_value, spec_param ) # get position! arg_position = p.get("target_argument_position", 0) target_prop = p.get("target_property", None) if target_prop is not None: final_input = inputs_dict.get(arg_position, {}) if "/" in target_prop: # This is case when slashes in target_prop separate # elements in nested maps. We ignore escaped slashes # (separate backslashes should be escaped as well). bck_slash = "\u244A" fwd_slash = "\u20EB" temp_string = target_prop.replace("\\\\", bck_slash) temp_string = temp_string.replace("\\/", fwd_slash) temp_path = [] for part in temp_string.split("/"): part = part.replace(bck_slash, "\\") part = part.replace(fwd_slash, "/") temp_path.append(part.encode("ascii", "ignore").decode("ascii")) temp_map = final_input temp_key = None # We're going along the path and creating intermediate # dictionaries. for temp_path_item in temp_path: if temp_key: if temp_key not in temp_map: temp_map[temp_key] = {} temp_map = temp_map[temp_key] temp_key = temp_path_item # temp_map points to deepest nested map now, temp_key is # the last item in the path temp_map[temp_key] = p_value else: final_input[target_prop] = p_value inputs_dict[arg_position] = final_input else: inputs_dict[arg_position] = p_value inputs_list = [] keys = sorted(inputs_dict.keys()) for k in keys: inputs_list.append(inputs_dict[k]) return inputs_list
def _map_inputs(self, input_mapping, params, spec_params): """ Maps the dictionary of parameters and inputs based on rules provided in the input_mapping. This iterates over the list of input_mappings, and uses them as a filter to apply to each parameter. Returns a list of inputs that can be passed directly to NJSW.run_job input_mapping is a list of dicts, as defined by NarrativeMethodStore.ServiceMethodInputMapping. params is a dict of key-value-pairs, each key is the input_parameter field of some parameter. """ inputs_dict = dict() for p in input_mapping: # 2 steps - figure out the proper value, then figure out the # proper position. value first! p_value = None input_param_id = None if 'input_parameter' in p: input_param_id = p['input_parameter'] p_value = params.get(input_param_id, None) if spec_params[input_param_id].get('type', '') == 'group': p_value = self._map_group_inputs(p_value, spec_params[input_param_id], spec_params) # turn empty strings into None if isinstance(p_value, basestring) and len(p_value) == 0: p_value = None elif 'narrative_system_variable' in p: p_value = system_variable(p['narrative_system_variable']) if 'constant_value' in p and p_value is None: p_value = p['constant_value'] if 'generated_value' in p and p_value is None: p_value = self._generate_input(p['generated_value']) spec_param = None if input_param_id: spec_param = spec_params[input_param_id] p_value = transform_param_value(p.get('target_type_transform'), p_value, spec_param) # get position! arg_position = p.get('target_argument_position', 0) target_prop = p.get('target_property', None) if target_prop is not None: final_input = inputs_dict.get(arg_position, dict()) if '/' in target_prop: # This is case when slashes in target_prop separeate # elements in nested maps. We ignore escaped slashes # (separate backslashes should be escaped as well). bck_slash = u"\u244A" fwd_slash = u"\u20EB" temp_string = target_prop.replace("\\\\", bck_slash) temp_string = temp_string.replace("\\/", fwd_slash) temp_path = [] for part in temp_string.split("/"): part = part.replace(bck_slash, "\\") part = part.replace(fwd_slash, "/") temp_path.append(part.encode('ascii', 'ignore')) temp_map = final_input temp_key = None # We're going along the path and creating intermediate # dictionaries. for temp_path_item in temp_path: if temp_key: if temp_key not in temp_map: temp_map[temp_key] = {} temp_map = temp_map[temp_key] temp_key = temp_path_item # temp_map points to deepest nested map now, temp_key is # the last item in the path temp_map[temp_key] = p_value else: final_input[target_prop] = p_value inputs_dict[arg_position] = final_input else: inputs_dict[arg_position] = p_value inputs_list = list() keys = sorted(inputs_dict.keys()) for k in keys: inputs_list.append(inputs_dict[k]) return inputs_list
def test_sys_var_workspace(self): os.environ['KB_WORKSPACE_ID'] = self.workspace self.assertEqual(system_variable('workspace'), self.workspace)