def _run_dynamic_service_internal(self, app_id, params, tag, version,
cell_id, run_id, **kwargs):
# Intro tests:
self.spec_manager.check_app(app_id, tag, raise_exception=True)
if version is not None and tag != "release":
raise ValueError(
"App versions only apply to released app modules!")
# Get the spec & params
spec = self.spec_manager.get_spec(app_id, tag)
if 'behavior' not in spec:
raise ValueError(
"This app appears invalid - it has no defined behavior")
behavior = spec['behavior']
if 'script_module' in behavior or 'script_name' in behavior:
# It's an old NJS script. These don't work anymore.
raise ValueError(
'This app relies on a service that is now obsolete. Please contact the administrator.'
)
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'username': system_variable('user_id'),
'ws': system_variable('workspace')
}
kblogging.log_event(self._log, "run_dynamic_service", log_info)
# Silly to keep this here, but we do not validate the incoming parameters.
# If they are provided by the UI (we have cell_id), they are constructed
# according to the spec, so are trusted;
# Otherwise, if they are the product of direct code cell entry, this is a mode we do not
# "support", so we can let it fail hard.
# In the future when code cell interaction is supported for users, we will need to provide
# robust validation and error reporting, but this may end up being (should be) provided by the
# sdk execution infrastructure anyway
input_vals = params
function_name = spec['behavior']['kb_service_name'] + '.' + spec[
'behavior']['kb_service_method']
try:
result = clients.get("service").sync_call(function_name,
input_vals,
service_version=tag)[0]
# if a ui call (a cell_id is defined) we send a result message, otherwise
# just the raw result for display in a code cell. This is how we "support"
# code cells for internal usage.
if cell_id:
self.send_cell_message('result', cell_id, run_id,
{'result': result})
else:
return result
except:
raise
def _run_widget_app_internal(self, app_id, tag, version, cell_id, run_id):
self._send_comm_message('run_status', {
'event': 'validating_app',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id
})
# Intro tests:
self.spec_manager.check_app(app_id, tag, raise_exception=True)
if version is not None and tag != "release":
raise ValueError("App versions only apply to released app modules!")
# Get the spec & params
spec = self.spec_manager.get_spec(app_id, tag)
if 'behavior' not in spec:
raise ValueError("This app appears invalid - it has no defined behavior")
behavior = spec['behavior']
if 'kb_service_input_mapping' in behavior:
# it's a service! Should run this with run_app!
raise ValueError('This app appears to be a long-running job! Please start it using the run_app function instead.')
if 'script_module' in behavior or 'script_name' in behavior:
# It's an old NJS script. These don't work anymore.
raise ValueError('This app relies on a service that is now obsolete. Please contact the administrator.')
# Here, we just deal with two behaviors:
# 1. None of the above - it's a viewer.
# 2. ***TODO*** python_class / python_function. Import and exec the python code.
# for now, just map the inputs to outputs.
# First, validate.
# Preflight check the params - all required ones are present, all values are the right type, all numerical values are in given ranges
#spec_params = self.spec_manager.app_params(spec)
#(params, ws_refs) = self._validate_parameters(app_id, tag, spec_params, kwargs)
log_info = {
'app_id': app_id,
'tag': tag,
'username': system_variable('user_id'),
'ws': system_variable('workspace')
}
kblogging.log_event(self._log, "run_widget_app", log_info)
self._send_comm_message('run_status', {
'event': 'success',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id
})
# now just map onto outputs.
custom_widget = spec.get('widgets', {}).get('input', None)
return WidgetManager().show_custom_widget(custom_widget, app_id, version, tag, spec, cell_id)
def _run_local_app_internal(self, app_id, params, widget_state, tag, version, cell_id, run_id):
self._send_comm_message('run_status', {
'event': 'validating_app',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id
})
spec = self._get_validated_app_spec(app_id, tag, False, version=version)
# Here, we just deal with two behaviors:
# 1. None of the above - it's a viewer.
# 2. ***TODO*** python_class / python_function.
# Import and exec the python code.
# for now, just map the inputs to outputs.
# First, validate.
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
(params, ws_refs) = validate_parameters(app_id, tag, spec_params, params)
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'username': system_variable('user_id'),
'ws': system_variable('workspace')
}
kblogging.log_event(self._log, "run_local_app", log_info)
self._send_comm_message('run_status', {
'event': 'success',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id
})
(output_widget, widget_params) = map_outputs_from_state([],
params,
spec)
# All a local app does is route the inputs to outputs through the
# spec's mapping, and then feed that into the specified output widget.
wm = WidgetManager()
if widget_state is not None:
return wm.show_advanced_viewer_widget(
output_widget, widget_params, widget_state, cell_id=cell_id, tag=tag
)
else:
return wm.show_output_widget(
output_widget, widget_params, cell_id=cell_id, tag=tag
)
def list_objects(obj_type=None, name=None, fuzzy_name=True):
"""
Returns a list of all objects in the current workspace with type=obj_type
obj_type is a string. if None, return all visible objects (no reports, data palettes, etc.)
name is a string. if None, then return everything. if not None, use that string to filter the search. if fuzzy_name is set to True, use that string
as a search filter. e.g., "foo" would match "Foobar" and "Bazfoo"
However, it doesn't go the other way. If name="Foobar" it will not match an object named "foo"
If fuzzy_name is False, only exact (case-insensitive) matches are allowed.
This has limited use, I know, but it's useful for fetching UPAs for objects you know, or names you're iterating over another way.
This first prototype just returns a list of dictionaries, where each dict contains 'type', 'upa', and 'name' keys for each object.
"""
ws_name = system_variable('workspace')
service = clients.get('service')
service_params = {'ws_name': ws_name}
if obj_type is not None:
# matches:
# foo.bar
# foo.bar-1.0
# doesn't match:
# foo
# foo.bar-
# foobar-
# foo.bar-1.2.0
if not re.match(r"[A-Za-z]+\.[A-Za-z]+(-\d+\.\d+)?$", obj_type):
raise ValueError(
'{} is not a valid type. Valid types are of the format "Module.Type" or "Module.Type-Version"'
.format(obj_type))
service_params['types'] = [obj_type]
all_obj = service.sync_call('NarrativeService.list_objects_with_sets',
[service_params])[0]
obj_list = list()
for obj in all_obj['data']:
# filtration!
# 1. ignore narratives
if 'KBaseNarrative.Narrative' in obj['object_info'][2]:
continue
# 2. name filter
if name is not None:
name = str(name).lower()
# if we're not strict, just search for the string
if fuzzy_name is True and name not in obj['object_info'][1].lower(
):
continue
elif fuzzy_name is False and name != obj['object_info'][1].lower():
continue
upa_prefix = '' # gavin's gonna wreck me.
if 'dp_info' in obj: # seriously.
upa_prefix = obj['dp_info'][
'ref'] + ';' # not like I want to support this, either...
info = obj['object_info']
obj_list.append({
"upa":
"{}{}/{}/{}".format(upa_prefix, info[6], info[0], info[4]),
"name":
info[1],
"type":
info[2]
})
return obj_list
def get_df(ws_ref, col_attributes=(), row_attributes=(), clustergrammer=False):
"""
Gets a dataframe from the WS object
:param ws_ref: The Workspace reference of the 2DMatrix containing object
:param col_attributes: Which column attributes should appear in the resulting DataFrame as a
multiIndex. Defaults to all attributes, pass None to use a simple index of only ID.
:param row_attributes: Which row attributes should appear in the resulting DataFrame as a
multiIndex. Defaults to all attributes, pass None to use a simple index of only ID.
:param clustergrammer: Returns a DataFrame with Clustergrammer compatible indices and columns.
Defaults to False.
:return: A Pandas DataFrame
"""
ws = clients.get('workspace')
if "/" not in ws_ref:
ws_ref = "{}/{}".format(system_variable('workspace'), ws_ref)
generic_data = ws.get_objects2({'objects': [{'ref': ws_ref}]})['data'][0]['data']
if not _is_compatible_matrix(generic_data):
raise ValueError("{} is not a compatible data type for this viewer. Data type must "
"contain a 'data' key with a FloatMatrix2D type value".format(ws_ref))
cols = _get_categories(generic_data['data']['col_ids'],
ws_ref,
generic_data.get('col_attributemapping_ref'),
generic_data.get('col_mapping'),
col_attributes,
clustergrammer)
rows = _get_categories(generic_data['data']['row_ids'],
ws_ref,
generic_data.get('row_attributemapping_ref'),
generic_data.get('row_mapping'),
row_attributes,
clustergrammer)
return pd.DataFrame(data=generic_data['data']['values'], columns=cols, index=rows)
def _run_dynamic_service_internal(self, app_id, params, tag, version,
cell_id, run_id):
spec = self._get_validated_app_spec(app_id,
tag,
False,
version=version)
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'username': system_variable('user_id'),
'ws': system_variable('workspace')
}
kblogging.log_event(self._log, "run_dynamic_service", log_info)
# Silly to keep this here, but we do not validate the incoming parameters.
# If they are provided by the UI (we have cell_id), they are constructed
# according to the spec, so are trusted;
# Otherwise, if they are the product of direct code cell entry, this is a mode we do not
# "support", so we can let it fail hard.
# In the future when code cell interaction is supported for users, we will need to provide
# robust validation and error reporting, but this may end up being (should be) provided by the
# sdk execution infrastructure anyway
input_vals = params
function_name = spec['behavior']['kb_service_name'] + '.' + spec[
'behavior']['kb_service_method']
try:
result = clients.get("service").sync_call(function_name,
input_vals,
service_version=tag)[0]
# if a ui call (a cell_id is defined) we send a result message, otherwise
# just the raw result for display in a code cell. This is how we "support"
# code cells for internal usage.
if cell_id:
self.send_cell_message('result', cell_id, run_id,
{'result': result})
else:
return result
except:
raise
def _run_dynamic_service_internal(self, app_id, params, tag, version, cell_id, run_id):
spec = self._get_validated_app_spec(app_id, tag, False, version=version)
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'username': system_variable('user_id'),
'ws': system_variable('workspace')
}
kblogging.log_event(self._log, "run_dynamic_service", log_info)
# Silly to keep this here, but we do not validate the incoming parameters.
# If they are provided by the UI (we have cell_id), they are constructed
# according to the spec, so are trusted;
# Otherwise, if they are the product of direct code cell entry, this is a mode we do not
# "support", so we can let it fail hard.
# In the future when code cell interaction is supported for users, we will need to provide
# robust validation and error reporting, but this may end up being (should be) provided by the
# sdk execution infrastructure anyway
input_vals = params
function_name = spec['behavior']['kb_service_name'] + '.' + spec['behavior']['kb_service_method']
try:
result = clients.get("service").sync_call(
function_name,
input_vals,
service_version=tag
)[0]
# if a ui call (a cell_id is defined) we send a result message, otherwise
# just the raw result for display in a code cell. This is how we "support"
# code cells for internal usage.
if cell_id:
self.send_cell_message('result', cell_id, run_id, {
'result': result
})
else:
return result
except:
raise
def initialize_jobs(self):
"""
Initializes this JobManager.
This is expected to be run by a running Narrative, and naturally linked to a workspace.
So it does the following steps.
1. app_util.system_variable('workspace_id')
2. get list of jobs with that ws id from UJS (also gets tag, cell_id, run_id)
3. initialize the Job objects by running NJS.get_job_params (also gets app_id)
4. start the status lookup loop.
"""
ws_id = system_variable("workspace_id")
job_states = dict()
kblogging.log_event(self._log, "JobManager.initialize_jobs",
{"ws_id": ws_id})
try:
job_states = clients.get("execution_engine2").check_workspace_jobs(
{
"workspace_id": ws_id,
"return_list": 0
})
self._running_jobs = dict()
except Exception as e:
kblogging.log_event(self._log, "init_error", {"err": str(e)})
new_e = transform_job_exception(e)
raise new_e
for job_id, job_state in job_states.items():
job_input = job_state.get("job_input", {})
job_meta = job_input.get("narrative_cell_info", {})
status = job_state.get("status")
job = Job.from_state(
job_id,
job_input,
job_state.get("user"),
app_id=job_input.get("app_id"),
tag=job_meta.get("tag", "release"),
cell_id=job_meta.get("cell_id", None),
run_id=job_meta.get("run_id", None),
token_id=job_meta.get("token_id", None),
meta=job_meta,
)
self._running_jobs[job_id] = {
"refresh":
1
if status not in ["completed", "errored", "terminated"] else 0,
"job":
job,
}
def initialize_jobs(self, cell_ids: List[str] = None) -> None:
"""
Initializes this JobManager.
This is expected to be run by a running Narrative, and naturally linked to a workspace.
So it does the following steps.
1. gets the current workspace ID from app_util.system_variable('workspace_id')
2. get list of jobs with that ws id from ee2 (also gets tag, cell_id, run_id)
3. initialize the Job objects and add them to the running jobs list
4. start the status lookup loop.
"""
ws_id = system_variable("workspace_id")
job_states = dict()
kblogging.log_event(self._log, "JobManager.initialize_jobs", {"ws_id": ws_id})
try:
job_states = clients.get("execution_engine2").check_workspace_jobs(
{
"workspace_id": ws_id,
"return_list": 0, # do not remove
"exclude_fields": JOB_INIT_EXCLUDED_JOB_STATE_FIELDS,
}
)
except Exception as e:
kblogging.log_event(self._log, "init_error", {"err": str(e)})
new_e = transform_job_exception(e, "Unable to initialize jobs")
raise new_e
self._running_jobs = dict()
job_states = self._reorder_parents_children(job_states)
for job_state in job_states.values():
child_jobs = None
if job_state.get("batch_job"):
child_jobs = [
self.get_job(child_id)
for child_id in job_state.get("child_jobs", [])
]
job = Job(job_state, children=child_jobs)
# Set to refresh when job is not in terminal state
# and when job is present in cells (if given)
# and when it is not part of a batch
refresh = not job.was_terminal() and not job.batch_id
if cell_ids is not None:
refresh = refresh and job.in_cells(cell_ids)
self.register_new_job(job, refresh)
def initialize_jobs(self):
"""
Initializes this JobManager.
This is expected to be run by a running Narrative, and naturally linked to a workspace.
So it does the following steps.
1. app_util.system_variable('workspace_id')
2. get list of jobs with that ws id from UJS (also gets tag, cell_id, run_id)
3. initialize the Job objects by running NJS.get_job_params (also gets app_id)
4. start the status lookup loop.
"""
ws_id = system_variable("workspace_id")
job_states = dict()
kblogging.log_event(self._log, "JobManager.initialize_jobs",
{'ws_id': ws_id})
try:
job_states = clients.get('execution_engine2').check_workspace_jobs(
{
'workspace_id': ws_id,
'return_list': 0
})
self._running_jobs = dict()
except Exception as e:
kblogging.log_event(self._log, 'init_error', {'err': str(e)})
new_e = transform_job_exception(e)
raise new_e
for job_id, job_state in job_states.items():
job_input = job_state.get('job_input', {})
job_meta = job_input.get('narrative_cell_info', {})
status = job_state.get('status')
job = Job.from_state(job_id,
job_input,
job_state.get('user'),
app_id=job_input.get('app_id'),
tag=job_meta.get('tag', 'release'),
cell_id=job_meta.get('cell_id', None),
run_id=job_meta.get('run_id', None),
token_id=job_meta.get('token_id', None),
meta=job_meta)
self._running_jobs[job_id] = {
'refresh':
1
if status not in ['completed', 'errored', 'terminated'] else 0,
'job':
job
}
def get_df(ws_ref, col_attributes=(), row_attributes=(), clustergrammer=False):
"""
Gets a dataframe from the WS object
:param ws_ref: The Workspace reference of the 2DMatrix containing object
:param col_attributes: Which column attributes should appear in the resulting DataFrame as a
multiIndex. Defaults to all attributes, pass None to use a simple index of only ID.
:param row_attributes: Which row attributes should appear in the resulting DataFrame as a
multiIndex. Defaults to all attributes, pass None to use a simple index of only ID.
:param clustergrammer: Returns a DataFrame with Clustergrammer compatible indices and columns.
Defaults to False.
:return: A Pandas DataFrame
"""
ws = clients.get("workspace")
if "/" not in ws_ref:
ws_ref = "{}/{}".format(system_variable("workspace"), ws_ref)
generic_data = ws.get_objects2({"objects": [{"ref": ws_ref}]})["data"][0]["data"]
if not _is_compatible_matrix(generic_data):
raise ValueError(
"{} is not a compatible data type for this viewer. Data type must "
"contain a 'data' key with a FloatMatrix2D type value".format(ws_ref)
)
cols = _get_categories(
generic_data["data"]["col_ids"],
ws_ref,
generic_data.get("col_attributemapping_ref"),
generic_data.get("col_mapping"),
col_attributes,
clustergrammer,
)
rows = _get_categories(
generic_data["data"]["row_ids"],
ws_ref,
generic_data.get("row_attributemapping_ref"),
generic_data.get("row_mapping"),
row_attributes,
clustergrammer,
)
return pd.DataFrame(data=generic_data["data"]["values"], columns=cols, index=rows)
def _transform_input(self, transform_type, value):
"""
Transforms an input according to the rules given in NarrativeMethodStore.ServiceMethodInputMapping
Really, there are three types of transforms possible:
1. ref - turns the input string into a workspace ref.
2. int - tries to coerce the input string into an int.
3. list<type> - turns the given list into a list of the given type.
(4.) none or None - doesn't transform.
Returns a transformed (or not) value.
"""
if transform_type == "none" or transform_type is None:
return value
elif transform_type == "ref":
# make a workspace ref
if value is not None:
value = system_variable('workspace') + '/' + value
return value
elif transform_type == "int":
# make it an integer, OR 0.
if value is None or len(str(value).strip()) == 0:
return None
return int(value)
elif transform_type.startswith("list<") and transform_type.endswith(">"):
# make it a list of transformed types.
list_type = transform_type[5:-1]
if isinstance(value, list):
ret = []
for pos in range(0, len(value)):
ret.append(self._transform_input(list_type, value[pos]))
return ret
else:
return [self._transform_input(list_type, value)]
else:
raise ValueError("Unsupported Transformation type: " + transform_type)
def _run_app_internal(self, app_id, params, tag, version,
cell_id, run_id, dry_run):
"""
Attemps to run the app, returns a Job with the running app info.
Should *hopefully* also inject that app into the Narrative's metadata.
Probably need some kind of JavaScript-foo to get that to work.
Parameters:
-----------
app_id - should be from the app spec, e.g. 'build_a_metabolic_model'
or 'MegaHit/run_megahit'.
params - a dictionary of parameters.
tag - optional, one of [release|beta|dev] (default=release)
version - optional, a semantic version string. Only released modules
have versions, so if the tag is not 'release', and a version
is given, a ValueError will be raised.
**kwargs - these are the set of parameters to be used with the app.
They can be found by using the app_usage function. If any
non-optional apps are missing, a ValueError will be raised.
"""
ws_id = strict_system_variable('workspace_id')
spec = self._get_validated_app_spec(app_id, tag, True, version=version)
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
spec_params_map = dict((spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params)))
ws_input_refs = extract_ws_refs(app_id, tag, spec_params, params)
input_vals = self._map_inputs(
spec['behavior']['kb_service_input_mapping'],
params,
spec_params_map)
service_method = spec['behavior']['kb_service_method']
service_name = spec['behavior']['kb_service_name']
service_ver = spec['behavior'].get('kb_service_version', None)
# Let the given version override the spec's version.
if version is not None:
service_ver = version
# This is what calls the function in the back end - Module.method
# This isn't the same as the app spec id.
function_name = service_name + '.' + service_method
job_meta = {'tag': tag}
if cell_id is not None:
job_meta['cell_id'] = cell_id
if run_id is not None:
job_meta['run_id'] = run_id
# This is the input set for NJSW.run_job. Now we need the workspace id
# and whatever fits in the metadata.
job_runner_inputs = {
'method': function_name,
'service_ver': service_ver,
'params': input_vals,
'app_id': app_id,
'wsid': ws_id,
'meta': job_meta
}
if len(ws_input_refs) > 0:
job_runner_inputs['source_ws_objects'] = ws_input_refs
if dry_run:
return job_runner_inputs
# We're now almost ready to run the job. Last, we need an agent token.
try:
token_name = 'KBApp_{}'.format(app_id)
token_name = token_name[:self.__MAX_TOKEN_NAME_LEN]
agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name)
except Exception as e:
raise
job_runner_inputs['meta']['token_id'] = agent_token['id']
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'version': service_ver,
'username': system_variable('user_id'),
'wsid': ws_id
}
kblogging.log_event(self._log, "run_app", log_info)
try:
job_id = clients.get("job_service", token=agent_token['token']).run_job(job_runner_inputs)
except Exception as e:
log_info.update({'err': str(e)})
kblogging.log_event(self._log, "run_app_error", log_info)
raise transform_job_exception(e)
new_job = Job(job_id,
app_id,
input_vals,
system_variable('user_id'),
tag=tag,
app_version=service_ver,
cell_id=cell_id,
run_id=run_id,
token_id=agent_token['id'])
self._send_comm_message('run_status', {
'event': 'launched_job',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id,
'job_id': job_id
})
JobManager().register_new_job(new_job)
if cell_id is not None:
return
else:
return new_job
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id, **kwargs):
"""
Attemps to run the app, returns a Job with the running app info.
Should *hopefully* also inject that app into the Narrative's metadata.
Probably need some kind of JavaScript-foo to get that to work.
Parameters:
-----------
app_id - should be from the app spec, e.g. 'build_a_metabolic_model'
or 'MegaHit/run_megahit'.
params - the dictionary of parameters.
tag - optional, one of [release|beta|dev] (default=release)
version - optional, a semantic version string. Only released modules have
versions, so if the tag is not 'release', and a version is given,
a ValueError will be raised.
**kwargs - these are the set of parameters to be used with the app.
They can be found by using the app_usage function. If any
non-optional apps are missing, a ValueError will be raised.
Example:
--------
my_job = mm.run_app('MegaHit/run_megahit', version=">=1.0.0", read_library_name="My_PE_Library", output_contigset_name="My_Contig_Assembly")
"""
### TODO: this needs restructuring so that we can send back validation failure
### messages. Perhaps a separate function and catch the errors, or return an
### error structure.
# Intro tests:
self.spec_manager.check_app(app_id, tag, raise_exception=True)
if version is not None and tag != "release":
if re.match(version, '\d+\.\d+\.\d+') is not None:
raise ValueError("Semantic versions only apply to released app modules. You can use a Git commit hash instead to specify a version.")
# Get the spec & params
spec = self.spec_manager.get_spec(app_id, tag)
# There's some branching to do here.
# Cases:
# app has behavior.kb_service_input_mapping -- is a valid long-running app.
# app only has behavior.output_mapping - not kb_service_input_mapping or script_module - it's a viewer and should return immediately
# app has other things besides kb_service_input_mapping -- not a valid app.
if 'behavior' not in spec:
raise Exception("This app appears invalid - it has no defined behavior")
if 'kb_service_input_mapping' not in spec['behavior']:
raise Exception("This app does not appear to be a long-running job! Please use 'run_local_app' to start this instead.")
# Preflight check the params - all required ones are present, all values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
(params, ws_input_refs) = self._validate_parameters(app_id, tag, spec_params, params)
ws_id = system_variable('workspace_id')
if ws_id is None:
raise ValueError('Unable to retrive current Narrative workspace information!')
input_vals = self._map_inputs(spec['behavior']['kb_service_input_mapping'], params)
service_method = spec['behavior']['kb_service_method']
service_name = spec['behavior']['kb_service_name']
service_ver = spec['behavior'].get('kb_service_version', None)
service_url = spec['behavior']['kb_service_url']
# Let the given version override the spec's version.
if version is not None:
service_ver = version
# This is what calls the function in the back end - Module.method
# This isn't the same as the app spec id.
function_name = service_name + '.' + service_method
job_meta = {'tag': tag}
if cell_id is not None:
job_meta['cell_id'] = cell_id
if run_id is not None:
job_meta['run_id'] = run_id
# This is the input set for NJSW.run_job. Now we need the worksapce id and whatever fits in the metadata.
job_runner_inputs = {
'method': function_name,
'service_ver': service_ver,
'params': input_vals,
'app_id': app_id,
'wsid': ws_id,
'meta': job_meta
}
if len(ws_input_refs) > 0:
job_runner_inputs['source_ws_objects'] = ws_input_refs
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'version': service_ver,
'username': system_variable('user_id'),
'wsid': ws_id
}
kblogging.log_event(self._log, "run_app", log_info)
try:
job_id = self.njs.run_job(job_runner_inputs)
except Exception as e:
log_info.update({'err': str(e)})
kblogging.log_event(self._log, "run_app_error", log_info)
raise transform_job_exception(e)
new_job = Job(job_id,
app_id,
[params],
system_variable('user_id'),
tag=tag,
app_version=service_ver,
cell_id=cell_id,
run_id=run_id)
self._send_comm_message('run_status', {
'event': 'launched_job',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id,
'job_id': job_id
})
JobManager().register_new_job(new_job)
if cell_id is not None:
return
else:
return new_job
def test_sys_var_user_none(self):
if 'KB_AUTH_TOKEN' in os.environ:
del os.environ['KB_AUTH_TOKEN']
self.assertIsNone(system_variable('user_id'))
def test_sys_var_bad_token(self):
if 'KB_AUTH_TOKEN' in os.environ:
del os.environ['KB_AUTH_TOKEN']
self.assertIsNone(system_variable('token'))
def test_sys_var_workspace_id(self):
os.environ['KB_WORKSPACE_ID'] = self.workspace
self.assertEquals(system_variable('workspace_id'), 12345)
def load_widget_info(self, tag="release", verbose=False):
"""
Loads widget info and mapping.
Eventually will fetch from kbase-ui, a kbase CDN, or the catalog service.
For now, it gets known vis widgets from all method specs.
This returns the a Dict where all keys are the name of a widget, and all values
contain widget information in this structure:
{
"params": {
"param_name": {
"is_constant": boolean,
"param_type": one of (string|boolean|dropdown),
"allowed_values": list of strings (exists when param_type==dropdown),
"allowed_types": list of data types (when param_type==string),
"param_value": something, mainly when is_constant==True
}
}
"""
check_tag(tag, raise_exception=True)
methods = self._sm.app_specs[tag].values()
all_widgets = dict()
# keys = widget names / namespaced require path / etc.
# Individual widget values should be:
# {params: {
# name1: {
# is_constant: boolean,
# value: (***something*** | None) (something = any structure),
# allowed: [ list of allowed values, optional ],
# type: (string, int, float, boolean, etc. list? hash?)
# allowed_types: [ list of allowed ws types, optional ]
# },
# name2: { is_constant, value }
# }
for method in methods:
if 'output' not in method['widgets']:
widget_name = self._default_output_widget
else:
widget_name = method['widgets']['output']
if widget_name == 'null':
if verbose:
print("Ignoring a widget named 'null' in {} - {}".format(tag, method['info']['id']))
continue
out_mapping = method['behavior'].get('kb_service_output_mapping', method['behavior'].get('output_mapping', None))
if out_mapping is not None:
params = {}
for p in out_mapping:
param_name = p['target_property']
allowed_values = set()
is_constant = False
param_value = None
param_type = 'string'
allowed_types = set()
if 'constant_value' in p:
# add this val to a set of constant values for that param in that widget.
# if more than one possible, this need to be optional
is_constant = True
allowed_values.add(p['constant_value'])
if 'input_parameter' in p:
# this is a user given input. look up what it expects from the
# associated parameter of that name
in_param = p['input_parameter']
for spec_param in method['parameters']:
if spec_param['id'] == in_param:
# want its:
# field_type = text, float, number, ...
in_type = spec_param['field_type']
if in_type == 'text':
param_type = 'string'
if spec_param.has_key('text_options'):
validate_as = spec_param['text_options'].get('validate_as', None)
if validate_as == 'int':
param_type = 'int'
elif validate_as == 'float':
param_type = 'float'
if spec_param['text_options'].has_key('valid_ws_types'):
allowed_types.update(spec_param['text_options']['valid_ws_types'])
elif param_type == 'textarea':
param_type = 'string'
elif param_type == 'checkbox':
param_type = 'boolean'
elif param_type == 'dropdown':
param_type = 'dropdown'
allowed_values.update([o['value'] for o in spec_param['dropdown_options']])
if 'narrative_system_variable' in p:
# this is something like the ws name or token that needs to get fetched
# by the system. Shouldn't be handled by the user.
is_constant = True
param_value = system_variable(p['narrative_system_variable'])
if 'service_method_output_path' in p:
param_type = 'from_service_output'
param_info = {
'is_constant': is_constant,
'param_type': param_type,
}
if allowed_values:
param_info['allowed_values'] = allowed_values
if allowed_types:
param_info['allowed_types'] = allowed_types
if param_value:
param_info['param_value'] = param_value
params[param_name] = param_info
if widget_name in all_widgets:
# if it's already there, just update the allowed_types and allowed_values for some params that have them
for p_name in params.keys():
if 'allowed_types' in params[p_name]:
if p_name not in all_widgets[widget_name]['params']:
all_widgets[widget_name]['params'][p_name] = params[p_name]
else:
widget_types = all_widgets[widget_name]['params'].get(p_name, {}).get('allowed_types', set())
widget_types.update(params[p_name]['allowed_types'])
all_widgets[widget_name]['params'][p_name]['allowed_types'] = widget_types
if 'allowed_values' in params[p_name]:
if p_name not in all_widgets[widget_name]['params']:
all_widgets[widget_name]['params'][p_name] = params[p_name]
else:
widget_vals = all_widgets[widget_name]['params'].get(p_name, {}).get('allowed_values', set())
widget_vals.update(params[p_name]['allowed_values'])
all_widgets[widget_name]['params'][p_name]['allowed_values'] = widget_vals
else:
all_widgets[widget_name] = { 'params': params }
# finally, turn all sets into lists
for w in all_widgets:
for p in all_widgets[w]["params"]:
if "allowed_types" in all_widgets[w]["params"][p]:
all_widgets[w]["params"][p]["allowed_types"] = list(all_widgets[w]["params"][p]["allowed_types"])
if "allowed_values" in all_widgets[w]['params'][p]:
all_widgets[w]["params"][p]["allowed_values"] = list(all_widgets[w]["params"][p]["allowed_values"])
return all_widgets
def run_local_app(
self,
app_id,
params,
tag="release",
version=None,
cell_id=None,
run_id=None,
widget_state=None,
):
"""
Attempts to run a local app. These do not return a Job object, but just
the result of the app. In most cases, this will be a Javascript display
of the result, but could be anything.
If the app_spec looks like it makes a service call, then this raises a
ValueError. Otherwise, it validates each parameter in **kwargs against
the app spec, executes it, and returns the result.
Parameters:
-----------
app_id - should be from the app spec, e.g. 'view_expression_profile'
params - the dictionary of parameters for the app. Should be key-value
pairs where they keys are strings. If any non-optional
parameters are missing, an informative string will be printed.
tag - optional, one of [release|beta|dev] (default=release)
version - optional, a semantic version string. Only released modules
have versions, so if the tag is not 'release', and a version
is given, a ValueError will be raised.
Example:
run_local_app('NarrativeViewers/view_expression_profile',
{
"input_expression_matrix": "MyMatrix",
"input_gene_ids": "1234"
},
version='0.0.1',
input_expression_matrix="MyMatrix")
"""
spec = self._get_validated_app_spec(app_id, tag, False, version=version)
# Here, we just deal with two behaviors:
# 1. None of the above - it's a viewer.
# 2. ***TODO*** python_class / python_function.
# Import and exec the python code.
# for now, just map the inputs to outputs.
# First, validate.
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
(params, ws_refs) = validate_parameters(app_id, tag, spec_params, params)
# Log that we're trying to run a job...
log_info = {
"app_id": app_id,
"tag": tag,
"username": system_variable("user_id"),
"ws": system_variable("workspace"),
}
kblogging.log_event(self._log, "run_local_app", log_info)
self._send_comm_message(
MESSAGE_TYPE["RUN_STATUS"],
{
"event": "success",
"event_at": timestamp(),
"cell_id": cell_id,
"run_id": run_id,
},
)
(output_widget, widget_params) = map_outputs_from_state([], params, spec)
# All a local app does is route the inputs to outputs through the
# spec's mapping, and then feed that into the specified output widget.
wm = WidgetManager()
if widget_state is not None:
return wm.show_advanced_viewer_widget(
output_widget, widget_params, widget_state, cell_id=cell_id, tag=tag
)
else:
return wm.show_output_widget(
output_widget, widget_params, cell_id=cell_id, tag=tag
)
def _run_local_app_internal(self, app_id, params, widget_state, tag,
version, cell_id, run_id):
self._send_comm_message(
'run_status', {
'event': 'validating_app',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id
})
spec = self._get_validated_app_spec(app_id,
tag,
False,
version=version)
# Here, we just deal with two behaviors:
# 1. None of the above - it's a viewer.
# 2. ***TODO*** python_class / python_function.
# Import and exec the python code.
# for now, just map the inputs to outputs.
# First, validate.
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
(params, ws_refs) = validate_parameters(app_id, tag, spec_params,
params)
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'username': system_variable('user_id'),
'ws': system_variable('workspace')
}
kblogging.log_event(self._log, "run_local_app", log_info)
self._send_comm_message(
'run_status', {
'event': 'success',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id
})
(output_widget, widget_params) = map_outputs_from_state([], params,
spec)
# All a local app does is route the inputs to outputs through the
# spec's mapping, and then feed that into the specified output widget.
wm = WidgetManager()
if widget_state is not None:
return wm.show_advanced_viewer_widget(output_widget,
widget_params,
widget_state,
cell_id=cell_id,
tag=tag)
else:
return wm.show_output_widget(output_widget,
widget_params,
cell_id=cell_id,
tag=tag)
def _run_app_internal(self, app_id, params, tag, version, cell_id, run_id,
dry_run):
"""
Attemps to run the app, returns a Job with the running app info.
Should *hopefully* also inject that app into the Narrative's metadata.
Probably need some kind of JavaScript-foo to get that to work.
Parameters:
-----------
app_id - should be from the app spec, e.g. 'build_a_metabolic_model'
or 'MegaHit/run_megahit'.
params - a dictionary of parameters.
tag - optional, one of [release|beta|dev] (default=release)
version - optional, a semantic version string. Only released modules
have versions, so if the tag is not 'release', and a version
is given, a ValueError will be raised.
**kwargs - these are the set of parameters to be used with the app.
They can be found by using the app_usage function. If any
non-optional apps are missing, a ValueError will be raised.
"""
ws_id = strict_system_variable('workspace_id')
spec = self._get_validated_app_spec(app_id, tag, True, version=version)
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
spec_params_map = dict((spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params)))
ws_input_refs = extract_ws_refs(app_id, tag, spec_params, params)
input_vals = self._map_inputs(
spec['behavior']['kb_service_input_mapping'], params,
spec_params_map)
service_method = spec['behavior']['kb_service_method']
service_name = spec['behavior']['kb_service_name']
service_ver = spec['behavior'].get('kb_service_version', None)
# Let the given version override the spec's version.
if version is not None:
service_ver = version
# This is what calls the function in the back end - Module.method
# This isn't the same as the app spec id.
function_name = service_name + '.' + service_method
job_meta = {'tag': tag}
if cell_id is not None:
job_meta['cell_id'] = cell_id
if run_id is not None:
job_meta['run_id'] = run_id
# This is the input set for NJSW.run_job. Now we need the workspace id
# and whatever fits in the metadata.
job_runner_inputs = {
'method': function_name,
'service_ver': service_ver,
'params': input_vals,
'app_id': app_id,
'wsid': ws_id,
'meta': job_meta
}
if len(ws_input_refs) > 0:
job_runner_inputs['source_ws_objects'] = ws_input_refs
if dry_run:
return job_runner_inputs
# We're now almost ready to run the job. Last, we need an agent token.
try:
token_name = 'KBApp_{}'.format(app_id)
token_name = token_name[:self.__MAX_TOKEN_NAME_LEN]
agent_token = auth.get_agent_token(auth.get_auth_token(),
token_name=token_name)
except Exception as e:
raise
job_runner_inputs['meta']['token_id'] = agent_token['id']
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': tag,
'version': service_ver,
'username': system_variable('user_id'),
'wsid': ws_id
}
kblogging.log_event(self._log, "run_app", log_info)
try:
job_id = clients.get(
"execution_engine2",
token=agent_token['token']).run_job(job_runner_inputs)
except Exception as e:
log_info.update({'err': str(e)})
kblogging.log_event(self._log, "run_app_error", log_info)
raise transform_job_exception(e)
new_job = Job(job_id,
app_id,
input_vals,
system_variable('user_id'),
tag=tag,
app_version=service_ver,
cell_id=cell_id,
run_id=run_id,
token_id=agent_token['id'])
self._send_comm_message(
'run_status', {
'event': 'launched_job',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id,
'job_id': job_id
})
self.register_new_job(new_job)
if cell_id is not None:
return
else:
return new_job
def _run_app_batch_internal(self, app_id, params, tag, version, cell_id,
run_id, dry_run):
batch_method = "kb_BatchApp.run_batch"
batch_app_id = "kb_BatchApp/run_batch"
batch_method_ver = "dev"
batch_method_tag = "dev"
ws_id = strict_system_variable('workspace_id')
spec = self._get_validated_app_spec(app_id, tag, True, version=version)
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
# A list of lists of UPAs, used for each subjob.
batch_ws_upas = list()
# The list of actual input values, post-mapping.
batch_run_inputs = list()
for param_set in params:
spec_params_map = dict((spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params)))
batch_ws_upas.append(
extract_ws_refs(app_id, tag, spec_params, param_set))
batch_run_inputs.append(
self._map_inputs(spec['behavior']['kb_service_input_mapping'],
param_set, spec_params_map))
service_method = spec['behavior']['kb_service_method']
service_name = spec['behavior']['kb_service_name']
service_ver = spec['behavior'].get('kb_service_version', None)
# Let the given version override the spec's version.
if version is not None:
service_ver = version
# This is what calls the function in the back end - Module.method
# This isn't the same as the app spec id.
job_meta = {
'tag': batch_method_tag,
'batch_app': app_id,
'batch_tag': tag,
'batch_size': len(params),
}
if cell_id is not None:
job_meta['cell_id'] = cell_id
if run_id is not None:
job_meta['run_id'] = run_id
# Now put these all together in a way that can be sent to the batch processing app.
batch_params = [{
"module_name":
service_name,
"method_name":
service_method,
"service_ver":
service_ver,
"wsid":
ws_id,
"meta":
job_meta,
"batch_params": [{
"params": batch_run_inputs[i],
"source_ws_objects": batch_ws_upas[i]
} for i in range(len(batch_run_inputs))],
}]
# We're now almost ready to run the job. Last, we need an agent token.
try:
token_name = 'KBApp_{}'.format(app_id)
token_name = token_name[:self.__MAX_TOKEN_NAME_LEN]
agent_token = auth.get_agent_token(auth.get_auth_token(),
token_name=token_name)
except Exception as e:
raise
job_meta['token_id'] = agent_token['id']
# This is the input set for NJSW.run_job. Now we need the workspace id
# and whatever fits in the metadata.
job_runner_inputs = {
'method': batch_method,
'service_ver': batch_method_ver,
'params': batch_params,
'app_id': batch_app_id,
'wsid': ws_id,
'meta': job_meta
}
# if len(ws_input_refs) > 0:
# job_runner_inputs['source_ws_objects'] = ws_input_refs
# if we're doing a dry run, just return the inputs that we made.
if dry_run:
return job_runner_inputs
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': batch_method_tag,
'version': service_ver,
'username': system_variable('user_id'),
'wsid': ws_id
}
kblogging.log_event(self._log, "run_batch_app", log_info)
try:
job_id = clients.get(
"execution_engine2",
token=agent_token['token']).run_job(job_runner_inputs)
except Exception as e:
log_info.update({'err': str(e)})
kblogging.log_event(self._log, "run_batch_app_error", log_info)
raise transform_job_exception(e)
new_job = Job(job_id,
batch_app_id,
batch_params,
system_variable('user_id'),
tag=batch_method_tag,
app_version=batch_method_ver,
cell_id=cell_id,
run_id=run_id,
token_id=agent_token['id'],
meta=job_meta)
self._send_comm_message(
'run_status', {
'event': 'launched_job',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id,
'job_id': job_id
})
self.register_new_job(new_job)
if cell_id is not None:
return
else:
return new_job
def initialize_jobs(self, start_lookup_thread=True):
"""
Initializes this JobManager.
This is expected to be run by a running Narrative, and naturally linked to a workspace.
So it does the following steps.
1. app_util.system_variable('workspace_id')
2. get list of jobs with that ws id from UJS (also gets tag, cell_id, run_id)
3. initialize the Job objects by running NJS.get_job_params (also gets app_id)
4. start the status lookup loop.
"""
the_time = int(round(time.time() * 1000))
self._send_comm_message('start', {'time': the_time})
ws_id = system_variable('workspace_id')
try:
nar_jobs = clients.get('user_and_job_state').list_jobs2({
'authstrat': 'kbaseworkspace',
'authparams': [str(ws_id)]
})
except Exception as e:
kblogging.log_event(self._log, 'init_error', {'err': str(e)})
new_e = transform_job_exception(e)
error = {
'error': 'Unable to get initial jobs list',
'message': getattr(new_e, 'message', 'Unknown reason'),
'code': getattr(new_e, 'code', -1),
'source': getattr(new_e, 'source', 'jobmanager'),
'name': getattr(new_e, 'name', type(e).__name__),
'service': 'user_and_job_state'
}
self._send_comm_message('job_init_err', error)
raise new_e
job_ids = [j[0] for j in nar_jobs]
job_states = clients.get('job_service').check_jobs({
'job_ids': job_ids, 'with_job_params': 1
})
job_param_info = job_states.get('job_params', {})
job_check_error = job_states.get('check_error', {})
error_jobs = dict()
for info in nar_jobs:
job_id = info[0]
user_info = info[1]
job_meta = info[10]
try:
if job_id in job_param_info:
job_info = job_param_info[job_id]
job = Job.from_state(job_id,
job_info,
user_info[0],
app_id=job_info.get('app_id'),
tag=job_meta.get('tag', 'release'),
cell_id=job_meta.get('cell_id', None),
run_id=job_meta.get('run_id', None),
token_id=job_meta.get('token_id', None),
meta=job_meta)
# Note that when jobs for this narrative are initially loaded,
# they are set to not be refreshed. Rather, if a client requests
# updates via the start_job_update message, the refresh flag will
# be set to True.
self._running_jobs[job_id] = {
'refresh': 0,
'job': job
}
elif job_id in job_check_error:
job_err_state = {
'job_state': 'error',
'error': {
'error': 'KBase execution engine returned an error while looking up this job.',
'message': job_check_error[job_id].get('message', 'No error message available'),
'name': 'Job Error',
'code': job_check_error[job_id].get('code', -999),
'exception': {
'error_message': 'Job lookup in execution engine failed',
'error_type': job_check_error[job_id].get('name', 'unknown'),
'error_stacktrace': job_check_error[job_id].get('error', '')
}
},
'cell_id': job_meta.get('cell_id', None),
'run_id': job_meta.get('run_id', None),
}
error_jobs[job_id] = job_err_state
except Exception as e:
kblogging.log_event(self._log, 'init_error', {'err': str(e)})
new_e = transform_job_exception(e)
error = {
'error': 'Unable to get job info on initial lookup',
'job_id': job_id,
'message': getattr(new_e, 'message', 'Unknown reason'),
'code': getattr(new_e, 'code', -1),
'source': getattr(new_e, 'source', 'jobmanager'),
'name': getattr(new_e, 'name', type(e).__name__),
'service': 'job_service'
}
self._send_comm_message('job_init_lookup_err', error)
raise new_e # should crash and burn on any of these.
if len(job_check_error):
err_str = 'Unable to find info for some jobs on initial lookup'
err_type = 'job_init_partial_err'
if len(job_check_error) == len(nar_jobs):
err_str = 'Unable to get info for any job on initial lookup'
err_type = 'job_init_lookup_err'
error = {
'error': err_str,
'job_errors': error_jobs,
'message': 'Job information was unavailable from the server',
'code': -2,
'source': 'jobmanager',
'name': 'jobmanager',
'service': 'job_service',
}
self._send_comm_message(err_type, error)
if not self._running_lookup_loop and start_lookup_thread:
# only keep one loop at a time in cause this gets called again!
if self._lookup_timer is not None:
self._lookup_timer.cancel()
self._running_lookup_loop = True
self._lookup_job_status_loop()
else:
self._lookup_all_job_status()
def test_sys_var_no_ws_id(self):
if 'KB_WORKSPACE_ID' in os.environ:
del os.environ['KB_WORKSPACE_ID']
self.assertIsNone(system_variable('workspace_id'))
def test_sys_var_workspace(self):
self.assertEquals(system_variable('workspace'), self.workspace)
def test_sys_var_workspace_id_except(self):
os.environ['KB_WORKSPACE_ID'] = 'invalid_workspace'
self.assertIsNone(system_variable('workspace_id'))
def test_sys_var_token(self):
os.environ['KB_AUTH_TOKEN'] = self.good_fake_token
self.assertEquals(system_variable('token'), self.good_fake_token)
def test_sys_var_user_bad(self):
biokbase.auth.set_environ_token(self.bad_fake_token)
self.assertIsNone(system_variable('user_id'))
def initialize_jobs(self):
"""
Initializes this JobManager.
This is expected to be run by a running Narrative, and naturally linked to a workspace.
So it does the following steps.
1. app_util.system_variable('workspace_id')
2. get list of jobs with that ws id from UJS (also gets tag, cell_id, run_id)
3. initialize the Job objects by running NJS.get_job_params on each of those (also gets app_id)
4. start the status lookup loop.
"""
ws_id = system_variable('workspace_id')
try:
nar_jobs = clients.get('user_and_job_state').list_jobs2({
'authstrat': 'kbaseworkspace',
'authparams': [str(ws_id)]
})
except Exception as e:
kblogging.log_event(self._log, 'init_error', {'err': str(e)})
new_e = transform_job_exception(e)
error = {
'error': 'Unable to get initial jobs list',
'message': getattr(new_e, 'message', 'Unknown reason'),
'code': getattr(new_e, 'code', -1),
'source': getattr(new_e, 'source', 'jobmanager'),
'name': getattr(new_e, 'name', type(e).__name__),
'service': 'user_and_job_state'
}
self._send_comm_message('job_init_err', error)
raise new_e
for info in nar_jobs:
job_id = info[0]
user_info = info[1]
job_meta = info[10]
try:
job_info = clients.get('job_service').get_job_params(job_id)[0]
self._running_jobs[job_id] = {
'refresh': True,
'job': Job.from_state(job_id,
job_info,
user_info[0],
app_id=job_info.get('app_id'),
tag=job_meta.get('tag', 'release'),
cell_id=job_meta.get('cell_id', None),
run_id=job_meta.get('run_id', None))
}
except Exception as e:
kblogging.log_event(self._log, 'init_error', {'err': str(e)})
new_e = transform_job_exception(e)
error = {
'error': 'Unable to get job info on initial lookup',
'job_id': job_id,
'message': getattr(new_e, 'message', 'Unknown reason'),
'code': getattr(new_e, 'code', -1),
'source': getattr(new_e, 'source', 'jobmanager'),
'name': getattr(new_e, 'name', type(e).__name__),
'service': 'job_service'
}
self._send_comm_message('job_init_lookup_err', error)
raise new_e # should crash and burn on any of these.
if not self._running_lookup_loop:
# only keep one loop at a time in cause this gets called again!
if self._lookup_timer is not None:
self._lookup_timer.cancel()
self._running_lookup_loop = True
self._lookup_job_status_loop()
else:
self._lookup_all_job_status()
def test_sys_var_bad(self):
self.assertIsNone(system_variable(self.bad_tag))
def load_widget_info(self, tag="release", verbose=False):
"""
Loads widget info and mapping.
Eventually will fetch from kbase-ui, a kbase CDN, or the catalog service.
For now, it gets known vis widgets from all method specs.
This returns the a Dict where all keys are the name of a widget, and all values
contain widget information in this structure:
{
"params": {
"param_name": {
"is_constant": boolean,
"param_type": one of (string|boolean|dropdown),
"allowed_values": list of strings (exists when param_type==dropdown),
"allowed_types": list of data types (when param_type==string),
"param_value": something, mainly when is_constant==True
}
}
"""
check_tag(tag, raise_exception=True)
methods = self._sm.app_specs[tag].values()
all_widgets = dict()
# keys = widget names / namespaced require path / etc.
# Individual widget values should be:
# {params: {
# name1: {
# is_constant: boolean,
# value: (***something*** | None) (something = any structure),
# allowed: [ list of allowed values, optional ],
# type: (string, int, float, boolean, etc. list? hash?)
# allowed_types: [ list of allowed ws types, optional ]
# },
# name2: { is_constant, value }
# }
for method in methods:
if 'output' not in method['widgets']:
widget_name = self._default_output_widget
else:
widget_name = method['widgets']['output']
if widget_name == 'null':
if verbose:
print("Ignoring a widget named 'null' in {} - {}".format(tag, method['info']['id']))
continue
out_mapping = method['behavior'].get('kb_service_output_mapping', method['behavior'].get('output_mapping', None))
if out_mapping is not None:
params = {}
for p in out_mapping:
param_name = p['target_property']
allowed_values = set()
is_constant = False
param_value = None
param_type = 'string'
allowed_types = set()
if 'constant_value' in p:
# add this val to a set of constant values for that param in that widget.
# if more than one possible, this need to be optional
is_constant = True
allowed_values.add(p['constant_value'])
if 'input_parameter' in p:
# this is a user given input. look up what it expects from the
# associated parameter of that name
in_param = p['input_parameter']
for spec_param in method['parameters']:
if spec_param['id'] == in_param:
# want its:
# field_type = text, float, number, ...
in_type = spec_param['field_type']
if in_type == 'text':
param_type = 'string'
if spec_param.has_key('text_options'):
validate_as = spec_param['text_options'].get('validate_as', None)
if validate_as == 'int':
param_type = 'int'
elif validate_as == 'float':
param_type = 'float'
if spec_param['text_options'].has_key('valid_ws_types'):
allowed_types.update(spec_param['text_options']['valid_ws_types'])
elif param_type == 'textarea':
param_type = 'string'
elif param_type == 'checkbox':
param_type = 'boolean'
elif param_type == 'dropdown':
param_type = 'dropdown'
allowed_values.update([o['value'] for o in spec_param['dropdown_options']])
if 'narrative_system_variable' in p:
# this is something like the ws name or token that needs to get fetched
# by the system. Shouldn't be handled by the user.
is_constant = True
param_value = system_variable(p['narrative_system_variable'])
if 'service_method_output_path' in p:
param_type = 'from_service_output'
param_info = {
'is_constant': is_constant,
'param_type': param_type,
}
if allowed_values:
param_info['allowed_values'] = allowed_values
if allowed_types:
param_info['allowed_types'] = allowed_types
if param_value:
param_info['param_value'] = param_value
params[param_name] = param_info
if widget_name in all_widgets:
# if it's already there, just update the allowed_types and allowed_values for some params that have them
for p_name in params.keys():
if 'allowed_types' in params[p_name]:
if p_name not in all_widgets[widget_name]['params']:
all_widgets[widget_name]['params'][p_name] = params[p_name]
else:
widget_types = all_widgets[widget_name]['params'].get(p_name, {}).get('allowed_types', set())
widget_types.update(params[p_name]['allowed_types'])
all_widgets[widget_name]['params'][p_name]['allowed_types'] = widget_types
if 'allowed_values' in params[p_name]:
if p_name not in all_widgets[widget_name]['params']:
all_widgets[widget_name]['params'][p_name] = params[p_name]
else:
widget_vals = all_widgets[widget_name]['params'].get(p_name, {}).get('allowed_values', set())
widget_vals.update(params[p_name]['allowed_values'])
all_widgets[widget_name]['params'][p_name]['allowed_values'] = widget_vals
else:
all_widgets[widget_name] = { 'params': params }
# finally, turn all sets into lists
for w in all_widgets:
for p in all_widgets[w]["params"]:
if "allowed_types" in all_widgets[w]["params"][p]:
all_widgets[w]["params"][p]["allowed_types"] = list(all_widgets[w]["params"][p]["allowed_types"])
if "allowed_values" in all_widgets[w]['params'][p]:
all_widgets[w]["params"][p]["allowed_values"] = list(all_widgets[w]["params"][p]["allowed_values"])
return all_widgets
def _validate_parameters(self, app_id, tag, spec_params, params):
"""
Validates the dict of params against the spec_params. If all is good, it updates a few
parameters that need it - checkboxes go from True/False to 1/0, and sets default values
where necessary.
Then it returns a tuple like this:
(dict_of_params, list_of_ws_refs)
where list_of_ws_refs is the list of workspace references for objects being passed into
the app.
If it fails, this will raise a ValueError with a description of the problem and a
(hopefully useful!) hint for the user as to what went wrong.
"""
spec_param_ids = [p['id'] for p in spec_params]
# First, test for presence.
missing_params = list()
for p in spec_params:
if not p['optional'] and not p['default'] and not params.get(p['id'], None):
missing_params.append(p['id'])
if len(missing_params):
raise ValueError('Missing required parameters {} - try executing app_usage("{}", tag="{}") for more information'.format(json.dumps(missing_params), app_id, tag))
# Next, test for extra params that don't make sense
extra_params = list()
for p in params.keys():
if p not in spec_param_ids:
extra_params.append(p)
if len(extra_params):
raise ValueError('Unknown parameters {} - maybe something was misspelled?\nexecute app_usage("{}", tag="{}") for more information'.format(json.dumps(extra_params), app_id, tag))
# Now, validate parameter values.
# Should also check if input (NOT OUTPUT) object variables are present in the current workspace
workspace = system_variable('workspace')
ws_id = system_variable('workspace_id')
if workspace is None or ws_id is None:
raise ValueError('Unable to retrive current Narrative workspace information! workspace={}, workspace_id={}'.format(workspace, ws_id))
param_errors = list()
# If they're workspace objects, track their refs in a list we'll pass to run_job as
# a separate param to track provenance.
ws_input_refs = list()
for p in spec_params:
if p['id'] in params:
(wsref, err) = self._check_parameter(p, params[p['id']], workspace)
if err is not None:
param_errors.append("{} - {}".format(p['id'], err))
if wsref is not None:
if isinstance(wsref, list):
for ref in wsref:
if ref is not None:
ws_input_refs.append(ref)
else:
ws_input_refs.append(wsref)
if len(param_errors):
raise ValueError('Parameter value errors found!\n{}'.format("\n".join(param_errors)))
# Hooray, parameters are validated. Set them up for transfer.
for p in spec_params:
# If any param is a checkbox, need to map from boolean to actual expected value in p['checkbox_map']
# note that True = 0th elem, False = 1st
if p['type'] == 'checkbox':
if p['id'] in params:
checkbox_idx = 0 if params[p['id']] else 1
params[p['id']] = p['checkbox_map'][checkbox_idx]
# While we're at it, set the default values for any unset parameters that have them
if p['default'] and p['id'] not in params:
params[p['id']] = p['default']
return (params, ws_input_refs)
def test_sys_var_time_ms(self):
cur_t = int(time.time() * 1000)
ts = system_variable('timestamp_epoch_ms')
self.assertTrue(cur_t <= ts)
self.assertTrue(ts - cur_t < 1000)
def _map_inputs(self, input_mapping, params):
"""
Maps the dictionary of parameters and inputs based on rules provided in the input_mapping.
This iterates over the list of input_mappings, and uses them as a filter to apply to each
parameter.
Returns a list of inputs that can be passed directly to NJSW.run_job
input_mapping is a list of dicts, as defined by NarrativeMethodStore.ServiceMethodInputMapping.
params is a dict of key-value-pairs, each key is the input_parameter field of some parameter.
"""
inputs_dict = dict()
for p in input_mapping:
# 2 steps - figure out the proper value, then figure out the proper position.
# value first!
p_value = None
if 'input_parameter' in p:
p_value = params.get(p['input_parameter'], None)
# turn empty strings into None
if isinstance(p_value, basestring) and len(p_value) == 0:
p_value = None
elif 'narrative_system_variable' in p:
p_value = system_variable(p['narrative_system_variable'])
if 'constant_value' in p and p_value is None:
p_value = p['constant_value']
if 'generated_value' in p and p_value is None:
p_value = self._generate_input(generated_value)
if 'target_type_transform' in p:
p_value = self._transform_input(p['target_type_transform'], p_value)
# get position!
arg_position = p.get('target_argument_position', 0)
target_prop = p.get('target_property', None)
if target_prop is not None:
final_input = inputs_dict.get(arg_position, dict())
if '/' in target_prop:
## This is case when slashes in target_prop separeate elements in nested maps.
## We ignore escaped slashes (separate backslashes should be escaped as well).
bck_slash = u"\u244A"
fwd_slash = u"\u20EB"
temp_string = target_prop.replace("\\\\", bck_slash).replace("\\/", fwd_slash)
temp_path = []
for part in temp_string.split("/"):
part = part.replace(bck_slash, "\\").replace(fwd_slash, "/")
temp_path.append(part.encode('ascii','ignore'))
temp_map = final_input
temp_key = None
## We're going along the path and creating intermediate dictionaries.
for temp_path_item in temp_path:
if temp_key:
if temp_key not in temp_map:
temp_map[temp_key] = {}
temp_map = temp_map[temp_key]
temp_key = temp_path_item
## temp_map points to deepest nested map now, temp_key is last item in path
temp_map[temp_key] = p_value
else:
final_input[target_prop] = p_value
inputs_dict[arg_position] = final_input
else:
inputs_dict[arg_position] = p_value
inputs_list = list()
keys = sorted(inputs_dict.keys())
for k in keys:
inputs_list.append(inputs_dict[k])
return inputs_list
def test_sys_var_time_sec(self):
cur_t = int(time.time())
ts = system_variable('timestamp_epoch_sec')
self.assertTrue(cur_t <= ts)
self.assertTrue(ts - cur_t < 1)
def _run_app_batch_internal(self, app_id, params, tag, version, cell_id, run_id, dry_run):
batch_method = "kb_BatchApp.run_batch"
batch_app_id = "kb_BatchApp/run_batch"
batch_method_ver = "dev"
batch_method_tag = "dev"
ws_id = strict_system_variable('workspace_id')
spec = self._get_validated_app_spec(app_id, tag, True, version=version)
# Preflight check the params - all required ones are present, all
# values are the right type, all numerical values are in given ranges
spec_params = self.spec_manager.app_params(spec)
# A list of lists of UPAs, used for each subjob.
batch_ws_upas = list()
# The list of actual input values, post-mapping.
batch_run_inputs = list()
for param_set in params:
spec_params_map = dict((spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params)))
batch_ws_upas.append(extract_ws_refs(app_id, tag, spec_params, param_set))
batch_run_inputs.append(self._map_inputs(
spec['behavior']['kb_service_input_mapping'],
param_set,
spec_params_map))
service_method = spec['behavior']['kb_service_method']
service_name = spec['behavior']['kb_service_name']
service_ver = spec['behavior'].get('kb_service_version', None)
# Let the given version override the spec's version.
if version is not None:
service_ver = version
# This is what calls the function in the back end - Module.method
# This isn't the same as the app spec id.
job_meta = {
'tag': batch_method_tag,
'batch_app': app_id,
'batch_tag': tag,
'batch_size': len(params),
}
if cell_id is not None:
job_meta['cell_id'] = cell_id
if run_id is not None:
job_meta['run_id'] = run_id
# Now put these all together in a way that can be sent to the batch processing app.
batch_params = [{
"module_name": service_name,
"method_name": service_method,
"service_ver": service_ver,
"wsid": ws_id,
"meta": job_meta,
"batch_params": [{
"params": batch_run_inputs[i],
"source_ws_objects": batch_ws_upas[i]
} for i in range(len(batch_run_inputs))],
}]
# We're now almost ready to run the job. Last, we need an agent token.
try:
token_name = 'KBApp_{}'.format(app_id)
token_name = token_name[:self.__MAX_TOKEN_NAME_LEN]
agent_token = auth.get_agent_token(auth.get_auth_token(), token_name=token_name)
except Exception as e:
raise
job_meta['token_id'] = agent_token['id']
# This is the input set for NJSW.run_job. Now we need the workspace id
# and whatever fits in the metadata.
job_runner_inputs = {
'method': batch_method,
'service_ver': batch_method_ver,
'params': batch_params,
'app_id': batch_app_id,
'wsid': ws_id,
'meta': job_meta
}
# if len(ws_input_refs) > 0:
# job_runner_inputs['source_ws_objects'] = ws_input_refs
# if we're doing a dry run, just return the inputs that we made.
if dry_run:
return job_runner_inputs
# Log that we're trying to run a job...
log_info = {
'app_id': app_id,
'tag': batch_method_tag,
'version': service_ver,
'username': system_variable('user_id'),
'wsid': ws_id
}
kblogging.log_event(self._log, "run_batch_app", log_info)
try:
job_id = clients.get("job_service", token=agent_token['token']).run_job(job_runner_inputs)
except Exception as e:
log_info.update({'err': str(e)})
kblogging.log_event(self._log, "run_batch_app_error", log_info)
raise transform_job_exception(e)
new_job = Job(job_id,
batch_app_id,
batch_params,
system_variable('user_id'),
tag=batch_method_tag,
app_version=batch_method_ver,
cell_id=cell_id,
run_id=run_id,
token_id=agent_token['id'],
meta=job_meta)
self._send_comm_message('run_status', {
'event': 'launched_job',
'event_at': datetime.datetime.utcnow().isoformat() + 'Z',
'cell_id': cell_id,
'run_id': run_id,
'job_id': job_id
})
JobManager().register_new_job(new_job)
if cell_id is not None:
return
else:
return new_job
def test_sys_var_token(self):
if (self.user_token):
biokbase.auth.set_environ_token(self.user_token)
self.assertEquals(system_variable('token'), self.user_token)
def test_sys_var_user_bad(self):
os.environ['KB_AUTH_TOKEN'] = self.bad_fake_token
self.assertIsNone(system_variable('user_id'))
def _map_inputs(self, input_mapping, params, spec_params):
"""
Maps the dictionary of parameters and inputs based on rules provided in
the input_mapping. This iterates over the list of input_mappings, and
uses them as a filter to apply to each parameter.
Returns a list of inputs that can be passed directly to NJSW.run_job
input_mapping is a list of dicts, as defined by
NarrativeMethodStore.ServiceMethodInputMapping.
params is a dict of key-value-pairs, each key is the input_parameter
field of some parameter.
"""
inputs_dict = {}
for p in input_mapping:
# 2 steps - figure out the proper value, then figure out the
# proper position. value first!
p_value = None
input_param_id = None
if "input_parameter" in p:
input_param_id = p["input_parameter"]
p_value = params.get(input_param_id, None)
if spec_params[input_param_id].get("type", "") == "group":
p_value = self._map_group_inputs(
p_value, spec_params[input_param_id], spec_params
)
# turn empty strings into None
if isinstance(p_value, str) and len(p_value) == 0:
p_value = None
elif "narrative_system_variable" in p:
p_value = system_variable(p["narrative_system_variable"])
if "constant_value" in p and p_value is None:
p_value = p["constant_value"]
if "generated_value" in p and p_value is None:
p_value = self._generate_input(p["generated_value"])
spec_param = None
if input_param_id:
spec_param = spec_params[input_param_id]
p_value = transform_param_value(
p.get("target_type_transform"), p_value, spec_param
)
# get position!
arg_position = p.get("target_argument_position", 0)
target_prop = p.get("target_property", None)
if target_prop is not None:
final_input = inputs_dict.get(arg_position, {})
if "/" in target_prop:
# This is case when slashes in target_prop separate
# elements in nested maps. We ignore escaped slashes
# (separate backslashes should be escaped as well).
bck_slash = "\u244A"
fwd_slash = "\u20EB"
temp_string = target_prop.replace("\\\\", bck_slash)
temp_string = temp_string.replace("\\/", fwd_slash)
temp_path = []
for part in temp_string.split("/"):
part = part.replace(bck_slash, "\\")
part = part.replace(fwd_slash, "/")
temp_path.append(part.encode("ascii", "ignore").decode("ascii"))
temp_map = final_input
temp_key = None
# We're going along the path and creating intermediate
# dictionaries.
for temp_path_item in temp_path:
if temp_key:
if temp_key not in temp_map:
temp_map[temp_key] = {}
temp_map = temp_map[temp_key]
temp_key = temp_path_item
# temp_map points to deepest nested map now, temp_key is
# the last item in the path
temp_map[temp_key] = p_value
else:
final_input[target_prop] = p_value
inputs_dict[arg_position] = final_input
else:
inputs_dict[arg_position] = p_value
inputs_list = []
keys = sorted(inputs_dict.keys())
for k in keys:
inputs_list.append(inputs_dict[k])
return inputs_list
def _map_inputs(self, input_mapping, params, spec_params):
"""
Maps the dictionary of parameters and inputs based on rules provided in
the input_mapping. This iterates over the list of input_mappings, and
uses them as a filter to apply to each parameter.
Returns a list of inputs that can be passed directly to NJSW.run_job
input_mapping is a list of dicts, as defined by
NarrativeMethodStore.ServiceMethodInputMapping.
params is a dict of key-value-pairs, each key is the input_parameter
field of some parameter.
"""
inputs_dict = dict()
for p in input_mapping:
# 2 steps - figure out the proper value, then figure out the
# proper position. value first!
p_value = None
input_param_id = None
if 'input_parameter' in p:
input_param_id = p['input_parameter']
p_value = params.get(input_param_id, None)
if spec_params[input_param_id].get('type', '') == 'group':
p_value = self._map_group_inputs(p_value, spec_params[input_param_id],
spec_params)
# turn empty strings into None
if isinstance(p_value, basestring) and len(p_value) == 0:
p_value = None
elif 'narrative_system_variable' in p:
p_value = system_variable(p['narrative_system_variable'])
if 'constant_value' in p and p_value is None:
p_value = p['constant_value']
if 'generated_value' in p and p_value is None:
p_value = self._generate_input(p['generated_value'])
spec_param = None
if input_param_id:
spec_param = spec_params[input_param_id]
p_value = transform_param_value(p.get('target_type_transform'), p_value, spec_param)
# get position!
arg_position = p.get('target_argument_position', 0)
target_prop = p.get('target_property', None)
if target_prop is not None:
final_input = inputs_dict.get(arg_position, dict())
if '/' in target_prop:
# This is case when slashes in target_prop separeate
# elements in nested maps. We ignore escaped slashes
# (separate backslashes should be escaped as well).
bck_slash = u"\u244A"
fwd_slash = u"\u20EB"
temp_string = target_prop.replace("\\\\", bck_slash)
temp_string = temp_string.replace("\\/", fwd_slash)
temp_path = []
for part in temp_string.split("/"):
part = part.replace(bck_slash, "\\")
part = part.replace(fwd_slash, "/")
temp_path.append(part.encode('ascii', 'ignore'))
temp_map = final_input
temp_key = None
# We're going along the path and creating intermediate
# dictionaries.
for temp_path_item in temp_path:
if temp_key:
if temp_key not in temp_map:
temp_map[temp_key] = {}
temp_map = temp_map[temp_key]
temp_key = temp_path_item
# temp_map points to deepest nested map now, temp_key is
# the last item in the path
temp_map[temp_key] = p_value
else:
final_input[target_prop] = p_value
inputs_dict[arg_position] = final_input
else:
inputs_dict[arg_position] = p_value
inputs_list = list()
keys = sorted(inputs_dict.keys())
for k in keys:
inputs_list.append(inputs_dict[k])
return inputs_list
def test_sys_var_workspace(self):
os.environ['KB_WORKSPACE_ID'] = self.workspace
self.assertEqual(system_variable('workspace'), self.workspace)
