def resolve_job_ref(jbor, job_outputs={}, should_resolve=True):
'''
:param jbor: a dict that is a valid job-based object reference
:type jbor: dict
:param job_outputs: a dict of finished local jobs to their output hashes
:type job_outputs: :class:`collections.OrderedDict`
:returns: the referenced value if present
:raises: :exc:`Exception` if the job-based object reference cannot be resolved
TODO: Support metadata references
'''
ref_job_id = get_job_from_jbor(jbor)
ref_job_field = get_field_from_jbor(jbor)
if is_localjob_id(ref_job_id):
if job_outputs.get(ref_job_id) is None:
if should_resolve:
raise Exception('Job ' + ref_job_id + ' not found in local finished jobs')
else:
return jbor
if ref_job_field not in job_outputs[ref_job_id]:
raise Exception('Cannot resolve a JBOR with job ID ' + ref_job_id + ' because field "' + ref_job_field + '" was not found in its output')
return job_outputs[ref_job_id][ref_job_field]
else:
dxjob = dxpy.DXJob(ref_job_id)
try:
dxjob.wait_on_done()
except Exception as e:
raise Exception('Could not wait for ' + ref_job_id + ' to finish: ' + str(e))
job_desc = dxjob.describe()
if ref_job_field not in job_desc['output']:
raise Exception('Cannot resolve a JBOR with job ID ' + ref_job_id + ' because field "' + ref_job_field + '" was not found in its output')
return job_desc['output'][ref_job_field]
def add(self, input_name, input_value):
if self.input_name_prefix is not None:
if input_name.startswith(self.input_name_prefix):
input_name = input_name[len(self.input_name_prefix):]
else: # Skip inputs that don't start with prefix
return
if ':' in input_name:
input_class = input_name[input_name.find(':') + 1:]
input_name = input_name[:input_name.find(':')]
else:
input_class = None
if self.input_spec is not None:
if input_name not in self.input_spec:
raise Exception('Input field called ' + input_name + ' was not found in the input spec')
input_class = self.input_spec[input_name]['class']
if input_class is None:
done = False
try:
# Resolve "job-xxxx:output-name" syntax into a canonical job ref
job_id, field = split_unescaped(':', input_value)
if is_job_id(job_id) or is_localjob_id(job_id):
input_value = {"job": job_id, "field": field}
done = True
except:
pass
if not done:
try:
parsed_input_value = json.loads(input_value, object_pairs_hook=collections.OrderedDict)
if type(parsed_input_value) in (collections.OrderedDict, list, int, long, float):
input_value = parsed_input_value
else:
raise Exception()
except:
# Not recognized JSON (list or dict), so resolve it as a name
try:
project, folderpath, entity_result = resolve_existing_path(input_value,
expected='entity')
except:
# If not possible, then leave it as a string
project, folderpath, entity_result = None, None, None
if entity_result is not None:
if is_hashid(input_value):
input_value = {'$dnanexus_link': entity_result['id']}
else:
input_value = {"$dnanexus_link": {"project": entity_result['describe']['project'],
"id": entity_result['id']}}
self.inputs[input_name].append(input_value)
else:
# Input class is known. Respect the "array" class.
input_value = parse_input_or_jbor(input_class, input_value)
if input_class.startswith('array:'):
self.inputs[input_name].append(input_value)
else:
self.inputs[input_name] = input_value
def parse_input_or_jbor(in_class, value):
val_substrings = split_unescaped(':', value)
if len(val_substrings) == 2 and (is_job_id(val_substrings[0]) or is_localjob_id(val_substrings[0])):
return {"job": val_substrings[0], "field": val_substrings[1]}
else:
if in_class.startswith('array:'):
in_class = in_class[6:]
return parse_input[in_class](value)
def parse_input_or_jbor(in_class, value):
val_substrings = split_unescaped(':', value)
if len(val_substrings) == 2 and (is_job_id(val_substrings[0])
or is_localjob_id(val_substrings[0])):
return {"job": val_substrings[0], "field": val_substrings[1]}
else:
if in_class.startswith('array:'):
in_class = in_class[6:]
return parse_input[in_class](value)
def resolve_job_ref(jbor, job_outputs={}, should_resolve=True):
'''
:param jbor: a dict that is a valid job-based object reference
:type jbor: dict
:param job_outputs: a dict of finished local jobs to their output hashes
:type job_outputs: :class:`collections.OrderedDict`
:returns: the referenced value if present
:raises: :exc:`Exception` if the job-based object reference cannot be resolved
TODO: Support metadata references
'''
ref_job_id = get_job_from_jbor(jbor)
ref_job_field = get_field_from_jbor(jbor)
ref_job_index = get_index_from_jbor(jbor)
def resolve_from_hash(output_hash):
if ref_job_index is None:
return output_hash[ref_job_field]
else:
return output_hash[ref_job_field][ref_job_index]
if is_localjob_id(ref_job_id):
if job_outputs.get(ref_job_id) is None:
if should_resolve:
raise Exception('Job ' + ref_job_id +
' not found in local finished jobs')
else:
return jbor
if ref_job_field not in job_outputs[ref_job_id]:
raise Exception('Cannot resolve a JBOR with job ID ' + ref_job_id +
' because field "' + ref_job_field +
'" was not found in its output')
return resolve_from_hash(job_outputs[ref_job_id])
else:
dxjob = dxpy.DXJob(ref_job_id)
try:
dxjob.wait_on_done()
except Exception as e:
raise Exception('Could not wait for ' + ref_job_id +
' to finish: ' + str(e))
job_desc = dxjob.describe()
if ref_job_field not in job_desc['output']:
raise Exception('Cannot resolve a JBOR with job ID ' + ref_job_id +
' because field "' + ref_job_field +
'" was not found in its output')
return resolve_from_hash(job_desc['output'])
def add(self, input_name, input_value):
if self.input_name_prefix is not None:
if input_name.startswith(self.input_name_prefix):
input_name = input_name[len(self.input_name_prefix):]
else: # Skip inputs that don't start with prefix
return
if ':' in input_name:
input_class = input_name[input_name.find(':') + 1:]
input_name = input_name[:input_name.find(':')]
else:
input_class = None
if self.input_spec is not None:
if input_name not in self.input_spec and self._desc.get(
'class') != 'workflow':
raise Exception('Input field called ' + input_name +
' was not found in the input spec')
elif input_name in self.input_spec:
input_class = self.input_spec[input_name]['class']
if input_class is None:
done = False
try:
# Resolve "job-xxxx:output-name" syntax into a canonical job ref
job_id, field = split_unescaped(':', input_value)
if is_job_id(job_id) or is_localjob_id(job_id):
input_value = {"job": job_id, "field": field}
done = True
except:
pass
if not done:
try:
parsed_input_value = json.loads(
input_value, object_pairs_hook=collections.OrderedDict)
if type(parsed_input_value) in (collections.OrderedDict,
list, int, long, float):
input_value = parsed_input_value
else:
raise Exception()
except:
# Not recognized JSON (list or dict), so resolve it as a name
try:
project, folderpath, entity_result = resolve_existing_path(
input_value, expected='entity')
except:
# If not possible, then leave it as a string
project, folderpath, entity_result = None, None, None
if entity_result is not None:
if is_hashid(input_value):
input_value = {
'$dnanexus_link': entity_result['id']
}
else:
input_value = {
"$dnanexus_link": {
"project":
entity_result['describe']['project'],
"id": entity_result['id']
}
}
if isinstance(self.inputs[input_name], list) and \
not isinstance(self.inputs[input_name], basestring):
self.inputs[input_name].append(input_value)
else:
self.inputs[input_name] = input_value
else:
# Input class is known. Respect the "array" class.
input_value = parse_input_or_jbor(input_class, input_value)
if input_class.startswith('array:'):
self.inputs[input_name].append(input_value)
else:
self.inputs[input_name] = input_value