def get_initial_value(self, trans, context):
rval = {}
child_context = ExpressionContext(rval, context)
for child_input in self.inputs.values():
rval[child_input.name] = child_input.get_initial_value(
trans, child_context)
return rval
def get_initial_value(self, trans, context):
if self.inputs is None:
raise Exception("Must set 'inputs' attribute to use.")
rval: Dict[str, Any] = {}
child_context = ExpressionContext(rval, context)
for child_input in self.inputs.values():
rval[child_input.name] = child_input.get_initial_value(
trans, child_context)
return rval
def populate_state(trans, inputs, state, incoming, prefix="", context=None ):
errors = dict()
context = ExpressionContext(state, context)
for input in inputs.itervalues():
key = prefix + input.name
if input.type == 'repeat':
group_state = state[input.name]
group_errors = []
rep_index = 0
del group_state[:]
while True:
rep_name = "%s_%d" % (key, rep_index)
if not any([incoming_key.startswith(rep_name) for incoming_key in incoming.keys()]):
break
if rep_index < input.max:
new_state = {}
new_state['__index__'] = rep_index
initialize_state(trans, input.inputs, new_state, context)
group_state.append(new_state)
group_errors.append({})
rep_errors = populate_state(trans, input.inputs, new_state, incoming, prefix=rep_name + "|", context=context)
if rep_errors:
group_errors[rep_index].update( rep_errors )
else:
group_errors[-1] = { '__index__': 'Cannot add repeat (max size=%i).' % input.max }
rep_index += 1
elif input.type == 'conditional':
group_state = state[input.name]
group_prefix = "%s|" % ( key )
test_param_key = group_prefix + input.test_param.name
default_value = incoming.get(test_param_key, group_state.get(input.test_param.name, None))
value, error = check_state(trans, input.test_param, default_value, context)
if error:
errors[input.name] = [error]
else:
current_case = input.get_current_case(value, trans)
group_state = state[input.name] = {}
initialize_state(trans, input.cases[current_case].inputs, group_state, context)
group_errors = populate_state( trans, input.cases[current_case].inputs, group_state, incoming, prefix=group_prefix, context=context)
if group_errors:
errors[input.name] = group_errors
group_state['__current_case__'] = current_case
group_state[input.test_param.name] = value
else:
default_value = incoming.get(key, state.get(input.name, None))
value, error = check_state(trans, input, default_value, context)
if error:
errors[input.name] = error
state[input.name] = value
return errors
def new_state(trans, tool, invalid=False):
"""Create a new `DefaultToolState` for the received tool. Only inputs on the first page will be initialized."""
state = galaxy.tools.DefaultToolState()
state.inputs = {}
if invalid:
# We're attempting to display a tool in the tool shed that has been determined to have errors, so is invalid.
return state
inputs = tool.inputs_by_page[0]
context = ExpressionContext(state.inputs, parent=None)
for input in inputs.itervalues():
try:
state.inputs[input.name] = input.get_initial_value(trans, context)
except:
state.inputs[input.name] = []
return state
def get_initial_value(self, trans, context):
# State for a conditional is a plain dictionary.
rval = {}
# Get the default value for the 'test element' and use it
# to determine the current case
test_value = self.test_param.get_initial_value(trans, context)
current_case = self.get_current_case(test_value)
# Store the current case in a special value
rval['__current_case__'] = current_case
# Store the value of the test element
rval[self.test_param.name] = test_value
# Fill in state for selected case
child_context = ExpressionContext(rval, context)
for child_input in self.cases[current_case].inputs.values():
rval[child_input.name] = child_input.get_initial_value(trans, child_context)
return rval
def iterate(group_inputs, inputs, tool_state, errors, other_values=None):
other_values = ExpressionContext( tool_state, other_values )
for input_index, input in enumerate( inputs.itervalues() ):
# create model dictionary
group_inputs[input_index] = input.to_dict(trans)
if group_inputs[input_index] is None:
continue
# identify stat for subsection/group
group_state = tool_state[input.name]
# iterate and update values
if input.type == 'repeat':
group_cache = group_inputs[input_index]['cache'] = {}
for i in range( len( group_state ) ):
group_cache[i] = {}
group_errors = errors[input.name][i] if input.name in errors else dict()
iterate( group_cache[i], input.inputs, group_state[i], group_errors, other_values )
elif input.type == 'conditional':
try:
test_param = group_inputs[input_index]['test_param']
test_param['value'] = convert(group_state[test_param['name']])
except Exception:
pass
i = group_state['__current_case__']
group_errors = errors.get( input.name, {} )
iterate(group_inputs[input_index]['cases'][i]['inputs'], input.cases[i].inputs, group_state, group_errors, other_values)
else:
# create input dictionary, try to pass other_values if to_dict function supports it e.g. dynamic options
try:
group_inputs[input_index] = input.to_dict(trans, other_values=other_values)
except Exception:
pass
# update input value from tool state
try:
group_inputs[input_index]['value'] = tool_state[group_inputs[input_index]['name']]
except Exception:
pass
# sanitize if value exists
if group_inputs[input_index]['value']:
group_inputs[input_index] = sanitize(group_inputs[input_index])
def new_state(trans, tool, invalid=False):
"""Create a new `DefaultToolState` for the received tool. Only inputs on the first page will be initialized."""
state = galaxy.tools.DefaultToolState()
state.inputs = {}
if invalid:
# We're attempting to display a tool in the tool shed that has been determined to have errors, so is invalid.
return state
try:
# Attempt to generate the tool state using the standard Galaxy-side code
return tool.new_state(trans)
except Exception as e:
# Fall back to building tool state as below
log.debug('Failed to build tool state for tool "%s" using standard method, will try to fall back on custom method: %s', tool.id, e)
inputs = tool.inputs_by_page[0]
context = ExpressionContext(state.inputs, parent=None)
for input in inputs.values():
try:
state.inputs[input.name] = input.get_initial_value(trans, context)
except Exception:
# FIXME: not all values should be an empty list
state.inputs[input.name] = []
return state
def finish( self, stdout, stderr ):
"""
Called to indicate that the associated command has been run. Updates
the output datasets based on stderr and stdout from the command, and
the contents of the output files.
"""
# default post job setup
self.sa_session.expunge_all()
job = self.get_job()
try:
self.reclaim_ownership()
except:
self.fail( job.info )
log.exception( '(%s) Failed to change ownership of %s, failing' % ( job.id, self.working_directory ) )
# if the job was deleted, don't finish it
if job.state == job.states.DELETED or job.state == job.states.ERROR:
#ERROR at this point means the job was deleted by an administrator.
return self.fail( job.info )
if stderr:
job.state = job.states.ERROR
else:
job.state = job.states.OK
if self.version_string_cmd:
version_filename = self.get_version_string_path()
if os.path.exists(version_filename):
self.version_string = open(version_filename).read()
os.unlink(version_filename)
if self.app.config.outputs_to_working_directory and not self.__link_file_check():
for dataset_path in self.get_output_fnames():
try:
shutil.move( dataset_path.false_path, dataset_path.real_path )
log.debug( "finish(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) )
except ( IOError, OSError ):
# this can happen if Galaxy is restarted during the job's
# finish method - the false_path file has already moved,
# and when the job is recovered, it won't be found.
if os.path.exists( dataset_path.real_path ) and os.stat( dataset_path.real_path ).st_size > 0:
log.warning( "finish(): %s not found, but %s is not empty, so it will be used instead" % ( dataset_path.false_path, dataset_path.real_path ) )
else:
return self.fail( "Job %s's output dataset(s) could not be read" % job.id )
job_context = ExpressionContext( dict( stdout = stdout, stderr = stderr ) )
job_tool = self.app.toolbox.tools_by_id.get( job.tool_id, None )
def in_directory( file, directory ):
# Make both absolute.
directory = os.path.abspath( directory )
file = os.path.abspath( file )
#Return true, if the common prefix of both is equal to directory
#e.g. /a/b/c/d.rst and directory is /a/b, the common prefix is /a/b
return os.path.commonprefix( [ file, directory ] ) == directory
for dataset_assoc in job.output_datasets + job.output_library_datasets:
context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset )
#should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur
for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: #need to update all associated output hdas, i.e. history was shared with job running
#
# If HDA is to be copied from the working directory, do it now so that other attributes are correctly set.
#
if isinstance( dataset, model.HistoryDatasetAssociation ):
joda = self.sa_session.query( model.JobToOutputDatasetAssociation ).filter_by( job=job, dataset=dataset ).first()
if joda and job_tool:
hda_tool_output = job_tool.outputs.get( joda.name, None )
if hda_tool_output and hda_tool_output.from_work_dir:
# Copy from working dir to HDA.
source_file = os.path.join( os.path.abspath( self.working_directory ), hda_tool_output.from_work_dir )
if in_directory( source_file, self.working_directory ):
try:
shutil.move( source_file, dataset.file_name )
log.debug( "finish(): Moved %s to %s as directed by from_work_dir" % ( source_file, dataset.file_name ) )
except ( IOError, OSError ):
log.debug( "finish(): Could not move %s to %s as directed by from_work_dir" % ( source_file, dataset.file_name ) )
else:
# Security violation.
log.exception( "from_work_dir specified a location not in the working directory: %s, %s" % ( source_file, self.working_directory ) )
dataset.blurb = 'done'
dataset.peek = 'no peek'
dataset.info = ( dataset.info or '' ) + context['stdout'] + context['stderr']
dataset.tool_version = self.version_string
dataset.set_size()
# Update (non-library) job output datasets through the object store
if dataset not in job.output_library_datasets:
self.app.object_store.update_from_file(dataset.dataset, create=True)
if context['stderr']:
dataset.blurb = "error"
elif dataset.has_data():
# If the tool was expected to set the extension, attempt to retrieve it
if dataset.ext == 'auto':
dataset.extension = context.get( 'ext', 'data' )
dataset.init_meta( copy_from=dataset )
#if a dataset was copied, it won't appear in our dictionary:
#either use the metadata from originating output dataset, or call set_meta on the copies
#it would be quicker to just copy the metadata from the originating output dataset,
#but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta()
if not self.app.config.set_metadata_externally or \
( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) \
and self.app.config.retry_metadata_internally ):
dataset.set_meta( overwrite = False )
elif not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and not context['stderr']:
dataset._state = model.Dataset.states.FAILED_METADATA
else:
#load metadata from file
#we need to no longer allow metadata to be edited while the job is still running,
#since if it is edited, the metadata changed on the running output will no longer match
#the metadata that was stored to disk for use via the external process,
#and the changes made by the user will be lost, without warning or notice
dataset.metadata.from_JSON_dict( self.external_output_metadata.get_output_filenames_by_dataset( dataset, self.sa_session ).filename_out )
try:
assert context.get( 'line_count', None ) is not None
if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte:
dataset.set_peek( line_count=context['line_count'], is_multi_byte=True )
else:
dataset.set_peek( line_count=context['line_count'] )
except:
if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte:
dataset.set_peek( is_multi_byte=True )
else:
dataset.set_peek()
try:
# set the name if provided by the tool
dataset.name = context['name']
except:
pass
else:
dataset.blurb = "empty"
if dataset.ext == 'auto':
dataset.extension = 'txt'
self.sa_session.add( dataset )
if context['stderr']:
dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR
else:
dataset_assoc.dataset.dataset.state = model.Dataset.states.OK
# If any of the rest of the finish method below raises an
# exception, the fail method will run and set the datasets to
# ERROR. The user will never see that the datasets are in error if
# they were flushed as OK here, since upon doing so, the history
# panel stops checking for updates. So allow the
# self.sa_session.flush() at the bottom of this method set
# the state instead.
for pja in job.post_job_actions:
ActionBox.execute(self.app, self.sa_session, pja.post_job_action, job)
# Flush all the dataset and job changes above. Dataset state changes
# will now be seen by the user.
self.sa_session.flush()
# Save stdout and stderr
if len( stdout ) > 32768:
log.error( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id )
job.stdout = stdout[:32768]
if len( stderr ) > 32768:
log.error( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id )
job.stderr = stderr[:32768]
# custom post process setup
inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] )
out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] )
inp_data.update( [ ( da.name, da.dataset ) for da in job.input_library_datasets ] )
out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] )
param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows
param_dict = self.tool.params_from_strings( param_dict, self.app )
# Check for and move associated_files
self.tool.collect_associated_files(out_data, self.working_directory)
gitd = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
if gitd:
self.tool.collect_associated_files({'' : gitd}, self.working_directory)
# Create generated output children and primary datasets and add to param_dict
collected_datasets = {'children':self.tool.collect_child_datasets(out_data, self.working_directory),'primary':self.tool.collect_primary_datasets(out_data, self.working_directory)}
param_dict.update({'__collected_datasets__':collected_datasets})
# Certain tools require tasks to be completed after job execution
# ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ).
self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict, job = job )
# Call 'exec_after_process' hook
self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data,
out_data=out_data, param_dict=param_dict,
tool=self.tool, stdout=stdout, stderr=stderr )
job.command_line = self.command_line
bytes = 0
# Once datasets are collected, set the total dataset size (includes extra files)
for dataset_assoc in job.output_datasets:
dataset_assoc.dataset.dataset.set_total_size()
bytes += dataset_assoc.dataset.dataset.get_total_size()
if job.user:
job.user.total_disk_usage += bytes
# fix permissions
for path in [ dp.real_path for dp in self.get_output_fnames() ]:
util.umask_fix_perms( path, self.app.config.umask, 0666, self.app.config.gid )
self.sa_session.flush()
log.debug( 'job %d ended' % self.job_id )
if self.app.config.cleanup_job == 'always' or ( not stderr and self.app.config.cleanup_job == 'onsuccess' ):
self.cleanup()
def populate_state(request_context,
inputs,
incoming,
state,
errors={},
prefix='',
context=None,
check=True):
"""
Populates nested state dict from incoming parameter values.
>>> from xml.etree.ElementTree import XML
>>> from galaxy.util.bunch import Bunch
>>> from galaxy.util.odict import odict
>>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter
>>> from galaxy.tools.parameters.grouping import Repeat
>>> trans = Bunch( workflow_building_mode=False )
>>> a = TextToolParameter( None, XML( '<param name="a"/>' ) )
>>> b = Repeat()
>>> b.min = 0
>>> b.max = 1
>>> c = TextToolParameter( None, XML( '<param name="c"/>' ) )
>>> d = Repeat()
>>> d.min = 0
>>> d.max = 1
>>> e = TextToolParameter( None, XML( '<param name="e"/>' ) )
>>> f = Conditional()
>>> g = BooleanToolParameter( None, XML( '<param name="g"/>' ) )
>>> h = TextToolParameter( None, XML( '<param name="h"/>' ) )
>>> i = TextToolParameter( None, XML( '<param name="i"/>' ) )
>>> b.name = 'b'
>>> b.inputs = odict([ ('c', c), ('d', d) ])
>>> d.name = 'd'
>>> d.inputs = odict([ ('e', e), ('f', f) ])
>>> f.test_param = g
>>> f.name = 'f'
>>> f.cases = [ Bunch( value='true', inputs= { 'h': h } ), Bunch( value='false', inputs= { 'i': i } ) ]
>>> inputs = odict([('a',a),('b',b)])
>>> flat = odict([ ('a', 1 ), ( 'b_0|c', 2 ), ( 'b_0|d_0|e', 3 ), ( 'b_0|d_0|f|h', 4 ), ( 'b_0|d_0|f|g', True ) ])
>>> state = odict()
>>> populate_state( trans, inputs, flat, state, check=False )
>>> print state[ 'a' ]
1
>>> print state[ 'b' ][ 0 ][ 'c' ]
2
>>> print state[ 'b' ][ 0 ][ 'd' ][ 0 ][ 'e' ]
3
>>> print state[ 'b' ][ 0 ][ 'd' ][ 0 ][ 'f' ][ 'h' ]
4
"""
context = ExpressionContext(state, context)
for input in inputs.values():
state[input.name] = input.get_initial_value(request_context, context)
key = prefix + input.name
group_state = state[input.name]
group_prefix = '%s|' % (key)
if input.type == 'repeat':
rep_index = 0
del group_state[:]
while True:
rep_prefix = '%s_%d' % (key, rep_index)
if not any(
incoming_key.startswith(rep_prefix) for incoming_key in
incoming.keys()) and rep_index >= input.min:
break
if rep_index < input.max:
new_state = {'__index__': rep_index}
group_state.append(new_state)
populate_state(request_context,
input.inputs,
incoming,
new_state,
errors,
prefix=rep_prefix + '|',
context=context,
check=check)
rep_index += 1
elif input.type == 'conditional':
if input.value_ref and not input.value_ref_in_group:
test_param_key = prefix + input.test_param.name
else:
test_param_key = group_prefix + input.test_param.name
test_param_value = incoming.get(
test_param_key, group_state.get(input.test_param.name))
value, error = check_param(
request_context, input.test_param, test_param_value,
context) if check else [test_param_value, None]
if error:
errors[test_param_key] = error
else:
try:
current_case = input.get_current_case(value)
group_state = state[input.name] = {}
populate_state(request_context,
input.cases[current_case].inputs,
incoming,
group_state,
errors,
prefix=group_prefix,
context=context,
check=check)
group_state['__current_case__'] = current_case
except Exception:
errors[
test_param_key] = 'The selected case is unavailable/invalid.'
pass
group_state[input.test_param.name] = value
elif input.type == 'section':
populate_state(request_context,
input.inputs,
incoming,
group_state,
errors,
prefix=group_prefix,
context=context,
check=check)
elif input.type == 'upload_dataset':
d_type = input.get_datatype(request_context, context=context)
writable_files = d_type.writable_files
while len(group_state) > len(writable_files):
del group_state[-1]
while len(writable_files) > len(group_state):
new_state = {'__index__': len(group_state)}
for upload_item in input.inputs.values():
new_state[
upload_item.name] = upload_item.get_initial_value(
request_context, context)
group_state.append(new_state)
for i, rep_state in enumerate(group_state):
rep_index = rep_state['__index__']
rep_prefix = '%s_%d|' % (key, rep_index)
populate_state(request_context,
input.inputs,
incoming,
rep_state,
errors,
prefix=rep_prefix,
context=context,
check=check)
else:
param_value = _get_incoming_value(incoming, key,
state.get(input.name))
value, error = check_param(
request_context, input, param_value,
context) if check else [param_value, None]
if error:
errors[key] = error
state[input.name] = value
def render_do_inputs(context,
inputs,
tool_state,
errors,
prefix,
other_values=None):
context.caller_stack._push_frame()
try:
def row_for_param(prefix, param, parent_state, parent_errors,
other_values):
return render_row_for_param(context, prefix, param, parent_state,
parent_errors, other_values)
def do_inputs(inputs, tool_state, errors, prefix, other_values=None):
return render_do_inputs(context, inputs, tool_state, errors,
prefix, other_values)
len = context.get('len', UNDEFINED)
range = context.get('range', UNDEFINED)
dict = context.get('dict', UNDEFINED)
str = context.get('str', UNDEFINED)
enumerate = context.get('enumerate', UNDEFINED)
trans = context.get('trans', UNDEFINED)
ExpressionContext = context.get('ExpressionContext', UNDEFINED)
__M_writer = context.writer()
# SOURCE LINE 77
__M_writer(u'\n ')
# SOURCE LINE 78
other_values = ExpressionContext(tool_state, other_values)
__M_writer(u'\n')
# SOURCE LINE 79
for input_index, input in enumerate(inputs.itervalues()):
# SOURCE LINE 80
if input.type == "repeat":
# SOURCE LINE 81
__M_writer(
u' <div class="repeat-group">\n <div class="form-title-row"><b>'
)
# SOURCE LINE 82
__M_writer(unicode(input.title_plural))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 83
repeat_state = tool_state[input.name]
__M_writer(u'\n')
# SOURCE LINE 84
for i in range(len(repeat_state)):
# SOURCE LINE 85
__M_writer(
u' <div class="repeat-group-item">\n '
)
# SOURCE LINE 86
if input.name in errors:
rep_errors = errors[input.name][i]
else:
rep_errors = dict()
index = repeat_state[i]['__index__']
# SOURCE LINE 92
__M_writer(
u'\n <div class="form-title-row"><b>'
)
# SOURCE LINE 93
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 94
__M_writer(
unicode(
do_inputs(
input.inputs, repeat_state[i], rep_errors,
prefix + input.name + "_" + str(index) + "|",
other_values)))
__M_writer(
u'\n <div class="form-row"><input type="submit" name="'
)
# SOURCE LINE 95
__M_writer(unicode(prefix))
__M_writer(unicode(input.name))
__M_writer(u'_')
__M_writer(unicode(index))
__M_writer(u'_remove" value="Remove ')
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'"></div>\n </div>\n')
# SOURCE LINE 98
__M_writer(
u' <div class="form-row"><input type="submit" name="'
)
__M_writer(unicode(prefix))
__M_writer(unicode(input.name))
__M_writer(u'_add" value="Add new ')
__M_writer(unicode(input.title))
__M_writer(u'"></div>\n </div>\n')
# SOURCE LINE 100
elif input.type == "conditional":
# SOURCE LINE 101
__M_writer(u' ')
group_state = tool_state[input.name]
group_errors = errors.get(input.name, {})
current_case = group_state['__current_case__']
group_prefix = prefix + input.name + "|"
# SOURCE LINE 106
__M_writer(u'\n')
# SOURCE LINE 107
if input.value_ref_in_group:
# SOURCE LINE 108
__M_writer(u' ')
__M_writer(
unicode(
row_for_param(group_prefix, input.test_param,
group_state, group_errors,
other_values)))
__M_writer(u'\n')
# SOURCE LINE 110
__M_writer(u' ')
__M_writer(
unicode(
do_inputs(input.cases[current_case].inputs,
group_state, group_errors, group_prefix,
other_values)))
__M_writer(u'\n')
# SOURCE LINE 111
elif input.type == "upload_dataset":
# SOURCE LINE 112
if input.get_datatype(
trans, other_values
).composite_type is None: #have non-composite upload appear as before
# SOURCE LINE 113
__M_writer(u' ')
if input.name in errors:
rep_errors = errors[input.name][0]
else:
rep_errors = dict()
# SOURCE LINE 118
__M_writer(u'\n ')
# SOURCE LINE 119
__M_writer(
unicode(
do_inputs(input.inputs, tool_state[input.name][0],
rep_errors,
prefix + input.name + "_" + str(0) + "|",
other_values)))
__M_writer(u'\n')
# SOURCE LINE 120
else:
# SOURCE LINE 121
__M_writer(
u' <div class="repeat-group">\n <div class="form-title-row"><b>'
)
# SOURCE LINE 122
__M_writer(unicode(input.group_title(other_values)))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 123
repeat_state = tool_state[input.name]
# SOURCE LINE 125
__M_writer(u'\n')
# SOURCE LINE 126
for i in range(len(repeat_state)):
# SOURCE LINE 127
__M_writer(
u' <div class="repeat-group-item">\n '
)
# SOURCE LINE 128
if input.name in errors:
rep_errors = errors[input.name][i]
else:
rep_errors = dict()
index = repeat_state[i]['__index__']
# SOURCE LINE 134
__M_writer(
u'\n <div class="form-title-row"><b>File Contents for '
)
# SOURCE LINE 135
__M_writer(
unicode(
input.title_by_index(trans, i, other_values)))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 136
__M_writer(
unicode(
do_inputs(
input.inputs, repeat_state[i], rep_errors,
prefix + input.name + "_" + str(index) +
"|", other_values)))
__M_writer(u'\n')
# SOURCE LINE 138
__M_writer(u' </div>\n')
# SOURCE LINE 141
__M_writer(u' </div>\n')
# SOURCE LINE 143
else:
# SOURCE LINE 144
__M_writer(u' ')
__M_writer(
unicode(
row_for_param(prefix, input, tool_state, errors,
other_values)))
__M_writer(u'\n')
# SOURCE LINE 147
__M_writer(u' ')
return ''
finally:
context.caller_stack._pop_frame()
def _populate_state_legacy(request_context,
inputs,
incoming,
state,
errors,
prefix='',
context=None,
check=True,
simple_errors=True):
context = ExpressionContext(state, context)
for input in inputs.values():
state[input.name] = input.get_initial_value(request_context, context)
key = prefix + input.name
group_state = state[input.name]
group_prefix = '%s|' % (key)
if input.type == 'repeat':
rep_index = 0
del group_state[:]
while True:
rep_prefix = '%s_%d' % (key, rep_index)
if not any(
incoming_key.startswith(rep_prefix) for incoming_key in
incoming.keys()) and rep_index >= input.min:
break
if rep_index < input.max:
new_state = {'__index__': rep_index}
group_state.append(new_state)
_populate_state_legacy(request_context,
input.inputs,
incoming,
new_state,
errors,
prefix=rep_prefix + '|',
context=context,
check=check,
simple_errors=simple_errors)
rep_index += 1
elif input.type == 'conditional':
if input.value_ref and not input.value_ref_in_group:
test_param_key = prefix + input.test_param.name
else:
test_param_key = group_prefix + input.test_param.name
test_param_value = incoming.get(
test_param_key, group_state.get(input.test_param.name))
value, error = check_param(
request_context,
input.test_param,
test_param_value,
context,
simple_errors=simple_errors) if check else [
test_param_value, None
]
if error:
errors[test_param_key] = error
else:
try:
current_case = input.get_current_case(value)
group_state = state[input.name] = {}
_populate_state_legacy(request_context,
input.cases[current_case].inputs,
incoming,
group_state,
errors,
prefix=group_prefix,
context=context,
check=check,
simple_errors=simple_errors)
group_state['__current_case__'] = current_case
except Exception:
errors[
test_param_key] = 'The selected case is unavailable/invalid.'
group_state[input.test_param.name] = value
elif input.type == 'section':
_populate_state_legacy(request_context,
input.inputs,
incoming,
group_state,
errors,
prefix=group_prefix,
context=context,
check=check,
simple_errors=simple_errors)
elif input.type == 'upload_dataset':
file_count = input.get_file_count(request_context, context)
while len(group_state) > file_count:
del group_state[-1]
while file_count > len(group_state):
new_state = {'__index__': len(group_state)}
for upload_item in input.inputs.values():
new_state[
upload_item.name] = upload_item.get_initial_value(
request_context, context)
group_state.append(new_state)
for rep_state in group_state:
rep_index = rep_state['__index__']
rep_prefix = '%s_%d|' % (key, rep_index)
_populate_state_legacy(request_context,
input.inputs,
incoming,
rep_state,
errors,
prefix=rep_prefix,
context=context,
check=check,
simple_errors=simple_errors)
else:
param_value = _get_incoming_value(incoming, key,
state.get(input.name))
value, error = check_param(
request_context,
input,
param_value,
context,
simple_errors=simple_errors) if check else [param_value, None]
if error:
errors[key] = error
state[input.name] = value
def visit_input_values(inputs,
input_values,
callback,
name_prefix='',
label_prefix='',
parent_prefix='',
context=None,
no_replacement_value=REPLACE_ON_TRUTHY):
"""
Given a tools parameter definition (`inputs`) and a specific set of
parameter `values`, call `callback` for each non-grouping parameter,
passing the parameter object, value, a constructed unique name,
and a display label.
If the callback returns a value, it will be replace the old value.
>>> from xml.etree.ElementTree import XML
>>> from galaxy.util.bunch import Bunch
>>> from galaxy.util.odict import odict
>>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter
>>> from galaxy.tools.parameters.grouping import Repeat
>>> a = TextToolParameter( None, XML( '<param name="a"/>' ) )
>>> b = Repeat()
>>> c = TextToolParameter( None, XML( '<param name="c"/>' ) )
>>> d = Repeat()
>>> e = TextToolParameter( None, XML( '<param name="e"/>' ) )
>>> f = Conditional()
>>> g = BooleanToolParameter( None, XML( '<param name="g"/>' ) )
>>> h = TextToolParameter( None, XML( '<param name="h"/>' ) )
>>> i = TextToolParameter( None, XML( '<param name="i"/>' ) )
>>> j = TextToolParameter( None, XML( '<param name="j"/>' ) )
>>> b.name = b.title = 'b'
>>> b.inputs = odict([ ('c', c), ('d', d) ])
>>> d.name = d.title = 'd'
>>> d.inputs = odict([ ('e', e), ('f', f) ])
>>> f.test_param = g
>>> f.name = 'f'
>>> f.cases = [ Bunch( value='true', inputs= { 'h': h } ), Bunch( value='false', inputs= { 'i': i } ) ]
>>>
>>> def visitor( input, value, prefix, prefixed_name, prefixed_label, error, **kwargs ):
... print 'name=%s, prefix=%s, prefixed_name=%s, prefixed_label=%s,value=%s' % ( input.name, prefix, prefixed_name, prefixed_label, value )
... if error:
... print error
>>> inputs = odict([('a', a),('b', b)])
>>> nested = odict([ ('a', 1), ('b', [ odict([('c', 3), ( 'd', [odict([ ('e', 5), ('f', odict([ ('g', True), ('h', 7) ])) ]) ])]) ]) ])
>>> visit_input_values( inputs, nested, visitor )
name=a, prefix=, prefixed_name=a, prefixed_label=a,value=1
name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c,value=3
name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e,value=5
name=g, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|g, prefixed_label=b 1 > d 1 > g,value=True
name=h, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|h, prefixed_label=b 1 > d 1 > h,value=7
>>> params_from_strings( inputs, params_to_strings( inputs, nested, None ), None )[ 'b' ][ 0 ][ 'd' ][ 0 ][ 'f' ][ 'g' ] is True
True
>>> # Conditional test parameter value does not match any case, warning is shown and child values are not visited
>>> f.test_param = j
>>> nested['b'][0]['d'][0]['f']['j'] = 'j'
>>> visit_input_values( inputs, nested, visitor )
name=a, prefix=, prefixed_name=a, prefixed_label=a,value=1
name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c,value=3
name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e,value=5
name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j,value=j
The selected case is unavailable/invalid.
>>> # Test parameter missing in state, value error
>>> del nested['b'][0]['d'][0]['f']['j']
>>> visit_input_values( inputs, nested, visitor )
name=a, prefix=, prefixed_name=a, prefixed_label=a,value=1
name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c,value=3
name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e,value=5
name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j,value=None
No value found for 'b 1 > d 1 > j'.
>>> # Conditional parameter missing in state, value error
>>> del nested['b'][0]['d'][0]['f']
>>> visit_input_values( inputs, nested, visitor )
name=a, prefix=, prefixed_name=a, prefixed_label=a,value=1
name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c,value=3
name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e,value=5
name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f|j, prefixed_label=b 1 > d 1 > j,value=None
No value found for 'b 1 > d 1 > j'.
>>> # Conditional input name has changed e.g. due to tool changes, key error
>>> f.name = 'f_1'
>>> visit_input_values( inputs, nested, visitor )
name=a, prefix=, prefixed_name=a, prefixed_label=a,value=1
name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c,value=3
name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e,value=5
name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f_1|j, prefixed_label=b 1 > d 1 > j,value=None
No value found for 'b 1 > d 1 > j'.
>>> # Other parameters are missing in state
>>> nested = odict([ ('b', [ odict([ ( 'd', [odict([ ('f', odict([ ('g', True), ('h', 7) ])) ]) ])]) ]) ])
>>> visit_input_values( inputs, nested, visitor )
name=a, prefix=, prefixed_name=a, prefixed_label=a,value=None
No value found for 'a'.
name=c, prefix=b_0|, prefixed_name=b_0|c, prefixed_label=b 1 > c,value=None
No value found for 'b 1 > c'.
name=e, prefix=b_0|d_0|, prefixed_name=b_0|d_0|e, prefixed_label=b 1 > d 1 > e,value=None
No value found for 'b 1 > d 1 > e'.
name=j, prefix=b_0|d_0|, prefixed_name=b_0|d_0|f_1|j, prefixed_label=b 1 > d 1 > j,value=None
No value found for 'b 1 > d 1 > j'.
"""
def callback_helper(input,
input_values,
name_prefix,
label_prefix,
parent_prefix,
context=None,
error=None):
args = {
'input': input,
'parent': input_values,
'value': input_values.get(input.name),
'prefixed_name': '%s%s' % (name_prefix, input.name),
'prefixed_label':
'%s%s' % (label_prefix, input.label or input.name),
'prefix': parent_prefix,
'context': context,
'error': error
}
if input.name not in input_values:
args['error'] = 'No value found for \'%s\'.' % args.get(
'prefixed_label')
new_value = callback(**args)
if no_replacement_value is REPLACE_ON_TRUTHY:
replace = bool(new_value)
else:
replace = new_value != no_replacement_value
if replace:
input_values[input.name] = new_value
def get_current_case(input, input_values):
try:
return input.get_current_case(input_values[input.test_param.name])
except (KeyError, ValueError):
return -1
context = ExpressionContext(input_values, context)
payload = {
'context': context,
'no_replacement_value': no_replacement_value
}
for input in inputs.values():
if isinstance(input, Repeat) or isinstance(input, UploadDataset):
values = input_values[input.name] = input_values.get(
input.name, [])
for i, d in enumerate(values):
d['__index__'] = i
new_name_prefix = name_prefix + '%s_%d|' % (input.name, i)
new_label_prefix = label_prefix + '%s %d > ' % (input.title,
i + 1)
visit_input_values(input.inputs,
d,
callback,
new_name_prefix,
new_label_prefix,
parent_prefix=new_name_prefix,
**payload)
elif isinstance(input, Conditional):
values = input_values[input.name] = input_values.get(
input.name, {})
new_name_prefix = name_prefix + input.name + '|'
case_error = None if get_current_case(
input,
values) >= 0 else 'The selected case is unavailable/invalid.'
callback_helper(input.test_param,
values,
new_name_prefix,
label_prefix,
parent_prefix=name_prefix,
context=context,
error=case_error)
values['__current_case__'] = get_current_case(input, values)
if values['__current_case__'] >= 0:
visit_input_values(
input.cases[values['__current_case__']].inputs,
values,
callback,
new_name_prefix,
label_prefix,
parent_prefix=name_prefix,
**payload)
elif isinstance(input, Section):
values = input_values[input.name] = input_values.get(
input.name, {})
new_name_prefix = name_prefix + input.name + '|'
visit_input_values(input.inputs,
values,
callback,
new_name_prefix,
label_prefix,
parent_prefix=name_prefix,
**payload)
else:
callback_helper(input,
input_values,
name_prefix,
label_prefix,
parent_prefix=parent_prefix,
context=context)
def render_do_inputs(context,
inputs,
tool_state,
errors,
prefix,
other_values=None):
context.caller_stack._push_frame()
try:
def row_for_param(prefix, param, parent_state, parent_errors,
other_values):
return render_row_for_param(context, prefix, param, parent_state,
parent_errors, other_values)
h = context.get('h', UNDEFINED)
def do_inputs(inputs, tool_state, errors, prefix, other_values=None):
return render_do_inputs(context, inputs, tool_state, errors,
prefix, other_values)
len = context.get('len', UNDEFINED)
range = context.get('range', UNDEFINED)
dict = context.get('dict', UNDEFINED)
str = context.get('str', UNDEFINED)
enumerate = context.get('enumerate', UNDEFINED)
trans = context.get('trans', UNDEFINED)
ExpressionContext = context.get('ExpressionContext', UNDEFINED)
__M_writer = context.writer()
# SOURCE LINE 80
__M_writer(u'\n ')
# SOURCE LINE 81
other_values = ExpressionContext(tool_state, other_values)
__M_writer(u'\n')
# SOURCE LINE 82
for input_index, input in enumerate(inputs.itervalues()):
# SOURCE LINE 83
if not input.visible:
# SOURCE LINE 84
__M_writer(u' ')
pass
__M_writer(u'\n')
# SOURCE LINE 85
elif input.type == "repeat":
# SOURCE LINE 86
__M_writer(
u' <div class="repeat-group">\n <div class="form-title-row"><b>'
)
# SOURCE LINE 87
__M_writer(unicode(input.title_plural))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 88
repeat_state = tool_state[input.name]
__M_writer(u'\n')
# SOURCE LINE 89
for i in range(len(repeat_state)):
# SOURCE LINE 90
__M_writer(
u' <div class="repeat-group-item">\n '
)
# SOURCE LINE 91
if input.name in errors:
rep_errors = errors[input.name][i]
else:
rep_errors = dict()
index = repeat_state[i]['__index__']
# SOURCE LINE 97
__M_writer(
u'\n <div class="form-title-row"><b>'
)
# SOURCE LINE 98
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 99
__M_writer(
unicode(
do_inputs(
input.inputs, repeat_state[i], rep_errors,
prefix + input.name + "_" + str(index) + "|",
other_values)))
__M_writer(
u'\n <div class="form-row"><input type="submit" name="'
)
# SOURCE LINE 100
__M_writer(unicode(prefix))
__M_writer(unicode(input.name))
__M_writer(u'_')
__M_writer(unicode(index))
__M_writer(u'_remove" value="Remove ')
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'"></div>\n </div>\n')
# SOURCE LINE 102
if rep_errors.has_key('__index__'):
# SOURCE LINE 103
__M_writer(
u' <div><img style="vertical-align: middle;" src="'
)
__M_writer(
unicode(
h.url_for('/static/style/error_small.png')))
__M_writer(
u'"> <span style="vertical-align: middle;">')
__M_writer(unicode(rep_errors['__index__']))
__M_writer(u'</span></div>\n')
pass
pass
# SOURCE LINE 106
__M_writer(
u' <div class="form-row"><input type="submit" name="'
)
__M_writer(unicode(prefix))
__M_writer(unicode(input.name))
__M_writer(u'_add" value="Add new ')
__M_writer(unicode(input.title))
__M_writer(u'"></div>\n </div>\n')
# SOURCE LINE 108
elif input.type == "conditional":
# SOURCE LINE 109
__M_writer(u' ')
group_state = tool_state[input.name]
group_errors = errors.get(input.name, {})
current_case = group_state['__current_case__']
group_prefix = prefix + input.name + "|"
# SOURCE LINE 114
__M_writer(u'\n')
# SOURCE LINE 115
if input.value_ref_in_group:
# SOURCE LINE 116
__M_writer(u' ')
__M_writer(
unicode(
row_for_param(group_prefix, input.test_param,
group_state, group_errors,
other_values)))
__M_writer(u'\n')
pass
# SOURCE LINE 118
__M_writer(u' ')
__M_writer(
unicode(
do_inputs(input.cases[current_case].inputs,
group_state, group_errors, group_prefix,
other_values)))
__M_writer(u'\n')
# SOURCE LINE 119
elif input.type == "upload_dataset":
# SOURCE LINE 120
if input.get_datatype(
trans, other_values
).composite_type is None: #have non-composite upload appear as before
# SOURCE LINE 121
__M_writer(u' ')
if input.name in errors:
rep_errors = errors[input.name][0]
else:
rep_errors = dict()
# SOURCE LINE 126
__M_writer(u'\n ')
# SOURCE LINE 127
__M_writer(
unicode(
do_inputs(input.inputs, tool_state[input.name][0],
rep_errors,
prefix + input.name + "_" + str(0) + "|",
other_values)))
__M_writer(u'\n')
# SOURCE LINE 128
else:
# SOURCE LINE 129
__M_writer(
u' <div class="repeat-group">\n <div class="form-title-row"><b>'
)
# SOURCE LINE 130
__M_writer(unicode(input.group_title(other_values)))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 131
repeat_state = tool_state[input.name]
# SOURCE LINE 133
__M_writer(u'\n')
# SOURCE LINE 134
for i in range(len(repeat_state)):
# SOURCE LINE 135
__M_writer(
u' <div class="repeat-group-item">\n '
)
# SOURCE LINE 136
if input.name in errors:
rep_errors = errors[input.name][i]
else:
rep_errors = dict()
index = repeat_state[i]['__index__']
# SOURCE LINE 142
__M_writer(
u'\n <div class="form-title-row"><b>File Contents for '
)
# SOURCE LINE 143
__M_writer(
unicode(
input.title_by_index(trans, i, other_values)))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 144
__M_writer(
unicode(
do_inputs(
input.inputs, repeat_state[i], rep_errors,
prefix + input.name + "_" + str(index) +
"|", other_values)))
__M_writer(u'\n')
# SOURCE LINE 146
__M_writer(u' </div>\n')
pass
# SOURCE LINE 149
__M_writer(u' </div>\n')
pass
# SOURCE LINE 151
else:
# SOURCE LINE 152
__M_writer(u' ')
__M_writer(
unicode(
row_for_param(prefix, input, tool_state, errors,
other_values)))
__M_writer(u'\n')
pass
pass
# SOURCE LINE 155
__M_writer(u' ')
return ''
finally:
context.caller_stack._pop_frame()
def populate_state(request_context,
inputs,
incoming,
state,
errors=None,
context=None,
check=True,
simple_errors=True,
input_format='legacy'):
"""
Populates nested state dict from incoming parameter values.
>>> from galaxy.util import XML
>>> from galaxy.util.bunch import Bunch
>>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter
>>> from galaxy.tools.parameters.grouping import Repeat
>>> trans = Bunch(workflow_building_mode=False)
>>> a = TextToolParameter(None, XML('<param name="a"/>'))
>>> b = Repeat()
>>> b.min = 0
>>> b.max = 1
>>> c = TextToolParameter(None, XML('<param name="c"/>'))
>>> d = Repeat()
>>> d.min = 0
>>> d.max = 1
>>> e = TextToolParameter(None, XML('<param name="e"/>'))
>>> f = Conditional()
>>> g = BooleanToolParameter(None, XML('<param name="g"/>'))
>>> h = TextToolParameter(None, XML('<param name="h"/>'))
>>> i = TextToolParameter(None, XML('<param name="i"/>'))
>>> b.name = 'b'
>>> b.inputs = dict([('c', c), ('d', d)])
>>> d.name = 'd'
>>> d.inputs = dict([('e', e), ('f', f)])
>>> f.test_param = g
>>> f.name = 'f'
>>> f.cases = [Bunch(value='true', inputs= { 'h': h }), Bunch(value='false', inputs= { 'i': i })]
>>> inputs = dict([('a',a),('b',b)])
>>> flat = dict([('a', 1), ('b_0|c', 2), ('b_0|d_0|e', 3), ('b_0|d_0|f|h', 4), ('b_0|d_0|f|g', True)])
>>> state = {}
>>> populate_state(trans, inputs, flat, state, check=False)
>>> print(state['a'])
1
>>> print(state['b'][0]['c'])
2
>>> print(state['b'][0]['d'][0]['e'])
3
>>> print(state['b'][0]['d'][0]['f']['h'])
4
>>> # now test with input_format='21.01'
>>> nested = {'a': 1, 'b': [{'c': 2, 'd': [{'e': 3, 'f': {'h': 4, 'g': True}}]}]}
>>> state_new = {}
>>> populate_state(trans, inputs, nested, state_new, check=False, input_format='21.01')
>>> print(state_new['a'])
1
>>> print(state_new['b'][0]['c'])
2
>>> print(state_new['b'][0]['d'][0]['e'])
3
>>> print(state_new['b'][0]['d'][0]['f']['h'])
4
"""
if errors is None:
errors = {}
if input_format == 'legacy':
_populate_state_legacy(request_context,
inputs,
incoming,
state,
errors=errors,
context=context,
check=check,
simple_errors=simple_errors)
return
elif input_format == '21.01':
context = ExpressionContext(state, context)
for input in inputs.values():
state[input.name] = input.get_initial_value(
request_context, context)
group_state = state[input.name]
if input.type == 'repeat':
if len(incoming[input.name]) > input.max or len(
incoming[input.name]) < input.min:
errors[
input.
name] = 'The number of repeat elements is outside the range specified by the tool.'
else:
del group_state[:]
for rep in incoming[input.name]:
new_state = {}
group_state.append(new_state)
new_errors = {}
populate_state(request_context,
input.inputs,
rep,
new_state,
new_errors,
context=context,
check=check,
simple_errors=simple_errors,
input_format=input_format)
if new_errors:
errors[input.name] = new_errors
elif input.type == 'conditional':
test_param_value = incoming.get(input.name,
{}).get(input.test_param.name)
value, error = check_param(
request_context,
input.test_param,
test_param_value,
context,
simple_errors=simple_errors) if check else [
test_param_value, None
]
if error:
errors[input.test_param.name] = error
else:
try:
current_case = input.get_current_case(value)
group_state = state[input.name] = {}
new_errors = {}
populate_state(request_context,
input.cases[current_case].inputs,
incoming.get(input.name),
group_state,
new_errors,
context=context,
check=check,
simple_errors=simple_errors,
input_format=input_format)
if new_errors:
errors[input.name] = new_errors
group_state['__current_case__'] = current_case
except Exception:
errors[
input.test_param.
name] = 'The selected case is unavailable/invalid.'
group_state[input.test_param.name] = value
elif input.type == 'section':
new_errors = {}
populate_state(request_context,
input.inputs,
incoming.get(input.name),
group_state,
new_errors,
context=context,
check=check,
simple_errors=simple_errors,
input_format=input_format)
if new_errors:
errors[input.name] = new_errors
elif input.type == 'upload_dataset':
raise NotImplementedError
else:
param_value = _get_incoming_value(incoming, input.name,
state.get(input.name))
value, error = check_param(
request_context,
input,
param_value,
context,
simple_errors=simple_errors) if check else [
param_value, None
]
if error:
errors[input.name] = error
state[input.name] = value
else:
raise Exception(
f'Input format {input_format} not recognized; input_format must be either legacy or 21.01.'
)
def set_metadata_portable():
import galaxy.model
tool_job_working_directory = os.path.abspath(os.getcwd())
metadata_tmp_files_dir = os.path.join(tool_job_working_directory,
"metadata")
galaxy.model.metadata.MetadataTempFile.tmp_dir = metadata_tmp_files_dir
metadata_params_path = os.path.join("metadata", "params.json")
try:
with open(metadata_params_path, "r") as f:
metadata_params = json.load(f)
except IOError:
raise Exception("Failed to find metadata/params.json from cwd [%s]" %
tool_job_working_directory)
datatypes_config = metadata_params["datatypes_config"]
job_metadata = metadata_params["job_metadata"]
provided_metadata_style = metadata_params.get("provided_metadata_style")
max_metadata_value_size = metadata_params.get(
"max_metadata_value_size") or 0
outputs = metadata_params["outputs"]
datatypes_registry = validate_and_load_datatypes_config(datatypes_config)
tool_provided_metadata = load_job_metadata(job_metadata,
provided_metadata_style)
def set_meta(new_dataset_instance, file_dict):
set_meta_with_tool_provided(new_dataset_instance, file_dict,
set_meta_kwds, datatypes_registry,
max_metadata_value_size)
object_store_conf_path = os.path.join("metadata", "object_store_conf.json")
extended_metadata_collection = os.path.exists(object_store_conf_path)
object_store = None
job_context = None
version_string = ""
export_store = None
if extended_metadata_collection:
from galaxy.tool_util.parser.stdio import ToolStdioRegex, ToolStdioExitCode
tool_dict = metadata_params["tool"]
stdio_exit_code_dicts, stdio_regex_dicts = tool_dict[
"stdio_exit_codes"], tool_dict["stdio_regexes"]
stdio_exit_codes = list(map(ToolStdioExitCode, stdio_exit_code_dicts))
stdio_regexes = list(map(ToolStdioRegex, stdio_regex_dicts))
with open(object_store_conf_path, "r") as f:
config_dict = json.load(f)
from galaxy.objectstore import build_object_store_from_config
assert config_dict is not None
object_store = build_object_store_from_config(None,
config_dict=config_dict)
galaxy.model.Dataset.object_store = object_store
outputs_directory = os.path.join(tool_job_working_directory, "outputs")
if not os.path.exists(outputs_directory):
outputs_directory = tool_job_working_directory
# TODO: constants...
if os.path.exists(os.path.join(outputs_directory, "tool_stdout")):
with open(os.path.join(outputs_directory, "tool_stdout"),
"rb") as f:
tool_stdout = f.read()
with open(os.path.join(outputs_directory, "tool_stderr"),
"rb") as f:
tool_stderr = f.read()
elif os.path.exists(os.path.join(outputs_directory, "stdout")):
# Puslar style working directory.
with open(os.path.join(outputs_directory, "stdout"), "rb") as f:
tool_stdout = f.read()
with open(os.path.join(outputs_directory, "stderr"), "rb") as f:
tool_stderr = f.read()
job_id_tag = metadata_params["job_id_tag"]
# TODO: this clearly needs to be refactored, nothing in runners should be imported here..
from galaxy.job_execution.output_collect import default_exit_code_file, read_exit_code_from
exit_code_file = default_exit_code_file(".", job_id_tag)
tool_exit_code = read_exit_code_from(exit_code_file, job_id_tag)
from galaxy.tool_util.output_checker import check_output, DETECTED_JOB_STATE
check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output(
stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr,
tool_exit_code, job_id_tag)
if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs(
):
final_job_state = galaxy.model.Job.states.OK
else:
final_job_state = galaxy.model.Job.states.ERROR
from pulsar.client.staging import COMMAND_VERSION_FILENAME
version_string = ""
if os.path.exists(COMMAND_VERSION_FILENAME):
version_string = open(COMMAND_VERSION_FILENAME).read()
# TODO: handle outputs_to_working_directory?
from galaxy.util.expressions import ExpressionContext
job_context = ExpressionContext(
dict(stdout=tool_stdout, stderr=tool_stderr))
# Load outputs.
import_model_store = store.imported_store_for_metadata(
'metadata/outputs_new', object_store=object_store)
export_store = store.DirectoryModelExportStore(
'metadata/outputs_populated',
serialize_dataset_objects=True,
for_edit=True)
for output_name, output_dict in outputs.items():
if extended_metadata_collection:
dataset_instance_id = output_dict["id"]
dataset = import_model_store.sa_session.query(
galaxy.model.HistoryDatasetAssociation).find(
dataset_instance_id)
assert dataset is not None
else:
filename_in = os.path.join("metadata/metadata_in_%s" % output_name)
dataset = cPickle.load(open(filename_in,
'rb')) # load DatasetInstance
filename_kwds = os.path.join("metadata/metadata_kwds_%s" % output_name)
filename_out = os.path.join("metadata/metadata_out_%s" % output_name)
filename_results_code = os.path.join("metadata/metadata_results_%s" %
output_name)
override_metadata = os.path.join("metadata/metadata_override_%s" %
output_name)
dataset_filename_override = output_dict["filename_override"]
# Same block as below...
set_meta_kwds = stringify_dictionary_keys(
json.load(open(filename_kwds))
) # load kwds; need to ensure our keywords are not unicode
try:
dataset.dataset.external_filename = dataset_filename_override
store_by = metadata_params.get("object_store_store_by", "id")
extra_files_dir_name = "dataset_%s_files" % getattr(
dataset.dataset, store_by)
files_path = os.path.abspath(
os.path.join(tool_job_working_directory, "working",
extra_files_dir_name))
dataset.dataset.external_extra_files_path = files_path
file_dict = tool_provided_metadata.get_dataset_meta(
output_name, dataset.dataset.id, dataset.dataset.uuid)
if 'ext' in file_dict:
dataset.extension = file_dict['ext']
# Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles
override_metadata = json.load(open(override_metadata))
for metadata_name, metadata_file_override in override_metadata:
if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(
metadata_file_override):
metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(
metadata_file_override)
setattr(dataset.metadata, metadata_name,
metadata_file_override)
if output_dict.get("validate", False):
set_validated_state(dataset)
set_meta(dataset, file_dict)
if extended_metadata_collection:
meta = tool_provided_metadata.get_dataset_meta(
output_name, dataset.dataset.id, dataset.dataset.uuid)
if meta:
context = ExpressionContext(meta, job_context)
else:
context = job_context
# Lazy and unattached
# if getattr(dataset, "hidden_beneath_collection_instance", None):
# dataset.visible = False
dataset.blurb = 'done'
dataset.peek = 'no peek'
dataset.info = (dataset.info or '')
if context['stdout'].strip():
# Ensure white space between entries
dataset.info = dataset.info.rstrip(
) + "\n" + context['stdout'].strip()
if context['stderr'].strip():
# Ensure white space between entries
dataset.info = dataset.info.rstrip(
) + "\n" + context['stderr'].strip()
dataset.tool_version = version_string
dataset.set_size()
if 'uuid' in context:
dataset.dataset.uuid = context['uuid']
object_store.update_from_file(dataset.dataset, create=True)
from galaxy.job_execution.output_collect import collect_extra_files
collect_extra_files(object_store, dataset, ".")
if galaxy.model.Job.states.ERROR == final_job_state:
dataset.blurb = "error"
dataset.mark_unhidden()
else:
# If the tool was expected to set the extension, attempt to retrieve it
if dataset.ext == 'auto':
dataset.extension = context.get('ext', 'data')
dataset.init_meta(copy_from=dataset)
# This has already been done:
# else:
# self.external_output_metadata.load_metadata(dataset, output_name, self.sa_session, working_directory=self.working_directory, remote_metadata_directory=remote_metadata_directory)
line_count = context.get('line_count', None)
try:
# Certain datatype's set_peek methods contain a line_count argument
dataset.set_peek(line_count=line_count)
except TypeError:
# ... and others don't
dataset.set_peek()
from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_KEYS
for context_key in TOOL_PROVIDED_JOB_METADATA_KEYS:
if context_key in context:
context_value = context[context_key]
setattr(dataset, context_key, context_value)
if extended_metadata_collection:
export_store.add_dataset(dataset)
else:
cPickle.dump(dataset, open(filename_out, 'wb+'))
else:
dataset.metadata.to_JSON_dict(
filename_out) # write out results of set_meta
json.dump((True, 'Metadata has been set successfully'),
open(filename_results_code,
'wt+')) # setting metadata has succeeded
except Exception:
json.dump((False, traceback.format_exc()),
open(filename_results_code,
'wt+')) # setting metadata has failed somehow
if extended_metadata_collection:
# discover extra outputs...
from galaxy.job_execution.output_collect import collect_dynamic_outputs, collect_primary_datasets, SessionlessJobContext
job_context = SessionlessJobContext(
metadata_params, tool_provided_metadata, object_store,
export_store, import_model_store,
os.path.join(tool_job_working_directory, "working"))
output_collections = {}
for name, output_collection in metadata_params[
"output_collections"].items():
output_collections[name] = import_model_store.sa_session.query(
galaxy.model.HistoryDatasetCollectionAssociation).find(
output_collection["id"])
outputs = {}
for name, output in metadata_params["outputs"].items():
outputs[name] = import_model_store.sa_session.query(
galaxy.model.HistoryDatasetAssociation).find(output["id"])
input_ext = json.loads(metadata_params["job_params"].get(
"__input_ext", '"data"'))
collect_primary_datasets(
job_context,
outputs,
input_ext=input_ext,
)
collect_dynamic_outputs(job_context, output_collections)
if export_store:
export_store._finalize()
write_job_metadata(tool_job_working_directory, job_metadata, set_meta,
tool_provided_metadata)
"""Create a new `DefaultToolState` for the received tool. Only inputs on the first page will be initialized."""
state = galaxy.tools.DefaultToolState()
state.inputs = {}
if invalid:
# We're attempting to display a tool in the tool shed that has been determined to have errors, so is invalid.
return state
try:
# Attempt to generate the tool state using the standard Galaxy-side code
return tool.new_state(trans)
except Exception, e:
# Fall back to building tool state as below
log.debug(
'Failed to build tool state for tool "%s" using standard method, will try to fall back on custom method: %s',
tool.id, e)
inputs = tool.inputs_by_page[0]
context = ExpressionContext(state.inputs, parent=None)
for input in inputs.itervalues():
try:
state.inputs[input.name] = input.get_initial_value(trans, context)
except:
# FIXME: not all values should be an empty list
state.inputs[input.name] = []
return state
def panel_entry_per_tool(tool_section_dict):
# Return True if tool_section_dict looks like this.
# {<Tool guid> :
# [{ tool_config : <tool_config_file>,
# id: <ToolSection id>,
# version : <ToolSection version>,
def render_do_inputs(context, inputs, values, errors, prefix, ctx=None):
context.caller_stack._push_frame()
try:
def row_for_param(param,
value,
error_dict,
prefix,
ctx,
allow_runtime=True):
return render_row_for_param(context, param, value, error_dict,
prefix, ctx, allow_runtime)
def do_inputs(inputs, values, errors, prefix, ctx=None):
return render_do_inputs(context, inputs, values, errors, prefix,
ctx)
len = context.get('len', UNDEFINED)
range = context.get('range', UNDEFINED)
dict = context.get('dict', UNDEFINED)
str = context.get('str', UNDEFINED)
enumerate = context.get('enumerate', UNDEFINED)
trans = context.get('trans', UNDEFINED)
ExpressionContext = context.get('ExpressionContext', UNDEFINED)
__M_writer = context.writer()
# SOURCE LINE 6
__M_writer(u'\n ')
# SOURCE LINE 7
ctx = ExpressionContext(values, ctx)
__M_writer(u'\n')
# SOURCE LINE 8
for input_index, input in enumerate(inputs.itervalues()):
# SOURCE LINE 9
if input.type == "repeat":
# SOURCE LINE 10
__M_writer(
u' <div class="repeat-group form-row">\n <label>'
)
# SOURCE LINE 11
__M_writer(unicode(input.title_plural))
__M_writer(u':</label>\n ')
# SOURCE LINE 12
repeat_values = values[input.name]
__M_writer(u'\n')
# SOURCE LINE 13
for i in range(len(repeat_values)):
# SOURCE LINE 14
__M_writer(u' ')
if input.name in errors:
rep_errors = errors[input.name][i]
else:
rep_errors = dict()
index = repeat_values[i]['__index__']
# SOURCE LINE 20
__M_writer(
u'\n <div class="repeat-group-item">\n <div class="form-title-row"><label>'
)
# SOURCE LINE 22
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'</label></div>\n ')
# SOURCE LINE 23
__M_writer(
unicode(
do_inputs(
input.inputs, repeat_values[i], rep_errors,
prefix + input.name + "_" + str(index) + "|",
ctx)))
__M_writer(
u'\n <div class="form-row"><input type="submit" name="'
)
# SOURCE LINE 24
__M_writer(unicode(prefix))
__M_writer(unicode(input.name))
__M_writer(u'_')
__M_writer(unicode(index))
__M_writer(u'_remove" value="Remove ')
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'"></div>\n </div>\n')
# SOURCE LINE 27
__M_writer(
u' <div class="form-row"><input type="submit" name="'
)
__M_writer(unicode(prefix))
__M_writer(unicode(input.name))
__M_writer(u'_add" value="Add new ')
__M_writer(unicode(input.title))
__M_writer(u'"></div>\n </div>\n')
# SOURCE LINE 29
elif input.type == "conditional":
# SOURCE LINE 30
__M_writer(u' ')
group_values = values[input.name]
__M_writer(u'\n ')
# SOURCE LINE 31
current_case = group_values['__current_case__']
__M_writer(u'\n ')
# SOURCE LINE 32
group_prefix = prefix + input.name + "|"
__M_writer(u'\n ')
# SOURCE LINE 33
group_errors = errors.get(input.name, {})
__M_writer(u'\n ')
# SOURCE LINE 34
__M_writer(
unicode(
row_for_param(input.test_param,
group_values[input.test_param.name],
group_errors,
group_prefix,
ctx,
allow_runtime=False)))
__M_writer(u'\n ')
# SOURCE LINE 35
__M_writer(
unicode(
do_inputs(input.cases[current_case].inputs,
group_values, group_errors, group_prefix,
ctx)))
__M_writer(u'\n')
# SOURCE LINE 36
else:
# SOURCE LINE 37
if input.name in values:
# SOURCE LINE 38
__M_writer(u' ')
__M_writer(
unicode(
row_for_param(input, values[input.name], errors,
prefix, ctx)))
__M_writer(u'\n')
# SOURCE LINE 39
else:
# SOURCE LINE 40
__M_writer(u' ')
errors[input.name] = 'Value not stored, displaying default'
__M_writer(u'\n ')
# SOURCE LINE 41
__M_writer(
unicode(
row_for_param(
input, input.get_initial_value(trans, values),
errors, prefix, ctx)))
__M_writer(u'\n')
return ''
finally:
context.caller_stack._pop_frame()
def render_do_inputs(context,
inputs,
values,
errors,
prefix,
step,
other_values=None,
already_used=None):
context.caller_stack._push_frame()
try:
def row_for_param(param, value, other_values, error_dict, prefix, step,
already_used):
return render_row_for_param(context, param, value, other_values,
error_dict, prefix, step, already_used)
def do_inputs(inputs,
values,
errors,
prefix,
step,
other_values=None,
already_used=None):
return render_do_inputs(context, inputs, values, errors, prefix,
step, other_values, already_used)
len = context.get('len', UNDEFINED)
range = context.get('range', UNDEFINED)
dict = context.get('dict', UNDEFINED)
str = context.get('str', UNDEFINED)
enumerate = context.get('enumerate', UNDEFINED)
__M_writer = context.writer()
# SOURCE LINE 325
__M_writer(u'\n ')
# SOURCE LINE 326
from galaxy.util.expressions import ExpressionContext
other_values = ExpressionContext(values, other_values)
# SOURCE LINE 329
__M_writer(u'\n')
# SOURCE LINE 330
for input_index, input in enumerate(inputs.itervalues()):
# SOURCE LINE 331
if input.type == "repeat":
# SOURCE LINE 332
__M_writer(
u' <div class="repeat-group">\n <div class="form-title-row"><b>'
)
# SOURCE LINE 333
__M_writer(unicode(input.title_plural))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 334
repeat_values = values[input.name]
__M_writer(u'\n')
# SOURCE LINE 335
for i in range(len(repeat_values)):
# SOURCE LINE 336
if input.name in errors:
# SOURCE LINE 337
__M_writer(u' ')
rep_errors = errors[input.name][i]
__M_writer(u'\n')
# SOURCE LINE 338
else:
# SOURCE LINE 339
__M_writer(u' ')
rep_errors = dict()
__M_writer(u'\n')
pass
# SOURCE LINE 341
__M_writer(
u' <div class="repeat-group-item">\n '
)
# SOURCE LINE 342
index = repeat_values[i]['__index__']
__M_writer(
u'\n <div class="form-title-row"><b>')
# SOURCE LINE 343
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'</b></div>\n ')
# SOURCE LINE 344
__M_writer(
unicode(
do_inputs(
input.inputs, repeat_values[i], rep_errors,
prefix + input.name + "_" + str(index) + "|",
step, other_values, already_used)))
__M_writer(u'\n')
# SOURCE LINE 346
__M_writer(u' </div>\n')
pass
# SOURCE LINE 349
__M_writer(u' </div>\n')
# SOURCE LINE 350
elif input.type == "conditional":
# SOURCE LINE 351
if input.is_job_resource_conditional:
# SOURCE LINE 352
__M_writer(u' ')
continue
__M_writer(u'\n')
pass
# SOURCE LINE 354
__M_writer(u' ')
group_values = values[input.name]
__M_writer(u'\n ')
# SOURCE LINE 355
current_case = group_values['__current_case__']
__M_writer(u'\n ')
# SOURCE LINE 356
new_prefix = prefix + input.name + "|"
__M_writer(u'\n ')
# SOURCE LINE 357
group_errors = errors.get(input.name, {})
__M_writer(
u'\n <span class="conditional-start"></span>\n ')
# SOURCE LINE 359
__M_writer(
unicode(
row_for_param(input.test_param,
group_values[input.test_param.name],
other_values, group_errors, prefix, step,
already_used)))
__M_writer(u'\n ')
# SOURCE LINE 360
__M_writer(
unicode(
do_inputs(input.cases[current_case].inputs,
group_values, group_errors, new_prefix, step,
other_values, already_used)))
__M_writer(u'\n')
# SOURCE LINE 361
else:
# SOURCE LINE 362
__M_writer(u' ')
__M_writer(
unicode(
row_for_param(input, values[input.name], other_values,
errors, prefix, step, already_used)))
__M_writer(u'\n')
pass
pass
return ''
finally:
context.caller_stack._pop_frame()
def get_dataset_finish_context( self, job_context, dataset ):
for meta in self.get_tool_provided_job_metadata():
if meta['type'] == 'dataset' and meta['dataset_id'] == dataset.id:
return ExpressionContext( meta, job_context )
return job_context
def render_do_inputs(context,
inputs,
tool_state,
errors,
prefix,
other_values=None):
context.caller_stack._push_frame()
try:
_import_ns = {}
_mako_get_namespace(context, '__anon_0x5a0dfd0')._populate(
_import_ns, [u'overlay'])
_mako_get_namespace(context, '__anon_0x5a0df10')._populate(
_import_ns, [u'render_msg'])
def row_for_param(prefix, param, parent_state, parent_errors,
other_values):
return render_row_for_param(context, prefix, param, parent_state,
parent_errors, other_values)
h = _import_ns.get('h', context.get('h', UNDEFINED))
def do_inputs(inputs, tool_state, errors, prefix, other_values=None):
return render_do_inputs(context, inputs, tool_state, errors,
prefix, other_values)
len = _import_ns.get('len', context.get('len', UNDEFINED))
range = _import_ns.get('range', context.get('range', UNDEFINED))
dict = _import_ns.get('dict', context.get('dict', UNDEFINED))
str = _import_ns.get('str', context.get('str', UNDEFINED))
enumerate = _import_ns.get('enumerate',
context.get('enumerate', UNDEFINED))
trans = _import_ns.get('trans', context.get('trans', UNDEFINED))
__M_writer = context.writer()
# SOURCE LINE 120
__M_writer(u'\n ')
# SOURCE LINE 121
from galaxy.util.expressions import ExpressionContext
other_values = ExpressionContext(tool_state, other_values)
# SOURCE LINE 124
__M_writer(u'\n')
# SOURCE LINE 125
for input_index, input in enumerate(inputs.itervalues()):
# SOURCE LINE 126
if not input.visible:
# SOURCE LINE 127
__M_writer(u' ')
pass
__M_writer(u'\n')
# SOURCE LINE 128
elif input.type == "repeat":
# SOURCE LINE 129
__M_writer(
u' <div class="repeat-group">\n <div class="form-title-row"><strong>'
)
# SOURCE LINE 130
__M_writer(unicode(input.title_plural))
__M_writer(u'</strong>\n')
# SOURCE LINE 131
if input.help:
# SOURCE LINE 132
__M_writer(
u' <div class="toolParamHelp" style="clear: both;">\n '
)
# SOURCE LINE 133
__M_writer(unicode(input.help))
__M_writer(u'\n </div>\n')
pass
# SOURCE LINE 136
__M_writer(u' </div>\n ')
# SOURCE LINE 137
repeat_state = tool_state[input.name]
__M_writer(u'\n')
# SOURCE LINE 138
for i in range(len(repeat_state)):
# SOURCE LINE 139
__M_writer(
u' <div class="repeat-group-item">\n '
)
# SOURCE LINE 140
if input.name in errors:
rep_errors = errors[input.name][i]
else:
rep_errors = dict()
index = repeat_state[i]['__index__']
# SOURCE LINE 146
__M_writer(
u'\n <div class="form-title-row"><strong>'
)
# SOURCE LINE 147
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'</strong></div>\n ')
# SOURCE LINE 148
__M_writer(
unicode(
do_inputs(
input.inputs, repeat_state[i], rep_errors,
prefix + input.name + "_" + str(index) + "|",
other_values)))
__M_writer(
u'\n <div class="form-row"><input type="submit" class="btn" name="'
)
# SOURCE LINE 149
__M_writer(unicode(prefix))
__M_writer(unicode(input.name))
__M_writer(u'_')
__M_writer(unicode(index))
__M_writer(u'_remove" value="Remove ')
__M_writer(unicode(input.title))
__M_writer(u' ')
__M_writer(unicode(i + 1))
__M_writer(u'"></div>\n </div>\n')
# SOURCE LINE 151
if rep_errors.has_key('__index__'):
# SOURCE LINE 152
__M_writer(
u' <div><img style="vertical-align: middle;" src="'
)
__M_writer(
unicode(
h.url_for('/static/style/error_small.png')))
__M_writer(
u'"> <span style="vertical-align: middle;">')
__M_writer(unicode(rep_errors['__index__']))
__M_writer(u'</span></div>\n')
pass
pass
# SOURCE LINE 155
__M_writer(
u' <div class="form-row"><input type="submit" class="btn" name="'
)
__M_writer(unicode(prefix))
__M_writer(unicode(input.name))
__M_writer(u'_add" value="Add new ')
__M_writer(unicode(input.title))
__M_writer(u'"></div>\n </div>\n')
# SOURCE LINE 157
elif input.type == "conditional":
# SOURCE LINE 158
__M_writer(u' ')
group_state = tool_state[input.name]
group_errors = errors.get(input.name, {})
current_case = group_state['__current_case__']
group_prefix = prefix + input.name + "|"
# SOURCE LINE 163
__M_writer(u'\n')
# SOURCE LINE 164
if input.value_ref_in_group:
# SOURCE LINE 165
__M_writer(u' ')
__M_writer(
unicode(
row_for_param(group_prefix, input.test_param,
group_state, group_errors,
other_values)))
__M_writer(u'\n')
pass
# SOURCE LINE 167
__M_writer(u' ')
__M_writer(
unicode(
do_inputs(input.cases[current_case].inputs,
group_state, group_errors, group_prefix,
other_values)))
__M_writer(u'\n')
# SOURCE LINE 168
elif input.type == "upload_dataset":
# SOURCE LINE 169
if input.get_datatype(
trans, other_values
).composite_type is None: #have non-composite upload appear as before
# SOURCE LINE 170
__M_writer(u' ')
if input.name in errors:
rep_errors = errors[input.name][0]
else:
rep_errors = dict()
# SOURCE LINE 175
__M_writer(u'\n ')
# SOURCE LINE 176
__M_writer(
unicode(
do_inputs(input.inputs, tool_state[input.name][0],
rep_errors,
prefix + input.name + "_" + str(0) + "|",
other_values)))
__M_writer(u'\n')
# SOURCE LINE 177
else:
# SOURCE LINE 178
__M_writer(
u' <div class="repeat-group">\n <div class="form-title-row"><strong>'
)
# SOURCE LINE 179
__M_writer(unicode(input.group_title(other_values)))
__M_writer(u'</strong></div>\n ')
# SOURCE LINE 180
repeat_state = tool_state[input.name]
# SOURCE LINE 182
__M_writer(u'\n')
# SOURCE LINE 183
for i in range(len(repeat_state)):
# SOURCE LINE 184
__M_writer(
u' <div class="repeat-group-item">\n '
)
# SOURCE LINE 185
if input.name in errors:
rep_errors = errors[input.name][i]
else:
rep_errors = dict()
index = repeat_state[i]['__index__']
# SOURCE LINE 191
__M_writer(
u'\n <div class="form-title-row"><strong>File Contents for '
)
# SOURCE LINE 192
__M_writer(
unicode(
input.title_by_index(trans, i, other_values)))
__M_writer(u'</strong></div>\n ')
# SOURCE LINE 193
__M_writer(
unicode(
do_inputs(
input.inputs, repeat_state[i], rep_errors,
prefix + input.name + "_" + str(index) +
"|", other_values)))
__M_writer(u'\n')
# SOURCE LINE 195
__M_writer(u' </div>\n')
pass
# SOURCE LINE 198
__M_writer(u' </div>\n')
pass
# SOURCE LINE 200
else:
# SOURCE LINE 201
__M_writer(u' ')
__M_writer(
unicode(
row_for_param(prefix, input, tool_state, errors,
other_values)))
__M_writer(u'\n')
pass
pass
return ''
finally:
context.caller_stack._pop_frame()
def set_metadata_portable():
tool_job_working_directory = os.path.abspath(os.getcwd())
metadata_tmp_files_dir = os.path.join(tool_job_working_directory,
"metadata")
MetadataTempFile.tmp_dir = metadata_tmp_files_dir
metadata_params_path = os.path.join("metadata", "params.json")
try:
with open(metadata_params_path) as f:
metadata_params = json.load(f)
except OSError:
raise Exception(
f"Failed to find metadata/params.json from cwd [{tool_job_working_directory}]"
)
datatypes_config = metadata_params["datatypes_config"]
job_metadata = metadata_params["job_metadata"]
provided_metadata_style = metadata_params.get("provided_metadata_style")
max_metadata_value_size = metadata_params.get(
"max_metadata_value_size") or 0
outputs = metadata_params["outputs"]
datatypes_registry = validate_and_load_datatypes_config(datatypes_config)
tool_provided_metadata = load_job_metadata(job_metadata,
provided_metadata_style)
def set_meta(new_dataset_instance, file_dict):
set_meta_with_tool_provided(new_dataset_instance, file_dict,
set_meta_kwds, datatypes_registry,
max_metadata_value_size)
object_store_conf_path = os.path.join("metadata", "object_store_conf.json")
extended_metadata_collection = os.path.exists(object_store_conf_path)
object_store = None
job_context = None
version_string = ""
export_store = None
final_job_state = Job.states.OK
if extended_metadata_collection:
tool_dict = metadata_params["tool"]
stdio_exit_code_dicts, stdio_regex_dicts = tool_dict[
"stdio_exit_codes"], tool_dict["stdio_regexes"]
stdio_exit_codes = list(map(ToolStdioExitCode, stdio_exit_code_dicts))
stdio_regexes = list(map(ToolStdioRegex, stdio_regex_dicts))
with open(object_store_conf_path) as f:
config_dict = json.load(f)
assert config_dict is not None
object_store = build_object_store_from_config(None,
config_dict=config_dict)
Dataset.object_store = object_store
outputs_directory = os.path.join(tool_job_working_directory, "outputs")
if not os.path.exists(outputs_directory):
outputs_directory = tool_job_working_directory
# TODO: constants...
if os.path.exists(os.path.join(outputs_directory, "tool_stdout")):
with open(os.path.join(outputs_directory, "tool_stdout"),
"rb") as f:
tool_stdout = f.read()
with open(os.path.join(outputs_directory, "tool_stderr"),
"rb") as f:
tool_stderr = f.read()
elif os.path.exists(os.path.join(tool_job_working_directory,
"stdout")):
with open(os.path.join(tool_job_working_directory, "stdout"),
"rb") as f:
tool_stdout = f.read()
with open(os.path.join(tool_job_working_directory, "stderr"),
"rb") as f:
tool_stderr = f.read()
elif os.path.exists(os.path.join(outputs_directory, "stdout")):
# Puslar style output directory? Was this ever used - did this ever work?
with open(os.path.join(outputs_directory, "stdout"), "rb") as f:
tool_stdout = f.read()
with open(os.path.join(outputs_directory, "stderr"), "rb") as f:
tool_stderr = f.read()
else:
wdc = os.listdir(tool_job_working_directory)
odc = os.listdir(outputs_directory)
error_desc = "Failed to find tool_stdout or tool_stderr for this job, cannot collect metadata"
error_extra = f"Working dir contents [{wdc}], output directory contents [{odc}]"
log.warn(f"{error_desc}. {error_extra}")
raise Exception(error_desc)
job_id_tag = metadata_params["job_id_tag"]
exit_code_file = default_exit_code_file(".", job_id_tag)
tool_exit_code = read_exit_code_from(exit_code_file, job_id_tag)
check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output(
stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr,
tool_exit_code, job_id_tag)
if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs(
):
final_job_state = Job.states.OK
else:
final_job_state = Job.states.ERROR
version_string = ""
if os.path.exists(COMMAND_VERSION_FILENAME):
version_string = open(COMMAND_VERSION_FILENAME).read()
expression_context = ExpressionContext(
dict(stdout=tool_stdout, stderr=tool_stderr))
# Load outputs.
export_store = store.DirectoryModelExportStore(
'metadata/outputs_populated',
serialize_dataset_objects=True,
for_edit=True,
strip_metadata_files=False,
serialize_jobs=False)
try:
import_model_store = store.imported_store_for_metadata(
'metadata/outputs_new', object_store=object_store)
except AssertionError:
# Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now
import_model_store = None
job_context = SessionlessJobContext(
metadata_params,
tool_provided_metadata,
object_store,
export_store,
import_model_store,
os.path.join(tool_job_working_directory, "working"),
final_job_state=final_job_state,
)
unnamed_id_to_path = {}
for unnamed_output_dict in job_context.tool_provided_metadata.get_unnamed_outputs(
):
destination = unnamed_output_dict["destination"]
elements = unnamed_output_dict["elements"]
destination_type = destination["type"]
if destination_type == 'hdas':
for element in elements:
filename = element.get('filename')
if filename:
unnamed_id_to_path[element['object_id']] = os.path.join(
job_context.job_working_directory, filename)
for output_name, output_dict in outputs.items():
dataset_instance_id = output_dict["id"]
klass = getattr(
galaxy.model,
output_dict.get('model_class', 'HistoryDatasetAssociation'))
dataset = None
if import_model_store:
dataset = import_model_store.sa_session.query(klass).find(
dataset_instance_id)
if dataset is None:
# legacy check for jobs that started before 21.01, remove on 21.05
filename_in = os.path.join(f"metadata/metadata_in_{output_name}")
import pickle
dataset = pickle.load(open(filename_in,
'rb')) # load DatasetInstance
assert dataset is not None
filename_kwds = os.path.join(f"metadata/metadata_kwds_{output_name}")
filename_out = os.path.join(f"metadata/metadata_out_{output_name}")
filename_results_code = os.path.join(
f"metadata/metadata_results_{output_name}")
override_metadata = os.path.join(
f"metadata/metadata_override_{output_name}")
dataset_filename_override = output_dict["filename_override"]
# pre-20.05 this was a per job parameter and not a per dataset parameter, drop in 21.XX
legacy_object_store_store_by = metadata_params.get(
"object_store_store_by", "id")
# Same block as below...
set_meta_kwds = stringify_dictionary_keys(
json.load(open(filename_kwds))
) # load kwds; need to ensure our keywords are not unicode
try:
dataset.dataset.external_filename = unnamed_id_to_path.get(
dataset_instance_id, dataset_filename_override)
store_by = output_dict.get("object_store_store_by",
legacy_object_store_store_by)
extra_files_dir_name = f"dataset_{getattr(dataset.dataset, store_by)}_files"
files_path = os.path.abspath(
os.path.join(tool_job_working_directory, "working",
extra_files_dir_name))
dataset.dataset.external_extra_files_path = files_path
file_dict = tool_provided_metadata.get_dataset_meta(
output_name, dataset.dataset.id, dataset.dataset.uuid)
if 'ext' in file_dict:
dataset.extension = file_dict['ext']
# Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles
override_metadata = json.load(open(override_metadata))
for metadata_name, metadata_file_override in override_metadata:
if MetadataTempFile.is_JSONified_value(metadata_file_override):
metadata_file_override = MetadataTempFile.from_JSON(
metadata_file_override)
setattr(dataset.metadata, metadata_name,
metadata_file_override)
if output_dict.get("validate", False):
set_validated_state(dataset)
if dataset_instance_id not in unnamed_id_to_path:
# We're going to run through set_metadata in collect_dynamic_outputs with more contextual metadata,
# so skip set_meta here.
set_meta(dataset, file_dict)
if extended_metadata_collection:
meta = tool_provided_metadata.get_dataset_meta(
output_name, dataset.dataset.id, dataset.dataset.uuid)
if meta:
context = ExpressionContext(meta, expression_context)
else:
context = expression_context
# Lazy and unattached
# if getattr(dataset, "hidden_beneath_collection_instance", None):
# dataset.visible = False
dataset.blurb = 'done'
dataset.peek = 'no peek'
dataset.info = (dataset.info or '')
if context['stdout'].strip():
# Ensure white space between entries
dataset.info = f"{dataset.info.rstrip()}\n{context['stdout'].strip()}"
if context['stderr'].strip():
# Ensure white space between entries
dataset.info = f"{dataset.info.rstrip()}\n{context['stderr'].strip()}"
dataset.tool_version = version_string
dataset.set_size()
if 'uuid' in context:
dataset.dataset.uuid = context['uuid']
if dataset_filename_override and dataset_filename_override != dataset.file_name:
# This has to be a job with outputs_to_working_directory set.
# We update the object store with the created output file.
object_store.update_from_file(
dataset.dataset,
file_name=dataset_filename_override,
create=True)
collect_extra_files(object_store, dataset, ".")
if Job.states.ERROR == final_job_state:
dataset.blurb = "error"
dataset.mark_unhidden()
else:
# If the tool was expected to set the extension, attempt to retrieve it
if dataset.ext == 'auto':
dataset.extension = context.get('ext', 'data')
dataset.init_meta(copy_from=dataset)
# This has already been done:
# else:
# self.external_output_metadata.load_metadata(dataset, output_name, self.sa_session, working_directory=self.working_directory, remote_metadata_directory=remote_metadata_directory)
line_count = context.get('line_count', None)
try:
# Certain datatype's set_peek methods contain a line_count argument
dataset.set_peek(line_count=line_count)
except TypeError:
# ... and others don't
dataset.set_peek()
for context_key in TOOL_PROVIDED_JOB_METADATA_KEYS:
if context_key in context:
context_value = context[context_key]
setattr(dataset, context_key, context_value)
# We never want to persist the external_filename.
dataset.dataset.external_filename = None
export_store.add_dataset(dataset)
else:
dataset.metadata.to_JSON_dict(
filename_out) # write out results of set_meta
json.dump((True, 'Metadata has been set successfully'),
open(filename_results_code,
'wt+')) # setting metadata has succeeded
except Exception:
json.dump((False, traceback.format_exc()),
open(filename_results_code,
'wt+')) # setting metadata has failed somehow
if extended_metadata_collection:
# discover extra outputs...
output_collections = {}
for name, output_collection in metadata_params[
"output_collections"].items():
output_collections[name] = import_model_store.sa_session.query(
HistoryDatasetCollectionAssociation).find(
output_collection["id"])
outputs = {}
for name, output in metadata_params["outputs"].items():
klass = getattr(
galaxy.model,
output.get('model_class', 'HistoryDatasetAssociation'))
outputs[name] = import_model_store.sa_session.query(klass).find(
output["id"])
input_ext = json.loads(metadata_params["job_params"].get(
"__input_ext", '"data"'))
collect_primary_datasets(
job_context,
outputs,
input_ext=input_ext,
)
collect_dynamic_outputs(job_context, output_collections)
if export_store:
export_store._finalize()
write_job_metadata(tool_job_working_directory, job_metadata, set_meta,
tool_provided_metadata)
def set_metadata_portable():
tool_job_working_directory = os.path.abspath(os.getcwd())
metadata_tmp_files_dir = os.path.join(tool_job_working_directory, "metadata")
MetadataTempFile.tmp_dir = metadata_tmp_files_dir
metadata_params = get_metadata_params(tool_job_working_directory)
datatypes_config = metadata_params["datatypes_config"]
job_metadata = metadata_params["job_metadata"]
provided_metadata_style = metadata_params.get("provided_metadata_style")
max_metadata_value_size = metadata_params.get("max_metadata_value_size") or 0
max_discovered_files = metadata_params.get("max_discovered_files")
outputs = metadata_params["outputs"]
datatypes_registry = validate_and_load_datatypes_config(datatypes_config)
tool_provided_metadata = load_job_metadata(job_metadata, provided_metadata_style)
def set_meta(new_dataset_instance, file_dict):
set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size)
try:
object_store = get_object_store(tool_job_working_directory=tool_job_working_directory)
except (FileNotFoundError, AssertionError):
object_store = None
extended_metadata_collection = bool(object_store)
job_context = None
version_string = None
export_store = None
final_job_state = Job.states.OK
job_messages = []
if extended_metadata_collection:
tool_dict = metadata_params["tool"]
stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"]
stdio_exit_codes = list(map(ToolStdioExitCode, stdio_exit_code_dicts))
stdio_regexes = list(map(ToolStdioRegex, stdio_regex_dicts))
outputs_directory = os.path.join(tool_job_working_directory, "outputs")
if not os.path.exists(outputs_directory):
outputs_directory = tool_job_working_directory
# TODO: constants...
locations = [
(outputs_directory, 'tool_'),
(tool_job_working_directory, ''),
(outputs_directory, ''), # # Pulsar style output directory? Was this ever used - did this ever work?
]
for directory, prefix in locations:
if os.path.exists(os.path.join(directory, f"{prefix}stdout")):
with open(os.path.join(directory, f"{prefix}stdout"), 'rb') as f:
tool_stdout = f.read(MAX_STDIO_READ_BYTES)
with open(os.path.join(directory, f"{prefix}stderr"), 'rb') as f:
tool_stderr = f.read(MAX_STDIO_READ_BYTES)
break
else:
if os.path.exists(os.path.join(tool_job_working_directory, 'task_0')):
# We have a task splitting job
tool_stdout = b''
tool_stderr = b''
paths = Path(tool_job_working_directory).glob('task_*')
for path in paths:
with open(path / 'outputs' / 'tool_stdout', 'rb') as f:
task_stdout = f.read(MAX_STDIO_READ_BYTES)
if task_stdout:
tool_stdout = b"%s[%s stdout]\n%s\n" % (tool_stdout, path.name.encode(), task_stdout)
with open(path / 'outputs' / 'tool_stderr', 'rb') as f:
task_stderr = f.read(MAX_STDIO_READ_BYTES)
if task_stderr:
tool_stderr = b"%s[%s stdout]\n%s\n" % (tool_stderr, path.name.encode(), task_stderr)
else:
wdc = os.listdir(tool_job_working_directory)
odc = os.listdir(outputs_directory)
error_desc = "Failed to find tool_stdout or tool_stderr for this job, cannot collect metadata"
error_extra = f"Working dir contents [{wdc}], output directory contents [{odc}]"
log.warn(f"{error_desc}. {error_extra}")
raise Exception(error_desc)
job_id_tag = metadata_params["job_id_tag"]
exit_code_file = default_exit_code_file(".", job_id_tag)
tool_exit_code = read_exit_code_from(exit_code_file, job_id_tag)
check_output_detected_state, tool_stdout, tool_stderr, job_messages = check_output(stdio_regexes, stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag)
if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs():
final_job_state = Job.states.OK
else:
final_job_state = Job.states.ERROR
version_string_path = os.path.join('outputs', COMMAND_VERSION_FILENAME)
version_string = collect_shrinked_content_from_path(version_string_path)
expression_context = ExpressionContext(dict(stdout=tool_stdout[:255], stderr=tool_stderr[:255]))
# Load outputs.
export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False, serialize_jobs=True)
try:
import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store)
except AssertionError:
# Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now
import_model_store = None
tool_script_file = os.path.join(tool_job_working_directory, 'tool_script.sh')
job = None
if import_model_store and export_store:
job = next(iter(import_model_store.sa_session.objects[Job].values()))
job_context = SessionlessJobContext(
metadata_params,
tool_provided_metadata,
object_store,
export_store,
import_model_store,
os.path.join(tool_job_working_directory, "working"),
final_job_state=final_job_state,
max_discovered_files=max_discovered_files,
)
if extended_metadata_collection:
# discover extra outputs...
output_collections = {}
for name, output_collection in metadata_params["output_collections"].items():
# TODO: remove HistoryDatasetCollectionAssociation fallback on 22.01, model_class used to not be serialized prior to 21.09
model_class = output_collection.get('model_class', 'HistoryDatasetCollectionAssociation')
collection = import_model_store.sa_session.query(getattr(galaxy.model, model_class)).find(output_collection["id"])
output_collections[name] = collection
output_instances = {}
for name, output in metadata_params["outputs"].items():
klass = getattr(galaxy.model, output.get('model_class', 'HistoryDatasetAssociation'))
output_instances[name] = import_model_store.sa_session.query(klass).find(output["id"])
input_ext = json.loads(metadata_params["job_params"].get("__input_ext") or '"data"')
try:
collect_primary_datasets(
job_context,
output_instances,
input_ext=input_ext,
)
collect_dynamic_outputs(job_context, output_collections)
except MaxDiscoveredFilesExceededError as e:
final_job_state = Job.states.ERROR
job_messages.append(str(e))
if job:
job.job_messages = job_messages
job.state = final_job_state
if os.path.exists(tool_script_file):
with open(tool_script_file) as command_fh:
command_line_lines = []
for i, line in enumerate(command_fh):
if i == 0 and line.endswith('COMMAND_VERSION 2>&1;'):
# Don't record version command as part of command line
continue
command_line_lines.append(line)
job.command_line = "".join(command_line_lines).strip()
export_store.export_job(job, include_job_data=False)
unnamed_id_to_path = {}
for unnamed_output_dict in job_context.tool_provided_metadata.get_unnamed_outputs():
destination = unnamed_output_dict["destination"]
elements = unnamed_output_dict["elements"]
destination_type = destination["type"]
if destination_type == 'hdas':
for element in elements:
filename = element.get('filename')
object_id = element.get('object_id')
if filename and object_id:
unnamed_id_to_path[object_id] = os.path.join(job_context.job_working_directory, filename)
for output_name, output_dict in outputs.items():
dataset_instance_id = output_dict["id"]
klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation'))
dataset = None
if import_model_store:
dataset = import_model_store.sa_session.query(klass).find(dataset_instance_id)
if dataset is None:
# legacy check for jobs that started before 21.01, remove on 21.05
filename_in = os.path.join(f"metadata/metadata_in_{output_name}")
import pickle
dataset = pickle.load(open(filename_in, 'rb')) # load DatasetInstance
assert dataset is not None
filename_kwds = os.path.join(f"metadata/metadata_kwds_{output_name}")
filename_out = os.path.join(f"metadata/metadata_out_{output_name}")
filename_results_code = os.path.join(f"metadata/metadata_results_{output_name}")
override_metadata = os.path.join(f"metadata/metadata_override_{output_name}")
dataset_filename_override = output_dict["filename_override"]
# pre-20.05 this was a per job parameter and not a per dataset parameter, drop in 21.XX
legacy_object_store_store_by = metadata_params.get("object_store_store_by", "id")
# Same block as below...
set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode
try:
external_filename = unnamed_id_to_path.get(dataset_instance_id, dataset_filename_override)
if not os.path.exists(external_filename):
matches = glob.glob(external_filename)
assert len(matches) == 1, f"More than one file matched by output glob '{external_filename}'"
external_filename = matches[0]
assert safe_contains(tool_job_working_directory, external_filename), f"Cannot collect output '{external_filename}' from outside of working directory"
created_from_basename = os.path.relpath(external_filename, os.path.join(tool_job_working_directory, 'working'))
dataset.dataset.created_from_basename = created_from_basename
# override filename if we're dealing with outputs to working directory and dataset is not linked to
link_data_only = metadata_params.get("link_data_only")
if not link_data_only:
# Only set external filename if we're dealing with files in job working directory.
# Fixes link_data_only uploads
dataset.dataset.external_filename = external_filename
store_by = output_dict.get("object_store_store_by", legacy_object_store_store_by)
extra_files_dir_name = f"dataset_{getattr(dataset.dataset, store_by)}_files"
files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name))
dataset.dataset.external_extra_files_path = files_path
file_dict = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid)
if 'ext' in file_dict:
dataset.extension = file_dict['ext']
# Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles
override_metadata = json.load(open(override_metadata))
for metadata_name, metadata_file_override in override_metadata:
if MetadataTempFile.is_JSONified_value(metadata_file_override):
metadata_file_override = MetadataTempFile.from_JSON(metadata_file_override)
setattr(dataset.metadata, metadata_name, metadata_file_override)
if output_dict.get("validate", False):
set_validated_state(dataset)
if dataset_instance_id not in unnamed_id_to_path:
# We're going to run through set_metadata in collect_dynamic_outputs with more contextual metadata,
# so skip set_meta here.
set_meta(dataset, file_dict)
if extended_metadata_collection:
collect_extra_files(object_store, dataset, ".")
dataset.state = dataset.dataset.state = final_job_state
if extended_metadata_collection:
if not link_data_only and os.path.getsize(external_filename):
# Here we might be updating a disk based objectstore when outputs_to_working_directory is used,
# or a remote object store from its cache path.
object_store.update_from_file(dataset.dataset, file_name=external_filename, create=True)
# TODO: merge expression_context into tool_provided_metadata so we don't have to special case this (here and in _finish_dataset)
meta = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid)
if meta:
context = ExpressionContext(meta, expression_context)
else:
context = expression_context
dataset.blurb = 'done'
dataset.peek = 'no peek'
dataset.info = (dataset.info or '')
if context['stdout'].strip():
# Ensure white space between entries
dataset.info = f"{dataset.info.rstrip()}\n{context['stdout'].strip()}"
if context['stderr'].strip():
# Ensure white space between entries
dataset.info = f"{dataset.info.rstrip()}\n{context['stderr'].strip()}"
dataset.tool_version = version_string
if 'uuid' in context:
dataset.dataset.uuid = context['uuid']
if not final_job_state == Job.states.ERROR:
line_count = context.get('line_count', None)
try:
# Certain datatype's set_peek methods contain a line_count argument
dataset.set_peek(line_count=line_count)
except TypeError:
# ... and others don't
dataset.set_peek()
for context_key in TOOL_PROVIDED_JOB_METADATA_KEYS:
if context_key in context:
context_value = context[context_key]
setattr(dataset, context_key, context_value)
# We only want to persist the external_filename if the dataset has been linked in.
if not link_data_only:
dataset.dataset.external_filename = None
dataset.dataset.extra_files_path = None
export_store.add_dataset(dataset)
else:
dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta
json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded
except Exception:
json.dump((False, traceback.format_exc()), open(filename_results_code, 'wt+')) # setting metadata has failed somehow
if export_store:
export_store._finalize()
write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_provided_metadata)
def initialize_state(trans, inputs, state, context=None):
context = ExpressionContext(state, context)
for input in inputs.itervalues():
state[input.name] = input.get_initial_value(trans, context)
