def process_move(self, data_table_name, column_name, source_base_path, relative_symlinks=False, **kwd):
if data_table_name in self.move_by_data_table_column and column_name in self.move_by_data_table_column[data_table_name]:
move_dict = self.move_by_data_table_column[data_table_name][column_name]
source = move_dict['source_base']
if source is None:
source = source_base_path
else:
source = fill_template(source, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd).strip()
if move_dict['source_value']:
source = os.path.join(source, fill_template(move_dict['source_value'], GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd).strip())
target = move_dict['target_base']
if target is None:
target = self.data_managers.app.config.galaxy_data_manager_data_path
else:
target = fill_template(target, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd).strip()
if move_dict['target_value']:
target = os.path.join(target, fill_template(move_dict['target_value'], GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd).strip())
if move_dict['type'] == 'file':
dirs = os.path.split(target)[0]
try:
os.makedirs(dirs)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
# moving a directory and the target already exists, we move the contents instead
if os.path.exists(source):
util.move_merge(source, target)
if move_dict.get('relativize_symlinks', False):
util.relativize_symlinks(target)
return True
return False
def test_fill_list_comprehension_template():
if sys.version_info.major > 2:
with pytest.raises(NotFound):
fill_template(LIST_COMPREHENSION_TEMPLATE, retry=0)
else:
template_str = fill_template(LIST_COMPREHENSION_TEMPLATE, retry=0)
assert template_str == 'echo 1\n'
def process_move( self, data_table_name, column_name, source_base_path, relative_symlinks=False, **kwd ):
if data_table_name in self.move_by_data_table_column and column_name in self.move_by_data_table_column[ data_table_name ]:
move_dict = self.move_by_data_table_column[ data_table_name ][ column_name ]
source = move_dict[ 'source_base' ]
if source is None:
source = source_base_path
else:
source = fill_template( source, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd )
if move_dict[ 'source_value' ]:
source = os.path.join( source, fill_template( move_dict[ 'source_value' ], GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd ) )
target = move_dict[ 'target_base' ]
if target is None:
target = self.data_managers.app.config.galaxy_data_manager_data_path
else:
target = fill_template( target, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd )
if move_dict[ 'target_value' ]:
target = os.path.join( target, fill_template( move_dict[ 'target_value' ], GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd ) )
if move_dict[ 'type' ] == 'file':
dirs = os.path.split( target )[0]
try:
os.makedirs( dirs )
except OSError as e:
if e.errno != errno.EEXIST:
raise e
# moving a directory and the target already exists, we move the contents instead
util.move_merge( source, target )
if move_dict.get( 'relativize_symlinks', False ):
util.relativize_symlinks( target )
return True
return False
def process_move(self, data_table_name, column_name, source_base_path, relative_symlinks=False, **kwd):
if (
data_table_name in self.move_by_data_table_column
and column_name in self.move_by_data_table_column[data_table_name]
):
move_dict = self.move_by_data_table_column[data_table_name][column_name]
source = move_dict["source_base"]
if source is None:
source = source_base_path
else:
source = fill_template(
source,
GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path,
**kwd
)
if move_dict["source_value"]:
source = os.path.join(
source,
fill_template(
move_dict["source_value"],
GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path,
**kwd
),
)
target = move_dict["target_base"]
if target is None:
target = self.data_managers.app.config.galaxy_data_manager_data_path
else:
target = fill_template(
target,
GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path,
**kwd
)
if move_dict["target_value"]:
target = os.path.join(
target,
fill_template(
move_dict["target_value"],
GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path,
**kwd
),
)
if move_dict["type"] == "file":
dirs, filename = os.path.split(target)
try:
os.makedirs(dirs)
except OSError, e:
if e.errno != errno.EEXIST:
raise e
# log.debug( 'Error creating directory "%s": %s' % ( dirs, e ) )
# moving a directory and the target already exists, we move the contents instead
util.move_merge(source, target)
if move_dict.get("relativize_symlinks", False):
util.relativize_symlinks(target)
return True
def __populate_interactivetools(self, param_dict):
"""
Populate InteractiveTools templated values.
"""
it = []
for ep in getattr(self.tool, 'ports', []):
ep_dict = {}
for key in 'port', 'name', 'url':
val = ep.get(key, None)
if val is not None:
val = fill_template(val,
context=param_dict,
python_template_version=self.tool.
python_template_version)
clean_val = []
for line in val.split('\n'):
clean_val.append(line.strip())
val = '\n'.join(clean_val)
val = val.replace("\n", " ").replace("\r", " ").strip()
ep_dict[key] = val
it.append(ep_dict)
self.interactivetools = it
it_man = getattr(self.app, "interactivetool_manager", None)
if it_man:
it_man.create_interactivetool(self.job, self.tool, it)
return it
def __build_command_line(self):
"""
Build command line to invoke this tool given a populated param_dict
"""
command = self.tool.command
param_dict = self.param_dict
interpreter = self.tool.interpreter
command_line = None
if not command:
return
try:
# Substituting parameters into the command
command_line = fill_template(command, context=param_dict, python_template_version=self.tool.python_template_version)
cleaned_command_line = []
# Remove leading and trailing whitespace from each line for readability.
for line in command_line.split('\n'):
cleaned_command_line.append(line.strip())
command_line = '\n'.join(cleaned_command_line)
# Remove newlines from command line, and any leading/trailing white space
command_line = command_line.replace("\n", " ").replace("\r", " ").strip()
except Exception:
# Modify exception message to be more clear
# e.args = ( 'Error substituting into command line. Params: %r, Command: %s' % ( param_dict, self.command ), )
raise
if interpreter:
# TODO: path munging for cluster/dataset server relocatability
executable = command_line.split()[0]
tool_dir = os.path.abspath(self.tool.tool_dir)
abs_executable = os.path.join(tool_dir, executable)
command_line = command_line.replace(executable, "{} {}".format(interpreter, shlex.quote(abs_executable)), 1)
self.command_line = command_line
def __build_command_line(self):
"""
Build command line to invoke this tool given a populated param_dict
"""
command = self.tool.command
param_dict = self.param_dict
interpreter = self.tool.interpreter
command_line = None
if not command:
return
try:
# Substituting parameters into the command
command_line = fill_template(command, context=param_dict)
# Remove newlines from command line, and any leading/trailing white space
command_line = command_line.replace("\n",
" ").replace("\r",
" ").strip()
except Exception:
# Modify exception message to be more clear
#e.args = ( 'Error substituting into command line. Params: %r, Command: %s' % ( param_dict, self.command ), )
raise
if interpreter:
# TODO: path munging for cluster/dataset server relocatability
executable = command_line.split()[0]
tool_dir = os.path.abspath(self.tool.tool_dir)
abs_executable = os.path.join(tool_dir, executable)
command_line = command_line.replace(executable, abs_executable, 1)
command_line = interpreter + " " + command_line
self.command_line = command_line
def get_output_name( self, output, dataset, tool, on_text, trans, incoming, history, params, job_params ):
if output.label:
params['tool'] = tool
params['on_string'] = on_text
return fill_template( output.label, context=params )
else:
return self._get_default_data_name( dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params )
def __build_command_line(self):
"""
Build command line to invoke this tool given a populated param_dict
"""
command = self.tool.command
param_dict = self.param_dict
interpreter = self.tool.interpreter
command_line = None
if not command:
return
try:
# Substituting parameters into the command
command_line = fill_template(command, context=param_dict)
cleaned_command_line = []
# Remove leading and trailing whitespace from each line for readability.
for line in command_line.split('\n'):
cleaned_command_line.append(line.strip())
command_line = '\n'.join(cleaned_command_line)
# Remove newlines from command line, and any leading/trailing white space
command_line = command_line.replace("\n", " ").replace("\r", " ").strip()
except Exception:
# Modify exception message to be more clear
# e.args = ( 'Error substituting into command line. Params: %r, Command: %s' % ( param_dict, self.command ), )
raise
if interpreter:
# TODO: path munging for cluster/dataset server relocatability
executable = command_line.split()[0]
tool_dir = os.path.abspath(self.tool.tool_dir)
abs_executable = os.path.join(tool_dir, executable)
command_line = command_line.replace(executable, abs_executable, 1)
command_line = interpreter + " " + command_line
self.command_line = command_line
def get_output_name(self, output, dataset, tool, on_text, trans, incoming, history, params, job_params):
if output.label:
params['tool'] = tool
params['on_string'] = on_text
return fill_template(output.label, context=params)
else:
return self._get_default_data_name(dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params)
def filter_by_dataset(self, data, trans):
context = self.get_inital_values(data, trans)
for filter_elem in self.filters:
if fill_template(filter_elem.text,
context=context) != filter_elem.get(
'value', 'True'):
return False
return True
def __write_workdir_file( self, config_filename, content, context, is_template=True ):
if is_template:
value = fill_template( content, context=context )
else:
value = content
with open( config_filename, "w" ) as f:
f.write( value )
# For running jobs as the actual user, ensure the config file is globally readable
os.chmod( config_filename, 0644 )
def __write_workdir_file( self, config_filename, content, context, is_template=True ):
if is_template:
value = fill_template( content, context=context )
else:
value = content
with open( config_filename, "w" ) as f:
f.write( value )
# For running jobs as the actual user, ensure the config file is globally readable
os.chmod( config_filename, 0644 )
def process_value_translation(self, data_table_name, column_name, **kwd):
value = kwd.get(column_name)
if data_table_name in self.value_translation_by_data_table_column and column_name in self.value_translation_by_data_table_column[data_table_name]:
for value_translation in self.value_translation_by_data_table_column[data_table_name][column_name]:
if isinstance(value_translation, string_types):
value = fill_template(value_translation, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd).strip()
else:
value = value_translation(value)
return value
def process_value_translation( self, data_table_name, column_name, **kwd ):
value = kwd.get( column_name )
if data_table_name in self.value_translation_by_data_table_column and column_name in self.value_translation_by_data_table_column[ data_table_name ]:
for value_translation in self.value_translation_by_data_table_column[ data_table_name ][ column_name ]:
if isinstance( value_translation, string_types ):
value = fill_template( value_translation, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd )
else:
value = value_translation( value )
return value
def parse_file_fields(self, filename, errors=None, here="__HERE__"):
try:
rgc = refgenconf.RefGenConf(filename,
writable=False,
skip_read_lock=True)
except refgenconf.exceptions.RefgenconfError as e:
log.error('Unable to load refgenie config file "%s": %s', filename,
e)
if errors is not None:
errors.append(e)
return []
rval = []
for genome in rgc.list_genomes_by_asset(self.rg_asset):
genome_attributes = rgc.get_genome_attributes(genome)
description = genome_attributes.get('genome_description', None)
if description:
description = f'{description} (refgenie: {genome})'
asset_list = rgc.list(genome, include_tags=True)[genome]
for tagged_asset in asset_list:
asset, tag = tagged_asset.rsplit(':', 1)
if asset != self.rg_asset:
continue
digest = rgc.id(genome, asset, tag=tag)
uuid = f'refgenie:{genome}/{self.rg_asset}:{tag}@{digest}'
display_name = description or f'{genome}/{tagged_asset}'
def _seek_key(key):
return rgc.seek(genome, asset, tag_name=tag, seek_key=key)
template_dict = {
'__REFGENIE_UUID__': uuid,
'__REFGENIE_GENOME__': genome,
'__REFGENIE_TAG__': tag,
'__REFGENIE_DISPLAY_NAME__': display_name,
'__REFGENIE_ASSET__': rgc.seek(genome, asset,
tag_name=tag),
'__REFGENIE_ASSET_NAME__': asset,
'__REFGENIE_DIGEST__': digest,
'__REFGENIE_GENOME_ATTRIBUTES__': genome_attributes,
'__REFGENIE__': rgc,
'__REFGENIE_SEEK_KEY__': _seek_key,
}
fields = [''] * (self.largest_index + 1)
for name, index in self.columns.items():
rg_value = self.key_map[name]
# Default is hard-coded value
if self.template_for_column.get(name, False):
rg_value = fill_template(rg_value, template_dict)
if self.strip_for_column.get(name, False):
rg_value = rg_value.strip()
fields[index] = rg_value
rval.append(fields)
log.debug("Loaded %i entries from refgenie '%s' asset '%s' for '%s'",
len(rval), filename, self.rg_asset, self.name)
return rval
def __write_workdir_file(self, config_filename, content, context, is_template=True, strip=False):
if is_template:
value = fill_template(content, context=context, python_template_version=self.tool.python_template_version)
else:
value = unicodify(content)
if strip:
value = value.strip()
with io.open(config_filename, "w", encoding='utf-8') as f:
f.write(value)
# For running jobs as the actual user, ensure the config file is globally readable
os.chmod(config_filename, 0o644)
def _evaluate_prop(self, prop_val, user_context):
rval = prop_val
if isinstance(prop_val, str) and "$" in prop_val:
template_context = dict(
user=user_context,
environ=os.environ,
config=self._file_sources_config,
)
rval = fill_template(prop_val, context=template_context, futurized=True)
return rval
def determine_output_format(output, parameter_context, input_datasets,
random_input_ext):
""" Determines the output format for a dataset based on an abstract
description of the output (galaxy.tools.ToolOutput), the parameter
wrappers, a map of the input datasets (name => HDA), and the last input
extensions in the tool form.
TODO: Don't deal with XML here - move this logic into ToolOutput.
TODO: Make the input extension used deterministic instead of random.
"""
# the type should match the input
ext = output.format
if ext == "input":
ext = random_input_ext
if output.format_source is not None and output.format_source in input_datasets:
try:
input_dataset = input_datasets[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception:
pass
#process change_format tags
if output.change_format:
for change_elem in output.change_format:
print change_elem
for when_elem in change_elem.findall('when'):
check = when_elem.get('input', None)
print check
if check is not None:
try:
if '$' not in check:
#allow a simple name or more complex specifications
check = '${%s}' % check
if str(fill_template(
check,
context=parameter_context)) == when_elem.get(
'value', None):
ext = when_elem.get('format', ext)
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get('input_dataset', None)
if check is not None:
check = input_datasets.get(check, None)
if check is not None:
if str(getattr(check, when_elem.get(
'attribute'))) == when_elem.get(
'value', None):
ext = when_elem.get('format', ext)
return ext
def __build_command_line( self ):
"""
Build command line to invoke this tool given a populated param_dict
"""
command = self.tool.command
# print "In eval.py: " + str(command)
param_dict = self.param_dict
#conn = object_session(self).connection()
#trans = conn.begin()
#print trans
#print " In eval.py param_dict"
#print param_dict
#print "Doner"
interpreter = self.tool.interpreter
command_line = None
if not command:
return
try:
# Substituting parameters into the command
command_line = fill_template( command, context=param_dict )
print "returned command_line: " + command_line
cleaned_command_line = []
# Remove leading and trailing whitespace from each line for readability.
for line in command_line.split( '\n' ):
cleaned_command_line.append( line.strip() )
command_line = '\n'.join( cleaned_command_line )
# Remove newlines from command line, and any leading/trailing white space
command_line = command_line.replace( "\n", " " ).replace( "\r", " " ).strip()
print "In eval.py cmd_line: " + str(command_line)
except Exception:
# Modify exception message to be more clear
# e.args = ( 'Error substituting into command line. Params: %r, Command: %s' % ( param_dict, self.command ), )
raise
if interpreter:
# TODO: path munging for cluster/dataset server relocatability
executable = command_line.split()[0]
tool_dir = os.path.abspath( self.tool.tool_dir )
abs_executable = os.path.join( tool_dir, executable )
command_line = command_line.replace(executable, abs_executable, 1)
command_line = interpreter + " " + command_line
# self.command_line = command_line
if command_line.find("JPCNn681vcGV4KuvuT16") != (-1):
start = command_line.find(' JPCNn681vcGV4KuvuT16 ')
end = start + len( 'JPCNn681vcGV4KuvuT16 ' )
index = end + 1
passVar = ''
while (index < len(command_line)) and (command_line[index] != ' '):
passVar = passVar + command_line[index]
index = index + 1
command_line = command_line.replace(passVar, '$PASS')
command_line = command_line.replace('JPCNn681vcGV4KuvuT16 ', '')
self.command_line = command_line
def __write_workdir_file(self, config_filename, content, context, is_template=True, strip=False):
parent_dir = os.path.dirname(config_filename)
if not os.path.exists(parent_dir):
safe_makedirs(parent_dir)
if is_template:
value = fill_template(content, context=context, python_template_version=self.tool.python_template_version)
else:
value = unicodify(content)
if strip:
value = value.strip()
with open(config_filename, "w", encoding='utf-8') as f:
f.write(value)
# For running jobs as the actual user, ensure the config file is globally readable
os.chmod(config_filename, RW_R__R__)
def set_metadata_defaults(self, output, dataset, tool, on_text, trans, incoming, history, params, job_params):
"""
This allows to map names of input files to metadata default values. Example:
<data format="tabular" name="output" label="Tabular output, aggregates data from individual_inputs" >
<actions>
<action name="column_names" type="metadata" default="${','.join([input.name for input in $individual_inputs ])}" />
</actions>
</data>
"""
if output.actions:
for action in output.actions.actions:
if action.tag == "metadata" and action.default:
metadata_new_value = fill_template(action.default, context=params).split(",")
dataset.metadata.__setattr__(str(action.name), metadata_new_value)
def set_metadata_defaults( self, output, dataset, tool, on_text, trans, incoming, history, params, job_params ):
"""
This allows to map names of input files to metadata default values. Example:
<data format="tabular" name="output" label="Tabular output, aggregates data from individual_inputs" >
<actions>
<action name="column_names" type="metadata" default="${','.join([input.name for input in $individual_inputs ])}" />
</actions>
</data>
"""
if output.actions:
for action in output.actions.actions:
if action.tag == "metadata" and action.default:
metadata_new_value = fill_template( action.default, context=params ).split(",")
dataset.metadata.__setattr__(str(action.name), metadata_new_value)
def determine_output_format(output, parameter_context, input_datasets, random_input_ext):
""" Determines the output format for a dataset based on an abstract
description of the output (galaxy.tools.ToolOutput), the parameter
wrappers, a map of the input datasets (name => HDA), and the last input
extensions in the tool form.
TODO: Don't deal with XML here - move this logic into ToolOutput.
TODO: Make the input extension used deterministic instead of random.
"""
# the type should match the input
ext = output.format
if ext == "input":
ext = random_input_ext
if output.format_source is not None and output.format_source in input_datasets:
try:
input_dataset = input_datasets[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception:
pass
#process change_format tags
if output.change_format:
for change_elem in output.change_format:
print change_elem
for when_elem in change_elem.findall( 'when' ):
check = when_elem.get( 'input', None )
print check
if check is not None:
try:
if '$' not in check:
#allow a simple name or more complex specifications
check = '${%s}' % check
if str( fill_template( check, context=parameter_context ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get( 'input_dataset', None )
if check is not None:
check = input_datasets.get( check, None )
if check is not None:
if str( getattr( check, when_elem.get( 'attribute' ) ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
return ext
def parse_file_fields(self, filename, errors=None, here="__HERE__"):
rgc = refgenconf.RefGenConf(filename)
rval = []
for genome in rgc.list_genomes_by_asset(self.rg_asset):
genome_attributes = rgc.get_genome_attributes(genome)
description = genome_attributes.get('description', None)
asset_list = rgc.list(genome, include_tags=True)[genome]
for tagged_asset in asset_list:
asset, tag = tagged_asset.rsplit(':', 1)
if asset != self.rg_asset:
continue
digest = rgc.id(genome, asset, tag=tag)
uuid = 'refgenie:%s/%s:%s@%s' % (genome, self.rg_asset, tag,
digest)
display_name = description or '%s/%s' % (genome, tagged_asset)
def _seek_key(key):
return rgc.seek(genome, asset, tag_name=tag, seek_key=key)
template_dict = {
'__REFGENIE_UUID__': uuid,
'__REFGENIE_GENOME__': genome,
'__REFGENIE_TAG__': tag,
'__REFGENIE_DISPLAY_NAME__': display_name,
'__REFGENIE_ASSET__': rgc.seek(genome, asset,
tag_name=tag),
'__REFGENIE_ASSET_NAME__': asset,
'__REFGENIE_DIGEST__': digest,
'__REFGENIE_GENOME_ATTRIBUTES__': genome_attributes,
'__REFGENIE__': rgc,
'__REFGENIE_SEEK_KEY__': _seek_key,
}
fields = [''] * (self.largest_index + 1)
for name, index in self.columns.items():
rg_value = self.key_map[name]
# Default is hard-coded value
if self.template_for_column.get(name, False):
rg_value = fill_template(rg_value, template_dict)
if self.strip_for_column.get(name, False):
rg_value = rg_value.strip()
fields[index] = rg_value
rval.append(fields)
log.debug("Loaded %i entries from refgenie '%s' asset '%s' for '%s'",
len(rval), filename, self.rg_asset, self.name)
return rval
def populate_interactivetools(self):
"""
Populate InteractiveTools templated values.
"""
it = []
for ep in getattr(self.tool, 'ports', []):
ep_dict = {}
for key in 'port', 'name', 'url', 'requires_domain':
val = ep.get(key, None)
if val is not None and not isinstance(val, bool):
val = fill_template(val, context=self.param_dict, python_template_version=self.tool.python_template_version)
clean_val = []
for line in val.split('\n'):
clean_val.append(line.strip())
val = '\n'.join(clean_val)
val = val.replace("\n", " ").replace("\r", " ").strip()
ep_dict[key] = val
it.append(ep_dict)
return it
def __build_config_files( self ):
"""
Build temporary file for file based parameter transfer if needed
"""
param_dict = self.param_dict
config_filenames = []
for name, filename, template_text in self.tool.config_files:
# If a particular filename was forced by the config use it
directory = self.local_working_directory
if filename is not None:
config_filename = os.path.join( directory, filename )
else:
fd, config_filename = tempfile.mkstemp( dir=directory )
os.close( fd )
f = open( config_filename, "wt" )
f.write( fill_template( template_text, context=param_dict ) )
f.close()
# For running jobs as the actual user, ensure the config file is globally readable
os.chmod( config_filename, 0644 )
self.__register_extra_file( name, config_filename )
config_filenames.append( config_filename )
return config_filenames
def __build_config_files(self):
"""
Build temporary file for file based parameter transfer if needed
"""
param_dict = self.param_dict
config_filenames = []
for name, filename, template_text in self.tool.config_files:
# If a particular filename was forced by the config use it
directory = self.local_working_directory
if filename is not None:
config_filename = os.path.join(directory, filename)
else:
fd, config_filename = tempfile.mkstemp(dir=directory)
os.close(fd)
f = open(config_filename, "wt")
f.write(fill_template(template_text, context=param_dict))
f.close()
# For running jobs as the actual user, ensure the config file is globally readable
os.chmod(config_filename, 0644)
self.__register_extra_file(name, config_filename)
config_filenames.append(config_filename)
return config_filenames
def test_fix_template_two_to_three():
template_str = fill_template(TWO_TO_THREE_TEMPLATE, python_template_version='2', retry=1)
assert template_str == 'a a 1'
def execute(self, tool, trans, incoming={}, set_output_hid=True ):
def make_dict_copy( from_dict ):
"""
Makes a copy of input dictionary from_dict such that all values that are dictionaries
result in creation of a new dictionary ( a sort of deepcopy ). We may need to handle
other complex types ( e.g., lists, etc ), but not sure...
"""
copy_from_dict = {}
for key, value in from_dict.items():
if type( value ).__name__ == 'dict':
copy_from_dict[ key ] = make_dict_copy( value )
else:
copy_from_dict[ key ] = value
return copy_from_dict
def wrap_values( inputs, input_values ):
# Wrap tool inputs as necessary
for input in inputs.itervalues():
if isinstance( input, Repeat ):
for d in input_values[ input.name ]:
wrap_values( input.inputs, d )
elif isinstance( input, Conditional ):
values = input_values[ input.name ]
current = values[ "__current_case__" ]
wrap_values( input.cases[current].inputs, values )
elif isinstance( input, DataToolParameter ):
input_values[ input.name ] = \
galaxy.tools.DatasetFilenameWrapper( input_values[ input.name ],
datatypes_registry = trans.app.datatypes_registry,
tool = tool,
name = input.name )
elif isinstance( input, SelectToolParameter ):
input_values[ input.name ] = galaxy.tools.SelectToolParameterWrapper( input, input_values[ input.name ], tool.app, other_values = incoming )
else:
input_values[ input.name ] = galaxy.tools.InputValueWrapper( input, input_values[ input.name ], incoming )
out_data = {}
# Collect any input datasets from the incoming parameters
inp_data = self.collect_input_datasets( tool, incoming, trans )
# Deal with input dataset names, 'dbkey' and types
input_names = []
input_ext = 'data'
input_dbkey = incoming.get( "dbkey", "?" )
for name, data in inp_data.items():
if data:
input_names.append( 'data %s' % data.hid )
input_ext = data.ext
else:
data = NoneDataset( datatypes_registry = trans.app.datatypes_registry )
if data.dbkey not in [None, '?']:
input_dbkey = data.dbkey
# Collect chromInfo dataset and add as parameters to incoming
db_datasets = {}
db_dataset = trans.db_dataset_for( input_dbkey )
if db_dataset:
db_datasets[ "chromInfo" ] = db_dataset
incoming[ "chromInfo" ] = db_dataset.file_name
else:
incoming[ "chromInfo" ] = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey )
inp_data.update( db_datasets )
# Determine output dataset permission/roles list
existing_datasets = [ inp for inp in inp_data.values() if inp ]
if existing_datasets:
output_permissions = trans.app.security_agent.guess_derived_permissions_for_datasets( existing_datasets )
else:
# No valid inputs, we will use history defaults
output_permissions = trans.app.security_agent.history_get_default_permissions( trans.history )
# Build name for output datasets based on tool name and input names
if len( input_names ) == 1:
on_text = input_names[0]
elif len( input_names ) == 2:
on_text = '%s and %s' % tuple(input_names[0:2])
elif len( input_names ) == 3:
on_text = '%s, %s, and %s' % tuple(input_names[0:3])
elif len( input_names ) > 3:
on_text = '%s, %s, and others' % tuple(input_names[0:2])
else:
on_text = ""
# Add the dbkey to the incoming parameters
incoming[ "dbkey" ] = input_dbkey
# Keep track of parent / child relationships, we'll create all the
# datasets first, then create the associations
parent_to_child_pairs = []
child_dataset_names = set()
for name, output in tool.outputs.items():
for filter in output.filters:
try:
if not eval( filter.text, globals(), incoming ):
break #do not create this dataset
except Exception, e:
log.debug( 'Dataset output filter failed: %s' % e )
else: #all filters passed
if output.parent:
parent_to_child_pairs.append( ( output.parent, name ) )
child_dataset_names.add( name )
## What is the following hack for? Need to document under what
## conditions can the following occur? ([email protected])
# HACK: the output data has already been created
# this happens i.e. as a result of the async controller
if name in incoming:
dataid = incoming[name]
data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dataid )
assert data != None
out_data[name] = data
else:
# the type should match the input
ext = output.format
if ext == "input":
ext = input_ext
#process change_format tags
if output.change_format:
for change_elem in output.change_format:
for when_elem in change_elem.findall( 'when' ):
check = incoming.get( when_elem.get( 'input' ), None )
if check is not None:
if check == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
else:
check = when_elem.get( 'input_dataset', None )
if check is not None:
check = inp_data.get( check, None )
if check is not None:
if str( getattr( check, when_elem.get( 'attribute' ) ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True, sa_session=trans.sa_session )
# Commit the dataset immediately so it gets database assigned unique id
trans.sa_session.add( data )
trans.sa_session.flush()
trans.app.security_agent.set_all_dataset_permissions( data.dataset, output_permissions )
# Create an empty file immediately
open( data.file_name, "w" ).close()
# Fix permissions
util.umask_fix_perms( data.file_name, trans.app.config.umask, 0666 )
# This may not be neccesary with the new parent/child associations
data.designation = name
# Copy metadata from one of the inputs if requested.
if output.metadata_source:
data.init_meta( copy_from=inp_data[output.metadata_source] )
else:
data.init_meta()
# Take dbkey from LAST input
data.dbkey = str(input_dbkey)
# Set state
# FIXME: shouldn't this be NEW until the job runner changes it?
data.state = data.states.QUEUED
data.blurb = "queued"
# Set output label
if output.label:
params = make_dict_copy( incoming )
# wrapping the params allows the tool config to contain things like
# <outputs>
# <data format="input" name="output" label="Blat on ${<input_param>.name}" />
# </outputs>
wrap_values( tool.inputs, params )
params['tool'] = tool
params['on_string'] = on_text
data.name = fill_template( output.label, context=params )
else:
data.name = tool.name
if on_text:
data.name += ( " on " + on_text )
# Store output
out_data[ name ] = data
# Store all changes to database
trans.sa_session.flush()
def display_url(self):
assert self.display_ready(), "Display is not yet ready, cannot generate display link"
return fill_template(self.link.url.text, context=self.parameters)
def display_url(self):
assert self.display_ready(
), 'Display is not yet ready, cannot generate display link'
return fill_template(self.link.url.text, context=self.parameters)
def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, set_output_history=True, history=None, job_params=None, rerun_remap_job_id=None):
"""
Executes a tool, creating job and tool outputs, associating them, and
submitting the job to the job queue. If history is not specified, use
trans.history as destination for tool's output datasets.
"""
# Set history.
if not history:
history = tool.get_default_history_by_trans( trans, create=True )
out_data = odict()
# Track input dataset collections - but replace with simply lists so collect
# input datasets can process these normally.
inp_dataset_collections = self.collect_input_dataset_collections( tool, incoming )
# Collect any input datasets from the incoming parameters
inp_data = self.collect_input_datasets( tool, incoming, trans )
# Deal with input dataset names, 'dbkey' and types
input_names = []
input_ext = 'data'
input_dbkey = incoming.get( "dbkey", "?" )
for name, data in inp_data.items():
if not data:
data = NoneDataset( datatypes_registry=trans.app.datatypes_registry )
continue
# Convert LDDA to an HDA.
if isinstance(data, LibraryDatasetDatasetAssociation):
data = data.to_history_dataset_association( None )
inp_data[name] = data
else: # HDA
if data.hid:
input_names.append( 'data %s' % data.hid )
input_ext = data.ext
if data.dbkey not in [None, '?']:
input_dbkey = data.dbkey
# Collect chromInfo dataset and add as parameters to incoming
db_datasets = {}
( chrom_info, db_dataset ) = trans.app.genome_builds.get_chrom_info( input_dbkey, trans=trans, custom_build_hack_get_len_from_fasta_conversion=tool.id!='CONVERTER_fasta_to_len' )
if db_dataset:
inp_data.update( { "chromInfo": db_dataset } )
incoming[ "chromInfo" ] = chrom_info
# Determine output dataset permission/roles list
existing_datasets = [ inp for inp in inp_data.values() if inp ]
if existing_datasets:
output_permissions = trans.app.security_agent.guess_derived_permissions_for_datasets( existing_datasets )
else:
# No valid inputs, we will use history defaults
output_permissions = trans.app.security_agent.history_get_default_permissions( history )
# Build name for output datasets based on tool name and input names
on_text = on_text_for_names( input_names )
# Add the dbkey to the incoming parameters
incoming[ "dbkey" ] = input_dbkey
# wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
wrapped_params = WrappedParameters( trans, tool, incoming )
# Keep track of parent / child relationships, we'll create all the
# datasets first, then create the associations
parent_to_child_pairs = []
child_dataset_names = set()
object_store_id = None
for name, output in tool.outputs.items():
for filter in output.filters:
try:
if not eval( filter.text.strip(), globals(), incoming ):
break # do not create this dataset
except Exception, e:
log.debug( 'Dataset output filter failed: %s' % e )
else: # all filters passed
if output.parent:
parent_to_child_pairs.append( ( output.parent, name ) )
child_dataset_names.add( name )
## What is the following hack for? Need to document under what
## conditions can the following occur? ([email protected])
# HACK: the output data has already been created
# this happens i.e. as a result of the async controller
if name in incoming:
dataid = incoming[name]
data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dataid )
assert data != None
out_data[name] = data
else:
# the type should match the input
ext = output.format
if ext == "input":
ext = input_ext
if output.format_source is not None and output.format_source in inp_data:
try:
input_dataset = inp_data[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception, e:
pass
#process change_format tags
if output.change_format:
for change_elem in output.change_format:
for when_elem in change_elem.findall( 'when' ):
check = when_elem.get( 'input', None )
if check is not None:
try:
if '$' not in check:
#allow a simple name or more complex specifications
check = '${%s}' % check
if str( fill_template( check, context=wrapped_params.params ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get( 'input_dataset', None )
if check is not None:
check = inp_data.get( check, None )
if check is not None:
if str( getattr( check, when_elem.get( 'attribute' ) ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True, sa_session=trans.sa_session )
if output.hidden:
data.visible = False
# Commit the dataset immediately so it gets database assigned unique id
trans.sa_session.add( data )
trans.sa_session.flush()
trans.app.security_agent.set_all_dataset_permissions( data.dataset, output_permissions )
# Create an empty file immediately. The first dataset will be
# created in the "default" store, all others will be created in
# the same store as the first.
data.dataset.object_store_id = object_store_id
try:
trans.app.object_store.create( data.dataset )
except ObjectInvalid:
raise Exception('Unable to create output dataset: object store is full')
object_store_id = data.dataset.object_store_id # these will be the same thing after the first output
# This may not be neccesary with the new parent/child associations
data.designation = name
# Copy metadata from one of the inputs if requested.
if output.metadata_source:
data.init_meta( copy_from=inp_data[output.metadata_source] )
else:
data.init_meta()
# Take dbkey from LAST input
data.dbkey = str(input_dbkey)
# Set state
# FIXME: shouldn't this be NEW until the job runner changes it?
data.state = data.states.QUEUED
data.blurb = "queued"
# Set output label
data.name = self.get_output_name( output, data, tool, on_text, trans, incoming, history, wrapped_params.params, job_params )
# Store output
out_data[ name ] = data
if output.actions:
#Apply pre-job tool-output-dataset actions; e.g. setting metadata, changing format
output_action_params = dict( out_data )
output_action_params.update( incoming )
output.actions.apply_action( data, output_action_params )
# Store all changes to database
trans.sa_session.flush()
def determine_output_format(output, parameter_context, input_datasets,
input_dataset_collections, random_input_ext):
""" Determines the output format for a dataset based on an abstract
description of the output (galaxy.tools.parser.ToolOutput), the parameter
wrappers, a map of the input datasets (name => HDA), and the last input
extensions in the tool form.
TODO: Don't deal with XML here - move this logic into ToolOutput.
TODO: Make the input extension used deterministic instead of random.
"""
# the type should match the input
ext = output.format
if ext == "input":
ext = random_input_ext
format_source = output.format_source
if format_source is not None and format_source in input_datasets:
try:
input_dataset = input_datasets[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception:
pass
elif format_source is not None:
if re.match(r"^[^\[\]]*\[[^\[\]]*\]$", format_source):
collection_name, element_index = format_source[0:-1].split("[")
# Treat as json to interpret "forward" vs 0 with type
# Make it feel more like Python, single quote better in XML also.
element_index = element_index.replace("'", '"')
element_index = json.loads(element_index)
if collection_name in input_dataset_collections:
try:
input_collection = input_dataset_collections[
collection_name][0][0]
input_dataset = input_collection.collection[
element_index].element_object
input_extension = input_dataset.ext
ext = input_extension
except Exception as e:
log.debug(
"Exception while trying to determine format_source: %s",
e)
pass
# process change_format tags
if output.change_format is not None:
new_format_set = False
for change_elem in output.change_format:
for when_elem in change_elem.findall('when'):
check = when_elem.get('input', None)
if check is not None:
try:
if '$' not in check:
# allow a simple name or more complex specifications
check = '${%s}' % check
if str(fill_template(
check,
context=parameter_context)) == when_elem.get(
'value', None):
ext = when_elem.get('format', ext)
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get('input_dataset', None)
if check is not None:
check = input_datasets.get(check, None)
# At this point check is a HistoryDatasetAssociation object.
check_format = when_elem.get('format', ext)
check_value = when_elem.get('value', None)
check_attribute = when_elem.get('attribute', None)
if check is not None and check_value is not None and check_attribute is not None:
# See if the attribute to be checked belongs to the HistoryDatasetAssociation object.
if hasattr(check, check_attribute):
if str(getattr(
check,
check_attribute)) == str(check_value):
ext = check_format
new_format_set = True
break
# See if the attribute to be checked belongs to the metadata associated with the
# HistoryDatasetAssociation object.
if check.metadata is not None:
metadata_value = check.metadata.get(
check_attribute, None)
if metadata_value is not None:
if str(metadata_value) == str(check_value):
ext = check_format
new_format_set = True
break
if new_format_set:
break
return ext
def get_value( self, param_dict ):
value = fill_template( self.text, context = param_dict )
if self.strip:
value = value.strip()
return value
def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, set_output_history=True, history=None, job_params=None ):
"""
Executes a tool, creating job and tool outputs, associating them, and
submitting the job to the job queue. If history is not specified, use
trans.history as destination for tool's output datasets.
"""
def make_dict_copy( from_dict ):
"""
Makes a copy of input dictionary from_dict such that all values that are dictionaries
result in creation of a new dictionary ( a sort of deepcopy ). We may need to handle
other complex types ( e.g., lists, etc ), but not sure...
Yes, we need to handle lists (and now are)...
"""
copy_from_dict = {}
for key, value in from_dict.items():
if type( value ).__name__ == 'dict':
copy_from_dict[ key ] = make_dict_copy( value )
elif isinstance( value, list ):
copy_from_dict[ key ] = make_list_copy( value )
else:
copy_from_dict[ key ] = value
return copy_from_dict
def make_list_copy( from_list ):
new_list = []
for value in from_list:
if isinstance( value, dict ):
new_list.append( make_dict_copy( value ) )
elif isinstance( value, list ):
new_list.append( make_list_copy( value ) )
else:
new_list.append( value )
return new_list
def wrap_values( inputs, input_values, skip_missing_values = False ):
# Wrap tool inputs as necessary
for input in inputs.itervalues():
if input.name not in input_values and skip_missing_values:
continue
if isinstance( input, Repeat ):
for d in input_values[ input.name ]:
wrap_values( input.inputs, d, skip_missing_values = skip_missing_values )
elif isinstance( input, Conditional ):
values = input_values[ input.name ]
current = values[ "__current_case__" ]
wrap_values( input.cases[current].inputs, values, skip_missing_values = skip_missing_values )
elif isinstance( input, DataToolParameter ) and input.multiple:
input_values[ input.name ] = \
galaxy.tools.DatasetListWrapper( input_values[ input.name ],
datatypes_registry = trans.app.datatypes_registry,
tool = tool,
name = input.name )
elif isinstance( input, DataToolParameter ):
input_values[ input.name ] = \
galaxy.tools.DatasetFilenameWrapper( input_values[ input.name ],
datatypes_registry = trans.app.datatypes_registry,
tool = tool,
name = input.name )
elif isinstance( input, SelectToolParameter ):
input_values[ input.name ] = galaxy.tools.SelectToolParameterWrapper( input, input_values[ input.name ], tool.app, other_values = incoming )
else:
input_values[ input.name ] = galaxy.tools.InputValueWrapper( input, input_values[ input.name ], incoming )
# Set history.
if not history:
history = trans.history
out_data = odict()
# Collect any input datasets from the incoming parameters
inp_data = self.collect_input_datasets( tool, incoming, trans )
# Deal with input dataset names, 'dbkey' and types
input_names = []
input_ext = 'data'
input_dbkey = incoming.get( "dbkey", "?" )
for name, data in inp_data.items():
if not data:
data = NoneDataset( datatypes_registry = trans.app.datatypes_registry )
continue
# Convert LDDA to an HDA.
if isinstance(data, LibraryDatasetDatasetAssociation):
data = data.to_history_dataset_association( None )
inp_data[name] = data
else: # HDA
if data.hid:
input_names.append( 'data %s' % data.hid )
input_ext = data.ext
if data.dbkey not in [None, '?']:
input_dbkey = data.dbkey
# Collect chromInfo dataset and add as parameters to incoming
db_datasets = {}
db_dataset = trans.db_dataset_for( input_dbkey )
if db_dataset:
db_datasets[ "chromInfo" ] = db_dataset
incoming[ "chromInfo" ] = db_dataset.file_name
else:
# For custom builds, chrom info resides in converted dataset; for built-in builds, chrom info resides in tool-data/shared.
chrom_info = None
if trans.user and ( 'dbkeys' in trans.user.preferences ) and ( input_dbkey in from_json_string( trans.user.preferences[ 'dbkeys' ] ) ):
# Custom build.
custom_build_dict = from_json_string( trans.user.preferences[ 'dbkeys' ] )[ input_dbkey ]
if 'fasta' in custom_build_dict:
build_fasta_dataset = trans.app.model.HistoryDatasetAssociation.get( custom_build_dict[ 'fasta' ] )
chrom_info = build_fasta_dataset.get_converted_dataset( trans, 'len' ).file_name
if not chrom_info:
# Default to built-in build.
chrom_info = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey )
incoming[ "chromInfo" ] = chrom_info
inp_data.update( db_datasets )
# Determine output dataset permission/roles list
existing_datasets = [ inp for inp in inp_data.values() if inp ]
if existing_datasets:
output_permissions = trans.app.security_agent.guess_derived_permissions_for_datasets( existing_datasets )
else:
# No valid inputs, we will use history defaults
output_permissions = trans.app.security_agent.history_get_default_permissions( history )
# Build name for output datasets based on tool name and input names
if len( input_names ) == 1:
on_text = input_names[0]
elif len( input_names ) == 2:
on_text = '%s and %s' % tuple(input_names[0:2])
elif len( input_names ) == 3:
on_text = '%s, %s, and %s' % tuple(input_names[0:3])
elif len( input_names ) > 3:
on_text = '%s, %s, and others' % tuple(input_names[0:2])
else:
on_text = ""
# Add the dbkey to the incoming parameters
incoming[ "dbkey" ] = input_dbkey
params = None #wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
# Keep track of parent / child relationships, we'll create all the
# datasets first, then create the associations
parent_to_child_pairs = []
child_dataset_names = set()
object_store_id = None
for name, output in tool.outputs.items():
for filter in output.filters:
try:
if not eval( filter.text.strip(), globals(), incoming ):
break #do not create this dataset
except Exception, e:
log.debug( 'Dataset output filter failed: %s' % e )
else: #all filters passed
if output.parent:
parent_to_child_pairs.append( ( output.parent, name ) )
child_dataset_names.add( name )
## What is the following hack for? Need to document under what
## conditions can the following occur? ([email protected])
# HACK: the output data has already been created
# this happens i.e. as a result of the async controller
if name in incoming:
dataid = incoming[name]
data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dataid )
assert data != None
out_data[name] = data
else:
# the type should match the input
ext = output.format
if ext == "input":
ext = input_ext
if output.format_source is not None and output.format_source in inp_data:
try:
ext = inp_data[output.format_source].ext
except Exception, e:
pass
#process change_format tags
if output.change_format:
if params is None:
params = make_dict_copy( incoming )
wrap_values( tool.inputs, params, skip_missing_values = not tool.check_values )
for change_elem in output.change_format:
for when_elem in change_elem.findall( 'when' ):
check = when_elem.get( 'input', None )
if check is not None:
try:
if '$' not in check:
#allow a simple name or more complex specifications
check = '${%s}' % check
if str( fill_template( check, context = params ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
except: #bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get( 'input_dataset', None )
if check is not None:
check = inp_data.get( check, None )
if check is not None:
if str( getattr( check, when_elem.get( 'attribute' ) ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True, sa_session=trans.sa_session )
if output.hidden:
data.visible = False
# Commit the dataset immediately so it gets database assigned unique id
trans.sa_session.add( data )
trans.sa_session.flush()
trans.app.security_agent.set_all_dataset_permissions( data.dataset, output_permissions )
# Create an empty file immediately. The first dataset will be
# created in the "default" store, all others will be created in
# the same store as the first.
data.dataset.object_store_id = object_store_id
try:
trans.app.object_store.create( data.dataset )
except ObjectInvalid:
raise Exception('Unable to create output dataset: object store is full')
object_store_id = data.dataset.object_store_id # these will be the same thing after the first output
# This may not be neccesary with the new parent/child associations
data.designation = name
# Copy metadata from one of the inputs if requested.
if output.metadata_source:
data.init_meta( copy_from=inp_data[output.metadata_source] )
else:
data.init_meta()
# Take dbkey from LAST input
data.dbkey = str(input_dbkey)
# Set state
# FIXME: shouldn't this be NEW until the job runner changes it?
data.state = data.states.QUEUED
data.blurb = "queued"
# Set output label
if output.label:
if params is None:
params = make_dict_copy( incoming )
# wrapping the params allows the tool config to contain things like
# <outputs>
# <data format="input" name="output" label="Blat on ${<input_param>.name}" />
# </outputs>
wrap_values( tool.inputs, params, skip_missing_values = not tool.check_values )
#tool (only needing to be set once) and on_string (set differently for each label) are overwritten for each output dataset label being determined
params['tool'] = tool
params['on_string'] = on_text
params['time']=logging.time.strftime('%X %x', logging.time.gmtime())
data.name = fill_template( output.label, context=params )
else:
data.name = tool.name
if on_text:
data.name += ( " on " + on_text )
# Store output
out_data[ name ] = data
if output.actions:
#Apply pre-job tool-output-dataset actions; e.g. setting metadata, changing format
output_action_params = dict( out_data )
output_action_params.update( incoming )
output.actions.apply_action( data, output_action_params )
# Store all changes to database
trans.sa_session.flush()
def execute(self,
tool,
trans,
incoming={},
return_job=False,
set_output_hid=True,
set_output_history=True,
history=None,
job_params=None,
rerun_remap_job_id=None):
"""
Executes a tool, creating job and tool outputs, associating them, and
submitting the job to the job queue. If history is not specified, use
trans.history as destination for tool's output datasets.
"""
# Set history.
if not history:
history = tool.get_default_history_by_trans(trans, create=True)
out_data = odict()
# Track input dataset collections - but replace with simply lists so collect
# input datasets can process these normally.
inp_dataset_collections = self.collect_input_dataset_collections(
tool, incoming, trans)
# Collect any input datasets from the incoming parameters
inp_data = self.collect_input_datasets(tool, incoming, trans)
# Deal with input dataset names, 'dbkey' and types
input_names = []
input_ext = 'data'
input_dbkey = incoming.get("dbkey", "?")
for name, data in inp_data.items():
if not data:
data = NoneDataset(
datatypes_registry=trans.app.datatypes_registry)
continue
# Convert LDDA to an HDA.
if isinstance(data, LibraryDatasetDatasetAssociation):
data = data.to_history_dataset_association(None)
inp_data[name] = data
else: # HDA
if data.hid:
input_names.append('data %s' % data.hid)
input_ext = data.ext
if data.dbkey not in [None, '?']:
input_dbkey = data.dbkey
# Collect chromInfo dataset and add as parameters to incoming
db_datasets = {}
(chrom_info, db_dataset) = trans.app.genome_builds.get_chrom_info(
input_dbkey,
trans=trans,
custom_build_hack_get_len_from_fasta_conversion=tool.id !=
'CONVERTER_fasta_to_len')
if db_dataset:
inp_data.update({"chromInfo": db_dataset})
incoming["chromInfo"] = chrom_info
# Determine output dataset permission/roles list
existing_datasets = [inp for inp in inp_data.values() if inp]
if existing_datasets:
output_permissions = trans.app.security_agent.guess_derived_permissions_for_datasets(
existing_datasets)
else:
# No valid inputs, we will use history defaults
output_permissions = trans.app.security_agent.history_get_default_permissions(
history)
# Build name for output datasets based on tool name and input names
on_text = on_text_for_names(input_names)
# Add the dbkey to the incoming parameters
incoming["dbkey"] = input_dbkey
# wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
wrapped_params = WrappedParameters(trans, tool, incoming)
# Keep track of parent / child relationships, we'll create all the
# datasets first, then create the associations
parent_to_child_pairs = []
child_dataset_names = set()
object_store_id = None
for name, output in tool.outputs.items():
for filter in output.filters:
try:
if not eval(filter.text.strip(), globals(), incoming):
break # do not create this dataset
except Exception, e:
log.debug('Dataset output filter failed: %s' % e)
else: # all filters passed
if output.parent:
parent_to_child_pairs.append((output.parent, name))
child_dataset_names.add(name)
## What is the following hack for? Need to document under what
## conditions can the following occur? ([email protected])
# HACK: the output data has already been created
# this happens i.e. as a result of the async controller
if name in incoming:
dataid = incoming[name]
data = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(dataid)
assert data != None
out_data[name] = data
else:
# the type should match the input
ext = output.format
if ext == "input":
ext = input_ext
if output.format_source is not None and output.format_source in inp_data:
try:
input_dataset = inp_data[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception, e:
pass
#process change_format tags
if output.change_format:
for change_elem in output.change_format:
for when_elem in change_elem.findall('when'):
check = when_elem.get('input', None)
if check is not None:
try:
if '$' not in check:
#allow a simple name or more complex specifications
check = '${%s}' % check
if str(
fill_template(
check,
context=wrapped_params.
params)) == when_elem.get(
'value', None):
ext = when_elem.get('format', ext)
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get(
'input_dataset', None)
if check is not None:
check = inp_data.get(check, None)
if check is not None:
if str(
getattr(
check,
when_elem.get(
'attribute'))
) == when_elem.get('value', None):
ext = when_elem.get(
'format', ext)
data = trans.app.model.HistoryDatasetAssociation(
extension=ext,
create_dataset=True,
sa_session=trans.sa_session)
if output.hidden:
data.visible = False
# Commit the dataset immediately so it gets database assigned unique id
trans.sa_session.add(data)
trans.sa_session.flush()
trans.app.security_agent.set_all_dataset_permissions(
data.dataset, output_permissions)
# Create an empty file immediately. The first dataset will be
# created in the "default" store, all others will be created in
# the same store as the first.
data.dataset.object_store_id = object_store_id
try:
trans.app.object_store.create(data.dataset)
except ObjectInvalid:
raise Exception(
'Unable to create output dataset: object store is full'
)
object_store_id = data.dataset.object_store_id # these will be the same thing after the first output
# This may not be neccesary with the new parent/child associations
data.designation = name
# Copy metadata from one of the inputs if requested.
if output.metadata_source:
data.init_meta(copy_from=inp_data[output.metadata_source])
else:
data.init_meta()
# Take dbkey from LAST input
data.dbkey = str(input_dbkey)
# Set state
# FIXME: shouldn't this be NEW until the job runner changes it?
data.state = data.states.QUEUED
data.blurb = "queued"
# Set output label
data.name = self.get_output_name(output, data, tool, on_text,
trans, incoming, history,
wrapped_params.params,
job_params)
# Store output
out_data[name] = data
if output.actions:
#Apply pre-job tool-output-dataset actions; e.g. setting metadata, changing format
output_action_params = dict(out_data)
output_action_params.update(incoming)
output.actions.apply_action(data, output_action_params)
# Store all changes to database
trans.sa_session.flush()
def test_set_comprehension():
with pytest.raises(NotFound):
fill_template(SET_COMPR_TEMPLATE, retry=1)
def test_gen_expr():
with pytest.raises(NotFound):
fill_template(GEN_EXPR_TEMPLATE, retry=1)
def build_url( self, other_values ):
return fill_template( self.url, context = other_values )
def install_and_build_package( app, tool_dependency, actions_dict ):
"""Install a Galaxy tool dependency package either via a url or a mercurial or git clone command."""
sa_session = app.model.context.current
install_dir = actions_dict[ 'install_dir' ]
package_name = actions_dict[ 'package_name' ]
actions = actions_dict.get( 'actions', None )
filtered_actions = []
env_shell_file_paths = []
if actions:
with make_tmp_dir() as work_dir:
with lcd( work_dir ):
# The first action in the list of actions will be the one that defines the installation process. There
# are currently only two supported processes; download_by_url and clone via a "shell_command" action type.
action_type, action_dict = actions[ 0 ]
if action_type == 'download_by_url':
# Eliminate the download_by_url action so remaining actions can be processed correctly.
filtered_actions = actions[ 1: ]
url = action_dict[ 'url' ]
if 'target_filename' in action_dict:
# Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this
# attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
downloaded_filename = action_dict[ 'target_filename' ]
else:
downloaded_filename = os.path.split( url )[ -1 ]
dir = common_util.url_download( work_dir, downloaded_filename, url, extract=True )
elif action_type == 'shell_command':
# <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action>
# Eliminate the shell_command clone action so remaining actions can be processed correctly.
filtered_actions = actions[ 1: ]
return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] )
if return_code:
return
dir = package_name
elif action_type == 'download_file':
# <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
# Download a single file to the working directory.
filtered_actions = actions[ 1: ]
url = action_dict[ 'url' ]
if 'target_filename' in action_dict:
# Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this
# attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
filename = action_dict[ 'target_filename' ]
else:
filename = url.split( '/' )[ -1 ]
common_util.url_download( work_dir, filename, url )
dir = os.path.curdir
else:
# We're handling a complex repository dependency where we only have a set_environment tag set.
# <action type="set_environment">
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
filtered_actions = [ a for a in actions ]
dir = install_dir
# We need to be careful in determining if the value of dir is a valid directory because we're dealing with 2 environments, the fabric local
# environment and the python environment. Checking the path as follows should work.
full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) )
if not os.path.exists( full_path_to_dir ):
os.makedirs( full_path_to_dir )
# The package has been down-loaded, so we can now perform all of the actions defined for building it.
for action_tup in filtered_actions:
current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
with lcd( current_dir ):
action_type, action_dict = action_tup
if action_type == 'make_directory':
common_util.make_directory( full_path=action_dict[ 'full_path' ] )
elif action_type == 'move_directory_files':
common_util.move_directory_files( current_dir=current_dir,
source_dir=os.path.join( action_dict[ 'source_directory' ] ),
destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) )
elif action_type == 'move_file':
# TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found.
# current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9'
common_util.move_file( current_dir=current_dir,
source=os.path.join( action_dict[ 'source' ] ),
destination_dir=os.path.join( action_dict[ 'destination' ] ) )
elif action_type == 'set_environment':
# Currently the only action supported in this category is "environment_variable".
# Build a command line from the prior_installation_required, in case an environment variable is referenced
# in the set_environment action.
cmds = []
for env_shell_file_path in env_shell_file_paths:
if os.path.exists( env_shell_file_path ):
for env_setting in open( env_shell_file_path ):
cmds.append( env_setting.strip( '\n' ) )
else:
log.debug( 'Invalid file %s specified, ignoring set_environment action.', env_shell_file_path )
env_var_dicts = action_dict[ 'environment_variable' ]
for env_var_dict in env_var_dicts:
# Check for the presence of the $ENV[] key string and populate it if possible.
env_var_dict = handle_environment_variables( app, tool_dependency, install_dir, env_var_dict, cmds )
env_command = common_util.create_or_update_env_shell_file( install_dir, env_var_dict )
return_code = handle_command( app, tool_dependency, install_dir, env_command )
if return_code:
return
elif action_type == 'set_environment_for_install':
# Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files,
# the environment setting in each of which will be injected into the environment for all <action type="shell_command">
# tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file.
env_shell_file_paths = action_dict[ 'env_shell_file_paths' ]
elif action_type == 'setup_virtualenv':
# TODO: maybe should be configurable
venv_src_directory = os.path.abspath( os.path.join( app.config.tool_dependency_dir, '__virtualenv_src' ) )
if not install_virtualenv( app, venv_src_directory ):
log.error( 'Unable to install virtualenv' )
return
requirements = action_dict[ 'requirements' ]
if os.path.exists( os.path.join( dir, requirements ) ):
# requirements specified as path to a file
requirements_path = requirements
else:
# requirements specified directly in XML, create a file with these for pip.
requirements_path = os.path.join( install_dir, "requirements.txt" )
with open( requirements_path, "w" ) as f:
f.write( requirements )
venv_directory = os.path.join( install_dir, "venv" )
# TODO: Consider making --no-site-packages optional.
setup_command = "python %s/virtualenv.py --no-site-packages '%s'" % (venv_src_directory, venv_directory)
# POSIXLY_CORRECT forces shell commands . and source to have the same
# and well defined behavior in bash/zsh.
activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate" )
install_command = "python '%s' install -r '%s'" % ( os.path.join( venv_directory, "bin", "pip" ), requirements_path )
full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command )
return_code = handle_command( app, tool_dependency, install_dir, full_setup_command )
if return_code:
return
site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( venv_directory, "bin", "python" )
output = handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True )
if output.return_code:
return
if not os.path.exists( output.stdout ):
log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout )
return
modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) )
return_code = handle_command( app, tool_dependency, install_dir, modify_env_command )
if return_code:
return
modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) )
return_code = handle_command( app, tool_dependency, install_dir, modify_env_command )
if return_code:
return
elif action_type == 'shell_command':
with settings( warn_only=True ):
cmd = ''
for env_shell_file_path in env_shell_file_paths:
if os.path.exists( env_shell_file_path ):
for env_setting in open( env_shell_file_path ):
cmd += '%s\n' % env_setting
else:
log.debug( 'Invalid file %s specified, ignoring shell_command action.', env_shell_file_path )
cmd += action_dict[ 'command' ]
return_code = handle_command( app, tool_dependency, install_dir, cmd )
if return_code:
return
elif action_type == 'template_command':
env_vars = dict()
for env_shell_file_path in env_shell_file_paths:
if os.path.exists( env_shell_file_path ):
for env_setting in open( env_shell_file_path ):
env_string = env_setting.split( ';' )[ 0 ]
env_name, env_path = env_string.split( '=' )
env_vars[ env_name ] = env_path
else:
log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path )
env_vars.update( common_util.get_env_var_values( install_dir ) )
language = action_dict[ 'language' ]
with settings( warn_only=True, **env_vars ):
if language == 'cheetah':
# We need to import fabric.api.env so that we can access all collected environment variables.
cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars )
return_code = handle_command( app, tool_dependency, install_dir, cmd )
if return_code:
return
elif action_type == 'download_file':
# Download a single file to the current working directory.
url = action_dict[ 'url' ]
if 'target_filename' in action_dict:
filename = action_dict[ 'target_filename' ]
else:
filename = url.split( '/' )[ -1 ]
extract = action_dict.get( 'extract', False )
common_util.url_download( current_dir, filename, url, extract=extract )
elif action_type == 'change_directory':
target_directory = os.path.realpath( os.path.normpath( os.path.join( current_dir, action_dict[ 'directory' ] ) ) )
if target_directory.startswith( os.path.realpath( current_dir ) ) and os.path.exists( target_directory ):
# Change directory to a directory within the current working directory.
dir = target_directory
elif target_directory.startswith( os.path.realpath( work_dir ) ) and os.path.exists( target_directory ):
# Change directory to a directory above the current working directory, but within the defined work_dir.
dir = target_directory.replace( os.path.realpath( work_dir ), '' ).lstrip( '/' )
else:
log.error( 'Invalid or nonexistent directory %s specified, ignoring change_directory action.', target_directory )
def build_template( self, param_dict ):
template = fill_template( self.text, context=param_dict )
return template
e)
pass
# process change_format tags
if output.change_format is not None:
new_format_set = False
for change_elem in output.change_format:
for when_elem in change_elem.findall('when'):
check = when_elem.get('input', None)
if check is not None:
try:
if '$' not in check:
# allow a simple name or more complex specifications
check = '${%s}' % check
if str(fill_template(
check,
context=parameter_context)) == when_elem.get(
'value', None):
ext = when_elem.get('format', ext)
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get('input_dataset', None)
if check is not None:
check = input_datasets.get(check, None)
# At this point check is a HistoryDatasetAssociation object.
check_format = when_elem.get('format', ext)
check_value = when_elem.get('value', None)
check_attribute = when_elem.get('attribute', None)
if check is not None and check_value is not None and check_attribute is not None:
# See if the attribute to be checked belongs to the HistoryDatasetAssociation object.
except Exception, e:
log.debug("Exception while trying to determine format_source: %s", e)
pass
# process change_format tags
if output.change_format is not None:
new_format_set = False
for change_elem in output.change_format:
for when_elem in change_elem.findall( 'when' ):
check = when_elem.get( 'input', None )
if check is not None:
try:
if '$' not in check:
# allow a simple name or more complex specifications
check = '${%s}' % check
if str( fill_template( check, context=parameter_context ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get( 'input_dataset', None )
if check is not None:
check = input_datasets.get( check, None )
# At this point check is a HistoryDatasetAssociation object.
check_format = when_elem.get( 'format', ext )
check_value = when_elem.get( 'value', None )
check_attribute = when_elem.get( 'attribute', None )
if check is not None and check_value is not None and check_attribute is not None:
# See if the attribute to be checked belongs to the HistoryDatasetAssociation object.
if hasattr( check, check_attribute ):
if str( getattr( check, check_attribute ) ) == str( check_value ):
def build_template(self, param_dict):
template = fill_template(self.text, context=param_dict)
return template
def get_value(self, other_values, dataset_hash, user_hash, trans):
value = fill_template(self.text, context=other_values)
if self.strip:
value = value.strip()
return DisplayParameterValueWrapper(value, self, other_values,
dataset_hash, user_hash, trans)
def execute(self, tool, trans, incoming={}, set_output_hid=True ):
def make_dict_copy( from_dict ):
"""
Makes a copy of input dictionary from_dict such that all values that are dictionaries
result in creation of a new dictionary ( a sort of deepcopy ). We may need to handle
other complex types ( e.g., lists, etc ), but not sure...
"""
copy_from_dict = {}
for key, value in from_dict.items():
if type( value ).__name__ == 'dict':
copy_from_dict[ key ] = make_dict_copy( value )
else:
copy_from_dict[ key ] = value
return copy_from_dict
def wrap_values( inputs, input_values ):
# Wrap tool inputs as necessary
for input in inputs.itervalues():
if isinstance( input, Repeat ):
for d in input_values[ input.name ]:
wrap_values( input.inputs, d )
elif isinstance( input, Conditional ):
values = input_values[ input.name ]
current = values[ "__current_case__" ]
wrap_values( input.cases[current].inputs, values )
elif isinstance( input, DataToolParameter ):
input_values[ input.name ] = \
galaxy.tools.DatasetFilenameWrapper( input_values[ input.name ],
datatypes_registry = trans.app.datatypes_registry,
tool = tool,
name = input.name )
elif isinstance( input, SelectToolParameter ):
input_values[ input.name ] = galaxy.tools.SelectToolParameterWrapper( input, input_values[ input.name ], tool.app, other_values = incoming )
else:
input_values[ input.name ] = galaxy.tools.InputValueWrapper( input, input_values[ input.name ], incoming )
out_data = {}
# Collect any input datasets from the incoming parameters
inp_data = self.collect_input_datasets( tool, incoming, trans )
# Deal with input dataset names, 'dbkey' and types
input_names = []
input_ext = 'data'
input_dbkey = incoming.get( "dbkey", "?" )
for name, data in inp_data.items():
if data:
input_names.append( 'data %s' % data.hid )
input_ext = data.ext
else:
data = NoneDataset( datatypes_registry = trans.app.datatypes_registry )
if data.dbkey not in [None, '?']:
input_dbkey = data.dbkey
# Collect chromInfo dataset and add as parameters to incoming
db_datasets = {}
db_dataset = trans.db_dataset_for( input_dbkey )
if db_dataset:
db_datasets[ "chromInfo" ] = db_dataset
incoming[ "chromInfo" ] = db_dataset.file_name
else:
incoming[ "chromInfo" ] = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey )
inp_data.update( db_datasets )
# Determine output dataset permission/roles list
existing_datasets = [ inp for inp in inp_data.values() if inp ]
if existing_datasets:
output_permissions = trans.app.security_agent.guess_derived_permissions_for_datasets( existing_datasets )
else:
# No valid inputs, we will use history defaults
output_permissions = trans.app.security_agent.history_get_default_permissions( trans.history )
# Build name for output datasets based on tool name and input names
if len( input_names ) == 1:
on_text = input_names[0]
elif len( input_names ) == 2:
on_text = '%s and %s' % tuple(input_names[0:2])
elif len( input_names ) == 3:
on_text = '%s, %s, and %s' % tuple(input_names[0:3])
elif len( input_names ) > 3:
on_text = '%s, %s, and others' % tuple(input_names[0:2])
else:
on_text = ""
# Add the dbkey to the incoming parameters
incoming[ "dbkey" ] = input_dbkey
# Keep track of parent / child relationships, we'll create all the
# datasets first, then create the associations
parent_to_child_pairs = []
child_dataset_names = set()
for name, output in tool.outputs.items():
if output.parent:
parent_to_child_pairs.append( ( output.parent, name ) )
child_dataset_names.add( name )
## What is the following hack for? Need to document under what
## conditions can the following occur? ([email protected])
# HACK: the output data has already been created
# this happens i.e. as a result of the async controller
if name in incoming:
dataid = incoming[name]
data = trans.app.model.HistoryDatasetAssociation.get( dataid )
assert data != None
out_data[name] = data
else:
# the type should match the input
ext = output.format
if ext == "input":
ext = input_ext
#process change_format tags
if output.change_format:
for change_elem in output.change_format:
for when_elem in change_elem.findall( 'when' ):
check = incoming.get( when_elem.get( 'input' ), None )
if check is not None:
if check == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
else:
check = when_elem.get( 'input_dataset', None )
if check is not None:
check = inp_data.get( check, None )
if check is not None:
if str( getattr( check, when_elem.get( 'attribute' ) ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True )
# Commit the dataset immediately so it gets database assigned unique id
data.flush()
trans.app.security_agent.set_all_dataset_permissions( data.dataset, output_permissions )
# Create an empty file immediately
open( data.file_name, "w" ).close()
# This may not be neccesary with the new parent/child associations
data.designation = name
# Copy metadata from one of the inputs if requested.
if output.metadata_source:
data.init_meta( copy_from=inp_data[output.metadata_source] )
else:
data.init_meta()
# Take dbkey from LAST input
data.dbkey = str(input_dbkey)
# Set state
# FIXME: shouldn't this be NEW until the job runner changes it?
data.state = data.states.QUEUED
data.blurb = "queued"
# Set output label
if output.label:
params = make_dict_copy( incoming )
# wrapping the params allows the tool config to contain things like
# <outputs>
# <data format="input" name="output" label="Blat on ${<input_param>.name}" />
# </outputs>
wrap_values( tool.inputs, params )
params['tool'] = tool
params['on_string'] = on_text
data.name = fill_template( output.label, context=params )
else:
data.name = tool.name
if on_text:
data.name += ( " on " + on_text )
# Store output
out_data[ name ] = data
# Store all changes to database
trans.app.model.flush()
# Add all the top-level (non-child) datasets to the history
for name in out_data.keys():
if name not in child_dataset_names and name not in incoming: #don't add children; or already existing datasets, i.e. async created
data = out_data[ name ]
trans.history.add_dataset( data, set_hid = set_output_hid )
data.flush()
# Add all the children to their parents
for parent_name, child_name in parent_to_child_pairs:
parent_dataset = out_data[ parent_name ]
child_dataset = out_data[ child_name ]
parent_dataset.children.append( child_dataset )
# Store data after custom code runs
trans.app.model.flush()
# Create the job object
job = trans.app.model.Job()
job.session_id = trans.get_galaxy_session().id
job.history_id = trans.history.id
job.tool_id = tool.id
try:
# For backward compatibility, some tools may not have versions yet.
job.tool_version = tool.version
except:
job.tool_version = "1.0.0"
# FIXME: Don't need all of incoming here, just the defined parameters
# from the tool. We need to deal with tools that pass all post
# parameters to the command as a special case.
for name, value in tool.params_to_strings( incoming, trans.app ).iteritems():
job.add_parameter( name, value )
for name, dataset in inp_data.iteritems():
if dataset:
# TODO, Nate: Make sure the permitted actions here are appropriate.
if not trans.app.security_agent.allow_action( trans.user, dataset.permitted_actions.DATASET_ACCESS, dataset=dataset ):
raise "User does not have permission to use a dataset (%s) provided for input." % data.id
job.add_input_dataset( name, dataset )
else:
job.add_input_dataset( name, None )
for name, dataset in out_data.iteritems():
job.add_output_dataset( name, dataset )
trans.app.model.flush()
# Some tools are not really executable, but jobs are still created for them ( for record keeping ).
# Examples include tools that redirect to other applications ( epigraph ). These special tools must
# include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job
# from being queued.
if 'REDIRECT_URL' in incoming:
# Get the dataset - there should only be 1
for name in inp_data.keys():
dataset = inp_data[ name ]
redirect_url = tool.parse_redirect_url( dataset, incoming )
# GALAXY_URL should be include in the tool params to enable the external application
# to send back to the current Galaxy instance
GALAXY_URL = incoming.get( 'GALAXY_URL', None )
assert GALAXY_URL is not None, "GALAXY_URL parameter missing in tool config."
redirect_url += "&GALAXY_URL=%s" % GALAXY_URL
# Job should not be queued, so set state to ok
job.state = JOB_OK
job.info = "Redirected to: %s" % redirect_url
job.flush()
trans.response.send_redirect( url_for( controller='tool_runner', action='redirect', redirect_url=redirect_url ) )
else:
# Queue the job for execution
trans.app.job_queue.put( job.id, tool )
trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id )
return out_data
def build_url(self, other_values):
return fill_template(self.url, context=other_values)
def execute(self,
tool,
trans,
incoming={},
return_job=False,
set_output_hid=True,
set_output_history=True,
history=None,
job_params=None,
rerun_remap_job_id=None):
"""
Executes a tool, creating job and tool outputs, associating them, and
submitting the job to the job queue. If history is not specified, use
trans.history as destination for tool's output datasets.
"""
def make_dict_copy(from_dict):
"""
Makes a copy of input dictionary from_dict such that all values that are dictionaries
result in creation of a new dictionary ( a sort of deepcopy ). We may need to handle
other complex types ( e.g., lists, etc ), but not sure...
Yes, we need to handle lists (and now are)...
"""
copy_from_dict = {}
for key, value in from_dict.items():
if type(value).__name__ == 'dict':
copy_from_dict[key] = make_dict_copy(value)
elif isinstance(value, list):
copy_from_dict[key] = make_list_copy(value)
else:
copy_from_dict[key] = value
return copy_from_dict
def make_list_copy(from_list):
new_list = []
for value in from_list:
if isinstance(value, dict):
new_list.append(make_dict_copy(value))
elif isinstance(value, list):
new_list.append(make_list_copy(value))
else:
new_list.append(value)
return new_list
def wrap_values(inputs, input_values, skip_missing_values=False):
# Wrap tool inputs as necessary
for input in inputs.itervalues():
if input.name not in input_values and skip_missing_values:
continue
if isinstance(input, Repeat):
for d in input_values[input.name]:
wrap_values(input.inputs,
d,
skip_missing_values=skip_missing_values)
elif isinstance(input, Conditional):
values = input_values[input.name]
current = values["__current_case__"]
wrap_values(input.cases[current].inputs,
values,
skip_missing_values=skip_missing_values)
elif isinstance(input, DataToolParameter) and input.multiple:
input_values[ input.name ] = \
galaxy.tools.DatasetListWrapper( input_values[ input.name ],
datatypes_registry = trans.app.datatypes_registry,
tool = tool,
name = input.name )
elif isinstance(input, DataToolParameter):
input_values[ input.name ] = \
galaxy.tools.DatasetFilenameWrapper( input_values[ input.name ],
datatypes_registry = trans.app.datatypes_registry,
tool = tool,
name = input.name )
elif isinstance(input, SelectToolParameter):
input_values[
input.name] = galaxy.tools.SelectToolParameterWrapper(
input,
input_values[input.name],
tool.app,
other_values=incoming)
else:
input_values[input.name] = galaxy.tools.InputValueWrapper(
input, input_values[input.name], incoming)
# Set history.
if not history:
history = tool.get_default_history_by_trans(trans, create=True)
out_data = odict()
# Collect any input datasets from the incoming parameters
inp_data = self.collect_input_datasets(tool, incoming, trans)
# Deal with input dataset names, 'dbkey' and types
input_names = []
input_ext = 'data'
input_dbkey = incoming.get("dbkey", "?")
for name, data in inp_data.items():
if not data:
data = NoneDataset(
datatypes_registry=trans.app.datatypes_registry)
continue
# Convert LDDA to an HDA.
if isinstance(data, LibraryDatasetDatasetAssociation):
data = data.to_history_dataset_association(None)
inp_data[name] = data
else: # HDA
if data.hid:
input_names.append('data %s' % data.hid)
input_ext = data.ext
if data.dbkey not in [None, '?']:
input_dbkey = data.dbkey
# Collect chromInfo dataset and add as parameters to incoming
db_datasets = {}
db_dataset = trans.db_dataset_for(input_dbkey)
if db_dataset:
db_datasets["chromInfo"] = db_dataset
incoming["chromInfo"] = db_dataset.file_name
else:
# For custom builds, chrom info resides in converted dataset; for built-in builds, chrom info resides in tool-data/shared.
chrom_info = None
if trans.user and ('dbkeys' in trans.user.preferences) and (
input_dbkey in from_json_string(
trans.user.preferences['dbkeys'])):
# Custom build.
custom_build_dict = from_json_string(
trans.user.preferences['dbkeys'])[input_dbkey]
if 'fasta' in custom_build_dict:
build_fasta_dataset = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(
custom_build_dict['fasta'])
chrom_info = build_fasta_dataset.get_converted_dataset(
trans, 'len').file_name
if not chrom_info:
# Default to built-in build.
chrom_info = os.path.join(trans.app.config.len_file_path,
"%s.len" % input_dbkey)
incoming["chromInfo"] = chrom_info
inp_data.update(db_datasets)
# Determine output dataset permission/roles list
existing_datasets = [inp for inp in inp_data.values() if inp]
if existing_datasets:
output_permissions = trans.app.security_agent.guess_derived_permissions_for_datasets(
existing_datasets)
else:
# No valid inputs, we will use history defaults
output_permissions = trans.app.security_agent.history_get_default_permissions(
history)
# Build name for output datasets based on tool name and input names
if len(input_names) == 1:
on_text = input_names[0]
elif len(input_names) == 2:
on_text = '%s and %s' % tuple(input_names[0:2])
elif len(input_names) == 3:
on_text = '%s, %s, and %s' % tuple(input_names[0:3])
elif len(input_names) > 3:
on_text = '%s, %s, and others' % tuple(input_names[0:2])
else:
on_text = ""
# Add the dbkey to the incoming parameters
incoming["dbkey"] = input_dbkey
params = None #wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
# Keep track of parent / child relationships, we'll create all the
# datasets first, then create the associations
parent_to_child_pairs = []
child_dataset_names = set()
object_store_id = None
for name, output in tool.outputs.items():
for filter in output.filters:
try:
if not eval(filter.text.strip(), globals(), incoming):
break #do not create this dataset
except Exception, e:
log.debug('Dataset output filter failed: %s' % e)
else: #all filters passed
if output.parent:
parent_to_child_pairs.append((output.parent, name))
child_dataset_names.add(name)
## What is the following hack for? Need to document under what
## conditions can the following occur? ([email protected])
# HACK: the output data has already been created
# this happens i.e. as a result of the async controller
if name in incoming:
dataid = incoming[name]
data = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(dataid)
assert data != None
out_data[name] = data
else:
# the type should match the input
ext = output.format
if ext == "input":
ext = input_ext
if output.format_source is not None and output.format_source in inp_data:
try:
input_dataset = inp_data[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception, e:
pass
#process change_format tags
if output.change_format:
if params is None:
params = make_dict_copy(incoming)
wrap_values(
tool.inputs,
params,
skip_missing_values=not tool.check_values)
for change_elem in output.change_format:
for when_elem in change_elem.findall('when'):
check = when_elem.get('input', None)
if check is not None:
try:
if '$' not in check:
#allow a simple name or more complex specifications
check = '${%s}' % check
if str(
fill_template(check,
context=params)
) == when_elem.get('value', None):
ext = when_elem.get('format', ext)
except: #bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get(
'input_dataset', None)
if check is not None:
check = inp_data.get(check, None)
if check is not None:
if str(
getattr(
check,
when_elem.get(
'attribute'))
) == when_elem.get('value', None):
ext = when_elem.get(
'format', ext)
data = trans.app.model.HistoryDatasetAssociation(
extension=ext,
create_dataset=True,
sa_session=trans.sa_session)
if output.hidden:
data.visible = False
# Commit the dataset immediately so it gets database assigned unique id
trans.sa_session.add(data)
trans.sa_session.flush()
trans.app.security_agent.set_all_dataset_permissions(
data.dataset, output_permissions)
# Create an empty file immediately. The first dataset will be
# created in the "default" store, all others will be created in
# the same store as the first.
data.dataset.object_store_id = object_store_id
try:
trans.app.object_store.create(data.dataset)
except ObjectInvalid:
raise Exception(
'Unable to create output dataset: object store is full'
)
object_store_id = data.dataset.object_store_id # these will be the same thing after the first output
# This may not be neccesary with the new parent/child associations
data.designation = name
# Copy metadata from one of the inputs if requested.
if output.metadata_source:
data.init_meta(copy_from=inp_data[output.metadata_source])
else:
data.init_meta()
# Take dbkey from LAST input
data.dbkey = str(input_dbkey)
# Set state
# FIXME: shouldn't this be NEW until the job runner changes it?
data.state = data.states.QUEUED
data.blurb = "queued"
# Set output label
if output.label:
if params is None:
params = make_dict_copy(incoming)
# wrapping the params allows the tool config to contain things like
# <outputs>
# <data format="input" name="output" label="Blat on ${<input_param>.name}" />
# </outputs>
wrap_values(tool.inputs,
params,
skip_missing_values=not tool.check_values)
#tool (only needing to be set once) and on_string (set differently for each label) are overwritten for each output dataset label being determined
params['tool'] = tool
params['on_string'] = on_text
data.name = fill_template(output.label, context=params)
else:
if params is None:
params = make_dict_copy(incoming)
wrap_values(tool.inputs,
params,
skip_missing_values=not tool.check_values)
data.name = self._get_default_data_name(
data,
tool,
on_text=on_text,
trans=trans,
incoming=incoming,
history=history,
params=params,
job_params=job_params)
# Store output
out_data[name] = data
if output.actions:
#Apply pre-job tool-output-dataset actions; e.g. setting metadata, changing format
output_action_params = dict(out_data)
output_action_params.update(incoming)
output.actions.apply_action(data, output_action_params)
# Store all changes to database
trans.sa_session.flush()
def determine_output_format(output, parameter_context, input_datasets,
random_input_ext):
""" Determines the output format for a dataset based on an abstract
description of the output (galaxy.tools.ToolOutput), the parameter
wrappers, a map of the input datasets (name => HDA), and the last input
extensions in the tool form.
TODO: Don't deal with XML here - move this logic into ToolOutput.
TODO: Make the input extension used deterministic instead of random.
"""
# the type should match the input
ext = output.format
if ext == "input":
ext = random_input_ext
if output.format_source is not None and output.format_source in input_datasets:
try:
input_dataset = input_datasets[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception:
pass
#process change_format tags
if output.change_format is not None:
new_format_set = False
for change_elem in output.change_format:
for when_elem in change_elem.findall('when'):
check = when_elem.get('input', None)
if check is not None:
try:
if '$' not in check:
#allow a simple name or more complex specifications
check = '${%s}' % check
if str(fill_template(
check,
context=parameter_context)) == when_elem.get(
'value', None):
ext = when_elem.get('format', ext)
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get('input_dataset', None)
if check is not None:
check = input_datasets.get(check, None)
# At this point check is a HistoryDatasetAssociation object.
check_format = when_elem.get('format', ext)
check_value = when_elem.get('value', None)
check_attribute = when_elem.get('attribute', None)
if check is not None and check_value is not None and check_attribute is not None:
# See if the attribute to be checked belongs to the HistoryDatasetAssociation object.
if hasattr(check, check_attribute):
if str(getattr(
check,
check_attribute)) == str(check_value):
ext = check_format
new_format_set = True
break
# See if the attribute to be checked belongs to the metadata associated with the
# HistoryDatasetAssociation object.
if check.metadata is not None:
metadata_value = check.metadata.get(
check_attribute, None)
if metadata_value is not None:
if str(metadata_value) == str(check_value):
ext = check_format
new_format_set = True
break
if new_format_set:
break
return ext
def filter_by_dataset(self, data, trans):
context = self.get_inital_values(data, trans)
for filter_elem in self.filters:
if fill_template(filter_elem.text, context=context) != filter_elem.get("value", "True"):
return False
return True
def get_value( self, other_values, dataset_hash, user_hash, trans ):
value = fill_template( self.text, context = other_values )
if self.strip:
value = value.strip()
return DisplayParameterValueWrapper( value, self, other_values, dataset_hash, user_hash, trans )
def determine_output_format(output, parameter_context, input_datasets, random_input_ext):
""" Determines the output format for a dataset based on an abstract
description of the output (galaxy.tools.ToolOutput), the parameter
wrappers, a map of the input datasets (name => HDA), and the last input
extensions in the tool form.
TODO: Don't deal with XML here - move this logic into ToolOutput.
TODO: Make the input extension used deterministic instead of random.
"""
# the type should match the input
ext = output.format
if ext == "input":
ext = random_input_ext
if output.format_source is not None and output.format_source in input_datasets:
try:
input_dataset = input_datasets[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception:
pass
# process change_format tags
if output.change_format is not None:
new_format_set = False
for change_elem in output.change_format:
for when_elem in change_elem.findall( 'when' ):
check = when_elem.get( 'input', None )
if check is not None:
try:
if '$' not in check:
# allow a simple name or more complex specifications
check = '${%s}' % check
if str( fill_template( check, context=parameter_context ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get( 'input_dataset', None )
if check is not None:
check = input_datasets.get( check, None )
# At this point check is a HistoryDatasetAssociation object.
check_format = when_elem.get( 'format', ext )
check_value = when_elem.get( 'value', None )
check_attribute = when_elem.get( 'attribute', None )
if check is not None and check_value is not None and check_attribute is not None:
# See if the attribute to be checked belongs to the HistoryDatasetAssociation object.
if hasattr( check, check_attribute ):
if str( getattr( check, check_attribute ) ) == str( check_value ):
ext = check_format
new_format_set = True
break
# See if the attribute to be checked belongs to the metadata associated with the
# HistoryDatasetAssociation object.
if check.metadata is not None:
metadata_value = check.metadata.get( check_attribute, None )
if metadata_value is not None:
if str( metadata_value ) == str( check_value ):
ext = check_format
new_format_set = True
break
if new_format_set:
break
return ext
def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext):
""" Determines the output format for a dataset based on an abstract
description of the output (galaxy.tools.parser.ToolOutput), the parameter
wrappers, a map of the input datasets (name => HDA), and the last input
extensions in the tool form.
TODO: Don't deal with XML here - move this logic into ToolOutput.
TODO: Make the input extension used deterministic instead of random.
"""
# the type should match the input
ext = output.format
if ext == "input":
ext = random_input_ext
format_source = output.format_source
if format_source is not None and format_source in input_datasets:
try:
input_dataset = input_datasets[output.format_source]
input_extension = input_dataset.ext
ext = input_extension
except Exception:
pass
elif format_source is not None:
if re.match(r"^[^\[\]]*\[[^\[\]]*\]$", format_source):
collection_name, element_index = format_source[0:-1].split("[")
# Treat as json to interpret "forward" vs 0 with type
# Make it feel more like Python, single quote better in XML also.
element_index = element_index.replace("'", '"')
element_index = json.loads(element_index)
if collection_name in input_dataset_collections:
try:
input_collection = input_dataset_collections[collection_name][0][0]
input_collection_collection = input_collection.collection
try:
input_element = input_collection_collection[element_index]
except KeyError:
for element in input_collection_collection.dataset_elements:
if element.element_identifier == element_index:
input_element = element
break
input_dataset = input_element.element_object
input_extension = input_dataset.ext
ext = input_extension
except Exception as e:
log.debug("Exception while trying to determine format_source: %s", e)
pass
# process change_format tags
if output.change_format is not None:
new_format_set = False
for change_elem in output.change_format:
for when_elem in change_elem.findall( 'when' ):
check = when_elem.get( 'input', None )
if check is not None:
try:
if '$' not in check:
# allow a simple name or more complex specifications
check = '${%s}' % check
if str( fill_template( check, context=parameter_context ) ) == when_elem.get( 'value', None ):
ext = when_elem.get( 'format', ext )
except: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
else:
check = when_elem.get( 'input_dataset', None )
if check is not None:
check = input_datasets.get( check, None )
# At this point check is a HistoryDatasetAssociation object.
check_format = when_elem.get( 'format', ext )
check_value = when_elem.get( 'value', None )
check_attribute = when_elem.get( 'attribute', None )
if check is not None and check_value is not None and check_attribute is not None:
# See if the attribute to be checked belongs to the HistoryDatasetAssociation object.
if hasattr( check, check_attribute ):
if str( getattr( check, check_attribute ) ) == str( check_value ):
ext = check_format
new_format_set = True
break
# See if the attribute to be checked belongs to the metadata associated with the
# HistoryDatasetAssociation object.
if check.metadata is not None:
metadata_value = check.metadata.get( check_attribute, None )
if metadata_value is not None:
if str( metadata_value ) == str( check_value ):
ext = check_format
new_format_set = True
break
if new_format_set:
break
return ext
