def get_latest_tool_config_revision_from_repository_manifest( self, repo, filename, changeset_revision ):
"""
Get the latest revision of a tool config file named filename from the repository
manifest up to the value of changeset_revision. This method is restricted to tool_config
files rather than any file since it is likely that, with the exception of tool config
files, multiple files will have the same name in various directories within the repository.
"""
stripped_filename = basic_util.strip_path( filename )
for changeset in hg_util.reversed_upper_bounded_changelog( repo, changeset_revision ):
manifest_ctx = repo.changectx( changeset )
for ctx_file in manifest_ctx.files():
ctx_file_name = basic_util.strip_path( ctx_file )
if ctx_file_name == stripped_filename:
try:
fctx = manifest_ctx[ ctx_file ]
except LookupError:
# The ctx_file may have been moved in the change set. For example,
# 'ncbi_blastp_wrapper.xml' was moved to 'tools/ncbi_blast_plus/ncbi_blastp_wrapper.xml',
# so keep looking for the file until we find the new location.
continue
fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-gltcrfrm" )
tmp_filename = fh.name
fh.close()
fh = open( tmp_filename, 'wb' )
fh.write( fctx.data() )
fh.close()
return tmp_filename
return None
def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups, sample_files_copied ):
"""
Inspect each tool to see if it has any input parameters that are dynamically
generated select lists that depend on a .loc file. This method is not called
from the tool shed, but from Galaxy when a repository is being installed.
"""
for index, repository_tools_tup in enumerate( repository_tools_tups ):
tup_path, guid, repository_tool = repository_tools_tup
params_with_missing_index_file = repository_tool.params_with_missing_index_file
for param in params_with_missing_index_file:
options = param.options
missing_file_name = basic_util.strip_path( options.missing_index_file )
if missing_file_name not in sample_files_copied:
# The repository must contain the required xxx.loc.sample file.
for sample_file in sample_files:
sample_file_name = basic_util.strip_path( sample_file )
if sample_file_name == '%s.sample' % missing_file_name:
copy_sample_file( app, sample_file )
if options.tool_data_table and options.tool_data_table.missing_index_file:
options.tool_data_table.handle_found_index_file( options.missing_index_file )
sample_files_copied.append( options.missing_index_file )
break
# Reload the tool into the local list of repository_tools_tups.
repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid, use_cached=False )
repository_tools_tups[ index ] = ( tup_path, guid, repository_tool )
return repository_tools_tups, sample_files_copied
def get_named_tmpfile_from_ctx( ctx, filename, dir ):
"""
Return a named temporary file created from a specified file with a given name included in a repository
changeset revision.
"""
filename = basic_util.strip_path( filename )
for ctx_file in ctx.files():
ctx_file_name = basic_util.strip_path( ctx_file )
if filename == ctx_file_name:
try:
# If the file was moved, its destination file contents will be returned here.
fctx = ctx[ ctx_file ]
except LookupError:
# Continue looking in case the file was moved.
fctx = None
continue
if fctx:
fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-gntfc", dir=dir )
tmp_filename = fh.name
fh.close()
fh = open( tmp_filename, 'wb' )
fh.write( fctx.data() )
fh.close()
return tmp_filename
return None
def get_named_tmpfile_from_ctx(ctx, filename, dir):
"""
Return a named temporary file created from a specified file with a given name included in a repository
changeset revision.
"""
filename = basic_util.strip_path(filename)
for ctx_file in ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
if filename == ctx_file_name:
try:
# If the file was moved, its destination file contents will be returned here.
fctx = ctx[ctx_file]
except LookupError:
# Continue looking in case the file was moved.
fctx = None
continue
if fctx:
fh = tempfile.NamedTemporaryFile('wb',
prefix="tmp-toolshed-gntfc",
dir=dir)
tmp_filename = fh.name
fh.close()
fh = open(tmp_filename, 'wb')
fh.write(fctx.data())
fh.close()
return tmp_filename
return None
def handle_missing_index_file(app, tool_path, sample_files,
repository_tools_tups, sample_files_copied):
"""
Inspect each tool to see if it has any input parameters that are dynamically
generated select lists that depend on a .loc file. This method is not called
from the tool shed, but from Galaxy when a repository is being installed.
"""
for index, repository_tools_tup in enumerate(repository_tools_tups):
tup_path, guid, repository_tool = repository_tools_tup
params_with_missing_index_file = repository_tool.params_with_missing_index_file
for param in params_with_missing_index_file:
options = param.options
missing_file_name = basic_util.strip_path(
options.missing_index_file)
if missing_file_name not in sample_files_copied:
# The repository must contain the required xxx.loc.sample file.
for sample_file in sample_files:
sample_file_name = basic_util.strip_path(sample_file)
if sample_file_name == '%s.sample' % missing_file_name:
copy_sample_file(app, sample_file)
if options.tool_data_table and options.tool_data_table.missing_index_file:
options.tool_data_table.handle_found_index_file(
options.missing_index_file)
sample_files_copied.append(options.missing_index_file)
break
# Reload the tool into the local list of repository_tools_tups.
repository_tool = app.toolbox.load_tool(os.path.join(
tool_path, tup_path),
guid=guid,
use_cached=False)
repository_tools_tups[index] = (tup_path, guid, repository_tool)
return repository_tools_tups, sample_files_copied
def check_tool_input_params(self, repo_dir, tool_config_name, tool,
sample_files):
"""
Check all of the tool's input parameters, looking for any that are dynamically
generated using external data files to make sure the files exist.
"""
invalid_files_and_errors_tups = []
correction_msg = ''
for input_param in tool.input_params:
if isinstance(input_param, parameters.basic.SelectToolParameter
) and input_param.is_dynamic:
# If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml,
# make sure all requirements exist.
options = input_param.dynamic_options or input_param.options
if options and isinstance(options,
dynamic_options.DynamicOptions):
if options.tool_data_table or options.missing_tool_data_table_name:
# Make sure the repository contains a tool_data_table_conf.xml.sample file.
sample_tool_data_table_conf = hg_util.get_config_from_disk(
'tool_data_table_conf.xml.sample', repo_dir)
if sample_tool_data_table_conf:
error, correction_msg = \
self.tdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf,
persist=False)
if error:
invalid_files_and_errors_tups.append(
('tool_data_table_conf.xml.sample',
correction_msg))
else:
correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. "
correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository "
correction_msg += "that includes the required entry to correct this error.<br/>"
invalid_tup = (tool_config_name, correction_msg)
if invalid_tup not in invalid_files_and_errors_tups:
invalid_files_and_errors_tups.append(
invalid_tup)
if options.index_file or options.missing_index_file:
# Make sure the repository contains the required xxx.loc.sample file.
index_file = options.index_file or options.missing_index_file
index_file_name = basic_util.strip_path(index_file)
sample_found = False
for sample_file in sample_files:
sample_file_name = basic_util.strip_path(
sample_file)
if sample_file_name == '%s.sample' % index_file_name:
options.index_file = index_file_name
if options.tool_data_table:
options.tool_data_table.missing_index_file = None
sample_found = True
break
if not sample_found:
correction_msg = "This file refers to a file named <b>%s</b>. " % str(
index_file_name)
correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % \
str(index_file_name)
invalid_files_and_errors_tups.append(
(tool_config_name, correction_msg))
return invalid_files_and_errors_tups
def check_tool_input_params(self, repo_dir, tool_config_name, tool, sample_files):
"""
Check all of the tool's input parameters, looking for any that are dynamically
generated using external data files to make sure the files exist.
"""
invalid_files_and_errors_tups = []
correction_msg = ""
for input_param in tool.input_params:
if isinstance(input_param, parameters.basic.SelectToolParameter) and input_param.is_dynamic:
# If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml,
# make sure all requirements exist.
options = input_param.dynamic_options or input_param.options
if options and isinstance(options, dynamic_options.DynamicOptions):
if options.tool_data_table or options.missing_tool_data_table_name:
# Make sure the repository contains a tool_data_table_conf.xml.sample file.
sample_tool_data_table_conf = hg_util.get_config_from_disk(
"tool_data_table_conf.xml.sample", repo_dir
)
if sample_tool_data_table_conf:
error, correction_msg = self.tdtm.handle_sample_tool_data_table_conf_file(
sample_tool_data_table_conf, persist=False
)
if error:
invalid_files_and_errors_tups.append(
("tool_data_table_conf.xml.sample", correction_msg)
)
else:
options.missing_tool_data_table_name = None
else:
correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. "
correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository "
correction_msg += "that includes the required entry to correct this error.<br/>"
invalid_tup = (tool_config_name, correction_msg)
if invalid_tup not in invalid_files_and_errors_tups:
invalid_files_and_errors_tups.append(invalid_tup)
if options.index_file or options.missing_index_file:
# Make sure the repository contains the required xxx.loc.sample file.
index_file = options.index_file or options.missing_index_file
index_file_name = basic_util.strip_path(index_file)
sample_found = False
for sample_file in sample_files:
sample_file_name = basic_util.strip_path(sample_file)
if sample_file_name == "%s.sample" % index_file_name:
options.index_file = index_file_name
options.missing_index_file = None
if options.tool_data_table:
options.tool_data_table.missing_index_file = None
sample_found = True
break
if not sample_found:
correction_msg = "This file refers to a file named <b>%s</b>. " % str(index_file_name)
correction_msg += (
"Upload a file named <b>%s.sample</b> to the repository to correct this error."
% str(index_file_name)
)
invalid_files_and_errors_tups.append((tool_config_name, correction_msg))
return invalid_files_and_errors_tups
def get_config( config_file, repo, ctx, dir ):
"""Return the latest version of config_filename from the repository manifest."""
config_file = basic_util.strip_path( config_file )
for changeset in reversed_upper_bounded_changelog( repo, ctx ):
changeset_ctx = repo.changectx( changeset )
for ctx_file in changeset_ctx.files():
ctx_file_name = basic_util.strip_path( ctx_file )
if ctx_file_name == config_file:
return get_named_tmpfile_from_ctx( changeset_ctx, ctx_file, dir )
return None
def get_config(config_file, repo, ctx, dir):
"""Return the latest version of config_filename from the repository manifest."""
config_file = basic_util.strip_path(config_file)
for changeset in reversed_upper_bounded_changelog(repo, ctx):
changeset_ctx = repo.changectx(changeset)
for ctx_file in changeset_ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
if ctx_file_name == config_file:
return get_named_tmpfile_from_ctx(changeset_ctx, ctx_file, dir)
return None
def get_ctx_file_path_from_manifest( filename, repo, changeset_revision ):
"""
Get the ctx file path for the latest revision of filename from the repository manifest up
to the value of changeset_revision.
"""
stripped_filename = basic_util.strip_path( filename )
for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ):
manifest_ctx = repo.changectx( changeset )
for ctx_file in manifest_ctx.files():
ctx_file_name = basic_util.strip_path( ctx_file )
if ctx_file_name == stripped_filename:
return manifest_ctx, ctx_file
return None, None
def get_ctx_file_path_from_manifest(filename, repo, changeset_revision):
"""
Get the ctx file path for the latest revision of filename from the repository manifest up
to the value of changeset_revision.
"""
stripped_filename = basic_util.strip_path(filename)
for changeset in reversed_upper_bounded_changelog(repo, changeset_revision):
manifest_ctx = repo.changectx(changeset)
for ctx_file in manifest_ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
if ctx_file_name == stripped_filename:
return manifest_ctx, ctx_file
return None, None
def get_converter_and_display_paths(self, registration_elem,
relative_install_dir):
"""
Find the relative path to data type converters and display applications included
in installed tool shed repositories.
"""
converter_path = None
display_path = None
for elem in registration_elem.findall('datatype'):
if not converter_path:
# If any of the <datatype> tag sets contain <converter> tags, set the converter_path
# if it is not already set. This requires developers to place all converters in the
# same subdirectory within the repository hierarchy.
for converter in elem.findall('converter'):
converter_config = converter.get('file', None)
if converter_config:
converter_config_file_name = basic_util.strip_path(
converter_config)
for root, dirs, files in os.walk(relative_install_dir):
if root.find('.hg') < 0:
for name in files:
if name == converter_config_file_name:
# The value of converter_path must be absolute due to job_working_directory.
converter_path = os.path.abspath(root)
break
if converter_path:
break
if not display_path:
# If any of the <datatype> tag sets contain <display> tags, set the display_path
# if it is not already set. This requires developers to place all display acpplications
# in the same subdirectory within the repository hierarchy.
for display_app in elem.findall('display'):
display_config = display_app.get('file', None)
if display_config:
display_config_file_name = basic_util.strip_path(
display_config)
for root, dirs, files in os.walk(relative_install_dir):
if root.find('.hg') < 0:
for name in files:
if name == display_config_file_name:
# The value of display_path must be absolute due to job_working_directory.
display_path = os.path.abspath(root)
break
if display_path:
break
if converter_path and display_path:
break
return converter_path, display_path
def get_guid( self, repository_clone_url, relative_install_dir, tool_config ):
if self.shed_config_dict.get( 'tool_path' ):
relative_install_dir = os.path.join( self.shed_config_dict[ 'tool_path' ], relative_install_dir )
tool_config_filename = basic_util.strip_path( tool_config )
for root, dirs, files in os.walk( relative_install_dir ):
if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0:
if '.hg' in dirs:
dirs.remove( '.hg' )
for name in files:
filename = basic_util.strip_path( name )
if filename == tool_config_filename:
full_path = str( os.path.abspath( os.path.join( root, name ) ) )
tool = self.toolbox.load_tool( full_path )
return suc.generate_tool_guid( repository_clone_url, tool )
# Not quite sure what should happen here, throw an exception or what?
return None
def get_guid( self, repository_clone_url, relative_install_dir, tool_config ):
if self.shed_config_dict.get( 'tool_path' ):
relative_install_dir = os.path.join( self.shed_config_dict[ 'tool_path' ], relative_install_dir )
tool_config_filename = basic_util.strip_path( tool_config )
for root, dirs, files in os.walk( relative_install_dir ):
if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0:
if '.hg' in dirs:
dirs.remove( '.hg' )
for name in files:
filename = basic_util.strip_path( name )
if filename == tool_config_filename:
full_path = str( os.path.abspath( os.path.join( root, name ) ) )
tool = self.toolbox.load_tool( full_path )
return suc.generate_tool_guid( repository_clone_url, tool )
# Not quite sure what should happen here, throw an exception or what?
return None
def generate_tool_panel_dict_from_shed_tool_conf_entries(self, repository):
"""
Keep track of the section in the tool panel in which this repository's
tools will be contained by parsing the shed_tool_conf in which the
repository's tools are defined and storing the tool panel definition
of each tool in the repository. This method is called only when the
repository is being deactivated or un-installed and allows for
activation or re-installation using the original layout.
"""
tool_panel_dict = {}
shed_tool_conf, tool_path, relative_install_dir = \
suc.get_tool_panel_config_tool_path_install_dir( self.app, repository )
metadata = repository.metadata
# Create a dictionary of tool guid and tool config file name for each tool in the repository.
guids_and_configs = {}
if 'tools' in metadata:
for tool_dict in metadata['tools']:
guid = tool_dict['guid']
tool_config = tool_dict['tool_config']
file_name = basic_util.strip_path(tool_config)
guids_and_configs[guid] = file_name
# Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict.
tree, error_message = xml_util.parse_xml(shed_tool_conf)
if tree is None:
return tool_panel_dict
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
guid = elem.get('guid')
if guid in guids_and_configs:
# The tool is displayed in the tool panel outside of any tool sections.
tool_section_dict = dict(
tool_config=guids_and_configs[guid],
id='',
name='',
version='')
if guid in tool_panel_dict:
tool_panel_dict[guid].append(tool_section_dict)
else:
tool_panel_dict[guid] = [tool_section_dict]
elif elem.tag == 'section':
section_id = elem.get('id') or ''
section_name = elem.get('name') or ''
section_version = elem.get('version') or ''
for section_elem in elem:
if section_elem.tag == 'tool':
guid = section_elem.get('guid')
if guid in guids_and_configs:
# The tool is displayed in the tool panel inside the current tool section.
tool_section_dict = dict(
tool_config=guids_and_configs[guid],
id=section_id,
name=section_name,
version=section_version)
if guid in tool_panel_dict:
tool_panel_dict[guid].append(tool_section_dict)
else:
tool_panel_dict[guid] = [tool_section_dict]
return tool_panel_dict
def get_converter_and_display_paths( self, registration_elem, relative_install_dir ):
"""
Find the relative path to data type converters and display applications included
in installed tool shed repositories.
"""
converter_path = None
display_path = None
for elem in registration_elem.findall( 'datatype' ):
if not converter_path:
# If any of the <datatype> tag sets contain <converter> tags, set the converter_path
# if it is not already set. This requires developers to place all converters in the
# same subdirectory within the repository hierarchy.
for converter in elem.findall( 'converter' ):
converter_config = converter.get( 'file', None )
if converter_config:
converter_config_file_name = basic_util.strip_path( converter_config )
for root, dirs, files in os.walk( relative_install_dir ):
if root.find( '.hg' ) < 0:
for name in files:
if name == converter_config_file_name:
# The value of converter_path must be absolute due to job_working_directory.
converter_path = os.path.abspath( root )
break
if converter_path:
break
if not display_path:
# If any of the <datatype> tag sets contain <display> tags, set the display_path
# if it is not already set. This requires developers to place all display acpplications
# in the same subdirectory within the repository hierarchy.
for display_app in elem.findall( 'display' ):
display_config = display_app.get( 'file', None )
if display_config:
display_config_file_name = basic_util.strip_path( display_config )
for root, dirs, files in os.walk( relative_install_dir ):
if root.find( '.hg' ) < 0:
for name in files:
if name == display_config_file_name:
# The value of display_path must be absolute due to job_working_directory.
display_path = os.path.abspath( root )
break
if display_path:
break
if converter_path and display_path:
break
return converter_path, display_path
def get_file_context_from_ctx(ctx, filename):
"""Return the mercurial file context for a specified file."""
# We have to be careful in determining if we found the correct file because multiple files with
# the same name may be in different directories within ctx if the files were moved within the change
# set. For example, in the following ctx.files() list, the former may have been moved to the latter:
# ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample'].
# Another scenario is that the file has been deleted.
deleted = False
filename = basic_util.strip_path(filename)
for ctx_file in ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
if filename == ctx_file_name:
try:
# If the file was moved, its destination will be returned here.
fctx = ctx[ctx_file]
return fctx
except LookupError, e:
# Set deleted for now, and continue looking in case the file was moved instead of deleted.
deleted = True
def get_file_context_from_ctx( ctx, filename ):
"""Return the mercurial file context for a specified file."""
# We have to be careful in determining if we found the correct file because multiple files with
# the same name may be in different directories within ctx if the files were moved within the change
# set. For example, in the following ctx.files() list, the former may have been moved to the latter:
# ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample'].
# Another scenario is that the file has been deleted.
deleted = False
filename = basic_util.strip_path( filename )
for ctx_file in ctx.files():
ctx_file_name = basic_util.strip_path( ctx_file )
if filename == ctx_file_name:
try:
# If the file was moved, its destination will be returned here.
fctx = ctx[ ctx_file ]
return fctx
except LookupError, e:
# Set deleted for now, and continue looking in case the file was moved instead of deleted.
deleted = True
def generate_tool_panel_dict_from_shed_tool_conf_entries( self, repository ):
"""
Keep track of the section in the tool panel in which this repository's
tools will be contained by parsing the shed_tool_conf in which the
repository's tools are defined and storing the tool panel definition
of each tool in the repository. This method is called only when the
repository is being deactivated or un-installed and allows for
activation or re-installation using the original layout.
"""
tool_panel_dict = {}
shed_tool_conf, tool_path, relative_install_dir = \
suc.get_tool_panel_config_tool_path_install_dir( self.app, repository )
metadata = repository.metadata
# Create a dictionary of tool guid and tool config file name for each tool in the repository.
guids_and_configs = {}
if 'tools' in metadata:
for tool_dict in metadata[ 'tools' ]:
guid = tool_dict[ 'guid' ]
tool_config = tool_dict[ 'tool_config' ]
file_name = basic_util.strip_path( tool_config )
guids_and_configs[ guid ] = file_name
# Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict.
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree is None:
return tool_panel_dict
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
guid = elem.get( 'guid' )
if guid in guids_and_configs:
# The tool is displayed in the tool panel outside of any tool sections.
tool_section_dict = dict( tool_config=guids_and_configs[ guid ], id='', name='', version='' )
if guid in tool_panel_dict:
tool_panel_dict[ guid ].append( tool_section_dict )
else:
tool_panel_dict[ guid ] = [ tool_section_dict ]
elif elem.tag == 'section':
section_id = elem.get( 'id' ) or ''
section_name = elem.get( 'name' ) or ''
section_version = elem.get( 'version' ) or ''
for section_elem in elem:
if section_elem.tag == 'tool':
guid = section_elem.get( 'guid' )
if guid in guids_and_configs:
# The tool is displayed in the tool panel inside the current tool section.
tool_section_dict = dict( tool_config=guids_and_configs[ guid ],
id=section_id,
name=section_name,
version=section_version )
if guid in tool_panel_dict:
tool_panel_dict[ guid ].append( tool_section_dict )
else:
tool_panel_dict[ guid ] = [ tool_section_dict ]
return tool_panel_dict
def get_shed_tool_conf_dict( self, shed_tool_conf ):
"""
Return the in-memory version of the shed_tool_conf file, which is stored in
the config_elems entry in the shed_tool_conf_dict associated with the file.
"""
for index, shed_tool_conf_dict in enumerate( self.app.toolbox.shed_tool_confs ):
if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]:
return index, shed_tool_conf_dict
else:
file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] )
if shed_tool_conf == file_name:
return index, shed_tool_conf_dict
def get_shed_tool_conf_dict( self, shed_tool_conf ):
"""
Return the in-memory version of the shed_tool_conf file, which is stored in
the config_elems entry in the shed_tool_conf_dict associated with the file.
"""
for shed_tool_conf_dict in self.app.toolbox.dynamic_confs( include_migrated_tool_conf=True ):
if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]:
return shed_tool_conf_dict
else:
file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] )
if shed_tool_conf == file_name:
return shed_tool_conf_dict
def get_tool_path_by_shed_tool_conf_filename(app, shed_tool_conf):
"""
Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the
dictionary whose config_filename key has a value matching the received shed_tool_conf.
"""
for shed_tool_conf_dict in app.toolbox.dynamic_confs(include_migrated_tool_conf=True):
config_filename = shed_tool_conf_dict['config_filename']
if config_filename == shed_tool_conf:
return shed_tool_conf_dict['tool_path']
else:
file_name = basic_util.strip_path(config_filename)
if file_name == shed_tool_conf:
return shed_tool_conf_dict['tool_path']
return None
def get_shed_tool_conf_dict(self, shed_tool_conf):
"""
Return the in-memory version of the shed_tool_conf file, which is stored in
the config_elems entry in the shed_tool_conf_dict associated with the file.
"""
for shed_tool_conf_dict in self.app.toolbox.dynamic_confs(
include_migrated_tool_conf=True):
if shed_tool_conf == shed_tool_conf_dict['config_filename']:
return shed_tool_conf_dict
else:
file_name = basic_util.strip_path(
shed_tool_conf_dict['config_filename'])
if shed_tool_conf == file_name:
return shed_tool_conf_dict
def get_shed_tool_conf_dict(self, shed_tool_conf):
"""
Return the in-memory version of the shed_tool_conf file, which is stored in
the config_elems entry in the shed_tool_conf_dict associated with the file.
"""
for index, shed_tool_conf_dict in enumerate(
self.app.toolbox.shed_tool_confs):
if shed_tool_conf == shed_tool_conf_dict['config_filename']:
return index, shed_tool_conf_dict
else:
file_name = basic_util.strip_path(
shed_tool_conf_dict['config_filename'])
if shed_tool_conf == file_name:
return index, shed_tool_conf_dict
def get_tool_path_by_shed_tool_conf_filename(app, shed_tool_conf):
"""
Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the
dictionary whose config_filename key has a value matching the received shed_tool_conf.
"""
for shed_tool_conf_dict in app.toolbox.dynamic_confs(include_migrated_tool_conf=True):
config_filename = shed_tool_conf_dict['config_filename']
if config_filename == shed_tool_conf:
return shed_tool_conf_dict['tool_path']
else:
file_name = basic_util.strip_path(config_filename)
if file_name == shed_tool_conf:
return shed_tool_conf_dict['tool_path']
return None
def get_shed_tool_conf_dict(self, shed_tool_conf):
"""
Return the in-memory version of the shed_tool_conf file, which is stored in
the config_elems entry in the shed_tool_conf_dict associated with the file.
"""
for shed_tool_conf_dict in self.app.toolbox.dynamic_confs(
include_migrated_tool_conf=True):
if shed_tool_conf == shed_tool_conf_dict['config_filename']:
return shed_tool_conf_dict
else:
file_name = basic_util.strip_path(
shed_tool_conf_dict['config_filename'])
if shed_tool_conf == file_name:
return shed_tool_conf_dict
raise RequestParameterInvalidException(
"Requested shed_tool_conf '%s' is not an active shed_tool_config_file"
% shed_tool_conf)
def generate_tool_panel_dict_for_tool_config( self, guid, tool_config, tool_sections=None ):
"""
Create a dictionary of the following type for a single tool config file name.
The intent is to call this method for every tool config in a repository and
append each of these as entries to a tool panel dictionary for the repository.
This enables each tool to be loaded into a different section in the tool panel.
{<Tool guid> :
[{ tool_config : <tool_config_file>,
id: <ToolSection id>,
version : <ToolSection version>,
name : <TooSection name>}]}
"""
tool_panel_dict = {}
file_name = basic_util.strip_path( tool_config )
tool_section_dicts = self. generate_tool_section_dicts( tool_config=file_name,
tool_sections=tool_sections )
tool_panel_dict[ guid ] = tool_section_dicts
return tool_panel_dict
def copy_sample_file( app, filename, dest_path=None ):
"""
Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path
is ~/tool-data.
"""
if dest_path is None:
dest_path = os.path.abspath( app.config.tool_data_path )
sample_file_name = basic_util.strip_path( filename )
copied_file = sample_file_name.replace( '.sample', '' )
full_source_path = os.path.abspath( filename )
full_destination_path = os.path.join( dest_path, sample_file_name )
# Don't copy a file to itself - not sure how this happens, but sometimes it does...
if full_source_path != full_destination_path:
# It's ok to overwrite the .sample version of the file.
shutil.copy( full_source_path, full_destination_path )
# Only create the .loc file if it does not yet exist. We don't overwrite it in case it
# contains stuff proprietary to the local instance.
if not os.path.exists( os.path.join( dest_path, copied_file ) ):
shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) )
def copy_sample_file(app, filename, dest_path=None):
"""
Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path
is ~/tool-data.
"""
if dest_path is None:
dest_path = os.path.abspath(app.config.tool_data_path)
sample_file_name = basic_util.strip_path(filename)
copied_file = sample_file_name.replace('.sample', '')
full_source_path = os.path.abspath(filename)
full_destination_path = os.path.join(dest_path, sample_file_name)
# Don't copy a file to itself - not sure how this happens, but sometimes it does...
if full_source_path != full_destination_path:
# It's ok to overwrite the .sample version of the file.
shutil.copy(full_source_path, full_destination_path)
# Only create the .loc file if it does not yet exist. We don't overwrite it in case it
# contains stuff proprietary to the local instance.
if not os.path.exists(os.path.join(dest_path, copied_file)):
shutil.copy(full_source_path, os.path.join(dest_path, copied_file))
def get_list_of_copied_sample_files(self, repo, changeset_revision, dir):
"""
Find all sample files (files in the repository with the special .sample extension)
in the reversed repository manifest up to changeset_revision. Copy each discovered file to dir and
return the list of filenames. If a .sample file was added in a changeset and then
deleted in a later changeset, it will be returned in the deleted_sample_files list.
The caller will set the value of app.config.tool_data_path to dir in order to load
the tools and generate metadata for them.
"""
deleted_sample_files = []
sample_files = []
for changeset in hg_util.reversed_upper_bounded_changelog(
repo, changeset_revision):
changeset_ctx = repo[changeset]
for ctx_file in changeset_ctx.files():
ctx_file = unicodify(ctx_file)
ctx_file_name = basic_util.strip_path(ctx_file)
# If we decide in the future that files deleted later in the changelog should
# not be used, we can use the following if statement. if ctx_file_name.endswith( '.sample' )
# and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files:
if ctx_file_name.endswith(
'.sample') and ctx_file_name not in sample_files:
fctx = hg_util.get_file_context_from_ctx(
changeset_ctx, ctx_file)
if fctx in ['DELETED']:
# Since the possibly future used if statement above is commented out, the
# same file that was initially added will be discovered in an earlier changeset
# in the change log and fall through to the else block below. In other words,
# if a file named blast2go.loc.sample was added in change set 0 and then deleted
# in changeset 3, the deleted file in changeset 3 will be handled here, but the
# later discovered file in changeset 0 will be handled in the else block below.
# In this way, the file contents will always be found for future tools even though
# the file was deleted.
if ctx_file_name not in deleted_sample_files:
deleted_sample_files.append(ctx_file_name)
else:
sample_files.append(ctx_file_name)
tmp_ctx_file_name = os.path.join(
dir, ctx_file_name.replace('.sample', ''))
with open(tmp_ctx_file_name, 'wb') as fh:
fh.write(fctx.data())
return sample_files, deleted_sample_files
def is_valid_for_type(self, app, repository, revisions_to_check=None):
"""
Inspect the received repository's contents to determine if they abide by the rules defined for the contents of this type.
If the received revisions_to_check is a list of changeset revisions, then inspection will be restricted to the revisions
in the list.
"""
repo = hg_util.get_repo_for_repository(app, repository=repository)
if revisions_to_check:
changeset_revisions = revisions_to_check
else:
changeset_revisions = repo.changelog
for changeset in changeset_revisions:
ctx = repo.changectx(changeset)
# Inspect all files in the changeset (in sorted order) to make sure there is only one and it is named tool_dependencies.xml.
files_changed_in_changeset = ctx.files()
for file_path in files_changed_in_changeset:
file_name = basic_util.strip_path(file_path)
if file_name not in self.valid_file_names:
return False
return True
def is_valid_for_type( self, app, repository, revisions_to_check=None ):
"""
Inspect the received repository's contents to determine if they abide by the rules defined for the contents of this type.
If the received revisions_to_check is a list of changeset revisions, then inspection will be restricted to the revisions
in the list.
"""
repo = hg.repository( ui.ui(), repository.repo_path( app ) )
if revisions_to_check:
changeset_revisions = revisions_to_check
else:
changeset_revisions = repo.changelog
for changeset in changeset_revisions:
ctx = repo.changectx( changeset )
# Inspect all files in the changeset (in sorted order) to make sure there is only one and it is named tool_dependencies.xml.
files_changed_in_changeset = ctx.files()
for file_path in files_changed_in_changeset:
file_name = basic_util.strip_path( file_path )
if file_name not in self.valid_file_names:
return False
return True
def generate_tool_panel_dict_for_tool_config(self,
guid,
tool_config,
tool_sections=None):
"""
Create a dictionary of the following type for a single tool config file name.
The intent is to call this method for every tool config in a repository and
append each of these as entries to a tool panel dictionary for the repository.
This enables each tool to be loaded into a different section in the tool panel.
{<Tool guid> :
[{ tool_config : <tool_config_file>,
id: <ToolSection id>,
version : <ToolSection version>,
name : <TooSection name>}]}
"""
tool_panel_dict = {}
file_name = basic_util.strip_path(tool_config)
tool_section_dicts = self.generate_tool_section_dicts(
tool_config=file_name, tool_sections=tool_sections)
tool_panel_dict[guid] = tool_section_dicts
return tool_panel_dict
def copy_sample_file(app, filename, dest_path=None):
"""
Copies a sample file at `filename` to `the dest_path`
directory and strips the '.sample' extensions from `filename`.
Returns the path to the copied file (with the .sample extension).
"""
if dest_path is None:
dest_path = os.path.abspath(app.config.tool_data_path)
sample_file_name = basic_util.strip_path(filename)
copied_file = sample_file_name.rsplit('.sample', 1)[0]
full_source_path = os.path.abspath(filename)
full_destination_path = os.path.join(dest_path, sample_file_name)
# Don't copy a file to itself - not sure how this happens, but sometimes it does...
if full_source_path != full_destination_path:
# It's ok to overwrite the .sample version of the file.
shutil.copy(full_source_path, full_destination_path)
# Only create the .loc file if it does not yet exist. We don't overwrite it in case it
# contains stuff proprietary to the local instance.
non_sample_path = os.path.join(dest_path, copied_file)
if not os.path.lexists(non_sample_path):
shutil.copy(full_source_path, os.path.join(dest_path, copied_file))
return non_sample_path
def can_use_tool_config_disk_file( self, repository, repo, file_path, changeset_revision ):
"""
Determine if repository's tool config file on disk can be used. This method
is restricted to tool config files since, with the exception of tool config
files, multiple files with the same name will likely be in various directories
in the repository and we're comparing file names only (not relative paths).
"""
if not file_path or not os.path.exists( file_path ):
# The file no longer exists on disk, so it must have been deleted at some previous
# point in the change log.
return False
if changeset_revision == repository.tip( self.app ):
return True
file_name = basic_util.strip_path( file_path )
latest_version_of_file = \
self.get_latest_tool_config_revision_from_repository_manifest( repo, file_name, changeset_revision )
can_use_disk_file = filecmp.cmp( file_path, latest_version_of_file )
try:
os.unlink( latest_version_of_file )
except:
pass
return can_use_disk_file
def copy_sample_file(app, filename, dest_path=None):
"""
Copies a sample file at `filename` to `the dest_path`
directory and strips the '.sample' extensions from `filename`.
Returns the path to the copied file (with the .sample extension).
"""
if dest_path is None:
dest_path = os.path.abspath(app.config.tool_data_path)
sample_file_name = basic_util.strip_path(filename)
copied_file = sample_file_name.rsplit('.sample', 1)[0]
full_source_path = os.path.abspath(filename)
full_destination_path = os.path.join(dest_path, sample_file_name)
# Don't copy a file to itself - not sure how this happens, but sometimes it does...
if full_source_path != full_destination_path:
# It's ok to overwrite the .sample version of the file.
shutil.copy(full_source_path, full_destination_path)
# Only create the .loc file if it does not yet exist. We don't overwrite it in case it
# contains stuff proprietary to the local instance.
non_sample_path = os.path.join(dest_path, copied_file)
if not os.path.lexists(non_sample_path):
shutil.copy(full_source_path, os.path.join(dest_path, copied_file))
return non_sample_path
def get_list_of_copied_sample_files( self, repo, ctx, dir ):
"""
Find all sample files (files in the repository with the special .sample extension)
in the reversed repository manifest up to ctx. Copy each discovered file to dir and
return the list of filenames. If a .sample file was added in a changeset and then
deleted in a later changeset, it will be returned in the deleted_sample_files list.
The caller will set the value of app.config.tool_data_path to dir in order to load
the tools and generate metadata for them.
"""
deleted_sample_files = []
sample_files = []
for changeset in hg_util.reversed_upper_bounded_changelog( repo, ctx ):
changeset_ctx = repo.changectx( changeset )
for ctx_file in changeset_ctx.files():
ctx_file_name = basic_util.strip_path( ctx_file )
# If we decide in the future that files deleted later in the changelog should
# not be used, we can use the following if statement. if ctx_file_name.endswith( '.sample' )
# and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files:
if ctx_file_name.endswith( '.sample' ) and ctx_file_name not in sample_files:
fctx = hg_util.get_file_context_from_ctx( changeset_ctx, ctx_file )
if fctx in [ 'DELETED' ]:
# Since the possibly future used if statement above is commented out, the
# same file that was initially added will be discovered in an earlier changeset
# in the change log and fall through to the else block below. In other words,
# if a file named blast2go.loc.sample was added in change set 0 and then deleted
# in changeset 3, the deleted file in changeset 3 will be handled here, but the
# later discovered file in changeset 0 will be handled in the else block below.
# In this way, the file contents will always be found for future tools even though
# the file was deleted.
if ctx_file_name not in deleted_sample_files:
deleted_sample_files.append( ctx_file_name )
else:
sample_files.append( ctx_file_name )
tmp_ctx_file_name = os.path.join( dir, ctx_file_name.replace( '.sample', '' ) )
fh = open( tmp_ctx_file_name, 'wb' )
fh.write( fctx.data() )
fh.close()
return sample_files, deleted_sample_files
