def guess_shed_config( self, app, default=None ):
tool_ids = []
metadata = self.metadata or {}
for tool in metadata.get( 'tools', [] ):
tool_ids.append( tool.get( 'guid' ) )
for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ):
name = shed_tool_conf_dict[ 'config_filename' ]
for elem in shed_tool_conf_dict[ 'config_elems' ]:
if elem.tag == 'tool':
for sub_elem in elem.findall( 'id' ):
tool_id = sub_elem.text.strip()
if tool_id in tool_ids:
self.shed_config_filename = name
return shed_tool_conf_dict
elif elem.tag == "section":
for tool_elem in elem.findall( 'tool' ):
for sub_elem in tool_elem.findall( 'id' ):
tool_id = sub_elem.text.strip()
if tool_id in tool_ids:
self.shed_config_filename = name
return shed_tool_conf_dict
if self.includes_datatypes:
# We need to search by file paths here, which is less desirable.
tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed )
for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ):
tool_path = shed_tool_conf_dict[ 'tool_path' ]
relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision )
if os.path.exists( relative_path ):
self.shed_config_filename = shed_tool_conf_dict[ 'config_filename' ]
return shed_tool_conf_dict
return default
def guess_shed_config( self, app, default=None ):
tool_ids = []
metadata = self.metadata or {}
for tool in metadata.get( 'tools', [] ):
tool_ids.append( tool.get( 'guid' ) )
for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ):
name = shed_tool_conf_dict[ 'config_filename' ]
for elem in shed_tool_conf_dict[ 'config_elems' ]:
if elem.tag == 'tool':
for sub_elem in elem.findall( 'id' ):
tool_id = sub_elem.text.strip()
if tool_id in tool_ids:
self.shed_config_filename = name
return shed_tool_conf_dict
elif elem.tag == "section":
for tool_elem in elem.findall( 'tool' ):
for sub_elem in tool_elem.findall( 'id' ):
tool_id = sub_elem.text.strip()
if tool_id in tool_ids:
self.shed_config_filename = name
return shed_tool_conf_dict
if self.includes_datatypes:
# We need to search by file paths here, which is less desirable.
tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed )
for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ):
tool_path = shed_tool_conf_dict[ 'tool_path' ]
relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision )
if os.path.exists( relative_path ):
self.shed_config_filename = shed_tool_conf_dict[ 'config_filename' ]
return shed_tool_conf_dict
return default
def repo_path( self, app ):
tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed )
for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ):
tool_path = shed_tool_conf_dict[ 'tool_path' ]
relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision )
if os.path.exists( relative_path ):
return relative_path
return None
def repo_path( self, app ):
tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed )
for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ):
tool_path = shed_tool_conf_dict[ 'tool_path' ]
relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision )
if os.path.exists( relative_path ):
return relative_path
return None
def repo_path( self, app ):
tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed )
for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ):
tool_path = shed_tool_conf_dict[ 'tool_path' ]
relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision )
if os.path.exists( relative_path ):
return relative_path
return None
def repo_path(self, app):
tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url(
self.tool_shed)
for index, shed_tool_conf_dict in enumerate(
app.toolbox.shed_tool_confs):
tool_path = shed_tool_conf_dict['tool_path']
relative_path = os.path.join(tool_path, tool_shed, 'repos',
self.owner, self.name,
self.installed_changeset_revision)
if os.path.exists(relative_path):
return relative_path
return None
def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config,
install_dependencies ):
"""
Check tool settings in tool_shed_install_config and install all repositories
that are not already installed. The tool panel configuration file is the received
migrated_tools_config, which is the reserved file named migrated_tools_conf.xml.
"""
self.app = app
self.toolbox = self.app.toolbox
self.migrated_tools_config = migrated_tools_config
# Initialize the ToolPanelManager.
self.tpm = tool_panel_manager.ToolPanelManager( self.app )
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt
# to install but print informative error message.
if install_dependencies and app.config.tool_dependency_dir is None:
message = 'You are attempting to install tool dependencies but do not have a value '
message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this '
message += 'location value to the path where you want tool dependencies installed and '
message += 'rerun the migration script.'
raise Exception( message )
# Get the local non-shed related tool panel configs (there can be more than one, and the
# default name is tool_conf.xml).
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number )
# Set the location where the repositories will be installed by retrieving the tool_path
# setting from migrated_tools_config.
tree, error_message = xml_util.parse_xml( migrated_tools_config )
if tree is None:
print error_message
else:
root = tree.getroot()
self.tool_path = root.get( 'tool_path' )
print "Repositories will be installed into configured tool_path location ", str( self.tool_path )
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree, error_message = xml_util.parse_xml( tool_shed_install_config )
if tree is None:
print error_message
else:
root = tree.getroot()
defined_tool_shed_url = root.get( 'name' )
self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url )
self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url )
self.repository_owner = common_util.REPOSITORY_OWNER
index, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config )
# Since tool migration scripts can be executed any number of times, we need to
# make sure the appropriate tools are defined in tool_conf.xml. If no tools
# associated with the migration stage are defined, no repositories will be installed
# on disk. The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app )
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = \
common_util.check_for_missing_tools( app,
tool_panel_configs,
latest_migration_script_number )
else:
# It doesn't matter if the tool shed is accessible since there are no migrated
# tools defined in the local Galaxy instance, but we have to set the value of
# tool_shed_accessible to True so that the value of migrate_tools.version can
# be correctly set in the database.
tool_shed_accessible = True
missing_tool_configs_dict = odict()
if tool_shed_accessible:
if len( self.proprietary_tool_confs ) == 1:
plural = ''
file_names = self.proprietary_tool_confs[ 0 ]
else:
plural = 's'
file_names = ', '.join( self.proprietary_tool_confs )
if missing_tool_configs_dict:
for proprietary_tool_conf in self.proprietary_tool_confs:
# Create a backup of the tool configuration in the un-migrated state.
shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % ( proprietary_tool_conf,
latest_migration_script_number ) )
for repository_elem in root:
# Make sure we have a valid repository tag.
if self.__is_valid_repository_tag( repository_elem ):
# Get all repository dependencies for the repository defined by the
# current repository_elem. Repository dependency definitions contained
# in tool shed repositories with migrated tools must never define a
# relationship to a repository dependency that contains a tool. The
# repository dependency can only contain items that are not loaded into
# the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes,
# etc). This restriction must be followed down the entire dependency hierarchy.
name = repository_elem.get( 'name' )
changeset_revision = repository_elem.get( 'changeset_revision' )
tool_shed_accessible, repository_dependencies_dict = \
common_util.get_repository_dependencies( app,
self.tool_shed_url,
name,
self.repository_owner,
changeset_revision )
# Make sure all repository dependency records exist (as tool_shed_repository
# table rows) in the Galaxy database.
created_tool_shed_repositories = \
self.create_or_update_tool_shed_repository_records( name,
changeset_revision,
repository_dependencies_dict )
# Order the repositories for proper installation. This process is similar to the
# process used when installing tool shed repositories, but does not handle managing
# tool panel sections and other components since repository dependency definitions
# contained in tool shed repositories with migrated tools must never define a relationship
# to a repository dependency that contains a tool.
ordered_tool_shed_repositories = \
self.order_repositories_for_installation( created_tool_shed_repositories,
repository_dependencies_dict )
for tool_shed_repository in ordered_tool_shed_repositories:
is_repository_dependency = self.__is_repository_dependency( name,
changeset_revision,
tool_shed_repository )
self.install_repository( repository_elem,
tool_shed_repository,
install_dependencies,
is_repository_dependency=is_repository_dependency )
else:
message = "\nNo tools associated with migration stage %s are defined in your " % \
str( latest_migration_script_number )
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \
( plural, file_names )
print message
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \
str( latest_migration_script_number )
message += "Try again later.\n"
print message
def __init__(self, app, latest_migration_script_number,
tool_shed_install_config, migrated_tools_config,
install_dependencies):
"""
Check tool settings in tool_shed_install_config and install all repositories
that are not already installed. The tool panel configuration file is the received
migrated_tools_config, which is the reserved file named migrated_tools_conf.xml.
"""
self.app = app
self.toolbox = self.app.toolbox
self.migrated_tools_config = migrated_tools_config
# Initialize the ToolPanelManager.
self.tpm = tool_panel_manager.ToolPanelManager(self.app)
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt
# to install but print informative error message.
if install_dependencies and app.config.tool_dependency_dir is None:
message = 'You are attempting to install tool dependencies but do not have a value '
message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this '
message += 'location value to the path where you want tool dependencies installed and '
message += 'rerun the migration script.'
raise Exception(message)
# Get the local non-shed related tool panel configs (there can be more than one, and the
# default name is tool_conf.xml).
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems(
latest_migration_script_number)
# Set the location where the repositories will be installed by retrieving the tool_path
# setting from migrated_tools_config.
tree, error_message = xml_util.parse_xml(migrated_tools_config)
if tree is None:
log.error(error_message)
else:
root = tree.getroot()
self.tool_path = root.get('tool_path')
log.debug(
"Repositories will be installed into configured tool_path location ",
str(self.tool_path))
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree, error_message = xml_util.parse_xml(tool_shed_install_config)
if tree is None:
log.error(error_message)
else:
root = tree.getroot()
defined_tool_shed_url = root.get('name')
self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, defined_tool_shed_url)
self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url(
self.tool_shed_url)
self.repository_owner = common_util.REPOSITORY_OWNER
self.shed_config_dict = self.tpm.get_shed_tool_conf_dict(
self.migrated_tools_config)
# Since tool migration scripts can be executed any number of times, we need to
# make sure the appropriate tools are defined in tool_conf.xml. If no tools
# associated with the migration stage are defined, no repositories will be installed
# on disk. The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs(
app)
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = \
common_util.check_for_missing_tools(app,
tool_panel_configs,
latest_migration_script_number)
else:
# It doesn't matter if the tool shed is accessible since there are no migrated
# tools defined in the local Galaxy instance, but we have to set the value of
# tool_shed_accessible to True so that the value of migrate_tools.version can
# be correctly set in the database.
tool_shed_accessible = True
missing_tool_configs_dict = odict()
if tool_shed_accessible:
if len(self.proprietary_tool_confs) == 1:
plural = ''
file_names = self.proprietary_tool_confs[0]
else:
plural = 's'
file_names = ', '.join(self.proprietary_tool_confs)
if missing_tool_configs_dict:
for proprietary_tool_conf in self.proprietary_tool_confs:
# Create a backup of the tool configuration in the un-migrated state.
shutil.copy(
proprietary_tool_conf, '%s-pre-stage-%04d' %
(proprietary_tool_conf,
latest_migration_script_number))
for repository_elem in root:
# Make sure we have a valid repository tag.
if self.__is_valid_repository_tag(repository_elem):
# Get all repository dependencies for the repository defined by the
# current repository_elem. Repository dependency definitions contained
# in tool shed repositories with migrated tools must never define a
# relationship to a repository dependency that contains a tool. The
# repository dependency can only contain items that are not loaded into
# the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes,
# etc). This restriction must be followed down the entire dependency hierarchy.
name = repository_elem.get('name')
changeset_revision = repository_elem.get(
'changeset_revision')
tool_shed_accessible, repository_dependencies_dict = \
common_util.get_repository_dependencies(app,
self.tool_shed_url,
name,
self.repository_owner,
changeset_revision)
# Make sure all repository dependency records exist (as tool_shed_repository
# table rows) in the Galaxy database.
created_tool_shed_repositories = \
self.create_or_update_tool_shed_repository_records(name,
changeset_revision,
repository_dependencies_dict)
# Order the repositories for proper installation. This process is similar to the
# process used when installing tool shed repositories, but does not handle managing
# tool panel sections and other components since repository dependency definitions
# contained in tool shed repositories with migrated tools must never define a relationship
# to a repository dependency that contains a tool.
ordered_tool_shed_repositories = \
self.order_repositories_for_installation(created_tool_shed_repositories,
repository_dependencies_dict)
for tool_shed_repository in ordered_tool_shed_repositories:
is_repository_dependency = self.__is_repository_dependency(
name, changeset_revision,
tool_shed_repository)
self.install_repository(
repository_elem,
tool_shed_repository,
install_dependencies,
is_repository_dependency=
is_repository_dependency)
else:
message = "\nNo tools associated with migration stage %s are defined in your " % \
str(latest_migration_script_number)
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \
(plural, file_names)
log.info(message)
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \
str(latest_migration_script_number)
message += "Try again later.\n"
log.error(message)
