def non_shed_tool_panel_configs(self):
return common_util.get_non_shed_tool_panel_configs(self.app)
def verify_tools(app, url, galaxy_config_file=None, engine_options={}):
# Check the value in the migrate_tools.version database table column to verify that the number is in
# sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions.
# Create engine and metadata
engine = create_engine(url, **engine_options)
meta = MetaData(bind=engine)
# The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py.
Table("migrate_tools", meta, autoload=True)
# Verify that the code and the database are in sync.
db_schema = schema.ControlledSchema(engine, migrate_repository)
latest_tool_migration_script_number = migrate_repository.versions.latest
if latest_tool_migration_script_number != db_schema.version:
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
if app.new_installation:
# New installations will not be missing tools, so we don't need to worry about them.
missing_tool_configs_dict = OrderedDict()
else:
tool_panel_configs = common_util.get_non_shed_tool_panel_configs(app)
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools(app,
tool_panel_configs,
latest_tool_migration_script_number)
else:
# It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but
# we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
# the database.
tool_shed_accessible = True
missing_tool_configs_dict = OrderedDict()
have_tool_dependencies = False
for k, v in missing_tool_configs_dict.items():
if v:
have_tool_dependencies = True
break
if not app.config.running_functional_tests:
if tool_shed_accessible:
# Automatically update the value of the migrate_tools.version database table column.
manage_tools = os.path.abspath(os.path.join(os.path.dirname(__file__), 'scripts', 'manage_tools.py'))
cmd = [get_executable(), manage_tools, 'upgrade', 'tools']
if galaxy_config_file:
cmd[2:2] = ['-c', galaxy_config_file]
proc = subprocess.Popen(args=cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
return_code = proc.wait()
output = proc.stdout.read(32768)
if return_code != 0:
raise Exception("Error attempting to update the value of migrate_tools.version: %s" % output)
elif missing_tool_configs_dict:
if len(tool_panel_configs) == 1:
plural = ''
tool_panel_config_file_names = tool_panel_configs[0]
else:
plural = 's'
tool_panel_config_file_names = ', '.join(tool_panel_configs)
msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n"
msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n"
msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n"
msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n"
msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n"
msg += "be removed from your file{} named {}.\n\n".format(plural, tool_panel_config_file_names)
msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n"
msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n"
msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="database/shed_tools">). The default location\n'
msg += "setting is 'database/shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n"
msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n"
msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n"
msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n"
msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n"
msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n"
msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n"
msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n"
if have_tool_dependencies:
msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n"
msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n"
msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n"
processed_tool_dependencies = []
for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items():
for tool_dependencies_tup in missing_tool_configs_dict[missing_tool_config]['tool_dependencies']:
if tool_dependencies_tup not in processed_tool_dependencies:
msg += "------------------------------------\n"
msg += "Tool Dependency\n"
msg += "------------------------------------\n"
msg += "Name: {}, Version: {}, Type: {}\n".format(tool_dependencies_tup[0],
tool_dependencies_tup[1],
tool_dependencies_tup[2])
if len(tool_dependencies_tup) >= 4:
msg += "Requirements and installation information:\n"
msg += "%s\n" % tool_dependencies_tup[3]
else:
msg += "\n"
msg += "------------------------------------\n"
processed_tool_dependencies.append(tool_dependencies_tup)
msg += "\n"
msg += "%s" % output.replace('done', '')
msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"
msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number
msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n"
if have_tool_dependencies:
msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n"
msg += "option to the above command like this:\n\n"
msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"
msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number
msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n"
msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n"
msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n"
msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n"
msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these tools have been\n"
msg += "removed from your file{} named {}.\n\n".format(plural, tool_panel_config_file_names)
for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items():
msg += "%s\n" % missing_tool_config
msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
raise Exception(msg)
else:
log.debug("The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup" % db_schema.version)
else:
log.info("At migrate_tools version %d" % db_schema.version)
def __init__(self, app, latest_migration_script_number,
tool_shed_install_config, migrated_tools_config,
install_dependencies):
"""
Check tool settings in tool_shed_install_config and install all repositories
that are not already installed. The tool panel configuration file is the received
migrated_tools_config, which is the reserved file named migrated_tools_conf.xml.
"""
self.app = app
self.toolbox = self.app.toolbox
self.migrated_tools_config = migrated_tools_config
# Initialize the ToolPanelManager.
self.tpm = tool_panel_manager.ToolPanelManager(self.app)
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt
# to install but print informative error message.
if install_dependencies and app.tool_dependency_dir is None:
message = 'You are attempting to install tool dependencies but do not have a value '
message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this '
message += 'location value to the path where you want tool dependencies installed and '
message += 'rerun the migration script.'
raise Exception(message)
# Get the local non-shed related tool panel configs (there can be more than one, and the
# default name is tool_conf.xml).
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems(
latest_migration_script_number)
# Set the location where the repositories will be installed by retrieving the tool_path
# setting from migrated_tools_config.
try:
tree, error_message = parse_xml(migrated_tools_config)
except OSError as exc:
if exc.errno == errno.ENOENT:
with open(migrated_tools_config, 'w') as fh:
fh.write(
MIGRATED_TOOLS_CONF_XML.format(
shed_tools_dir=self.app.config.shed_tools_dir))
tree, error_message = parse_xml(migrated_tools_config)
else:
raise
if tree is None:
log.error(error_message)
else:
root = tree.getroot()
self.tool_path = root.get('tool_path')
log.debug(
"Repositories will be installed into configured tool_path location ",
str(self.tool_path))
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree, error_message = parse_xml(tool_shed_install_config)
if tree is None:
log.error(error_message)
else:
root = tree.getroot()
defined_tool_shed_url = root.get('name')
self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, defined_tool_shed_url)
self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url(
self.tool_shed_url)
self.repository_owner = common_util.REPOSITORY_OWNER
self.shed_config_dict = self.tpm.get_shed_tool_conf_dict(
self.migrated_tools_config)
# Since tool migration scripts can be executed any number of times, we need to
# make sure the appropriate tools are defined in tool_conf.xml. If no tools
# associated with the migration stage are defined, no repositories will be installed
# on disk. The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs(
app)
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = \
common_util.check_for_missing_tools(app,
tool_panel_configs,
latest_migration_script_number)
else:
# It doesn't matter if the tool shed is accessible since there are no migrated
# tools defined in the local Galaxy instance, but we have to set the value of
# tool_shed_accessible to True so that the value of migrate_tools.version can
# be correctly set in the database.
tool_shed_accessible = True
missing_tool_configs_dict = {}
if tool_shed_accessible:
if len(self.proprietary_tool_confs) == 1:
plural = ''
file_names = self.proprietary_tool_confs[0]
else:
plural = 's'
file_names = ', '.join(self.proprietary_tool_confs)
if missing_tool_configs_dict:
for proprietary_tool_conf in self.proprietary_tool_confs:
# Create a backup of the tool configuration in the un-migrated state.
shutil.copy(
proprietary_tool_conf, '%s-pre-stage-%04d' %
(proprietary_tool_conf,
latest_migration_script_number))
for repository_elem in root:
# Make sure we have a valid repository tag.
if self.__is_valid_repository_tag(repository_elem):
# Get all repository dependencies for the repository defined by the
# current repository_elem. Repository dependency definitions contained
# in tool shed repositories with migrated tools must never define a
# relationship to a repository dependency that contains a tool. The
# repository dependency can only contain items that are not loaded into
# the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes,
# etc). This restriction must be followed down the entire dependency hierarchy.
name = repository_elem.get('name')
changeset_revision = repository_elem.get(
'changeset_revision')
tool_shed_accessible, repository_dependencies_dict = \
common_util.get_repository_dependencies(app,
self.tool_shed_url,
name,
self.repository_owner,
changeset_revision)
# Make sure all repository dependency records exist (as tool_shed_repository
# table rows) in the Galaxy database.
created_tool_shed_repositories = \
self.create_or_update_tool_shed_repository_records(name=name,
changeset_revision=changeset_revision,
repository_dependencies_dict=repository_dependencies_dict)
# Order the repositories for proper installation. This process is similar to the
# process used when installing tool shed repositories, but does not handle managing
# tool panel sections and other components since repository dependency definitions
# contained in tool shed repositories with migrated tools must never define a relationship
# to a repository dependency that contains a tool.
ordered_tool_shed_repositories = \
self.order_repositories_for_installation(created_tool_shed_repositories,
repository_dependencies_dict)
for tool_shed_repository in ordered_tool_shed_repositories:
is_repository_dependency = self.__is_repository_dependency(
name, changeset_revision,
tool_shed_repository)
self.install_repository(
repository_elem,
tool_shed_repository,
install_dependencies,
is_repository_dependency=
is_repository_dependency)
else:
message = "\nNo tools associated with migration stage %s are defined in your " % \
str(latest_migration_script_number)
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \
(plural, file_names)
log.info(message)
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \
str(latest_migration_script_number)
message += "Try again later.\n"
log.error(message)