def __init__(self,
app,
tpm=None,
repository=None,
changeset_revision=None,
repository_clone_url=None,
shed_config_dict=None,
relative_install_dir=None,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=False,
metadata_dict=None):
super().__init__(app,
repository,
changeset_revision,
repository_clone_url,
shed_config_dict,
relative_install_dir,
repository_files_dir,
resetting_all_metadata_on_repository,
updating_installed_repository,
persist,
metadata_dict=metadata_dict,
user=None)
if tpm is None:
self.tpm = tool_panel_manager.ToolPanelManager(self.app)
else:
self.tpm = tpm
def create_repository_dependency_objects(
self,
tool_path,
tool_shed_url,
repo_info_dicts,
install_repository_dependencies=False,
no_changes_checked=False,
tool_panel_section_id=None,
new_tool_panel_section_label=None):
"""
Discover all repository dependencies and make sure all tool_shed_repository and
associated repository_dependency records exist as well as the dependency relationships
between installed repositories. This method is called when uninstalled repositories
are being reinstalled. If the user elected to install repository dependencies, all
items in the all_repo_info_dicts list will be processed. However, if repository
dependencies are not to be installed, only those items contained in the received
repo_info_dicts list will be processed.
"""
install_model = self.app.install_model
log.debug("Creating repository dependency objects...")
# The following list will be maintained within this method to contain all created
# or updated tool shed repositories, including repository dependencies that may not
# be installed.
all_created_or_updated_tool_shed_repositories = []
# There will be a one-to-one mapping between items in 3 lists:
# created_or_updated_tool_shed_repositories, tool_panel_section_keys
# and filtered_repo_info_dicts. The 3 lists will filter out repository
# dependencies that are not to be installed.
created_or_updated_tool_shed_repositories = []
tool_panel_section_keys = []
# Repositories will be filtered (e.g., if already installed, if elected
# to not be installed, etc), so filter the associated repo_info_dicts accordingly.
filtered_repo_info_dicts = []
# Discover all repository dependencies and retrieve information for installing
# them. Even if the user elected to not install repository dependencies we have
# to make sure all repository dependency objects exist so that the appropriate
# repository dependency relationships can be built.
all_required_repo_info_dict = self.get_required_repo_info_dicts(
tool_shed_url, repo_info_dicts)
all_repo_info_dicts = all_required_repo_info_dict.get(
'all_repo_info_dicts', [])
if not all_repo_info_dicts:
# No repository dependencies were discovered so process the received repositories.
all_repo_info_dicts = [rid for rid in repo_info_dicts]
for repo_info_dict in all_repo_info_dicts:
# If the user elected to install repository dependencies, all items in the
# all_repo_info_dicts list will be processed. However, if repository dependencies
# are not to be installed, only those items contained in the received repo_info_dicts
# list will be processed but the all_repo_info_dicts list will be used to create all
# defined repository dependency relationships.
if self.is_in_repo_info_dicts(
repo_info_dict,
repo_info_dicts) or install_repository_dependencies:
for name, repo_info_tuple in repo_info_dict.items():
can_update_db_record = False
clear_metadata = True
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
repository_util.get_repo_info_tuple_contents(repo_info_tuple)
# See if the repository has an existing record in the database.
repository_db_record, installed_changeset_revision = \
repository_util.repository_was_previously_installed(self.app, tool_shed_url, name, repo_info_tuple, from_tip=False)
if repository_db_record:
if (installed_changeset_revision != changeset_revision
and repository_db_record.status
== install_model.ToolShedRepository.
installation_status.INSTALLED):
log.info(
"Repository '%s' already present at revision %s, will be updated to revision %s",
repository_db_record.name,
installed_changeset_revision,
changeset_revision)
can_update_db_record = True
clear_metadata = False
elif repository_db_record.status in [
install_model.ToolShedRepository.
installation_status.INSTALLED, install_model.
ToolShedRepository.installation_status.CLONING,
install_model.ToolShedRepository.
installation_status.SETTING_TOOL_VERSIONS,
install_model.ToolShedRepository.
installation_status.
INSTALLING_REPOSITORY_DEPENDENCIES,
install_model.ToolShedRepository.
installation_status.
INSTALLING_TOOL_DEPENDENCIES, install_model.
ToolShedRepository.installation_status.
LOADING_PROPRIETARY_DATATYPES
]:
info_msg = "Skipping installation of revision %s of repository '%s' because it was installed " % \
(changeset_revision, repository_db_record.name)
info_msg += "with the (possibly updated) revision %s and its current installation status is '%s'." % \
(installed_changeset_revision, repository_db_record.status)
log.info(info_msg)
can_update_db_record = False
else:
if repository_db_record.status in [
install_model.ToolShedRepository.
installation_status.ERROR, install_model.
ToolShedRepository.installation_status.NEW,
install_model.ToolShedRepository.
installation_status.UNINSTALLED
]:
# The current tool shed repository is not currently installed, so we can update its
# record in the database.
name = repository_db_record.name
installed_changeset_revision = repository_db_record.installed_changeset_revision
can_update_db_record = True
elif repository_db_record.status in [
install_model.ToolShedRepository.
installation_status.DEACTIVATED
]:
# The current tool shed repository is deactivated, so updating its database record
# is not necessary - just activate it.
log.info(
"Reactivating deactivated tool_shed_repository '%s'."
% str(repository_db_record.name))
self.app.installed_repository_manager.activate_repository(
repository_db_record)
# No additional updates to the database record are necessary.
can_update_db_record = False
elif repository_db_record.status not in [
install_model.ToolShedRepository.
installation_status.NEW
]:
# Set changeset_revision here so repository_util.create_or_update_tool_shed_repository will find
# the previously installed and uninstalled repository instead of creating a new record.
changeset_revision = repository_db_record.installed_changeset_revision
self.reset_previously_installed_repository(
repository_db_record)
can_update_db_record = True
else:
# No record exists in the database for the repository currently being processed.
installed_changeset_revision = changeset_revision
can_update_db_record = True
if can_update_db_record:
# The database record for the tool shed repository currently being processed can be updated.
# Get the repository metadata to see where it was previously located in the tool panel.
tpm = tool_panel_manager.ToolPanelManager(self.app)
if repository_db_record and repository_db_record.metadata:
_, tool_panel_section_key = \
tpm.handle_tool_panel_selection(toolbox=self.app.toolbox,
metadata=repository_db_record.metadata,
no_changes_checked=no_changes_checked,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label)
else:
# We're installing a new tool shed repository that does not yet have a database record.
tool_panel_section_key, _ = \
tpm.handle_tool_panel_section(self.app.toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label)
metadata_dict = {} if clear_metadata else None
current_changeset_revision = changeset_revision if clear_metadata else None
tool_shed_repository = \
repository_util.create_or_update_tool_shed_repository(app=self.app,
name=name,
description=description,
installed_changeset_revision=installed_changeset_revision,
ctx_rev=ctx_rev,
repository_clone_url=repository_clone_url,
status=install_model.ToolShedRepository.installation_status.NEW,
metadata_dict=metadata_dict,
current_changeset_revision=current_changeset_revision,
owner=repository_owner,
dist_to_shed=False)
if tool_shed_repository not in all_created_or_updated_tool_shed_repositories:
all_created_or_updated_tool_shed_repositories.append(
tool_shed_repository)
# Only append the tool shed repository to the list of created_or_updated_tool_shed_repositories if
# it is supposed to be installed.
if install_repository_dependencies or self.is_in_repo_info_dicts(
repo_info_dict, repo_info_dicts):
if tool_shed_repository not in created_or_updated_tool_shed_repositories:
# Keep the one-to-one mapping between items in 3 lists.
created_or_updated_tool_shed_repositories.append(
tool_shed_repository)
tool_panel_section_keys.append(
tool_panel_section_key)
filtered_repo_info_dicts.append(repo_info_dict)
# Build repository dependency relationships even if the user chose to not install repository dependencies.
self.build_repository_dependency_relationships(
all_repo_info_dicts, all_created_or_updated_tool_shed_repositories)
return created_or_updated_tool_shed_repositories, tool_panel_section_keys, all_repo_info_dicts, filtered_repo_info_dicts
def __init__(self, app, latest_migration_script_number,
tool_shed_install_config, migrated_tools_config,
install_dependencies):
"""
Check tool settings in tool_shed_install_config and install all repositories
that are not already installed. The tool panel configuration file is the received
migrated_tools_config, which is the reserved file named migrated_tools_conf.xml.
"""
self.app = app
self.toolbox = self.app.toolbox
self.migrated_tools_config = migrated_tools_config
# Initialize the ToolPanelManager.
self.tpm = tool_panel_manager.ToolPanelManager(self.app)
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt
# to install but print informative error message.
if install_dependencies and app.tool_dependency_dir is None:
message = 'You are attempting to install tool dependencies but do not have a value '
message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this '
message += 'location value to the path where you want tool dependencies installed and '
message += 'rerun the migration script.'
raise Exception(message)
# Get the local non-shed related tool panel configs (there can be more than one, and the
# default name is tool_conf.xml).
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems(
latest_migration_script_number)
# Set the location where the repositories will be installed by retrieving the tool_path
# setting from migrated_tools_config.
try:
tree, error_message = parse_xml(migrated_tools_config)
except OSError as exc:
if exc.errno == errno.ENOENT:
with open(migrated_tools_config, 'w') as fh:
fh.write(
MIGRATED_TOOLS_CONF_XML.format(
shed_tools_dir=self.app.config.shed_tools_dir))
tree, error_message = parse_xml(migrated_tools_config)
else:
raise
if tree is None:
log.error(error_message)
else:
root = tree.getroot()
self.tool_path = root.get('tool_path')
log.debug(
"Repositories will be installed into configured tool_path location ",
str(self.tool_path))
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree, error_message = parse_xml(tool_shed_install_config)
if tree is None:
log.error(error_message)
else:
root = tree.getroot()
defined_tool_shed_url = root.get('name')
self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, defined_tool_shed_url)
self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url(
self.tool_shed_url)
self.repository_owner = common_util.REPOSITORY_OWNER
self.shed_config_dict = self.tpm.get_shed_tool_conf_dict(
self.migrated_tools_config)
# Since tool migration scripts can be executed any number of times, we need to
# make sure the appropriate tools are defined in tool_conf.xml. If no tools
# associated with the migration stage are defined, no repositories will be installed
# on disk. The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs(
app)
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = \
common_util.check_for_missing_tools(app,
tool_panel_configs,
latest_migration_script_number)
else:
# It doesn't matter if the tool shed is accessible since there are no migrated
# tools defined in the local Galaxy instance, but we have to set the value of
# tool_shed_accessible to True so that the value of migrate_tools.version can
# be correctly set in the database.
tool_shed_accessible = True
missing_tool_configs_dict = {}
if tool_shed_accessible:
if len(self.proprietary_tool_confs) == 1:
plural = ''
file_names = self.proprietary_tool_confs[0]
else:
plural = 's'
file_names = ', '.join(self.proprietary_tool_confs)
if missing_tool_configs_dict:
for proprietary_tool_conf in self.proprietary_tool_confs:
# Create a backup of the tool configuration in the un-migrated state.
shutil.copy(
proprietary_tool_conf, '%s-pre-stage-%04d' %
(proprietary_tool_conf,
latest_migration_script_number))
for repository_elem in root:
# Make sure we have a valid repository tag.
if self.__is_valid_repository_tag(repository_elem):
# Get all repository dependencies for the repository defined by the
# current repository_elem. Repository dependency definitions contained
# in tool shed repositories with migrated tools must never define a
# relationship to a repository dependency that contains a tool. The
# repository dependency can only contain items that are not loaded into
# the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes,
# etc). This restriction must be followed down the entire dependency hierarchy.
name = repository_elem.get('name')
changeset_revision = repository_elem.get(
'changeset_revision')
tool_shed_accessible, repository_dependencies_dict = \
common_util.get_repository_dependencies(app,
self.tool_shed_url,
name,
self.repository_owner,
changeset_revision)
# Make sure all repository dependency records exist (as tool_shed_repository
# table rows) in the Galaxy database.
created_tool_shed_repositories = \
self.create_or_update_tool_shed_repository_records(name=name,
changeset_revision=changeset_revision,
repository_dependencies_dict=repository_dependencies_dict)
# Order the repositories for proper installation. This process is similar to the
# process used when installing tool shed repositories, but does not handle managing
# tool panel sections and other components since repository dependency definitions
# contained in tool shed repositories with migrated tools must never define a relationship
# to a repository dependency that contains a tool.
ordered_tool_shed_repositories = \
self.order_repositories_for_installation(created_tool_shed_repositories,
repository_dependencies_dict)
for tool_shed_repository in ordered_tool_shed_repositories:
is_repository_dependency = self.__is_repository_dependency(
name, changeset_revision,
tool_shed_repository)
self.install_repository(
repository_elem,
tool_shed_repository,
install_dependencies,
is_repository_dependency=
is_repository_dependency)
else:
message = "\nNo tools associated with migration stage %s are defined in your " % \
str(latest_migration_script_number)
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \
(plural, file_names)
log.info(message)
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \
str(latest_migration_script_number)
message += "Try again later.\n"
log.error(message)
def tpm(self):
return tool_panel_manager.ToolPanelManager(self.app)
