def sync_database_with_file_system(self,
app,
tool_shed_repository,
tool_dependency_name,
tool_dependency_version,
tool_dependency_install_dir,
tool_dependency_type='package'):
"""
The installation directory defined by the received tool_dependency_install_dir exists, so check for
the presence of INSTALLATION_LOG. If the files exists, we'll assume the tool dependency is installed,
but not necessarily successfully (it could be in an error state on disk. However, we can justifiably
assume here that no matter the state, an associated database record will exist.
"""
# This method should be reached very rarely. It implies that either the Galaxy environment
# became corrupted (i.e., the database records for installed tool dependencies is not synchronized
# with tool dependencies on disk) or the Tool Shed's install and test framework is running. The Tool
# Shed's install and test framework installs repositories in 2 stages, those of type tool_dependency_definition
# followed by those containing valid tools and tool functional test components.
log.debug("Synchronizing the database with the file system...")
try:
log.debug(
"The value of app.config.running_functional_tests is: %s" %
str(app.config.running_functional_tests))
except Exception:
pass
sa_session = app.install_model.context
can_install_tool_dependency = False
tool_dependency = \
tool_dependency_util.get_tool_dependency_by_name_version_type_repository(app,
tool_shed_repository,
tool_dependency_name,
tool_dependency_version,
tool_dependency_type)
if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLING:
# The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool
# dependency is being installed by someone else, we don't want to interfere with that. This assumes
# the installation by "someone else" is not hung in an Installing state, which is a weakness if that
# "someone else" never repaired it.
log.debug(
'Skipping installation of tool dependency %s version %s because it has a status of %s'
% (str(tool_dependency.name), str(
tool_dependency.version), str(tool_dependency.status)))
else:
# We have a pre-existing installation directory on the file system, but our associated database record is
# in a state that allowed us to arrive here. At this point, we'll inspect the installation directory to
# see if we have a "proper installation" and if so, synchronize the database record rather than reinstalling
# the dependency if we're "running_functional_tests". If we're not "running_functional_tests, we'll set
# the tool dependency's installation status to ERROR.
tool_dependency_installation_directory_contents = os.listdir(
tool_dependency_install_dir)
if INSTALLATION_LOG in tool_dependency_installation_directory_contents:
# Since this tool dependency's installation directory contains an installation log, we consider it to be
# installed. In some cases the record may be missing from the database due to some activity outside of
# the control of the Tool Shed. Since a new record was created for it and we don't know the state of the
# files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see
# below).
log.debug(
'Skipping installation of tool dependency %s version %s because it is installed in %s'
% (str(tool_dependency.name), str(tool_dependency.version),
str(tool_dependency_install_dir)))
if app.config.running_functional_tests:
# If we are running functional tests, the state will be set to Installed because previously compiled
# tool dependencies are not deleted by default, from the "install and test" framework..
tool_dependency.status = app.install_model.ToolDependency.installation_status.INSTALLED
else:
error_message = 'The installation directory for this tool dependency had contents but the database had no record. '
error_message += 'The installation log may show this tool dependency to be correctly installed, but due to the '
error_message += 'missing database record it is now being set to Error.'
tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR
tool_dependency.error_message = error_message
else:
error_message = '\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s' % \
(str(tool_dependency_install_dir),
str(tool_dependency_name),
str(tool_dependency_version),
str(INSTALLATION_LOG))
error_message += ' is missing. This indicates an installation error so the tool dependency is being'
error_message += ' prepared for re-installation.'
log.error(error_message)
tool_dependency.status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
remove_dir(tool_dependency_install_dir)
can_install_tool_dependency = True
sa_session.add(tool_dependency)
sa_session.flush()
try:
log.debug(
"Returning from sync_database_with_file_system with tool_dependency %s, can_install_tool_dependency %s."
%
(str(tool_dependency.name), str(can_install_tool_dependency)))
except Exception as e:
log.debug(str(e))
return tool_dependency, can_install_tool_dependency
def handle_repository_contents(self,
tool_shed_repository,
repository_clone_url,
relative_install_dir,
repository_elem,
install_dependencies,
is_repository_dependency=False):
"""
Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository
contains tools that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each
of the tools defined in the received repository_elem from all non-shed-related tool panel configuration files since the
entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process.
"""
tool_configs_to_filter = []
tool_panel_dict_for_display = {}
if self.tool_path:
repo_install_dir = os.path.join(self.tool_path,
relative_install_dir)
else:
repo_install_dir = relative_install_dir
if not is_repository_dependency:
for tool_elem in repository_elem:
# The tool_elem looks something like this:
# <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" />
tool_config = tool_elem.get('file')
guid = self.get_guid(repository_clone_url,
relative_install_dir, tool_config)
# See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems.
is_displayed, tool_sections = self.get_containing_tool_sections(
tool_config)
if is_displayed:
tool_panel_dict_for_tool_config = \
self.tpm.generate_tool_panel_dict_for_tool_config(guid,
tool_config,
tool_sections=tool_sections)
# The tool-panel_dict has the following structure.
# {<Tool guid> : [{ tool_config : <tool_config_file>,
# id: <ToolSection id>,
# version : <ToolSection version>,
# name : <TooSection name>}]}
# TODO: this code need to be revised
for k, v in tool_panel_dict_for_tool_config.items():
tool_panel_dict_for_display[k] = v
for _tool_panel_dict in v:
# Keep track of tool config file names associated with entries that have been made to the
# migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs.
if tool_config not in tool_configs_to_filter:
tool_configs_to_filter.append(tool_config)
else:
log.error(
'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).'
% (guid, tool_config, ", ".join(
self.proprietary_tool_confs or [])))
if tool_configs_to_filter:
lock = threading.Lock()
lock.acquire(True)
try:
self.filter_and_persist_proprietary_tool_panel_configs(
tool_configs_to_filter)
except Exception:
log.exception(
"Exception attempting to filter and persist non-shed-related tool panel configs"
)
finally:
lock.release()
irmm = InstalledRepositoryMetadataManager(
app=self.app,
tpm=self.tpm,
repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict=self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True)
irmm.generate_metadata_for_changeset_revision()
irmm_metadata_dict = irmm.get_metadata_dict()
tool_shed_repository.metadata = irmm_metadata_dict
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
has_tool_dependencies = self.__has_tool_dependencies(
irmm_metadata_dict)
if has_tool_dependencies:
# All tool_dependency objects must be created before the tools are processed even if no
# tool dependencies will be installed.
tool_dependencies = create_tool_dependency_objects(
self.app,
tool_shed_repository,
relative_install_dir,
set_status=True)
else:
tool_dependencies = None
if 'tools' in irmm_metadata_dict:
tdtm = data_table_manager.ToolDataTableManager(self.app)
sample_files = irmm_metadata_dict.get('sample_files', [])
sample_files = [str(s) for s in sample_files]
tool_index_sample_files = tdtm.get_tool_index_sample_files(
sample_files)
tool_util.copy_sample_files(self.app,
tool_index_sample_files,
tool_path=self.tool_path)
sample_files_copied = [s for s in tool_index_sample_files]
repository_tools_tups = irmm.get_repository_tools_tups()
if repository_tools_tups:
# Handle missing data table entries for tool parameters that are dynamically
# generated select lists.
repository_tools_tups = tdtm.handle_missing_data_table_entry(
relative_install_dir, self.tool_path,
repository_tools_tups)
# Handle missing index files for tool parameters that are dynamically generated select lists.
repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file(
self.app, self.tool_path, sample_files,
repository_tools_tups, sample_files_copied)
# Copy remaining sample files included in the repository to the ~/tool-data
# directory of the local Galaxy instance.
tool_util.copy_sample_files(
self.app,
sample_files,
tool_path=self.tool_path,
sample_files_copied=sample_files_copied)
if not is_repository_dependency:
self.tpm.add_to_tool_panel(
tool_shed_repository.name,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True)
if install_dependencies and tool_dependencies and has_tool_dependencies:
# Install tool dependencies.
irm = install_manager.InstallRepositoryManager(self.app, self.tpm)
itdm = install_manager.InstallToolDependencyManager(self.app)
irm.update_tool_shed_repository_status(
tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.
INSTALLING_TOOL_DEPENDENCIES)
# Get the tool_dependencies.xml file from disk.
tool_dependencies_config = get_config_from_disk(
'tool_dependencies.xml', repo_install_dir)
installed_tool_dependencies = itdm.install_specified_tool_dependencies(
tool_shed_repository=tool_shed_repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=tool_dependencies,
from_tool_migration_manager=True)
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR:
log.error(
'The ToolMigrationManager returned the following error while installing tool dependency %s: %s',
installed_tool_dependency.name,
installed_tool_dependency.error_message)
if 'datatypes' in irmm_metadata_dict:
cdl = custom_datatype_manager.CustomDatatypeLoader(self.app)
tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
tool_shed_repository.includes_datatypes = True
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-hrc")
datatypes_config = get_config_from_disk(
suc.DATATYPES_CONFIG_FILENAME, repo_install_dir)
# Load proprietary data types required by tools. The value of override is not
# important here since the Galaxy server will be started after this installation
# completes.
converter_path, display_path = \
cdl.alter_config_and_load_prorietary_datatypes(datatypes_config,
repo_install_dir,
override=False)
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = \
cdl.create_repository_dict_for_proprietary_datatypes(tool_shed=self.tool_shed_url,
name=tool_shed_repository.name,
owner=self.repository_owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dicts=irmm_metadata_dict.get('tools', []),
converter_path=converter_path,
display_path=display_path)
if converter_path:
# Load proprietary datatype converters
self.app.datatypes_registry.load_datatype_converters(
self.toolbox, installed_repository_dict=repository_dict)
if display_path:
# Load proprietary datatype display applications
self.app.datatypes_registry.load_display_applications(
self.app, installed_repository_dict=repository_dict)
remove_dir(work_dir)