def load_installed_datatypes(self,
repository,
relative_install_dir,
deactivate=False):
"""
Load proprietary datatypes and return information needed for loading custom
datatypes converters and display applications later.
"""
metadata = repository.metadata
repository_dict = None
datatypes_config = hg_util.get_config_from_disk(
suc.DATATYPES_CONFIG_FILENAME, relative_install_dir)
if datatypes_config:
converter_path, display_path = \
self.alter_config_and_load_prorietary_datatypes(datatypes_config,
relative_install_dir,
deactivate=deactivate)
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = \
self.create_repository_dict_for_proprietary_datatypes(tool_shed=repository.tool_shed,
name=repository.name,
owner=repository.owner,
installed_changeset_revision=repository.installed_changeset_revision,
tool_dicts=metadata.get('tools', []),
converter_path=converter_path,
display_path=display_path)
return repository_dict
def handle_missing_data_table_entry(self, relative_install_dir, tool_path,
repository_tools_tups):
"""
Inspect each tool to see if any have input parameters that are dynamically
generated select lists that require entries in the tool_data_table_conf.xml
file. This method is called only from Galaxy (not the tool shed) when a
repository is being installed or reinstalled.
"""
missing_data_table_entry = False
for index, repository_tools_tup in enumerate(repository_tools_tups):
tup_path, guid, repository_tool = repository_tools_tup
if repository_tool.params_with_missing_data_table_entry:
missing_data_table_entry = True
break
if missing_data_table_entry:
# The repository must contain a tool_data_table_conf.xml.sample file that includes
# all required entries for all tools in the repository.
sample_tool_data_table_conf = hg_util.get_config_from_disk(
'tool_data_table_conf.xml.sample', relative_install_dir)
if sample_tool_data_table_conf:
# Add entries to the ToolDataTableManager's in-memory data_tables dictionary.
error, message = self.handle_sample_tool_data_table_conf_file(
sample_tool_data_table_conf, persist=True)
if error:
# TODO: Do more here than logging an exception.
log.debug(message)
# Reload the tool into the local list of repository_tools_tups.
repository_tool = self.app.toolbox.load_tool(os.path.join(
tool_path, tup_path),
guid=guid,
use_cached=False)
repository_tools_tups[index] = (tup_path, guid, repository_tool)
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
self.reset_tool_data_tables()
return repository_tools_tups
def check_tool_input_params(self, repo_dir, tool_config_name, tool,
sample_files):
"""
Check all of the tool's input parameters, looking for any that are dynamically
generated using external data files to make sure the files exist.
"""
invalid_files_and_errors_tups = []
correction_msg = ''
for input_param in tool.input_params:
if isinstance(input_param, parameters.basic.SelectToolParameter
) and input_param.is_dynamic:
# If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml,
# make sure all requirements exist.
options = input_param.dynamic_options or input_param.options
if options and isinstance(options,
dynamic_options.DynamicOptions):
if options.tool_data_table or options.missing_tool_data_table_name:
# Make sure the repository contains a tool_data_table_conf.xml.sample file.
sample_tool_data_table_conf = hg_util.get_config_from_disk(
'tool_data_table_conf.xml.sample', repo_dir)
if sample_tool_data_table_conf:
error, correction_msg = \
self.tdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf,
persist=False)
if error:
invalid_files_and_errors_tups.append(
('tool_data_table_conf.xml.sample',
correction_msg))
else:
correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. "
correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository "
correction_msg += "that includes the required entry to correct this error.<br/>"
invalid_tup = (tool_config_name, correction_msg)
if invalid_tup not in invalid_files_and_errors_tups:
invalid_files_and_errors_tups.append(
invalid_tup)
if options.index_file or options.missing_index_file:
# Make sure the repository contains the required xxx.loc.sample file.
index_file = options.index_file or options.missing_index_file
index_file_name = basic_util.strip_path(index_file)
sample_found = False
for sample_file in sample_files:
sample_file_name = basic_util.strip_path(
sample_file)
if sample_file_name == '%s.sample' % index_file_name:
options.index_file = index_file_name
if options.tool_data_table:
options.tool_data_table.missing_index_file = None
sample_found = True
break
if not sample_found:
correction_msg = "This file refers to a file named <b>%s</b>. " % str(
index_file_name)
correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % \
str(index_file_name)
invalid_files_and_errors_tups.append(
(tool_config_name, correction_msg))
return invalid_files_and_errors_tups
def create_tool_dependency_objects(app,
tool_shed_repository,
relative_install_dir,
set_status=True):
"""
Create or update a ToolDependency for each entry in tool_dependencies_config. This method is called when
installing a new tool_shed_repository.
"""
tool_dependency_objects = []
shed_config_dict = tool_shed_repository.get_shed_config_dict(app)
if shed_config_dict.get('tool_path'):
relative_install_dir = os.path.join(shed_config_dict.get('tool_path'),
relative_install_dir)
# Get the tool_dependencies.xml file from the repository.
tool_dependencies_config = hg_util.get_config_from_disk(
'tool_dependencies.xml', relative_install_dir)
tree, error_message = xml_util.parse_xml(tool_dependencies_config)
if tree is None:
return tool_dependency_objects
root = tree.getroot()
for elem in root:
tool_dependency_type = elem.tag
if tool_dependency_type == 'package':
name = elem.get('name', None)
version = elem.get('version', None)
if name and version:
status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
tool_dependency = create_or_update_tool_dependency(
app,
tool_shed_repository,
name=name,
version=version,
type=tool_dependency_type,
status=status,
set_status=set_status)
tool_dependency_objects.append(tool_dependency)
elif tool_dependency_type == 'set_environment':
for env_elem in elem:
# <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
name = env_elem.get('name', None)
action = env_elem.get('action', None)
if name and action:
status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
tool_dependency = create_or_update_tool_dependency(
app,
tool_shed_repository,
name=name,
version=None,
type=tool_dependency_type,
status=status,
set_status=set_status)
tool_dependency_objects.append(tool_dependency)
return tool_dependency_objects
for tool_dependency in repository.missing_tool_dependencies:
if tool_dependency.status in [
self.app.install_model.ToolDependency.
installation_status.ERROR, self.app.install_model.
ToolDependency.installation_status.INSTALLING
]:
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED )
# Install tool dependencies.
irm.update_tool_shed_repository_status(
repository, self.app.install_model.ToolShedRepository.
installation_status.INSTALLING_TOOL_DEPENDENCIES)
# Get the tool_dependencies.xml file from the repository.
tool_dependencies_config = hg_util.get_config_from_disk(
'tool_dependencies.xml', repository.repo_path(self.app))
itdm = install_manager.InstallToolDependencyManager(self.app)
installed_tool_dependencies = itdm.install_specified_tool_dependencies(
tool_shed_repository=repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=repository.tool_dependencies,
from_tool_migration_manager=False)
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status in [
self.app.install_model.ToolDependency.
installation_status.ERROR
]:
repair_dict = add_repair_dict_entry(
repository.name,
installed_tool_dependency.error_message)
basic_util.remove_dir(work_dir)
def handle_repository_contents(self,
tool_shed_repository,
repository_clone_url,
relative_install_dir,
repository_elem,
install_dependencies,
is_repository_dependency=False):
"""
Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository
contains tools that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each
of the tools defined in the received repository_elem from all non-shed-related tool panel configuration files since the
entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process.
"""
tool_configs_to_filter = []
tool_panel_dict_for_display = odict()
if self.tool_path:
repo_install_dir = os.path.join(self.tool_path,
relative_install_dir)
else:
repo_install_dir = relative_install_dir
if not is_repository_dependency:
for tool_elem in repository_elem:
# The tool_elem looks something like this:
# <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" />
tool_config = tool_elem.get('file')
guid = self.get_guid(repository_clone_url,
relative_install_dir, tool_config)
# See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems.
is_displayed, tool_sections = self.get_containing_tool_sections(
tool_config)
if is_displayed:
tool_panel_dict_for_tool_config = \
self.tpm.generate_tool_panel_dict_for_tool_config(guid,
tool_config,
tool_sections=tool_sections)
# The tool-panel_dict has the following structure.
# {<Tool guid> : [{ tool_config : <tool_config_file>,
# id: <ToolSection id>,
# version : <ToolSection version>,
# name : <TooSection name>}]}
for k, v in tool_panel_dict_for_tool_config.items():
tool_panel_dict_for_display[k] = v
for tool_panel_dict in v:
# Keep track of tool config file names associated with entries that have been made to the
# migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs.
if tool_config not in tool_configs_to_filter:
tool_configs_to_filter.append(tool_config)
else:
log.error(
'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).'
% (guid, tool_config, ", ".join(
self.proprietary_tool_confs or [])))
if tool_configs_to_filter:
lock = threading.Lock()
lock.acquire(True)
try:
self.filter_and_persist_proprietary_tool_panel_configs(
tool_configs_to_filter)
except Exception:
log.exception(
"Exception attempting to filter and persist non-shed-related tool panel configs"
)
finally:
lock.release()
irmm = InstalledRepositoryMetadataManager(
app=self.app,
tpm=self.tpm,
repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict=self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True)
irmm.generate_metadata_for_changeset_revision()
irmm_metadata_dict = irmm.get_metadata_dict()
tool_shed_repository.metadata = irmm_metadata_dict
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
has_tool_dependencies = self.__has_tool_dependencies(
irmm_metadata_dict)
if has_tool_dependencies:
# All tool_dependency objects must be created before the tools are processed even if no
# tool dependencies will be installed.
tool_dependencies = tool_dependency_util.create_tool_dependency_objects(
self.app,
tool_shed_repository,
relative_install_dir,
set_status=True)
else:
tool_dependencies = None
if 'tools' in irmm_metadata_dict:
tdtm = data_table_manager.ToolDataTableManager(self.app)
sample_files = irmm_metadata_dict.get('sample_files', [])
sample_files = [str(s) for s in sample_files]
tool_index_sample_files = tdtm.get_tool_index_sample_files(
sample_files)
tool_util.copy_sample_files(self.app,
tool_index_sample_files,
tool_path=self.tool_path)
sample_files_copied = [s for s in tool_index_sample_files]
repository_tools_tups = irmm.get_repository_tools_tups()
if repository_tools_tups:
# Handle missing data table entries for tool parameters that are dynamically
# generated select lists.
repository_tools_tups = tdtm.handle_missing_data_table_entry(
relative_install_dir, self.tool_path,
repository_tools_tups)
# Handle missing index files for tool parameters that are dynamically generated select lists.
repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file(
self.app, self.tool_path, sample_files,
repository_tools_tups, sample_files_copied)
# Copy remaining sample files included in the repository to the ~/tool-data
# directory of the local Galaxy instance.
tool_util.copy_sample_files(
self.app,
sample_files,
tool_path=self.tool_path,
sample_files_copied=sample_files_copied)
if not is_repository_dependency:
self.tpm.add_to_tool_panel(
tool_shed_repository.name,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True)
if install_dependencies and tool_dependencies and has_tool_dependencies:
# Install tool dependencies.
irm = install_manager.InstallRepositoryManager(self.app, self.tpm)
itdm = install_manager.InstallToolDependencyManager(self.app)
irm.update_tool_shed_repository_status(
tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.
INSTALLING_TOOL_DEPENDENCIES)
# Get the tool_dependencies.xml file from disk.
tool_dependencies_config = hg_util.get_config_from_disk(
'tool_dependencies.xml', repo_install_dir)
installed_tool_dependencies = itdm.install_specified_tool_dependencies(
tool_shed_repository=tool_shed_repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=tool_dependencies,
from_tool_migration_manager=True)
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR:
log.error(
'The ToolMigrationManager returned the following error while installing tool dependency %s: %s',
installed_tool_dependency.name,
installed_tool_dependency.error_message)
if 'datatypes' in irmm_metadata_dict:
cdl = custom_datatype_manager.CustomDatatypeLoader(self.app)
tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
tool_shed_repository.includes_datatypes = True
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-hrc")
datatypes_config = hg_util.get_config_from_disk(
suc.DATATYPES_CONFIG_FILENAME, repo_install_dir)
# Load proprietary data types required by tools. The value of override is not
# important here since the Galaxy server will be started after this installation
# completes.
converter_path, display_path = \
cdl.alter_config_and_load_prorietary_datatypes(datatypes_config,
repo_install_dir,
override=False)
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = \
cdl.create_repository_dict_for_proprietary_datatypes(tool_shed=self.tool_shed_url,
name=tool_shed_repository.name,
owner=self.repository_owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dicts=irmm_metadata_dict.get('tools', []),
converter_path=converter_path,
display_path=display_path)
if converter_path:
# Load proprietary datatype converters
self.app.datatypes_registry.load_datatype_converters(
self.toolbox, installed_repository_dict=repository_dict)
if display_path:
# Load proprietary datatype display applications
self.app.datatypes_registry.load_display_applications(
self.app, installed_repository_dict=repository_dict)
basic_util.remove_dir(work_dir)
def repair_tool_shed_repository(self, repository, repo_info_dict):
def add_repair_dict_entry(repository_name, error_message):
if repository_name in repair_dict:
repair_dict[repository_name].append(error_message)
else:
repair_dict[repository_name] = [error_message]
return repair_dict
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, repository.tool_shed)
metadata = repository.metadata
# The repository.metadata contains dependency information that corresponds to the current changeset revision,
# which may be different from what is stored in the database
# If any of these repository-repository dependency associations is obsolete, clean_dependency_relationships removes them.
suc.clean_dependency_relationships(self.app, metadata, repository,
tool_shed_url)
repair_dict = {}
tpm = tool_panel_manager.ToolPanelManager(self.app)
if repository.status in [
self.app.install_model.ToolShedRepository.installation_status.
DEACTIVATED
]:
try:
self.app.installed_repository_manager.activate_repository(
repository)
except Exception as e:
error_message = "Error activating repository %s: %s" % (
repository.name, str(e))
log.debug(error_message)
repair_dict[repository.name] = error_message
elif repository.status not in [
self.app.install_model.ToolShedRepository.installation_status.
INSTALLED
]:
shed_tool_conf, tool_path, relative_install_dir = \
suc.get_tool_panel_config_tool_path_install_dir( self.app, repository )
# Reset the repository attributes to the New state for installation.
if metadata:
_, tool_panel_section_key = \
tpm.handle_tool_panel_selection( self.app.toolbox,
metadata,
no_changes_checked=True,
tool_panel_section_id=None,
new_tool_panel_section_label=None )
else:
# The tools will be loaded outside of any sections in the tool panel.
tool_panel_section_key = None
repository_util.set_repository_attributes(
self.app,
repository,
status=self.app.install_model.ToolShedRepository.
installation_status.NEW,
error_message=None,
deleted=False,
uninstalled=False,
remove_from_disk=True)
irm = install_manager.InstallRepositoryManager(self.app, tpm)
irm.install_tool_shed_repository(
repository,
repo_info_dict,
tool_panel_section_key,
shed_tool_conf,
tool_path,
install_tool_dependencies=True,
install_resolver_dependencies=
False, # Assuming repairs are only necessary toolshed packages
reinstalling=True)
if repository.status in [
self.app.install_model.ToolShedRepository.
installation_status.ERROR
]:
repair_dict = add_repair_dict_entry(repository.name,
repository.error_message)
else:
irm = install_manager.InstallRepositoryManager(self.app, tpm)
# We have an installed tool shed repository, so handle tool dependencies if necessary.
if repository.missing_tool_dependencies and metadata and 'tool_dependencies' in metadata:
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-itdep")
# Reset missing tool dependencies.
for tool_dependency in repository.missing_tool_dependencies:
if tool_dependency.status in [
self.app.install_model.ToolDependency.
installation_status.ERROR, self.app.install_model.
ToolDependency.installation_status.INSTALLING
]:
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED )
# Install tool dependencies.
irm.update_tool_shed_repository_status(
repository, self.app.install_model.ToolShedRepository.
installation_status.INSTALLING_TOOL_DEPENDENCIES)
# Get the tool_dependencies.xml file from the repository.
tool_dependencies_config = hg_util.get_config_from_disk(
'tool_dependencies.xml', repository.repo_path(self.app))
itdm = install_manager.InstallToolDependencyManager(self.app)
installed_tool_dependencies = itdm.install_specified_tool_dependencies(
tool_shed_repository=repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=repository.tool_dependencies,
from_tool_migration_manager=False)
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status in [
self.app.install_model.ToolDependency.
installation_status.ERROR
]:
repair_dict = add_repair_dict_entry(
repository.name,
installed_tool_dependency.error_message)
basic_util.remove_dir(work_dir)
irm.update_tool_shed_repository_status(
repository, self.app.install_model.ToolShedRepository.
installation_status.INSTALLED)
return repair_dict
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True)
if install_dependencies and tool_dependencies and has_tool_dependencies:
# Install tool dependencies.
irm = install_manager.InstallRepositoryManager(self.app, self.tpm)
itdm = install_manager.InstallToolDependencyManager(self.app)
irm.update_tool_shed_repository_status(
tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.
INSTALLING_TOOL_DEPENDENCIES)
# Get the tool_dependencies.xml file from disk.
tool_dependencies_config = hg_util.get_config_from_disk(
'tool_dependencies.xml', repo_install_dir)
installed_tool_dependencies = itdm.install_specified_tool_dependencies(
tool_shed_repository=tool_shed_repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=tool_dependencies,
from_tool_migration_manager=True)
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR:
log.error(
'The ToolMigrationManager returned the following error while installing tool dependency %s: %s',
installed_tool_dependency.name,
installed_tool_dependency.error_message)
if 'datatypes' in irmm_metadata_dict:
cdl = custom_datatype_manager.CustomDatatypeLoader(self.app)
tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
