def handle_repository_contents(self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies, is_repository_dependency=False): """ Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository contains tools that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each of the tools defined in the received repository_elem from all non-shed-related tool panel configuration files since the entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process. """ tool_configs_to_filter = [] tool_panel_dict_for_display = odict() if self.tool_path: repo_install_dir = os.path.join(self.tool_path, relative_install_dir) else: repo_install_dir = relative_install_dir if not is_repository_dependency: for tool_elem in repository_elem: # The tool_elem looks something like this: # <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> tool_config = tool_elem.get('file') guid = self.get_guid(repository_clone_url, relative_install_dir, tool_config) # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. is_displayed, tool_sections = self.get_containing_tool_sections( tool_config) if is_displayed: tool_panel_dict_for_tool_config = \ self.tpm.generate_tool_panel_dict_for_tool_config(guid, tool_config, tool_sections=tool_sections) # The tool-panel_dict has the following structure. # {<Tool guid> : [{ tool_config : <tool_config_file>, # id: <ToolSection id>, # version : <ToolSection version>, # name : <TooSection name>}]} for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[k] = v for tool_panel_dict in v: # Keep track of tool config file names associated with entries that have been made to the # migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs. if tool_config not in tool_configs_to_filter: tool_configs_to_filter.append(tool_config) else: log.error( 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' % (guid, tool_config, ", ".join( self.proprietary_tool_confs or []))) if tool_configs_to_filter: lock = threading.Lock() lock.acquire(True) try: self.filter_and_persist_proprietary_tool_panel_configs( tool_configs_to_filter) except Exception: log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs" ) finally: lock.release() irmm = InstalledRepositoryMetadataManager( app=self.app, tpm=self.tpm, repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, repository_clone_url=repository_clone_url, shed_config_dict=self.shed_config_dict, relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=True) irmm.generate_metadata_for_changeset_revision() irmm_metadata_dict = irmm.get_metadata_dict() tool_shed_repository.metadata = irmm_metadata_dict self.app.install_model.context.add(tool_shed_repository) self.app.install_model.context.flush() has_tool_dependencies = self.__has_tool_dependencies( irmm_metadata_dict) if has_tool_dependencies: # All tool_dependency objects must be created before the tools are processed even if no # tool dependencies will be installed. tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True) else: tool_dependencies = None if 'tools' in irmm_metadata_dict: tdtm = data_table_manager.ToolDataTableManager(self.app) sample_files = irmm_metadata_dict.get('sample_files', []) sample_files = [str(s) for s in sample_files] tool_index_sample_files = tdtm.get_tool_index_sample_files( sample_files) tool_util.copy_sample_files(self.app, tool_index_sample_files, tool_path=self.tool_path) sample_files_copied = [s for s in tool_index_sample_files] repository_tools_tups = irmm.get_repository_tools_tups() if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically # generated select lists. repository_tools_tups = tdtm.handle_missing_data_table_entry( relative_install_dir, self.tool_path, repository_tools_tups) # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups, sample_files_copied) # Copy remaining sample files included in the repository to the ~/tool-data # directory of the local Galaxy instance. tool_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, sample_files_copied=sample_files_copied) if not is_repository_dependency: self.tpm.add_to_tool_panel( tool_shed_repository.name, repository_clone_url, tool_shed_repository.installed_changeset_revision, repository_tools_tups, self.repository_owner, self.migrated_tools_config, tool_panel_dict=tool_panel_dict_for_display, new_install=True) if install_dependencies and tool_dependencies and has_tool_dependencies: # Install tool dependencies. irm = install_manager.InstallRepositoryManager(self.app, self.tpm) itdm = install_manager.InstallToolDependencyManager(self.app) irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status. INSTALLING_TOOL_DEPENDENCIES) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=tool_dependencies, from_tool_migration_manager=True) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR: log.error( 'The ToolMigrationManager returned the following error while installing tool dependency %s: %s', installed_tool_dependency.name, installed_tool_dependency.error_message) if 'datatypes' in irmm_metadata_dict: cdl = custom_datatype_manager.CustomDatatypeLoader(self.app) tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True self.app.install_model.context.add(tool_shed_repository) self.app.install_model.context.flush() work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-hrc") datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir) # Load proprietary data types required by tools. The value of override is not # important here since the Galaxy server will be started after this installation # completes. converter_path, display_path = \ cdl.alter_config_and_load_prorietary_datatypes(datatypes_config, repo_install_dir, override=False) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ cdl.create_repository_dict_for_proprietary_datatypes(tool_shed=self.tool_shed_url, name=tool_shed_repository.name, owner=self.repository_owner, installed_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dicts=irmm_metadata_dict.get('tools', []), converter_path=converter_path, display_path=display_path) if converter_path: # Load proprietary datatype converters self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict) if display_path: # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( self.app, installed_repository_dict=repository_dict) basic_util.remove_dir(work_dir)
self.app.install_model.ToolDependency. installation_status.ERROR, self.app.install_model. ToolDependency.installation_status.INSTALLING ]: tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED ) # Install tool dependencies. irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository. installation_status.INSTALLING_TOOL_DEPENDENCIES) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path(self.app)) itdm = install_manager.InstallToolDependencyManager(self.app) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=repository.tool_dependencies, from_tool_migration_manager=False) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status in [ self.app.install_model.ToolDependency. installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message) basic_util.remove_dir(work_dir) irm.update_tool_shed_repository_status(
def repair_tool_shed_repository(self, repository, repo_info_dict): def add_repair_dict_entry(repository_name, error_message): if repository_name in repair_dict: repair_dict[repository_name].append(error_message) else: repair_dict[repository_name] = [error_message] return repair_dict tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, repository.tool_shed) metadata = repository.metadata # The repository.metadata contains dependency information that corresponds to the current changeset revision, # which may be different from what is stored in the database # If any of these repository-repository dependency associations is obsolete, clean_dependency_relationships removes them. suc.clean_dependency_relationships(self.app, metadata, repository, tool_shed_url) repair_dict = {} tpm = tool_panel_manager.ToolPanelManager(self.app) if repository.status in [ self.app.install_model.ToolShedRepository.installation_status. DEACTIVATED ]: try: self.app.installed_repository_manager.activate_repository( repository) except Exception as e: error_message = "Error activating repository %s: %s" % ( repository.name, str(e)) log.debug(error_message) repair_dict[repository.name] = error_message elif repository.status not in [ self.app.install_model.ToolShedRepository.installation_status. INSTALLED ]: shed_tool_conf, tool_path, relative_install_dir = \ suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) # Reset the repository attributes to the New state for installation. if metadata: _, tool_panel_section_key = \ tpm.handle_tool_panel_selection( self.app.toolbox, metadata, no_changes_checked=True, tool_panel_section_id=None, new_tool_panel_section_label=None ) else: # The tools will be loaded outside of any sections in the tool panel. tool_panel_section_key = None repository_util.set_repository_attributes( self.app, repository, status=self.app.install_model.ToolShedRepository. installation_status.NEW, error_message=None, deleted=False, uninstalled=False, remove_from_disk=True) irm = install_manager.InstallRepositoryManager(self.app, tpm) irm.install_tool_shed_repository( repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies=True, install_resolver_dependencies= False, # Assuming repairs are only necessary toolshed packages reinstalling=True) if repository.status in [ self.app.install_model.ToolShedRepository. installation_status.ERROR ]: repair_dict = add_repair_dict_entry(repository.name, repository.error_message) else: irm = install_manager.InstallRepositoryManager(self.app, tpm) # We have an installed tool shed repository, so handle tool dependencies if necessary. if repository.missing_tool_dependencies and metadata and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-itdep") # Reset missing tool dependencies. for tool_dependency in repository.missing_tool_dependencies: if tool_dependency.status in [ self.app.install_model.ToolDependency. installation_status.ERROR, self.app.install_model. ToolDependency.installation_status.INSTALLING ]: tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED ) # Install tool dependencies. irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository. installation_status.INSTALLING_TOOL_DEPENDENCIES) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path(self.app)) itdm = install_manager.InstallToolDependencyManager(self.app) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=repository.tool_dependencies, from_tool_migration_manager=False) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status in [ self.app.install_model.ToolDependency. installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message) basic_util.remove_dir(work_dir) irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository. installation_status.INSTALLED) return repair_dict