def load_installed_datatypes(self, repository, relative_install_dir, deactivate=False): """ Load proprietary datatypes and return information needed for loading custom datatypes converters and display applications later. """ metadata = repository.metadata repository_dict = None datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir) if datatypes_config: converter_path, display_path = \ self.alter_config_and_load_prorietary_datatypes(datatypes_config, relative_install_dir, deactivate=deactivate) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ self.create_repository_dict_for_proprietary_datatypes(tool_shed=repository.tool_shed, name=repository.name, owner=repository.owner, installed_changeset_revision=repository.installed_changeset_revision, tool_dicts=metadata.get('tools', []), converter_path=converter_path, display_path=display_path) return repository_dict
def handle_missing_data_table_entry(self, relative_install_dir, tool_path, repository_tools_tups): """ Inspect each tool to see if any have input parameters that are dynamically generated select lists that require entries in the tool_data_table_conf.xml file. This method is called only from Galaxy (not the tool shed) when a repository is being installed or reinstalled. """ missing_data_table_entry = False for index, repository_tools_tup in enumerate(repository_tools_tups): tup_path, guid, repository_tool = repository_tools_tup if repository_tool.params_with_missing_data_table_entry: missing_data_table_entry = True break if missing_data_table_entry: # The repository must contain a tool_data_table_conf.xml.sample file that includes # all required entries for all tools in the repository. sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', relative_install_dir) if sample_tool_data_table_conf: # Add entries to the ToolDataTableManager's in-memory data_tables dictionary. error, message = self.handle_sample_tool_data_table_conf_file( sample_tool_data_table_conf, persist=True) if error: # TODO: Do more here than logging an exception. log.debug(message) # Reload the tool into the local list of repository_tools_tups. repository_tool = self.app.toolbox.load_tool(os.path.join( tool_path, tup_path), guid=guid, use_cached=False) repository_tools_tups[index] = (tup_path, guid, repository_tool) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.reset_tool_data_tables() return repository_tools_tups
def check_tool_input_params(self, repo_dir, tool_config_name, tool, sample_files): """ Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make sure the files exist. """ invalid_files_and_errors_tups = [] correction_msg = '' for input_param in tool.input_params: if isinstance(input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, # make sure all requirements exist. options = input_param.dynamic_options or input_param.options if options and isinstance(options, dynamic_options.DynamicOptions): if options.tool_data_table or options.missing_tool_data_table_name: # Make sure the repository contains a tool_data_table_conf.xml.sample file. sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir) if sample_tool_data_table_conf: error, correction_msg = \ self.tdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf, persist=False) if error: invalid_files_and_errors_tups.append( ('tool_data_table_conf.xml.sample', correction_msg)) else: correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. " correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository " correction_msg += "that includes the required entry to correct this error.<br/>" invalid_tup = (tool_config_name, correction_msg) if invalid_tup not in invalid_files_and_errors_tups: invalid_files_and_errors_tups.append( invalid_tup) if options.index_file or options.missing_index_file: # Make sure the repository contains the required xxx.loc.sample file. index_file = options.index_file or options.missing_index_file index_file_name = basic_util.strip_path(index_file) sample_found = False for sample_file in sample_files: sample_file_name = basic_util.strip_path( sample_file) if sample_file_name == '%s.sample' % index_file_name: options.index_file = index_file_name if options.tool_data_table: options.tool_data_table.missing_index_file = None sample_found = True break if not sample_found: correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file_name) correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % \ str(index_file_name) invalid_files_and_errors_tups.append( (tool_config_name, correction_msg)) return invalid_files_and_errors_tups
def create_tool_dependency_objects(app, tool_shed_repository, relative_install_dir, set_status=True): """ Create or update a ToolDependency for each entry in tool_dependencies_config. This method is called when installing a new tool_shed_repository. """ tool_dependency_objects = [] shed_config_dict = tool_shed_repository.get_shed_config_dict(app) if shed_config_dict.get('tool_path'): relative_install_dir = os.path.join(shed_config_dict.get('tool_path'), relative_install_dir) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', relative_install_dir) tree, error_message = xml_util.parse_xml(tool_dependencies_config) if tree is None: return tool_dependency_objects root = tree.getroot() for elem in root: tool_dependency_type = elem.tag if tool_dependency_type == 'package': name = elem.get('name', None) version = elem.get('version', None) if name and version: status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED tool_dependency = create_or_update_tool_dependency( app, tool_shed_repository, name=name, version=version, type=tool_dependency_type, status=status, set_status=set_status) tool_dependency_objects.append(tool_dependency) elif tool_dependency_type == 'set_environment': for env_elem in elem: # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> name = env_elem.get('name', None) action = env_elem.get('action', None) if name and action: status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED tool_dependency = create_or_update_tool_dependency( app, tool_shed_repository, name=name, version=None, type=tool_dependency_type, status=status, set_status=set_status) tool_dependency_objects.append(tool_dependency) return tool_dependency_objects
for tool_dependency in repository.missing_tool_dependencies: if tool_dependency.status in [ self.app.install_model.ToolDependency. installation_status.ERROR, self.app.install_model. ToolDependency.installation_status.INSTALLING ]: tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED ) # Install tool dependencies. irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository. installation_status.INSTALLING_TOOL_DEPENDENCIES) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path(self.app)) itdm = install_manager.InstallToolDependencyManager(self.app) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=repository.tool_dependencies, from_tool_migration_manager=False) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status in [ self.app.install_model.ToolDependency. installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message) basic_util.remove_dir(work_dir)
def handle_repository_contents(self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies, is_repository_dependency=False): """ Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository contains tools that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each of the tools defined in the received repository_elem from all non-shed-related tool panel configuration files since the entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process. """ tool_configs_to_filter = [] tool_panel_dict_for_display = odict() if self.tool_path: repo_install_dir = os.path.join(self.tool_path, relative_install_dir) else: repo_install_dir = relative_install_dir if not is_repository_dependency: for tool_elem in repository_elem: # The tool_elem looks something like this: # <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> tool_config = tool_elem.get('file') guid = self.get_guid(repository_clone_url, relative_install_dir, tool_config) # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. is_displayed, tool_sections = self.get_containing_tool_sections( tool_config) if is_displayed: tool_panel_dict_for_tool_config = \ self.tpm.generate_tool_panel_dict_for_tool_config(guid, tool_config, tool_sections=tool_sections) # The tool-panel_dict has the following structure. # {<Tool guid> : [{ tool_config : <tool_config_file>, # id: <ToolSection id>, # version : <ToolSection version>, # name : <TooSection name>}]} for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[k] = v for tool_panel_dict in v: # Keep track of tool config file names associated with entries that have been made to the # migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs. if tool_config not in tool_configs_to_filter: tool_configs_to_filter.append(tool_config) else: log.error( 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' % (guid, tool_config, ", ".join( self.proprietary_tool_confs or []))) if tool_configs_to_filter: lock = threading.Lock() lock.acquire(True) try: self.filter_and_persist_proprietary_tool_panel_configs( tool_configs_to_filter) except Exception: log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs" ) finally: lock.release() irmm = InstalledRepositoryMetadataManager( app=self.app, tpm=self.tpm, repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, repository_clone_url=repository_clone_url, shed_config_dict=self.shed_config_dict, relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=True) irmm.generate_metadata_for_changeset_revision() irmm_metadata_dict = irmm.get_metadata_dict() tool_shed_repository.metadata = irmm_metadata_dict self.app.install_model.context.add(tool_shed_repository) self.app.install_model.context.flush() has_tool_dependencies = self.__has_tool_dependencies( irmm_metadata_dict) if has_tool_dependencies: # All tool_dependency objects must be created before the tools are processed even if no # tool dependencies will be installed. tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True) else: tool_dependencies = None if 'tools' in irmm_metadata_dict: tdtm = data_table_manager.ToolDataTableManager(self.app) sample_files = irmm_metadata_dict.get('sample_files', []) sample_files = [str(s) for s in sample_files] tool_index_sample_files = tdtm.get_tool_index_sample_files( sample_files) tool_util.copy_sample_files(self.app, tool_index_sample_files, tool_path=self.tool_path) sample_files_copied = [s for s in tool_index_sample_files] repository_tools_tups = irmm.get_repository_tools_tups() if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically # generated select lists. repository_tools_tups = tdtm.handle_missing_data_table_entry( relative_install_dir, self.tool_path, repository_tools_tups) # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups, sample_files_copied) # Copy remaining sample files included in the repository to the ~/tool-data # directory of the local Galaxy instance. tool_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, sample_files_copied=sample_files_copied) if not is_repository_dependency: self.tpm.add_to_tool_panel( tool_shed_repository.name, repository_clone_url, tool_shed_repository.installed_changeset_revision, repository_tools_tups, self.repository_owner, self.migrated_tools_config, tool_panel_dict=tool_panel_dict_for_display, new_install=True) if install_dependencies and tool_dependencies and has_tool_dependencies: # Install tool dependencies. irm = install_manager.InstallRepositoryManager(self.app, self.tpm) itdm = install_manager.InstallToolDependencyManager(self.app) irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status. INSTALLING_TOOL_DEPENDENCIES) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=tool_dependencies, from_tool_migration_manager=True) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR: log.error( 'The ToolMigrationManager returned the following error while installing tool dependency %s: %s', installed_tool_dependency.name, installed_tool_dependency.error_message) if 'datatypes' in irmm_metadata_dict: cdl = custom_datatype_manager.CustomDatatypeLoader(self.app) tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True self.app.install_model.context.add(tool_shed_repository) self.app.install_model.context.flush() work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-hrc") datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir) # Load proprietary data types required by tools. The value of override is not # important here since the Galaxy server will be started after this installation # completes. converter_path, display_path = \ cdl.alter_config_and_load_prorietary_datatypes(datatypes_config, repo_install_dir, override=False) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ cdl.create_repository_dict_for_proprietary_datatypes(tool_shed=self.tool_shed_url, name=tool_shed_repository.name, owner=self.repository_owner, installed_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dicts=irmm_metadata_dict.get('tools', []), converter_path=converter_path, display_path=display_path) if converter_path: # Load proprietary datatype converters self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict) if display_path: # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( self.app, installed_repository_dict=repository_dict) basic_util.remove_dir(work_dir)
def repair_tool_shed_repository(self, repository, repo_info_dict): def add_repair_dict_entry(repository_name, error_message): if repository_name in repair_dict: repair_dict[repository_name].append(error_message) else: repair_dict[repository_name] = [error_message] return repair_dict tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, repository.tool_shed) metadata = repository.metadata # The repository.metadata contains dependency information that corresponds to the current changeset revision, # which may be different from what is stored in the database # If any of these repository-repository dependency associations is obsolete, clean_dependency_relationships removes them. suc.clean_dependency_relationships(self.app, metadata, repository, tool_shed_url) repair_dict = {} tpm = tool_panel_manager.ToolPanelManager(self.app) if repository.status in [ self.app.install_model.ToolShedRepository.installation_status. DEACTIVATED ]: try: self.app.installed_repository_manager.activate_repository( repository) except Exception as e: error_message = "Error activating repository %s: %s" % ( repository.name, str(e)) log.debug(error_message) repair_dict[repository.name] = error_message elif repository.status not in [ self.app.install_model.ToolShedRepository.installation_status. INSTALLED ]: shed_tool_conf, tool_path, relative_install_dir = \ suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) # Reset the repository attributes to the New state for installation. if metadata: _, tool_panel_section_key = \ tpm.handle_tool_panel_selection( self.app.toolbox, metadata, no_changes_checked=True, tool_panel_section_id=None, new_tool_panel_section_label=None ) else: # The tools will be loaded outside of any sections in the tool panel. tool_panel_section_key = None repository_util.set_repository_attributes( self.app, repository, status=self.app.install_model.ToolShedRepository. installation_status.NEW, error_message=None, deleted=False, uninstalled=False, remove_from_disk=True) irm = install_manager.InstallRepositoryManager(self.app, tpm) irm.install_tool_shed_repository( repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies=True, install_resolver_dependencies= False, # Assuming repairs are only necessary toolshed packages reinstalling=True) if repository.status in [ self.app.install_model.ToolShedRepository. installation_status.ERROR ]: repair_dict = add_repair_dict_entry(repository.name, repository.error_message) else: irm = install_manager.InstallRepositoryManager(self.app, tpm) # We have an installed tool shed repository, so handle tool dependencies if necessary. if repository.missing_tool_dependencies and metadata and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-itdep") # Reset missing tool dependencies. for tool_dependency in repository.missing_tool_dependencies: if tool_dependency.status in [ self.app.install_model.ToolDependency. installation_status.ERROR, self.app.install_model. ToolDependency.installation_status.INSTALLING ]: tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED ) # Install tool dependencies. irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository. installation_status.INSTALLING_TOOL_DEPENDENCIES) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path(self.app)) itdm = install_manager.InstallToolDependencyManager(self.app) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=repository.tool_dependencies, from_tool_migration_manager=False) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status in [ self.app.install_model.ToolDependency. installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message) basic_util.remove_dir(work_dir) irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository. installation_status.INSTALLED) return repair_dict
tool_shed_repository.installed_changeset_revision, repository_tools_tups, self.repository_owner, self.migrated_tools_config, tool_panel_dict=tool_panel_dict_for_display, new_install=True) if install_dependencies and tool_dependencies and has_tool_dependencies: # Install tool dependencies. irm = install_manager.InstallRepositoryManager(self.app, self.tpm) itdm = install_manager.InstallToolDependencyManager(self.app) irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status. INSTALLING_TOOL_DEPENDENCIES) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=tool_dependencies, from_tool_migration_manager=True) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR: log.error( 'The ToolMigrationManager returned the following error while installing tool dependency %s: %s', installed_tool_dependency.name, installed_tool_dependency.error_message) if 'datatypes' in irmm_metadata_dict: cdl = custom_datatype_manager.CustomDatatypeLoader(self.app) tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: