def guess_shed_config( self, app, default=None ): tool_ids = [] metadata = self.metadata or {} for tool in metadata.get( 'tools', [] ): tool_ids.append( tool.get( 'guid' ) ) for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): name = shed_tool_conf_dict[ 'config_filename' ] for elem in shed_tool_conf_dict[ 'config_elems' ]: if elem.tag == 'tool': for sub_elem in elem.findall( 'id' ): tool_id = sub_elem.text.strip() if tool_id in tool_ids: self.shed_config_filename = name return shed_tool_conf_dict elif elem.tag == "section": for tool_elem in elem.findall( 'tool' ): for sub_elem in tool_elem.findall( 'id' ): tool_id = sub_elem.text.strip() if tool_id in tool_ids: self.shed_config_filename = name return shed_tool_conf_dict if self.includes_datatypes: # We need to search by file paths here, which is less desirable. tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): self.shed_config_filename = shed_tool_conf_dict[ 'config_filename' ] return shed_tool_conf_dict return default
def repo_path( self, app ): tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): return relative_path return None
def repo_path( self, app ): tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): return relative_path return None
def repo_path(self, app): tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed) for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs): tool_path = shed_tool_conf_dict['tool_path'] relative_path = os.path.join(tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision) if os.path.exists(relative_path): return relative_path return None
def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ): """ Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. """ self.app = app self.toolbox = self.app.toolbox self.migrated_tools_config = migrated_tools_config # Initialize the ToolPanelManager. self.tpm = tool_panel_manager.ToolPanelManager( self.app ) # If install_dependencies is True but tool_dependency_dir is not set, do not attempt # to install but print informative error message. if install_dependencies and app.config.tool_dependency_dir is None: message = 'You are attempting to install tool dependencies but do not have a value ' message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this ' message += 'location value to the path where you want tool dependencies installed and ' message += 'rerun the migration script.' raise Exception( message ) # Get the local non-shed related tool panel configs (there can be more than one, and the # default name is tool_conf.xml). self.proprietary_tool_confs = self.non_shed_tool_panel_configs self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number ) # Set the location where the repositories will be installed by retrieving the tool_path # setting from migrated_tools_config. tree, error_message = xml_util.parse_xml( migrated_tools_config ) if tree is None: print error_message else: root = tree.getroot() self.tool_path = root.get( 'tool_path' ) print "Repositories will be installed into configured tool_path location ", str( self.tool_path ) # Parse tool_shed_install_config to check each of the tools. self.tool_shed_install_config = tool_shed_install_config tree, error_message = xml_util.parse_xml( tool_shed_install_config ) if tree is None: print error_message else: root = tree.getroot() defined_tool_shed_url = root.get( 'name' ) self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER index, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed # on disk. The default behavior is that the tool shed is down. tool_shed_accessible = False tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} tool_shed_accessible, missing_tool_configs_dict = \ common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number ) else: # It doesn't matter if the tool shed is accessible since there are no migrated # tools defined in the local Galaxy instance, but we have to set the value of # tool_shed_accessible to True so that the value of migrate_tools.version can # be correctly set in the database. tool_shed_accessible = True missing_tool_configs_dict = odict() if tool_shed_accessible: if len( self.proprietary_tool_confs ) == 1: plural = '' file_names = self.proprietary_tool_confs[ 0 ] else: plural = 's' file_names = ', '.join( self.proprietary_tool_confs ) if missing_tool_configs_dict: for proprietary_tool_conf in self.proprietary_tool_confs: # Create a backup of the tool configuration in the un-migrated state. shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % ( proprietary_tool_conf, latest_migration_script_number ) ) for repository_elem in root: # Make sure we have a valid repository tag. if self.__is_valid_repository_tag( repository_elem ): # Get all repository dependencies for the repository defined by the # current repository_elem. Repository dependency definitions contained # in tool shed repositories with migrated tools must never define a # relationship to a repository dependency that contains a tool. The # repository dependency can only contain items that are not loaded into # the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes, # etc). This restriction must be followed down the entire dependency hierarchy. name = repository_elem.get( 'name' ) changeset_revision = repository_elem.get( 'changeset_revision' ) tool_shed_accessible, repository_dependencies_dict = \ common_util.get_repository_dependencies( app, self.tool_shed_url, name, self.repository_owner, changeset_revision ) # Make sure all repository dependency records exist (as tool_shed_repository # table rows) in the Galaxy database. created_tool_shed_repositories = \ self.create_or_update_tool_shed_repository_records( name, changeset_revision, repository_dependencies_dict ) # Order the repositories for proper installation. This process is similar to the # process used when installing tool shed repositories, but does not handle managing # tool panel sections and other components since repository dependency definitions # contained in tool shed repositories with migrated tools must never define a relationship # to a repository dependency that contains a tool. ordered_tool_shed_repositories = \ self.order_repositories_for_installation( created_tool_shed_repositories, repository_dependencies_dict ) for tool_shed_repository in ordered_tool_shed_repositories: is_repository_dependency = self.__is_repository_dependency( name, changeset_revision, tool_shed_repository ) self.install_repository( repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=is_repository_dependency ) else: message = "\nNo tools associated with migration stage %s are defined in your " % \ str( latest_migration_script_number ) message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \ ( plural, file_names ) print message else: message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \ str( latest_migration_script_number ) message += "Try again later.\n" print message
def __init__(self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies): """ Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. """ self.app = app self.toolbox = self.app.toolbox self.migrated_tools_config = migrated_tools_config # Initialize the ToolPanelManager. self.tpm = tool_panel_manager.ToolPanelManager(self.app) # If install_dependencies is True but tool_dependency_dir is not set, do not attempt # to install but print informative error message. if install_dependencies and app.config.tool_dependency_dir is None: message = 'You are attempting to install tool dependencies but do not have a value ' message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this ' message += 'location value to the path where you want tool dependencies installed and ' message += 'rerun the migration script.' raise Exception(message) # Get the local non-shed related tool panel configs (there can be more than one, and the # default name is tool_conf.xml). self.proprietary_tool_confs = self.non_shed_tool_panel_configs self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number) # Set the location where the repositories will be installed by retrieving the tool_path # setting from migrated_tools_config. tree, error_message = xml_util.parse_xml(migrated_tools_config) if tree is None: log.error(error_message) else: root = tree.getroot() self.tool_path = root.get('tool_path') log.debug( "Repositories will be installed into configured tool_path location ", str(self.tool_path)) # Parse tool_shed_install_config to check each of the tools. self.tool_shed_install_config = tool_shed_install_config tree, error_message = xml_util.parse_xml(tool_shed_install_config) if tree is None: log.error(error_message) else: root = tree.getroot() defined_tool_shed_url = root.get('name') self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url) self.repository_owner = common_util.REPOSITORY_OWNER self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed # on disk. The default behavior is that the tool shed is down. tool_shed_accessible = False tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} tool_shed_accessible, missing_tool_configs_dict = \ common_util.check_for_missing_tools(app, tool_panel_configs, latest_migration_script_number) else: # It doesn't matter if the tool shed is accessible since there are no migrated # tools defined in the local Galaxy instance, but we have to set the value of # tool_shed_accessible to True so that the value of migrate_tools.version can # be correctly set in the database. tool_shed_accessible = True missing_tool_configs_dict = odict() if tool_shed_accessible: if len(self.proprietary_tool_confs) == 1: plural = '' file_names = self.proprietary_tool_confs[0] else: plural = 's' file_names = ', '.join(self.proprietary_tool_confs) if missing_tool_configs_dict: for proprietary_tool_conf in self.proprietary_tool_confs: # Create a backup of the tool configuration in the un-migrated state. shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % (proprietary_tool_conf, latest_migration_script_number)) for repository_elem in root: # Make sure we have a valid repository tag. if self.__is_valid_repository_tag(repository_elem): # Get all repository dependencies for the repository defined by the # current repository_elem. Repository dependency definitions contained # in tool shed repositories with migrated tools must never define a # relationship to a repository dependency that contains a tool. The # repository dependency can only contain items that are not loaded into # the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes, # etc). This restriction must be followed down the entire dependency hierarchy. name = repository_elem.get('name') changeset_revision = repository_elem.get( 'changeset_revision') tool_shed_accessible, repository_dependencies_dict = \ common_util.get_repository_dependencies(app, self.tool_shed_url, name, self.repository_owner, changeset_revision) # Make sure all repository dependency records exist (as tool_shed_repository # table rows) in the Galaxy database. created_tool_shed_repositories = \ self.create_or_update_tool_shed_repository_records(name, changeset_revision, repository_dependencies_dict) # Order the repositories for proper installation. This process is similar to the # process used when installing tool shed repositories, but does not handle managing # tool panel sections and other components since repository dependency definitions # contained in tool shed repositories with migrated tools must never define a relationship # to a repository dependency that contains a tool. ordered_tool_shed_repositories = \ self.order_repositories_for_installation(created_tool_shed_repositories, repository_dependencies_dict) for tool_shed_repository in ordered_tool_shed_repositories: is_repository_dependency = self.__is_repository_dependency( name, changeset_revision, tool_shed_repository) self.install_repository( repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency= is_repository_dependency) else: message = "\nNo tools associated with migration stage %s are defined in your " % \ str(latest_migration_script_number) message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \ (plural, file_names) log.info(message) else: message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \ str(latest_migration_script_number) message += "Try again later.\n" log.error(message)