def tool_shed_is_this_tool_shed(toolshed_base_url): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url) cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url( str(url_for('/', qualified=True))) return cleaned_toolshed_base_url == cleaned_tool_shed
def remove_repository_dependency_reference_to_self( self, key_rd_dicts ): """Remove all repository dependencies that point to a revision within its own repository.""" clean_key_rd_dicts = [] key = next(iter(key_rd_dicts[ 0 ])) repository_tup = key.split( container_util.STRSEP ) rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \ rd_prior_installation_required, \ rd_only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( repository_tup ) cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) for key_rd_dict in key_rd_dicts: k = next(iter(key_rd_dict)) repository_dependency = key_rd_dict[ k ] toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( repository_dependency ) cleaned_toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) if cleaned_rd_toolshed == cleaned_toolshed and rd_name == name and rd_owner == owner: debug_msg = "Removing repository dependency for repository %s owned by %s " % ( name, owner ) debug_msg += 'since it refers to a revision within itself.' log.debug( debug_msg ) else: new_key_rd_dict = {} new_key_rd_dict[ key ] = repository_dependency clean_key_rd_dicts.append( new_key_rd_dict ) return clean_key_rd_dicts
def remove_repository_dependency_reference_to_self(self, key_rd_dicts): """Remove all repository dependencies that point to a revision within its own repository.""" clean_key_rd_dicts = [] key = next(iter(key_rd_dicts[0])) repository_tup = key.split(container_util.STRSEP) rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \ rd_prior_installation_required, \ rd_only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple(repository_tup) cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url(rd_toolshed) for key_rd_dict in key_rd_dicts: k = next(iter(key_rd_dict)) repository_dependency = key_rd_dict[k] toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple(repository_dependency) cleaned_toolshed = common_util.remove_protocol_from_tool_shed_url(toolshed) if cleaned_rd_toolshed == cleaned_toolshed and rd_name == name and rd_owner == owner: debug_msg = "Removing repository dependency for repository {} owned by {} ".format(name, owner) debug_msg += 'since it refers to a revision within itself.' log.debug(debug_msg) else: new_key_rd_dict = {} new_key_rd_dict[key] = repository_dependency clean_key_rd_dicts.append(new_key_rd_dict) return clean_key_rd_dicts
def get_prior_installation_required_and_only_if_compiling_contained_td( self ): """ This method is called from the tool shed and never Galaxy. If self.all_repository_dependencies contains a repository dependency tuple that is associated with self.repository, return the value of the tuple's prior_installation_required component. """ cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( self.tool_shed_url ) if self.all_repository_dependencies: for rd_key, rd_tups in self.all_repository_dependencies.items(): if rd_key in [ 'root_key', 'description' ]: continue for rd_tup in rd_tups: rd_toolshed, \ rd_name, \ rd_owner, \ rd_changeset_revision, \ rd_prior_installation_required, \ rd_only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ rd_name == self.repository.name and \ rd_owner == self.repository.user.username and \ rd_changeset_revision == self.repository_metadata.changeset_revision: return rd_prior_installation_required, rd_only_if_compiling_contained_td elif self.repository_metadata: # Get the list of changeset revisions from the tool shed to which self.repository may be updated. metadata = self.repository_metadata.metadata current_changeset_revision = str( self.repository_metadata.changeset_revision ) # Get the changeset revision to which the current value of required_repository_changeset_revision # should be updated if it's not current. text = suc.get_updated_changeset_revisions( self.app, name=str( self.repository.name ), owner=str( self.repository.user.username ), changeset_revision=current_changeset_revision ) if text: valid_changeset_revisions = listify( text ) if current_changeset_revision not in valid_changeset_revisions: valid_changeset_revisions.append( current_changeset_revision ) else: valid_changeset_revisions = [ current_changeset_revision ] repository_dependencies_dict = metadata[ 'repository_dependencies' ] rd_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) for rd_tup in rd_tups: rd_toolshed, \ rd_name, \ rd_owner, \ rd_changeset_revision, \ rd_prior_installation_required, \ rd_only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed ) if cleaned_rd_toolshed == cleaned_toolshed_base_url and \ rd_name == self.repository.name and \ rd_owner == self.repository.user.username and \ rd_changeset_revision in valid_changeset_revisions: return rd_prior_installation_required, rd_only_if_compiling_contained_td # Default both prior_installation_required and only_if_compiling_contained_td to False. return 'False', 'False'
def get_or_create_tool_shed_repository( self, tool_shed, name, owner, changeset_revision ): """ Return a tool shed repository database record defined by the combination of tool shed, repository name, repository owner and changeset_revision or installed_changeset_revision. A new tool shed repository record will be created if one is not located. """ install_model = self.app.install_model # We store the port in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) # This method is used only in Galaxy, not the tool shed. repository = suc.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision ) if not repository: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed ) repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) ctx_rev = suc.get_ctx_rev( self.app, tool_shed_url, name, owner, changeset_revision ) repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=None, installed_changeset_revision=changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=install_model.ToolShedRepository.installation_status.NEW, current_changeset_revision=None, owner=owner, dist_to_shed=False ) return repository
def password_manager_for_url(self, url): """ If the tool shed is using external auth, the client to the tool shed must authenticate to that as well. This provides access to the six.moves.urllib.request.HTTPPasswordMgrWithdefaultRealm() object for the url passed in. Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at some stage... """ url_sans_protocol = common_util.remove_protocol_from_tool_shed_url(url) for shed_name, shed_url in self.tool_sheds.items(): shed_url_sans_protocol = common_util.remove_protocol_from_tool_shed_url(shed_url) if url_sans_protocol.startswith(shed_url_sans_protocol): return self.tool_sheds_auth[shed_name] log.debug("Invalid url '%s' received by tool shed registry's password_manager_for_url method." % str(url)) return None
def load_from_element( self, elem, tool_path ): assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag ) self.declared_id = elem.get( 'id', None ) self.guid = elem.get( 'guid', None ) path = elem.get( 'tool_file', None ) self.version = elem.get( 'version', self.version ) tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find( 'tool' ) assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) ) path = tool_elem.get( "file", None ) tool_guid = tool_elem.get( "guid", None ) # need to determine repository info so that dependencies will work correctly tool_shed_url = tool_elem.find( 'tool_shed' ).text # Handle protocol changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url ) # The protocol is not stored in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url ) repository_name = tool_elem.find( 'repository_name' ).text repository_owner = tool_elem.find( 'repository_owner' ).text installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision ) tool_shed_repository = \ suc.get_installed_repository( self.data_managers.app, tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision ) if tool_shed_repository is None: log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' ) # we'll set tool_path manually here from shed_conf_file tool_shed_repository_id = None try: tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path ) except Exception, e: log.error( 'Error determining tool_path for Data Manager during testing: %s', e ) else: tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) # use shed_conf_file to determine tool_path shed_conf_file = elem.get( "shed_conf_file", None ) if shed_conf_file: shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None ) if shed_conf: tool_path = shed_conf.get( "tool_path", tool_path )
def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td): """ Assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported. """ # The tool_shed portion of the key must be the value that is stored in the tool_shed_repository.tool_shed column # of the Galaxy database for an installed repository. This value does not include the protocol, but does include # the port if there is one. tool_shed = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url) return '%s%s%s%s%s%s%s%s%s%s%s' % ( tool_shed, STRSEP, str(repository_name), STRSEP, str(repository_owner), STRSEP, str(changeset_revision), STRSEP, str(prior_installation_required), STRSEP, str(only_if_compiling_contained_td))
def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ): """ Assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported. """ # The tool_shed portion of the key must be the value that is stored in the tool_shed_repository.tool_shed column # of the Galaxy database for an installed repository. This value does not include the protocol, but does include # the port if there is one. tool_shed = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) return '%s%s%s%s%s%s%s%s%s%s%s' % ( tool_shed, STRSEP, str( repository_name ), STRSEP, str( repository_owner ), STRSEP, str( changeset_revision ), STRSEP, str( prior_installation_required ), STRSEP, str( only_if_compiling_contained_td ) )
def load_from_element(self, elem, tool_path): assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % (elem.tag) self.declared_id = elem.get('id', None) self.guid = elem.get('guid', None) path = elem.get('tool_file', None) self.version = elem.get('version', self.version) tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find('tool') assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % (util.xml_to_string(elem)) path = tool_elem.get("file", None) tool_guid = tool_elem.get("guid", None) # need to determine repository info so that dependencies will work correctly if hasattr(self.data_managers.app, 'tool_cache') and tool_guid in self.data_managers.app.tool_cache._tool_paths_by_id: path = self.data_managers.app.tool_cache._tool_paths_by_id[tool_guid] tool = self.data_managers.app.tool_cache.get_tool(path) tool_shed_repository = tool.tool_shed_repository self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed_repository.tool_shed, name=tool_shed_repository.name, owner=tool_shed_repository.owner, installed_changeset_revision=tool_shed_repository.installed_changeset_revision) tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id) tool_path = "" else: tool_shed_url = tool_elem.find('tool_shed').text # Handle protocol changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.data_managers.app, tool_shed_url) # The protocol is not stored in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed_url) repository_name = tool_elem.find('repository_name').text repository_owner = tool_elem.find('repository_owner').text installed_changeset_revision = tool_elem.find('installed_changeset_revision').text self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision) tool_shed_repository = \ repository_util.get_installed_repository(self.data_managers.app, tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision) if tool_shed_repository is None: log.warning('Could not determine tool shed repository from database. This should only ever happen when running tests.') # we'll set tool_path manually here from shed_conf_file tool_shed_repository_id = None try: tool_path = util.parse_xml(elem.get('shed_conf_file')).getroot().get('tool_path', tool_path) except Exception as e: log.error('Error determining tool_path for Data Manager during testing: %s', e) else: tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id) # use shed_conf_file to determine tool_path shed_conf_file = elem.get("shed_conf_file", None) if shed_conf_file: shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename(shed_conf_file, None) if shed_conf: tool_path = shed_conf.get("tool_path", tool_path) assert path is not None, "A tool file path could not be determined:\n%s" % (util.xml_to_string(elem)) self.load_tool(os.path.join(tool_path, path), guid=tool_guid, data_manager_id=self.id, tool_shed_repository_id=tool_shed_repository_id) self.name = elem.get('name', self.tool.name) self.description = elem.get('description', self.tool.description) self.undeclared_tables = util.asbool(elem.get('undeclared_tables', self.undeclared_tables)) for data_table_elem in elem.findall('data_table'): data_table_name = data_table_elem.get("name") assert data_table_name is not None, "A name is required for a data table entry" if data_table_name not in self.data_tables: self.data_tables[data_table_name] = odict() output_elem = data_table_elem.find('output') if output_elem is not None: for column_elem in output_elem.findall('column'): column_name = column_elem.get('name', None) assert column_name is not None, "Name is required for column entry" data_table_coumn_name = column_elem.get('data_table_name', column_name) self.data_tables[data_table_name][data_table_coumn_name] = column_name output_ref = column_elem.get('output_ref', None) if output_ref is not None: if data_table_name not in self.output_ref_by_data_table: self.output_ref_by_data_table[data_table_name] = {} self.output_ref_by_data_table[data_table_name][data_table_coumn_name] = output_ref value_translation_elems = column_elem.findall('value_translation') if value_translation_elems is not None: for value_translation_elem in value_translation_elems: value_translation = value_translation_elem.text if value_translation is not None: value_translation_type = value_translation_elem.get('type', DEFAULT_VALUE_TRANSLATION_TYPE) if data_table_name not in self.value_translation_by_data_table_column: self.value_translation_by_data_table_column[data_table_name] = {} if data_table_coumn_name not in self.value_translation_by_data_table_column[data_table_name]: self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name] = [] if value_translation_type == 'function': if value_translation in VALUE_TRANSLATION_FUNCTIONS: value_translation = VALUE_TRANSLATION_FUNCTIONS[value_translation] else: raise ValueError("Unsupported value translation function: '%s'" % (value_translation)) else: assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError("Unsupported value translation type: '%s'" % (value_translation_type)) self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name].append(value_translation) for move_elem in column_elem.findall('move'): move_type = move_elem.get('type', 'directory') relativize_symlinks = move_elem.get('relativize_symlinks', False) # TODO: should we instead always relativize links? source_elem = move_elem.find('source') if source_elem is None: source_base = None source_value = '' else: source_base = source_elem.get('base', None) source_value = source_elem.text target_elem = move_elem.find('target') if target_elem is None: target_base = None target_value = '' else: target_base = target_elem.get('base', None) target_value = target_elem.text if data_table_name not in self.move_by_data_table_column: self.move_by_data_table_column[data_table_name] = {} self.move_by_data_table_column[data_table_name][data_table_coumn_name] = \ dict(type=move_type, source_base=source_base, source_value=source_value, target_base=target_base, target_value=target_value, relativize_symlinks=relativize_symlinks)
def load_from_element(self, elem, tool_path): assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag) self.declared_id = elem.get('id', None) self.guid = elem.get('guid', None) path = elem.get('tool_file', None) self.version = elem.get('version', self.version) tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find('tool') assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string(elem)) path = tool_elem.get("file", None) tool_guid = tool_elem.get("guid", None) #need to determine repository info so that dependencies will work correctly tool_shed_url = tool_elem.find('tool_shed').text # Handle protocol changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url) # The protocol is not stored in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url) repository_name = tool_elem.find('repository_name').text repository_owner = tool_elem.find('repository_owner').text installed_changeset_revision = tool_elem.find( 'installed_changeset_revision').text self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision) tool_shed_repository = \ suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.data_managers.app, tool_shed, repository_name, repository_owner, installed_changeset_revision ) if tool_shed_repository is None: log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' ) #we'll set tool_path manually here from shed_conf_file tool_shed_repository_id = None try: tool_path = util.parse_xml( elem.get('shed_conf_file')).getroot().get( 'tool_path', tool_path) except Exception, e: log.error( 'Error determining tool_path for Data Manager during testing: %s', e) else: tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id) #use shed_conf_file to determine tool_path shed_conf_file = elem.get("shed_conf_file", None) if shed_conf_file: shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None) if shed_conf: tool_path = shed_conf.get("tool_path", tool_path)
def get_env_shell_file_paths( self, elem ): # Currently only the following tag set is supported. # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> # <package name="numpy" version="1.7.1" /> # </repository> env_shell_file_paths = [] toolshed = elem.get( 'toolshed', None ) repository_name = elem.get( 'name', None ) repository_owner = elem.get( 'owner', None ) changeset_revision = elem.get( 'changeset_revision', None ) if toolshed and repository_name and repository_owner and changeset_revision: # The protocol is not stored, but the port is if it exists. toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) repository = suc.get_repository_for_dependency_relationship( self.app, toolshed, repository_name, repository_owner, changeset_revision ) if repository: for sub_elem in elem: tool_dependency_type = sub_elem.tag tool_dependency_name = sub_elem.get( 'name' ) tool_dependency_version = sub_elem.get( 'version' ) if tool_dependency_type and tool_dependency_name and tool_dependency_version: # Get the tool_dependency so we can get its installation directory. tool_dependency = None for tool_dependency in repository.tool_dependencies: if tool_dependency.type == tool_dependency_type and \ tool_dependency.name == tool_dependency_name and \ tool_dependency.version == tool_dependency_version: break if tool_dependency: installation_directory = tool_dependency.installation_directory( self.app ) env_shell_file_path = self.get_env_shell_file_path( installation_directory ) if env_shell_file_path: env_shell_file_paths.append( env_shell_file_path ) else: error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " error_message += "type %s, name %s, version %s for repository %s" % \ ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) log.debug( error_message ) continue else: error_message = "Skipping tool dependency definition because unable to locate tool dependency " error_message += "type %s, name %s, version %s for repository %s" % \ ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) log.debug( error_message ) continue else: error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) log.debug( error_message ) continue else: error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) log.debug( error_message ) else: error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) log.debug( error_message ) return env_shell_file_paths
def tool_shed_is_this_tool_shed(toolshed_base_url): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url(toolshed_base_url) cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url(str(url_for('/', qualified=True))) return cleaned_toolshed_base_url == cleaned_tool_shed
def handle_complex_repository_dependency_for_package( self, elem, package_name, package_version, tool_shed_repository, from_tool_migration_manager=False ): """ Inspect the repository defined by a complex repository dependency definition and take certain steps to enable installation of the received package name and version to proceed. The received elem is the <repository> tag set which defines the complex repository dependency. The received tool_shed_repository is the installed tool shed repository for which the tool dependency defined by the received package_name and package_version is being installed. """ handled_tool_dependencies = [] tool_shed_url = elem.attrib[ 'toolshed' ] required_repository_name = elem.attrib[ 'name' ] required_repository_owner = elem.attrib[ 'owner' ] default_required_repository_changeset_revision = elem.attrib[ 'changeset_revision' ] required_repository = repository_util.get_repository_for_dependency_relationship( self.app, tool_shed_url, required_repository_name, required_repository_owner, default_required_repository_changeset_revision ) tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url ) tmp_filename = None if required_repository: required_repository_changeset_revision = required_repository.installed_changeset_revision # Define the installation directory for the required tool dependency package in the required repository. required_repository_package_install_dir = \ tool_dependency_util.get_tool_dependency_install_dir( app=self.app, repository_name=required_repository_name, repository_owner=required_repository_owner, repository_changeset_revision=required_repository_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version ) # Define this dependent repository's tool dependency installation directory that will contain # the env.sh file with a path to the required repository's installed tool dependency package. dependent_install_dir = \ tool_dependency_util.get_tool_dependency_install_dir( app=self.app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version ) if os.path.exists( dependent_install_dir ): # The install manager handles tool migration stages and the sync_database_with_file_system() # method handles two scenarios: (1) where a Galaxy file system environment related to installed # Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy # database tables associated with these installed items, and (2) the Tool Shed's install and test # framework which installs repositories in 2 stages, those of type tool_dependency_definition # followed by those containing valid tools and tool functional test components. Neither of these # scenarios apply when the install manager is running. if from_tool_migration_manager: can_install_tool_dependency = True else: # Notice that we'll throw away the following tool_dependency if it can be installed. tool_dependency, can_install_tool_dependency = self.sync_database_with_file_system( self.app, tool_shed_repository, package_name, package_version, dependent_install_dir, tool_dependency_type='package' ) if not can_install_tool_dependency: log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), " "so appending it to the list of handled tool dependencies.", str( tool_dependency.name), str( tool_dependency.version ) ) handled_tool_dependencies.append( tool_dependency ) else: can_install_tool_dependency = True if can_install_tool_dependency: # Set this dependent repository's tool dependency env.sh file with a path to the required repository's # installed tool dependency package. We can get everything we need from the discovered installed # required_repository. if required_repository.is_deactivated_or_installed: if not os.path.exists( required_repository_package_install_dir ): log.error( 'Missing required tool dependency directory %s' % str( required_repository_package_install_dir ) ) repo_files_dir = required_repository.repo_files_directory( self.app ) if not repo_files_dir: message = "Unable to locate the repository directory for revision %s of installed repository %s owned by %s." % \ ( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) ) raise Exception( message ) tool_dependencies_config = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml' ) if tool_dependencies_config: config_to_use = tool_dependencies_config else: message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \ ( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) ) raise Exception( message ) else: # Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision # should be updated if it's not current. text = metadata_util.get_updated_changeset_revisions_from_tool_shed( app=self.app, tool_shed_url=tool_shed, name=required_repository_name, owner=required_repository_owner, changeset_revision=required_repository_changeset_revision ) if text: updated_changeset_revisions = listify( text ) # The list of changeset revisions is in reverse order, so the newest will be first. required_repository_changeset_revision = updated_changeset_revisions[ 0 ] # Make a call to the tool shed to get the required repository's tool_dependencies.xml file. tmp_filename = self.create_temporary_tool_dependencies_config( tool_shed, required_repository_name, required_repository_owner, required_repository_changeset_revision ) config_to_use = tmp_filename handled_tool_dependencies = \ self.create_tool_dependency_with_initialized_env_sh_file( dependent_install_dir=dependent_install_dir, tool_shed_repository=tool_shed_repository, required_repository=required_repository, package_name=package_name, package_version=package_version, tool_dependencies_config=config_to_use ) self.remove_file( tmp_filename ) else: message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \ ( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) ) raise Exception( message ) return handled_tool_dependencies
def get_env_shell_file_paths(self, elem): # Currently only the following tag set is supported. # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> # <package name="numpy" version="1.7.1" /> # </repository> env_shell_file_paths = [] toolshed = elem.get('toolshed', None) repository_name = elem.get('name', None) repository_owner = elem.get('owner', None) changeset_revision = elem.get('changeset_revision', None) if toolshed and repository_name and repository_owner and changeset_revision: # The protocol is not stored, but the port is if it exists. toolshed = common_util.remove_protocol_from_tool_shed_url(toolshed) repository = repository_util.get_repository_for_dependency_relationship( self.app, toolshed, repository_name, repository_owner, changeset_revision) if repository: for sub_elem in elem: tool_dependency_type = sub_elem.tag tool_dependency_name = sub_elem.get('name') tool_dependency_version = sub_elem.get('version') if tool_dependency_type and tool_dependency_name and tool_dependency_version: # Get the tool_dependency so we can get its installation directory. tool_dependency = None for tool_dependency in repository.tool_dependencies: if tool_dependency.type == tool_dependency_type and \ tool_dependency.name == tool_dependency_name and \ tool_dependency.version == tool_dependency_version: break if tool_dependency: installation_directory = tool_dependency.installation_directory( self.app) env_shell_file_path = self.get_env_shell_file_path( installation_directory) if env_shell_file_path: env_shell_file_paths.append( env_shell_file_path) else: error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " error_message += "type %s, name %s, version %s for repository %s" % \ (str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version), str(repository.name)) log.debug(error_message) continue else: error_message = "Skipping tool dependency definition because unable to locate tool dependency " error_message += "type %s, name %s, version %s for repository %s" % \ (str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version), str(repository.name)) log.debug(error_message) continue else: error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ (str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version)) log.debug(error_message) continue else: error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ (str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision)) log.debug(error_message) else: error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ (str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision)) log.debug(error_message) return env_shell_file_paths
def load_from_element( self, elem, tool_path ): assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag ) self.declared_id = elem.get( 'id', None ) self.guid = elem.get( 'guid', None ) path = elem.get( 'tool_file', None ) self.version = elem.get( 'version', self.version ) tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find( 'tool' ) assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) ) path = tool_elem.get( "file", None ) tool_guid = tool_elem.get( "guid", None ) # need to determine repository info so that dependencies will work correctly tool_shed_url = tool_elem.find( 'tool_shed' ).text # Handle protocol changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url ) # The protocol is not stored in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url ) repository_name = tool_elem.find( 'repository_name' ).text repository_owner = tool_elem.find( 'repository_owner' ).text installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision ) tool_shed_repository = \ repository_util.get_installed_repository( self.data_managers.app, tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision ) if tool_shed_repository is None: log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' ) # we'll set tool_path manually here from shed_conf_file tool_shed_repository_id = None try: tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path ) except Exception as e: log.error( 'Error determining tool_path for Data Manager during testing: %s', e ) else: tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) # use shed_conf_file to determine tool_path shed_conf_file = elem.get( "shed_conf_file", None ) if shed_conf_file: shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None ) if shed_conf: tool_path = shed_conf.get( "tool_path", tool_path ) assert path is not None, "A tool file path could not be determined:\n%s" % ( util.xml_to_string( elem ) ) self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id, tool_shed_repository_id=tool_shed_repository_id ) self.name = elem.get( 'name', self.tool.name ) self.description = elem.get( 'description', self.tool.description ) self.undeclared_tables = util.asbool( elem.get( 'undeclared_tables', self.undeclared_tables ) ) for data_table_elem in elem.findall( 'data_table' ): data_table_name = data_table_elem.get( "name" ) assert data_table_name is not None, "A name is required for a data table entry" if data_table_name not in self.data_tables: self.data_tables[ data_table_name ] = odict() output_elem = data_table_elem.find( 'output' ) if output_elem is not None: for column_elem in output_elem.findall( 'column' ): column_name = column_elem.get( 'name', None ) assert column_name is not None, "Name is required for column entry" data_table_coumn_name = column_elem.get( 'data_table_name', column_name ) self.data_tables[ data_table_name ][ data_table_coumn_name ] = column_name output_ref = column_elem.get( 'output_ref', None ) if output_ref is not None: if data_table_name not in self.output_ref_by_data_table: self.output_ref_by_data_table[ data_table_name ] = {} self.output_ref_by_data_table[ data_table_name ][ data_table_coumn_name ] = output_ref value_translation_elems = column_elem.findall( 'value_translation' ) if value_translation_elems is not None: for value_translation_elem in value_translation_elems: value_translation = value_translation_elem.text if value_translation is not None: value_translation_type = value_translation_elem.get( 'type', DEFAULT_VALUE_TRANSLATION_TYPE ) if data_table_name not in self.value_translation_by_data_table_column: self.value_translation_by_data_table_column[ data_table_name ] = {} if data_table_coumn_name not in self.value_translation_by_data_table_column[ data_table_name ]: self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = [] if value_translation_type == 'function': if value_translation in VALUE_TRANSLATION_FUNCTIONS: value_translation = VALUE_TRANSLATION_FUNCTIONS[ value_translation ] else: raise ValueError( "Unsupported value translation function: '%s'" % ( value_translation ) ) else: assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError( "Unsupported value translation type: '%s'" % ( value_translation_type ) ) self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ].append( value_translation ) for move_elem in column_elem.findall( 'move' ): move_type = move_elem.get( 'type', 'directory' ) relativize_symlinks = move_elem.get( 'relativize_symlinks', False ) # TODO: should we instead always relativize links? source_elem = move_elem.find( 'source' ) if source_elem is None: source_base = None source_value = '' else: source_base = source_elem.get( 'base', None ) source_value = source_elem.text target_elem = move_elem.find( 'target' ) if target_elem is None: target_base = None target_value = '' else: target_base = target_elem.get( 'base', None ) target_value = target_elem.text if data_table_name not in self.move_by_data_table_column: self.move_by_data_table_column[ data_table_name ] = {} self.move_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = \ dict( type=move_type, source_base=source_base, source_value=source_value, target_base=target_base, target_value=target_value, relativize_symlinks=relativize_symlinks )