def get_installed_repositories_from_repository_dependencies( self, repository_dependencies_dict ): installed_repositories = [] if repository_dependencies_dict and isinstance( repository_dependencies_dict, dict ): for rd_key, rd_vals in repository_dependencies_dict.items(): if rd_key in [ 'root_key', 'description' ]: continue # rd_key is something like: 'http://localhost:9009__ESEP__package_rdkit_2012_12__ESEP__test__ESEP__d635ffb9c665__ESEP__True' # rd_val is something like: [['http://localhost:9009', 'package_numpy_1_7', 'test', 'cddd64ecd985', 'True']] repository_components_tuple = container_util.get_components_from_key( rd_key ) components_list = repository_util.extract_components_from_tuple( repository_components_tuple ) tool_shed, name, owner, changeset_revision = components_list[ 0:4 ] installed_repository = repository_util.get_installed_repository( self.app, tool_shed=tool_shed, name=name, owner=owner, changeset_revision=changeset_revision ) if ( installed_repository ) and ( installed_repository not in installed_repositories ): installed_repositories.append( installed_repository ) for rd_val in rd_vals: tool_shed, name, owner, changeset_revision = rd_val[ 0:4 ] installed_repository = repository_util.get_installed_repository( self.app, tool_shed=tool_shed, name=name, owner=owner, changeset_revision=changeset_revision ) if ( installed_repository ) and ( installed_repository not in installed_repositories ): installed_repositories.append( installed_repository ) return installed_repositories
def uninstall_repository(self, trans, id=None, **kwd): """ DELETE /api/tool_shed_repositories/id DELETE /api/tool_shed_repositories/ :param id: encoded repository id. Either id or name, owner, changeset_revision and tool_shed_url need to be supplied :param kwd: 'remove_from_disk' : Remove repository from disk or deactivate repository. Defaults to `True` (= remove repository from disk). 'name' : Repository name 'owner' : Repository owner 'changeset_revision': Changeset revision to uninstall 'tool_shed_url' : Tool Shed URL """ if id: try: repository = repository_util.get_tool_shed_repository_by_id( self.app, id) except ValueError: raise HTTPBadRequest( detail="No repository with id '%s' found" % id) else: tsr_arguments = [ 'name', 'owner', 'changeset_revision', 'tool_shed_url' ] try: tsr_arguments = {key: kwd[key] for key in tsr_arguments} except KeyError as e: raise HTTPBadRequest(detail="Missing required parameter '%s'" % e.args[0]) repository = repository_util.get_installed_repository( app=self.app, tool_shed=tsr_arguments['tool_shed_url'], name=tsr_arguments['name'], owner=tsr_arguments['owner'], changeset_revision=tsr_arguments['changeset_revision']) if not repository: raise HTTPBadRequest(detail="Repository not found") irm = InstalledRepositoryManager(app=self.app) errors = irm.uninstall_repository(repository=repository, remove_from_disk=kwd.get( 'remove_from_disk', True)) if not errors: action = 'removed' if kwd.get('remove_from_disk', True) else 'deactivated' return { 'message': 'The repository named %s has been %s.' % (repository.name, action) } else: raise Exception( 'Attempting to uninstall tool dependencies for repository named %s resulted in errors: %s' % (repository.name, errors))
def repair_repository_revision(self, trans, payload, **kwd): """ POST /api/tool_shed_repositories/repair_repository_revision Repair a specified repository revision previously installed into Galaxy. :param key: the current Galaxy admin user's API key The following parameters are included in the payload. :param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed :param name (required): the name of the Repository :param owner (required): the owner of the Repository :param changeset_revision (required): the changeset_revision of the RepositoryMetadata object associated with the Repository """ # Get the information about the repository to be installed from the payload. tool_shed_url, name, owner, changeset_revision = self.__parse_repository_from_payload( payload, include_changeset=True) tool_shed_repositories = [] tool_shed_repository = repository_util.get_installed_repository( self.app, tool_shed=tool_shed_url, name=name, owner=owner, changeset_revision=changeset_revision) rrm = RepairRepositoryManager(self.app) repair_dict = rrm.get_repair_dict(tool_shed_repository) ordered_tsr_ids = repair_dict.get('ordered_tsr_ids', []) ordered_repo_info_dicts = repair_dict.get('ordered_repo_info_dicts', []) if ordered_tsr_ids and ordered_repo_info_dicts: for index, tsr_id in enumerate(ordered_tsr_ids): repository = trans.install_model.context.query( trans.install_model.ToolShedRepository).get( trans.security.decode_id(tsr_id)) repo_info_dict = ordered_repo_info_dicts[index] # TODO: handle errors in repair_dict. repair_dict = rrm.repair_tool_shed_repository( repository, encoding_util.tool_shed_encode(repo_info_dict)) repository_dict = repository.to_dict( value_mapper=self.__get_value_mapper(trans, repository)) repository_dict['url'] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id(repository.id)) if repair_dict: errors = repair_dict.get(repository.name, []) repository_dict['errors_attempting_repair'] = ' '.join( errors) tool_shed_repositories.append(repository_dict) # Display the list of repaired repositories. return tool_shed_repositories
def repair_repository_revision( self, trans, payload, **kwd ): """ POST /api/tool_shed_repositories/repair_repository_revision Repair a specified repository revision previously installed into Galaxy. :param key: the current Galaxy admin user's API key The following parameters are included in the payload. :param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed :param name (required): the name of the Repository :param owner (required): the owner of the Repository :param changeset_revision (required): the changeset_revision of the RepositoryMetadata object associated with the Repository """ # Get the information about the repository to be installed from the payload. tool_shed_url, name, owner, changeset_revision = self.__parse_repository_from_payload( payload, include_changeset=True ) tool_shed_repositories = [] tool_shed_repository = repository_util.get_installed_repository( self.app, tool_shed=tool_shed_url, name=name, owner=owner, changeset_revision=changeset_revision ) rrm = RepairRepositoryManager( self.app ) repair_dict = rrm.get_repair_dict( tool_shed_repository ) ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) if ordered_tsr_ids and ordered_repo_info_dicts: for index, tsr_id in enumerate( ordered_tsr_ids ): repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) repo_info_dict = ordered_repo_info_dicts[ index ] # TODO: handle errors in repair_dict. repair_dict = rrm.repair_tool_shed_repository( repository, encoding_util.tool_shed_encode( repo_info_dict ) ) repository_dict = repository.to_dict( value_mapper=self.__get_value_mapper( trans, repository ) ) repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( repository.id ) ) if repair_dict: errors = repair_dict.get( repository.name, [] ) repository_dict[ 'errors_attempting_repair' ] = ' '.join( errors ) tool_shed_repositories.append( repository_dict ) # Display the list of repaired repositories. return tool_shed_repositories
def uninstall_repository(self, trans, id=None, **kwd): """ DELETE /api/tool_shed_repositories/id DELETE /api/tool_shed_repositories/ :param id: encoded repository id. Either id or name, owner, changeset_revision and tool_shed_url need to be supplied :param kwd: 'remove_from_disk' : Remove repository from disk or deactivate repository. Defaults to `True` (= remove repository from disk). 'name' : Repository name 'owner' : Repository owner 'changeset_revision': Changeset revision to uninstall 'tool_shed_url' : Tool Shed URL """ if id: try: repository = repository_util.get_tool_shed_repository_by_id(self.app, id) except ValueError: raise HTTPBadRequest(detail="No repository with id '%s' found" % id) else: tsr_arguments = ['name', 'owner', 'changeset_revision', 'tool_shed_url'] try: tsr_arguments = {key: kwd[key] for key in tsr_arguments} except KeyError as e: raise HTTPBadRequest(detail="Missing required parameter '%s'" % e.args[0]) repository = repository_util.get_installed_repository(app=self.app, tool_shed=tsr_arguments['tool_shed_url'], name=tsr_arguments['name'], owner=tsr_arguments['owner'], changeset_revision=tsr_arguments['changeset_revision']) if not repository: raise HTTPBadRequest(detail="Repository not found") irm = InstalledRepositoryManager(app=self.app) errors = irm.uninstall_repository(repository=repository, remove_from_disk=kwd.get('remove_from_disk', True)) if not errors: action = 'removed' if kwd.get('remove_from_disk', True) else 'deactivated' return {'message': 'The repository named %s has been %s.' % (repository.name, action)} else: raise Exception('Attempting to uninstall tool dependencies for repository named %s resulted in errors: %s' % (repository.name, errors))
def load_from_element(self, elem, tool_path): assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % (elem.tag) self.declared_id = elem.get('id', None) self.guid = elem.get('guid', None) path = elem.get('tool_file', None) self.version = elem.get('version', self.version) tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find('tool') assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % (util.xml_to_string(elem)) path = tool_elem.get("file", None) tool_guid = tool_elem.get("guid", None) # need to determine repository info so that dependencies will work correctly if hasattr(self.data_managers.app, 'tool_cache') and tool_guid in self.data_managers.app.tool_cache._tool_paths_by_id: path = self.data_managers.app.tool_cache._tool_paths_by_id[tool_guid] tool = self.data_managers.app.tool_cache.get_tool(path) tool_shed_repository = tool.tool_shed_repository self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed_repository.tool_shed, name=tool_shed_repository.name, owner=tool_shed_repository.owner, installed_changeset_revision=tool_shed_repository.installed_changeset_revision) tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id) tool_path = "" else: tool_shed_url = tool_elem.find('tool_shed').text # Handle protocol changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.data_managers.app, tool_shed_url) # The protocol is not stored in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed_url) repository_name = tool_elem.find('repository_name').text repository_owner = tool_elem.find('repository_owner').text installed_changeset_revision = tool_elem.find('installed_changeset_revision').text self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision) tool_shed_repository = \ repository_util.get_installed_repository(self.data_managers.app, tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision) if tool_shed_repository is None: log.warning('Could not determine tool shed repository from database. This should only ever happen when running tests.') # we'll set tool_path manually here from shed_conf_file tool_shed_repository_id = None try: tool_path = util.parse_xml(elem.get('shed_conf_file')).getroot().get('tool_path', tool_path) except Exception as e: log.error('Error determining tool_path for Data Manager during testing: %s', e) else: tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id) # use shed_conf_file to determine tool_path shed_conf_file = elem.get("shed_conf_file", None) if shed_conf_file: shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename(shed_conf_file, None) if shed_conf: tool_path = shed_conf.get("tool_path", tool_path) assert path is not None, "A tool file path could not be determined:\n%s" % (util.xml_to_string(elem)) self.load_tool(os.path.join(tool_path, path), guid=tool_guid, data_manager_id=self.id, tool_shed_repository_id=tool_shed_repository_id) self.name = elem.get('name', self.tool.name) self.description = elem.get('description', self.tool.description) self.undeclared_tables = util.asbool(elem.get('undeclared_tables', self.undeclared_tables)) for data_table_elem in elem.findall('data_table'): data_table_name = data_table_elem.get("name") assert data_table_name is not None, "A name is required for a data table entry" if data_table_name not in self.data_tables: self.data_tables[data_table_name] = odict() output_elem = data_table_elem.find('output') if output_elem is not None: for column_elem in output_elem.findall('column'): column_name = column_elem.get('name', None) assert column_name is not None, "Name is required for column entry" data_table_coumn_name = column_elem.get('data_table_name', column_name) self.data_tables[data_table_name][data_table_coumn_name] = column_name output_ref = column_elem.get('output_ref', None) if output_ref is not None: if data_table_name not in self.output_ref_by_data_table: self.output_ref_by_data_table[data_table_name] = {} self.output_ref_by_data_table[data_table_name][data_table_coumn_name] = output_ref value_translation_elems = column_elem.findall('value_translation') if value_translation_elems is not None: for value_translation_elem in value_translation_elems: value_translation = value_translation_elem.text if value_translation is not None: value_translation_type = value_translation_elem.get('type', DEFAULT_VALUE_TRANSLATION_TYPE) if data_table_name not in self.value_translation_by_data_table_column: self.value_translation_by_data_table_column[data_table_name] = {} if data_table_coumn_name not in self.value_translation_by_data_table_column[data_table_name]: self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name] = [] if value_translation_type == 'function': if value_translation in VALUE_TRANSLATION_FUNCTIONS: value_translation = VALUE_TRANSLATION_FUNCTIONS[value_translation] else: raise ValueError("Unsupported value translation function: '%s'" % (value_translation)) else: assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError("Unsupported value translation type: '%s'" % (value_translation_type)) self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name].append(value_translation) for move_elem in column_elem.findall('move'): move_type = move_elem.get('type', 'directory') relativize_symlinks = move_elem.get('relativize_symlinks', False) # TODO: should we instead always relativize links? source_elem = move_elem.find('source') if source_elem is None: source_base = None source_value = '' else: source_base = source_elem.get('base', None) source_value = source_elem.text target_elem = move_elem.find('target') if target_elem is None: target_base = None target_value = '' else: target_base = target_elem.get('base', None) target_value = target_elem.text if data_table_name not in self.move_by_data_table_column: self.move_by_data_table_column[data_table_name] = {} self.move_by_data_table_column[data_table_name][data_table_coumn_name] = \ dict(type=move_type, source_base=source_base, source_value=source_value, target_base=target_base, target_value=target_value, relativize_symlinks=relativize_symlinks)
def load_from_element( self, elem, tool_path ): assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag ) self.declared_id = elem.get( 'id', None ) self.guid = elem.get( 'guid', None ) path = elem.get( 'tool_file', None ) self.version = elem.get( 'version', self.version ) tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find( 'tool' ) assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) ) path = tool_elem.get( "file", None ) tool_guid = tool_elem.get( "guid", None ) # need to determine repository info so that dependencies will work correctly tool_shed_url = tool_elem.find( 'tool_shed' ).text # Handle protocol changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url ) # The protocol is not stored in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url ) repository_name = tool_elem.find( 'repository_name' ).text repository_owner = tool_elem.find( 'repository_owner' ).text installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision ) tool_shed_repository = \ repository_util.get_installed_repository( self.data_managers.app, tool_shed=tool_shed, name=repository_name, owner=repository_owner, installed_changeset_revision=installed_changeset_revision ) if tool_shed_repository is None: log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' ) # we'll set tool_path manually here from shed_conf_file tool_shed_repository_id = None try: tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path ) except Exception as e: log.error( 'Error determining tool_path for Data Manager during testing: %s', e ) else: tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) # use shed_conf_file to determine tool_path shed_conf_file = elem.get( "shed_conf_file", None ) if shed_conf_file: shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None ) if shed_conf: tool_path = shed_conf.get( "tool_path", tool_path ) assert path is not None, "A tool file path could not be determined:\n%s" % ( util.xml_to_string( elem ) ) self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id, tool_shed_repository_id=tool_shed_repository_id ) self.name = elem.get( 'name', self.tool.name ) self.description = elem.get( 'description', self.tool.description ) self.undeclared_tables = util.asbool( elem.get( 'undeclared_tables', self.undeclared_tables ) ) for data_table_elem in elem.findall( 'data_table' ): data_table_name = data_table_elem.get( "name" ) assert data_table_name is not None, "A name is required for a data table entry" if data_table_name not in self.data_tables: self.data_tables[ data_table_name ] = odict() output_elem = data_table_elem.find( 'output' ) if output_elem is not None: for column_elem in output_elem.findall( 'column' ): column_name = column_elem.get( 'name', None ) assert column_name is not None, "Name is required for column entry" data_table_coumn_name = column_elem.get( 'data_table_name', column_name ) self.data_tables[ data_table_name ][ data_table_coumn_name ] = column_name output_ref = column_elem.get( 'output_ref', None ) if output_ref is not None: if data_table_name not in self.output_ref_by_data_table: self.output_ref_by_data_table[ data_table_name ] = {} self.output_ref_by_data_table[ data_table_name ][ data_table_coumn_name ] = output_ref value_translation_elems = column_elem.findall( 'value_translation' ) if value_translation_elems is not None: for value_translation_elem in value_translation_elems: value_translation = value_translation_elem.text if value_translation is not None: value_translation_type = value_translation_elem.get( 'type', DEFAULT_VALUE_TRANSLATION_TYPE ) if data_table_name not in self.value_translation_by_data_table_column: self.value_translation_by_data_table_column[ data_table_name ] = {} if data_table_coumn_name not in self.value_translation_by_data_table_column[ data_table_name ]: self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = [] if value_translation_type == 'function': if value_translation in VALUE_TRANSLATION_FUNCTIONS: value_translation = VALUE_TRANSLATION_FUNCTIONS[ value_translation ] else: raise ValueError( "Unsupported value translation function: '%s'" % ( value_translation ) ) else: assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError( "Unsupported value translation type: '%s'" % ( value_translation_type ) ) self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ].append( value_translation ) for move_elem in column_elem.findall( 'move' ): move_type = move_elem.get( 'type', 'directory' ) relativize_symlinks = move_elem.get( 'relativize_symlinks', False ) # TODO: should we instead always relativize links? source_elem = move_elem.find( 'source' ) if source_elem is None: source_base = None source_value = '' else: source_base = source_elem.get( 'base', None ) source_value = source_elem.text target_elem = move_elem.find( 'target' ) if target_elem is None: target_base = None target_value = '' else: target_base = target_elem.get( 'base', None ) target_value = target_elem.text if data_table_name not in self.move_by_data_table_column: self.move_by_data_table_column[ data_table_name ] = {} self.move_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = \ dict( type=move_type, source_base=source_base, source_value=source_value, target_base=target_base, target_value=target_value, relativize_symlinks=relativize_symlinks )