def get_installed_repositories_from_repository_dependencies( self, repository_dependencies_dict ):
installed_repositories = []
if repository_dependencies_dict and isinstance( repository_dependencies_dict, dict ):
for rd_key, rd_vals in repository_dependencies_dict.items():
if rd_key in [ 'root_key', 'description' ]:
continue
# rd_key is something like: 'http://localhost:9009__ESEP__package_rdkit_2012_12__ESEP__test__ESEP__d635ffb9c665__ESEP__True'
# rd_val is something like: [['http://localhost:9009', 'package_numpy_1_7', 'test', 'cddd64ecd985', 'True']]
repository_components_tuple = container_util.get_components_from_key( rd_key )
components_list = repository_util.extract_components_from_tuple( repository_components_tuple )
tool_shed, name, owner, changeset_revision = components_list[ 0:4 ]
installed_repository = repository_util.get_installed_repository( self.app,
tool_shed=tool_shed,
name=name,
owner=owner,
changeset_revision=changeset_revision )
if ( installed_repository ) and ( installed_repository not in installed_repositories ):
installed_repositories.append( installed_repository )
for rd_val in rd_vals:
tool_shed, name, owner, changeset_revision = rd_val[ 0:4 ]
installed_repository = repository_util.get_installed_repository( self.app,
tool_shed=tool_shed,
name=name,
owner=owner,
changeset_revision=changeset_revision )
if ( installed_repository ) and ( installed_repository not in installed_repositories ):
installed_repositories.append( installed_repository )
return installed_repositories
def uninstall_repository(self, trans, id=None, **kwd):
"""
DELETE /api/tool_shed_repositories/id
DELETE /api/tool_shed_repositories/
:param id: encoded repository id. Either id or name, owner, changeset_revision and tool_shed_url need to be supplied
:param kwd: 'remove_from_disk' : Remove repository from disk or deactivate repository.
Defaults to `True` (= remove repository from disk).
'name' : Repository name
'owner' : Repository owner
'changeset_revision': Changeset revision to uninstall
'tool_shed_url' : Tool Shed URL
"""
if id:
try:
repository = repository_util.get_tool_shed_repository_by_id(
self.app, id)
except ValueError:
raise HTTPBadRequest(
detail="No repository with id '%s' found" % id)
else:
tsr_arguments = [
'name', 'owner', 'changeset_revision', 'tool_shed_url'
]
try:
tsr_arguments = {key: kwd[key] for key in tsr_arguments}
except KeyError as e:
raise HTTPBadRequest(detail="Missing required parameter '%s'" %
e.args[0])
repository = repository_util.get_installed_repository(
app=self.app,
tool_shed=tsr_arguments['tool_shed_url'],
name=tsr_arguments['name'],
owner=tsr_arguments['owner'],
changeset_revision=tsr_arguments['changeset_revision'])
if not repository:
raise HTTPBadRequest(detail="Repository not found")
irm = InstalledRepositoryManager(app=self.app)
errors = irm.uninstall_repository(repository=repository,
remove_from_disk=kwd.get(
'remove_from_disk', True))
if not errors:
action = 'removed' if kwd.get('remove_from_disk',
True) else 'deactivated'
return {
'message':
'The repository named %s has been %s.' %
(repository.name, action)
}
else:
raise Exception(
'Attempting to uninstall tool dependencies for repository named %s resulted in errors: %s'
% (repository.name, errors))
def repair_repository_revision(self, trans, payload, **kwd):
"""
POST /api/tool_shed_repositories/repair_repository_revision
Repair a specified repository revision previously installed into Galaxy.
:param key: the current Galaxy admin user's API key
The following parameters are included in the payload.
:param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed
:param name (required): the name of the Repository
:param owner (required): the owner of the Repository
:param changeset_revision (required): the changeset_revision of the RepositoryMetadata object associated with the Repository
"""
# Get the information about the repository to be installed from the payload.
tool_shed_url, name, owner, changeset_revision = self.__parse_repository_from_payload(
payload, include_changeset=True)
tool_shed_repositories = []
tool_shed_repository = repository_util.get_installed_repository(
self.app,
tool_shed=tool_shed_url,
name=name,
owner=owner,
changeset_revision=changeset_revision)
rrm = RepairRepositoryManager(self.app)
repair_dict = rrm.get_repair_dict(tool_shed_repository)
ordered_tsr_ids = repair_dict.get('ordered_tsr_ids', [])
ordered_repo_info_dicts = repair_dict.get('ordered_repo_info_dicts',
[])
if ordered_tsr_ids and ordered_repo_info_dicts:
for index, tsr_id in enumerate(ordered_tsr_ids):
repository = trans.install_model.context.query(
trans.install_model.ToolShedRepository).get(
trans.security.decode_id(tsr_id))
repo_info_dict = ordered_repo_info_dicts[index]
# TODO: handle errors in repair_dict.
repair_dict = rrm.repair_tool_shed_repository(
repository, encoding_util.tool_shed_encode(repo_info_dict))
repository_dict = repository.to_dict(
value_mapper=self.__get_value_mapper(trans, repository))
repository_dict['url'] = web.url_for(
controller='tool_shed_repositories',
action='show',
id=trans.security.encode_id(repository.id))
if repair_dict:
errors = repair_dict.get(repository.name, [])
repository_dict['errors_attempting_repair'] = ' '.join(
errors)
tool_shed_repositories.append(repository_dict)
# Display the list of repaired repositories.
return tool_shed_repositories
def repair_repository_revision( self, trans, payload, **kwd ):
"""
POST /api/tool_shed_repositories/repair_repository_revision
Repair a specified repository revision previously installed into Galaxy.
:param key: the current Galaxy admin user's API key
The following parameters are included in the payload.
:param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed
:param name (required): the name of the Repository
:param owner (required): the owner of the Repository
:param changeset_revision (required): the changeset_revision of the RepositoryMetadata object associated with the Repository
"""
# Get the information about the repository to be installed from the payload.
tool_shed_url, name, owner, changeset_revision = self.__parse_repository_from_payload( payload, include_changeset=True )
tool_shed_repositories = []
tool_shed_repository = repository_util.get_installed_repository( self.app,
tool_shed=tool_shed_url,
name=name,
owner=owner,
changeset_revision=changeset_revision )
rrm = RepairRepositoryManager( self.app )
repair_dict = rrm.get_repair_dict( tool_shed_repository )
ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] )
ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] )
if ordered_tsr_ids and ordered_repo_info_dicts:
for index, tsr_id in enumerate( ordered_tsr_ids ):
repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) )
repo_info_dict = ordered_repo_info_dicts[ index ]
# TODO: handle errors in repair_dict.
repair_dict = rrm.repair_tool_shed_repository( repository,
encoding_util.tool_shed_encode( repo_info_dict ) )
repository_dict = repository.to_dict( value_mapper=self.__get_value_mapper( trans, repository ) )
repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
action='show',
id=trans.security.encode_id( repository.id ) )
if repair_dict:
errors = repair_dict.get( repository.name, [] )
repository_dict[ 'errors_attempting_repair' ] = ' '.join( errors )
tool_shed_repositories.append( repository_dict )
# Display the list of repaired repositories.
return tool_shed_repositories
def uninstall_repository(self, trans, id=None, **kwd):
"""
DELETE /api/tool_shed_repositories/id
DELETE /api/tool_shed_repositories/
:param id: encoded repository id. Either id or name, owner, changeset_revision and tool_shed_url need to be supplied
:param kwd: 'remove_from_disk' : Remove repository from disk or deactivate repository.
Defaults to `True` (= remove repository from disk).
'name' : Repository name
'owner' : Repository owner
'changeset_revision': Changeset revision to uninstall
'tool_shed_url' : Tool Shed URL
"""
if id:
try:
repository = repository_util.get_tool_shed_repository_by_id(self.app, id)
except ValueError:
raise HTTPBadRequest(detail="No repository with id '%s' found" % id)
else:
tsr_arguments = ['name', 'owner', 'changeset_revision', 'tool_shed_url']
try:
tsr_arguments = {key: kwd[key] for key in tsr_arguments}
except KeyError as e:
raise HTTPBadRequest(detail="Missing required parameter '%s'" % e.args[0])
repository = repository_util.get_installed_repository(app=self.app,
tool_shed=tsr_arguments['tool_shed_url'],
name=tsr_arguments['name'],
owner=tsr_arguments['owner'],
changeset_revision=tsr_arguments['changeset_revision'])
if not repository:
raise HTTPBadRequest(detail="Repository not found")
irm = InstalledRepositoryManager(app=self.app)
errors = irm.uninstall_repository(repository=repository, remove_from_disk=kwd.get('remove_from_disk', True))
if not errors:
action = 'removed' if kwd.get('remove_from_disk', True) else 'deactivated'
return {'message': 'The repository named %s has been %s.' % (repository.name, action)}
else:
raise Exception('Attempting to uninstall tool dependencies for repository named %s resulted in errors: %s' % (repository.name, errors))
def load_from_element(self, elem, tool_path):
assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % (elem.tag)
self.declared_id = elem.get('id', None)
self.guid = elem.get('guid', None)
path = elem.get('tool_file', None)
self.version = elem.get('version', self.version)
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find('tool')
assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % (util.xml_to_string(elem))
path = tool_elem.get("file", None)
tool_guid = tool_elem.get("guid", None)
# need to determine repository info so that dependencies will work correctly
if hasattr(self.data_managers.app, 'tool_cache') and tool_guid in self.data_managers.app.tool_cache._tool_paths_by_id:
path = self.data_managers.app.tool_cache._tool_paths_by_id[tool_guid]
tool = self.data_managers.app.tool_cache.get_tool(path)
tool_shed_repository = tool.tool_shed_repository
self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed_repository.tool_shed,
name=tool_shed_repository.name,
owner=tool_shed_repository.owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision)
tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id)
tool_path = ""
else:
tool_shed_url = tool_elem.find('tool_shed').text
# Handle protocol changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.data_managers.app, tool_shed_url)
# The protocol is not stored in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed_url)
repository_name = tool_elem.find('repository_name').text
repository_owner = tool_elem.find('repository_owner').text
installed_changeset_revision = tool_elem.find('installed_changeset_revision').text
self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision)
tool_shed_repository = \
repository_util.get_installed_repository(self.data_managers.app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision)
if tool_shed_repository is None:
log.warning('Could not determine tool shed repository from database. This should only ever happen when running tests.')
# we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml(elem.get('shed_conf_file')).getroot().get('tool_path', tool_path)
except Exception as e:
log.error('Error determining tool_path for Data Manager during testing: %s', e)
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id)
# use shed_conf_file to determine tool_path
shed_conf_file = elem.get("shed_conf_file", None)
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename(shed_conf_file, None)
if shed_conf:
tool_path = shed_conf.get("tool_path", tool_path)
assert path is not None, "A tool file path could not be determined:\n%s" % (util.xml_to_string(elem))
self.load_tool(os.path.join(tool_path, path),
guid=tool_guid,
data_manager_id=self.id,
tool_shed_repository_id=tool_shed_repository_id)
self.name = elem.get('name', self.tool.name)
self.description = elem.get('description', self.tool.description)
self.undeclared_tables = util.asbool(elem.get('undeclared_tables', self.undeclared_tables))
for data_table_elem in elem.findall('data_table'):
data_table_name = data_table_elem.get("name")
assert data_table_name is not None, "A name is required for a data table entry"
if data_table_name not in self.data_tables:
self.data_tables[data_table_name] = odict()
output_elem = data_table_elem.find('output')
if output_elem is not None:
for column_elem in output_elem.findall('column'):
column_name = column_elem.get('name', None)
assert column_name is not None, "Name is required for column entry"
data_table_coumn_name = column_elem.get('data_table_name', column_name)
self.data_tables[data_table_name][data_table_coumn_name] = column_name
output_ref = column_elem.get('output_ref', None)
if output_ref is not None:
if data_table_name not in self.output_ref_by_data_table:
self.output_ref_by_data_table[data_table_name] = {}
self.output_ref_by_data_table[data_table_name][data_table_coumn_name] = output_ref
value_translation_elems = column_elem.findall('value_translation')
if value_translation_elems is not None:
for value_translation_elem in value_translation_elems:
value_translation = value_translation_elem.text
if value_translation is not None:
value_translation_type = value_translation_elem.get('type', DEFAULT_VALUE_TRANSLATION_TYPE)
if data_table_name not in self.value_translation_by_data_table_column:
self.value_translation_by_data_table_column[data_table_name] = {}
if data_table_coumn_name not in self.value_translation_by_data_table_column[data_table_name]:
self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name] = []
if value_translation_type == 'function':
if value_translation in VALUE_TRANSLATION_FUNCTIONS:
value_translation = VALUE_TRANSLATION_FUNCTIONS[value_translation]
else:
raise ValueError("Unsupported value translation function: '%s'" % (value_translation))
else:
assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError("Unsupported value translation type: '%s'" % (value_translation_type))
self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name].append(value_translation)
for move_elem in column_elem.findall('move'):
move_type = move_elem.get('type', 'directory')
relativize_symlinks = move_elem.get('relativize_symlinks', False) # TODO: should we instead always relativize links?
source_elem = move_elem.find('source')
if source_elem is None:
source_base = None
source_value = ''
else:
source_base = source_elem.get('base', None)
source_value = source_elem.text
target_elem = move_elem.find('target')
if target_elem is None:
target_base = None
target_value = ''
else:
target_base = target_elem.get('base', None)
target_value = target_elem.text
if data_table_name not in self.move_by_data_table_column:
self.move_by_data_table_column[data_table_name] = {}
self.move_by_data_table_column[data_table_name][data_table_coumn_name] = \
dict(type=move_type,
source_base=source_base,
source_value=source_value,
target_base=target_base,
target_value=target_value,
relativize_symlinks=relativize_symlinks)
def load_from_element( self, elem, tool_path ):
assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag )
self.declared_id = elem.get( 'id', None )
self.guid = elem.get( 'guid', None )
path = elem.get( 'tool_file', None )
self.version = elem.get( 'version', self.version )
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find( 'tool' )
assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) )
path = tool_elem.get( "file", None )
tool_guid = tool_elem.get( "guid", None )
# need to determine repository info so that dependencies will work correctly
tool_shed_url = tool_elem.find( 'tool_shed' ).text
# Handle protocol changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url )
# The protocol is not stored in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url )
repository_name = tool_elem.find( 'repository_name' ).text
repository_owner = tool_elem.find( 'repository_owner' ).text
installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text
self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision )
tool_shed_repository = \
repository_util.get_installed_repository( self.data_managers.app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision )
if tool_shed_repository is None:
log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' )
# we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path )
except Exception as e:
log.error( 'Error determining tool_path for Data Manager during testing: %s', e )
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id )
# use shed_conf_file to determine tool_path
shed_conf_file = elem.get( "shed_conf_file", None )
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None )
if shed_conf:
tool_path = shed_conf.get( "tool_path", tool_path )
assert path is not None, "A tool file path could not be determined:\n%s" % ( util.xml_to_string( elem ) )
self.load_tool( os.path.join( tool_path, path ),
guid=tool_guid,
data_manager_id=self.id,
tool_shed_repository_id=tool_shed_repository_id )
self.name = elem.get( 'name', self.tool.name )
self.description = elem.get( 'description', self.tool.description )
self.undeclared_tables = util.asbool( elem.get( 'undeclared_tables', self.undeclared_tables ) )
for data_table_elem in elem.findall( 'data_table' ):
data_table_name = data_table_elem.get( "name" )
assert data_table_name is not None, "A name is required for a data table entry"
if data_table_name not in self.data_tables:
self.data_tables[ data_table_name ] = odict()
output_elem = data_table_elem.find( 'output' )
if output_elem is not None:
for column_elem in output_elem.findall( 'column' ):
column_name = column_elem.get( 'name', None )
assert column_name is not None, "Name is required for column entry"
data_table_coumn_name = column_elem.get( 'data_table_name', column_name )
self.data_tables[ data_table_name ][ data_table_coumn_name ] = column_name
output_ref = column_elem.get( 'output_ref', None )
if output_ref is not None:
if data_table_name not in self.output_ref_by_data_table:
self.output_ref_by_data_table[ data_table_name ] = {}
self.output_ref_by_data_table[ data_table_name ][ data_table_coumn_name ] = output_ref
value_translation_elems = column_elem.findall( 'value_translation' )
if value_translation_elems is not None:
for value_translation_elem in value_translation_elems:
value_translation = value_translation_elem.text
if value_translation is not None:
value_translation_type = value_translation_elem.get( 'type', DEFAULT_VALUE_TRANSLATION_TYPE )
if data_table_name not in self.value_translation_by_data_table_column:
self.value_translation_by_data_table_column[ data_table_name ] = {}
if data_table_coumn_name not in self.value_translation_by_data_table_column[ data_table_name ]:
self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = []
if value_translation_type == 'function':
if value_translation in VALUE_TRANSLATION_FUNCTIONS:
value_translation = VALUE_TRANSLATION_FUNCTIONS[ value_translation ]
else:
raise ValueError( "Unsupported value translation function: '%s'" % ( value_translation ) )
else:
assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError( "Unsupported value translation type: '%s'" % ( value_translation_type ) )
self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ].append( value_translation )
for move_elem in column_elem.findall( 'move' ):
move_type = move_elem.get( 'type', 'directory' )
relativize_symlinks = move_elem.get( 'relativize_symlinks', False ) # TODO: should we instead always relativize links?
source_elem = move_elem.find( 'source' )
if source_elem is None:
source_base = None
source_value = ''
else:
source_base = source_elem.get( 'base', None )
source_value = source_elem.text
target_elem = move_elem.find( 'target' )
if target_elem is None:
target_base = None
target_value = ''
else:
target_base = target_elem.get( 'base', None )
target_value = target_elem.text
if data_table_name not in self.move_by_data_table_column:
self.move_by_data_table_column[ data_table_name ] = {}
self.move_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = \
dict( type=move_type,
source_base=source_base,
source_value=source_value,
target_base=target_base,
target_value=target_value,
relativize_symlinks=relativize_symlinks )