def tool_shed_is_this_tool_shed(toolshed_base_url):
"""Determine if a tool shed is the current tool shed."""
cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url(
toolshed_base_url)
cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url(
str(url_for('/', qualified=True)))
return cleaned_toolshed_base_url == cleaned_tool_shed
def remove_repository_dependency_reference_to_self( self, key_rd_dicts ):
"""Remove all repository dependencies that point to a revision within its own repository."""
clean_key_rd_dicts = []
key = next(iter(key_rd_dicts[ 0 ]))
repository_tup = key.split( container_util.STRSEP )
rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( repository_tup )
cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed )
for key_rd_dict in key_rd_dicts:
k = next(iter(key_rd_dict))
repository_dependency = key_rd_dict[ k ]
toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( repository_dependency )
cleaned_toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed )
if cleaned_rd_toolshed == cleaned_toolshed and rd_name == name and rd_owner == owner:
debug_msg = "Removing repository dependency for repository %s owned by %s " % ( name, owner )
debug_msg += 'since it refers to a revision within itself.'
log.debug( debug_msg )
else:
new_key_rd_dict = {}
new_key_rd_dict[ key ] = repository_dependency
clean_key_rd_dicts.append( new_key_rd_dict )
return clean_key_rd_dicts
def remove_repository_dependency_reference_to_self(self, key_rd_dicts):
"""Remove all repository dependencies that point to a revision within its own repository."""
clean_key_rd_dicts = []
key = next(iter(key_rd_dicts[0]))
repository_tup = key.split(container_util.STRSEP)
rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple(repository_tup)
cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url(rd_toolshed)
for key_rd_dict in key_rd_dicts:
k = next(iter(key_rd_dict))
repository_dependency = key_rd_dict[k]
toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple(repository_dependency)
cleaned_toolshed = common_util.remove_protocol_from_tool_shed_url(toolshed)
if cleaned_rd_toolshed == cleaned_toolshed and rd_name == name and rd_owner == owner:
debug_msg = "Removing repository dependency for repository {} owned by {} ".format(name, owner)
debug_msg += 'since it refers to a revision within itself.'
log.debug(debug_msg)
else:
new_key_rd_dict = {}
new_key_rd_dict[key] = repository_dependency
clean_key_rd_dicts.append(new_key_rd_dict)
return clean_key_rd_dicts
def get_prior_installation_required_and_only_if_compiling_contained_td( self ):
"""
This method is called from the tool shed and never Galaxy. If self.all_repository_dependencies
contains a repository dependency tuple that is associated with self.repository, return the
value of the tuple's prior_installation_required component.
"""
cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( self.tool_shed_url )
if self.all_repository_dependencies:
for rd_key, rd_tups in self.all_repository_dependencies.items():
if rd_key in [ 'root_key', 'description' ]:
continue
for rd_tup in rd_tups:
rd_toolshed, \
rd_name, \
rd_owner, \
rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( rd_tup )
cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed )
if cleaned_rd_toolshed == cleaned_toolshed_base_url and \
rd_name == self.repository.name and \
rd_owner == self.repository.user.username and \
rd_changeset_revision == self.repository_metadata.changeset_revision:
return rd_prior_installation_required, rd_only_if_compiling_contained_td
elif self.repository_metadata:
# Get the list of changeset revisions from the tool shed to which self.repository may be updated.
metadata = self.repository_metadata.metadata
current_changeset_revision = str( self.repository_metadata.changeset_revision )
# Get the changeset revision to which the current value of required_repository_changeset_revision
# should be updated if it's not current.
text = suc.get_updated_changeset_revisions( self.app,
name=str( self.repository.name ),
owner=str( self.repository.user.username ),
changeset_revision=current_changeset_revision )
if text:
valid_changeset_revisions = listify( text )
if current_changeset_revision not in valid_changeset_revisions:
valid_changeset_revisions.append( current_changeset_revision )
else:
valid_changeset_revisions = [ current_changeset_revision ]
repository_dependencies_dict = metadata[ 'repository_dependencies' ]
rd_tups = repository_dependencies_dict.get( 'repository_dependencies', [] )
for rd_tup in rd_tups:
rd_toolshed, \
rd_name, \
rd_owner, \
rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( rd_tup )
cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed )
if cleaned_rd_toolshed == cleaned_toolshed_base_url and \
rd_name == self.repository.name and \
rd_owner == self.repository.user.username and \
rd_changeset_revision in valid_changeset_revisions:
return rd_prior_installation_required, rd_only_if_compiling_contained_td
# Default both prior_installation_required and only_if_compiling_contained_td to False.
return 'False', 'False'
def get_prior_installation_required_and_only_if_compiling_contained_td( self ):
"""
This method is called from the tool shed and never Galaxy. If self.all_repository_dependencies
contains a repository dependency tuple that is associated with self.repository, return the
value of the tuple's prior_installation_required component.
"""
cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( self.tool_shed_url )
if self.all_repository_dependencies:
for rd_key, rd_tups in self.all_repository_dependencies.items():
if rd_key in [ 'root_key', 'description' ]:
continue
for rd_tup in rd_tups:
rd_toolshed, \
rd_name, \
rd_owner, \
rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( rd_tup )
cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed )
if cleaned_rd_toolshed == cleaned_toolshed_base_url and \
rd_name == self.repository.name and \
rd_owner == self.repository.user.username and \
rd_changeset_revision == self.repository_metadata.changeset_revision:
return rd_prior_installation_required, rd_only_if_compiling_contained_td
elif self.repository_metadata:
# Get the list of changeset revisions from the tool shed to which self.repository may be updated.
metadata = self.repository_metadata.metadata
current_changeset_revision = str( self.repository_metadata.changeset_revision )
# Get the changeset revision to which the current value of required_repository_changeset_revision
# should be updated if it's not current.
text = suc.get_updated_changeset_revisions( self.app,
name=str( self.repository.name ),
owner=str( self.repository.user.username ),
changeset_revision=current_changeset_revision )
if text:
valid_changeset_revisions = listify( text )
if current_changeset_revision not in valid_changeset_revisions:
valid_changeset_revisions.append( current_changeset_revision )
else:
valid_changeset_revisions = [ current_changeset_revision ]
repository_dependencies_dict = metadata[ 'repository_dependencies' ]
rd_tups = repository_dependencies_dict.get( 'repository_dependencies', [] )
for rd_tup in rd_tups:
rd_toolshed, \
rd_name, \
rd_owner, \
rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( rd_tup )
cleaned_rd_toolshed = common_util.remove_protocol_from_tool_shed_url( rd_toolshed )
if cleaned_rd_toolshed == cleaned_toolshed_base_url and \
rd_name == self.repository.name and \
rd_owner == self.repository.user.username and \
rd_changeset_revision in valid_changeset_revisions:
return rd_prior_installation_required, rd_only_if_compiling_contained_td
# Default both prior_installation_required and only_if_compiling_contained_td to False.
return 'False', 'False'
def get_or_create_tool_shed_repository( self, tool_shed, name, owner, changeset_revision ):
"""
Return a tool shed repository database record defined by the combination of
tool shed, repository name, repository owner and changeset_revision or
installed_changeset_revision. A new tool shed repository record will be
created if one is not located.
"""
install_model = self.app.install_model
# We store the port in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed )
# This method is used only in Galaxy, not the tool shed.
repository = suc.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision )
if not repository:
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed )
repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name )
ctx_rev = suc.get_ctx_rev( self.app, tool_shed_url, name, owner, changeset_revision )
repository = suc.create_or_update_tool_shed_repository( app=self.app,
name=name,
description=None,
installed_changeset_revision=changeset_revision,
ctx_rev=ctx_rev,
repository_clone_url=repository_clone_url,
metadata_dict={},
status=install_model.ToolShedRepository.installation_status.NEW,
current_changeset_revision=None,
owner=owner,
dist_to_shed=False )
return repository
def get_or_create_tool_shed_repository( self, tool_shed, name, owner, changeset_revision ):
"""
Return a tool shed repository database record defined by the combination of
tool shed, repository name, repository owner and changeset_revision or
installed_changeset_revision. A new tool shed repository record will be
created if one is not located.
"""
install_model = self.app.install_model
# We store the port in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed )
# This method is used only in Galaxy, not the tool shed.
repository = suc.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision )
if not repository:
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed )
repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name )
ctx_rev = suc.get_ctx_rev( self.app, tool_shed_url, name, owner, changeset_revision )
repository = suc.create_or_update_tool_shed_repository( app=self.app,
name=name,
description=None,
installed_changeset_revision=changeset_revision,
ctx_rev=ctx_rev,
repository_clone_url=repository_clone_url,
metadata_dict={},
status=install_model.ToolShedRepository.installation_status.NEW,
current_changeset_revision=None,
owner=owner,
dist_to_shed=False )
return repository
def password_manager_for_url(self, url):
"""
If the tool shed is using external auth, the client to the tool shed must authenticate to that
as well. This provides access to the six.moves.urllib.request.HTTPPasswordMgrWithdefaultRealm() object for the
url passed in.
Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at
some stage...
"""
url_sans_protocol = common_util.remove_protocol_from_tool_shed_url(url)
for shed_name, shed_url in self.tool_sheds.items():
shed_url_sans_protocol = common_util.remove_protocol_from_tool_shed_url(shed_url)
if url_sans_protocol.startswith(shed_url_sans_protocol):
return self.tool_sheds_auth[shed_name]
log.debug("Invalid url '%s' received by tool shed registry's password_manager_for_url method." % str(url))
return None
def load_from_element( self, elem, tool_path ):
assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag )
self.declared_id = elem.get( 'id', None )
self.guid = elem.get( 'guid', None )
path = elem.get( 'tool_file', None )
self.version = elem.get( 'version', self.version )
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find( 'tool' )
assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) )
path = tool_elem.get( "file", None )
tool_guid = tool_elem.get( "guid", None )
# need to determine repository info so that dependencies will work correctly
tool_shed_url = tool_elem.find( 'tool_shed' ).text
# Handle protocol changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url )
# The protocol is not stored in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url )
repository_name = tool_elem.find( 'repository_name' ).text
repository_owner = tool_elem.find( 'repository_owner' ).text
installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text
self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision )
tool_shed_repository = \
suc.get_installed_repository( self.data_managers.app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision )
if tool_shed_repository is None:
log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' )
# we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path )
except Exception, e:
log.error( 'Error determining tool_path for Data Manager during testing: %s', e )
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id )
# use shed_conf_file to determine tool_path
shed_conf_file = elem.get( "shed_conf_file", None )
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None )
if shed_conf:
tool_path = shed_conf.get( "tool_path", tool_path )
def generate_repository_dependencies_key_for_repository(
toolshed_base_url, repository_name, repository_owner,
changeset_revision, prior_installation_required,
only_if_compiling_contained_td):
"""
Assumes tool shed is current tool shed since repository dependencies across tool sheds
is not yet supported.
"""
# The tool_shed portion of the key must be the value that is stored in the tool_shed_repository.tool_shed column
# of the Galaxy database for an installed repository. This value does not include the protocol, but does include
# the port if there is one.
tool_shed = common_util.remove_protocol_from_tool_shed_url(
toolshed_base_url)
return '%s%s%s%s%s%s%s%s%s%s%s' % (
tool_shed, STRSEP, str(repository_name), STRSEP, str(repository_owner),
STRSEP, str(changeset_revision), STRSEP,
str(prior_installation_required), STRSEP,
str(only_if_compiling_contained_td))
def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner,
changeset_revision, prior_installation_required,
only_if_compiling_contained_td ):
"""
Assumes tool shed is current tool shed since repository dependencies across tool sheds
is not yet supported.
"""
# The tool_shed portion of the key must be the value that is stored in the tool_shed_repository.tool_shed column
# of the Galaxy database for an installed repository. This value does not include the protocol, but does include
# the port if there is one.
tool_shed = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url )
return '%s%s%s%s%s%s%s%s%s%s%s' % ( tool_shed,
STRSEP,
str( repository_name ),
STRSEP,
str( repository_owner ),
STRSEP,
str( changeset_revision ),
STRSEP,
str( prior_installation_required ),
STRSEP,
str( only_if_compiling_contained_td ) )
def load_from_element(self, elem, tool_path):
assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % (elem.tag)
self.declared_id = elem.get('id', None)
self.guid = elem.get('guid', None)
path = elem.get('tool_file', None)
self.version = elem.get('version', self.version)
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find('tool')
assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % (util.xml_to_string(elem))
path = tool_elem.get("file", None)
tool_guid = tool_elem.get("guid", None)
# need to determine repository info so that dependencies will work correctly
if hasattr(self.data_managers.app, 'tool_cache') and tool_guid in self.data_managers.app.tool_cache._tool_paths_by_id:
path = self.data_managers.app.tool_cache._tool_paths_by_id[tool_guid]
tool = self.data_managers.app.tool_cache.get_tool(path)
tool_shed_repository = tool.tool_shed_repository
self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed_repository.tool_shed,
name=tool_shed_repository.name,
owner=tool_shed_repository.owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision)
tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id)
tool_path = ""
else:
tool_shed_url = tool_elem.find('tool_shed').text
# Handle protocol changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.data_managers.app, tool_shed_url)
# The protocol is not stored in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed_url)
repository_name = tool_elem.find('repository_name').text
repository_owner = tool_elem.find('repository_owner').text
installed_changeset_revision = tool_elem.find('installed_changeset_revision').text
self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision)
tool_shed_repository = \
repository_util.get_installed_repository(self.data_managers.app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision)
if tool_shed_repository is None:
log.warning('Could not determine tool shed repository from database. This should only ever happen when running tests.')
# we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml(elem.get('shed_conf_file')).getroot().get('tool_path', tool_path)
except Exception as e:
log.error('Error determining tool_path for Data Manager during testing: %s', e)
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id)
# use shed_conf_file to determine tool_path
shed_conf_file = elem.get("shed_conf_file", None)
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename(shed_conf_file, None)
if shed_conf:
tool_path = shed_conf.get("tool_path", tool_path)
assert path is not None, "A tool file path could not be determined:\n%s" % (util.xml_to_string(elem))
self.load_tool(os.path.join(tool_path, path),
guid=tool_guid,
data_manager_id=self.id,
tool_shed_repository_id=tool_shed_repository_id)
self.name = elem.get('name', self.tool.name)
self.description = elem.get('description', self.tool.description)
self.undeclared_tables = util.asbool(elem.get('undeclared_tables', self.undeclared_tables))
for data_table_elem in elem.findall('data_table'):
data_table_name = data_table_elem.get("name")
assert data_table_name is not None, "A name is required for a data table entry"
if data_table_name not in self.data_tables:
self.data_tables[data_table_name] = odict()
output_elem = data_table_elem.find('output')
if output_elem is not None:
for column_elem in output_elem.findall('column'):
column_name = column_elem.get('name', None)
assert column_name is not None, "Name is required for column entry"
data_table_coumn_name = column_elem.get('data_table_name', column_name)
self.data_tables[data_table_name][data_table_coumn_name] = column_name
output_ref = column_elem.get('output_ref', None)
if output_ref is not None:
if data_table_name not in self.output_ref_by_data_table:
self.output_ref_by_data_table[data_table_name] = {}
self.output_ref_by_data_table[data_table_name][data_table_coumn_name] = output_ref
value_translation_elems = column_elem.findall('value_translation')
if value_translation_elems is not None:
for value_translation_elem in value_translation_elems:
value_translation = value_translation_elem.text
if value_translation is not None:
value_translation_type = value_translation_elem.get('type', DEFAULT_VALUE_TRANSLATION_TYPE)
if data_table_name not in self.value_translation_by_data_table_column:
self.value_translation_by_data_table_column[data_table_name] = {}
if data_table_coumn_name not in self.value_translation_by_data_table_column[data_table_name]:
self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name] = []
if value_translation_type == 'function':
if value_translation in VALUE_TRANSLATION_FUNCTIONS:
value_translation = VALUE_TRANSLATION_FUNCTIONS[value_translation]
else:
raise ValueError("Unsupported value translation function: '%s'" % (value_translation))
else:
assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError("Unsupported value translation type: '%s'" % (value_translation_type))
self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name].append(value_translation)
for move_elem in column_elem.findall('move'):
move_type = move_elem.get('type', 'directory')
relativize_symlinks = move_elem.get('relativize_symlinks', False) # TODO: should we instead always relativize links?
source_elem = move_elem.find('source')
if source_elem is None:
source_base = None
source_value = ''
else:
source_base = source_elem.get('base', None)
source_value = source_elem.text
target_elem = move_elem.find('target')
if target_elem is None:
target_base = None
target_value = ''
else:
target_base = target_elem.get('base', None)
target_value = target_elem.text
if data_table_name not in self.move_by_data_table_column:
self.move_by_data_table_column[data_table_name] = {}
self.move_by_data_table_column[data_table_name][data_table_coumn_name] = \
dict(type=move_type,
source_base=source_base,
source_value=source_value,
target_base=target_base,
target_value=target_value,
relativize_symlinks=relativize_symlinks)
def load_from_element(self, elem, tool_path):
assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % (
elem.tag)
self.declared_id = elem.get('id', None)
self.guid = elem.get('guid', None)
path = elem.get('tool_file', None)
self.version = elem.get('version', self.version)
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find('tool')
assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % (
util.xml_to_string(elem))
path = tool_elem.get("file", None)
tool_guid = tool_elem.get("guid", None)
#need to determine repository info so that dependencies will work correctly
tool_shed_url = tool_elem.find('tool_shed').text
# Handle protocol changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.data_managers.app, tool_shed_url)
# The protocol is not stored in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(
tool_shed_url)
repository_name = tool_elem.find('repository_name').text
repository_owner = tool_elem.find('repository_owner').text
installed_changeset_revision = tool_elem.find(
'installed_changeset_revision').text
self.tool_shed_repository_info_dict = dict(
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision)
tool_shed_repository = \
suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.data_managers.app,
tool_shed,
repository_name,
repository_owner,
installed_changeset_revision )
if tool_shed_repository is None:
log.warning(
'Could not determine tool shed repository from database. This should only ever happen when running tests.'
)
#we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml(
elem.get('shed_conf_file')).getroot().get(
'tool_path', tool_path)
except Exception, e:
log.error(
'Error determining tool_path for Data Manager during testing: %s',
e)
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id(
tool_shed_repository.id)
#use shed_conf_file to determine tool_path
shed_conf_file = elem.get("shed_conf_file", None)
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename(
shed_conf_file, None)
if shed_conf:
tool_path = shed_conf.get("tool_path", tool_path)
def get_env_shell_file_paths( self, elem ):
# Currently only the following tag set is supported.
# <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056">
# <package name="numpy" version="1.7.1" />
# </repository>
env_shell_file_paths = []
toolshed = elem.get( 'toolshed', None )
repository_name = elem.get( 'name', None )
repository_owner = elem.get( 'owner', None )
changeset_revision = elem.get( 'changeset_revision', None )
if toolshed and repository_name and repository_owner and changeset_revision:
# The protocol is not stored, but the port is if it exists.
toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed )
repository = suc.get_repository_for_dependency_relationship( self.app,
toolshed,
repository_name,
repository_owner,
changeset_revision )
if repository:
for sub_elem in elem:
tool_dependency_type = sub_elem.tag
tool_dependency_name = sub_elem.get( 'name' )
tool_dependency_version = sub_elem.get( 'version' )
if tool_dependency_type and tool_dependency_name and tool_dependency_version:
# Get the tool_dependency so we can get its installation directory.
tool_dependency = None
for tool_dependency in repository.tool_dependencies:
if tool_dependency.type == tool_dependency_type and \
tool_dependency.name == tool_dependency_name and \
tool_dependency.version == tool_dependency_version:
break
if tool_dependency:
installation_directory = tool_dependency.installation_directory( self.app )
env_shell_file_path = self.get_env_shell_file_path( installation_directory )
if env_shell_file_path:
env_shell_file_paths.append( env_shell_file_path )
else:
error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency "
error_message += "type %s, name %s, version %s for repository %s" % \
( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) )
log.debug( error_message )
continue
else:
error_message = "Skipping tool dependency definition because unable to locate tool dependency "
error_message += "type %s, name %s, version %s for repository %s" % \
( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) )
log.debug( error_message )
continue
else:
error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \
( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) )
log.debug( error_message )
continue
else:
error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: "
error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \
( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) )
log.debug( error_message )
else:
error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \
( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) )
log.debug( error_message )
return env_shell_file_paths
def tool_shed_is_this_tool_shed(toolshed_base_url):
"""Determine if a tool shed is the current tool shed."""
cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url(toolshed_base_url)
cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url(str(url_for('/', qualified=True)))
return cleaned_toolshed_base_url == cleaned_tool_shed
def handle_complex_repository_dependency_for_package( self, elem, package_name, package_version, tool_shed_repository,
from_tool_migration_manager=False ):
"""
Inspect the repository defined by a complex repository dependency definition and take certain steps to
enable installation of the received package name and version to proceed. The received elem is the
<repository> tag set which defines the complex repository dependency. The received tool_shed_repository
is the installed tool shed repository for which the tool dependency defined by the received package_name
and package_version is being installed.
"""
handled_tool_dependencies = []
tool_shed_url = elem.attrib[ 'toolshed' ]
required_repository_name = elem.attrib[ 'name' ]
required_repository_owner = elem.attrib[ 'owner' ]
default_required_repository_changeset_revision = elem.attrib[ 'changeset_revision' ]
required_repository = repository_util.get_repository_for_dependency_relationship( self.app, tool_shed_url,
required_repository_name,
required_repository_owner,
default_required_repository_changeset_revision )
tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url )
tmp_filename = None
if required_repository:
required_repository_changeset_revision = required_repository.installed_changeset_revision
# Define the installation directory for the required tool dependency package in the required repository.
required_repository_package_install_dir = \
tool_dependency_util.get_tool_dependency_install_dir( app=self.app,
repository_name=required_repository_name,
repository_owner=required_repository_owner,
repository_changeset_revision=required_repository_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
# Define this dependent repository's tool dependency installation directory that will contain
# the env.sh file with a path to the required repository's installed tool dependency package.
dependent_install_dir = \
tool_dependency_util.get_tool_dependency_install_dir( app=self.app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
if os.path.exists( dependent_install_dir ):
# The install manager handles tool migration stages and the sync_database_with_file_system()
# method handles two scenarios: (1) where a Galaxy file system environment related to installed
# Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy
# database tables associated with these installed items, and (2) the Tool Shed's install and test
# framework which installs repositories in 2 stages, those of type tool_dependency_definition
# followed by those containing valid tools and tool functional test components. Neither of these
# scenarios apply when the install manager is running.
if from_tool_migration_manager:
can_install_tool_dependency = True
else:
# Notice that we'll throw away the following tool_dependency if it can be installed.
tool_dependency, can_install_tool_dependency = self.sync_database_with_file_system( self.app,
tool_shed_repository,
package_name,
package_version,
dependent_install_dir,
tool_dependency_type='package' )
if not can_install_tool_dependency:
log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), "
"so appending it to the list of handled tool dependencies.",
str( tool_dependency.name), str( tool_dependency.version ) )
handled_tool_dependencies.append( tool_dependency )
else:
can_install_tool_dependency = True
if can_install_tool_dependency:
# Set this dependent repository's tool dependency env.sh file with a path to the required repository's
# installed tool dependency package. We can get everything we need from the discovered installed
# required_repository.
if required_repository.is_deactivated_or_installed:
if not os.path.exists( required_repository_package_install_dir ):
log.error( 'Missing required tool dependency directory %s' % str( required_repository_package_install_dir ) )
repo_files_dir = required_repository.repo_files_directory( self.app )
if not repo_files_dir:
message = "Unable to locate the repository directory for revision %s of installed repository %s owned by %s." % \
( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) )
raise Exception( message )
tool_dependencies_config = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml' )
if tool_dependencies_config:
config_to_use = tool_dependencies_config
else:
message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \
( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) )
raise Exception( message )
else:
# Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision
# should be updated if it's not current.
text = metadata_util.get_updated_changeset_revisions_from_tool_shed( app=self.app,
tool_shed_url=tool_shed,
name=required_repository_name,
owner=required_repository_owner,
changeset_revision=required_repository_changeset_revision )
if text:
updated_changeset_revisions = listify( text )
# The list of changeset revisions is in reverse order, so the newest will be first.
required_repository_changeset_revision = updated_changeset_revisions[ 0 ]
# Make a call to the tool shed to get the required repository's tool_dependencies.xml file.
tmp_filename = self.create_temporary_tool_dependencies_config( tool_shed,
required_repository_name,
required_repository_owner,
required_repository_changeset_revision )
config_to_use = tmp_filename
handled_tool_dependencies = \
self.create_tool_dependency_with_initialized_env_sh_file( dependent_install_dir=dependent_install_dir,
tool_shed_repository=tool_shed_repository,
required_repository=required_repository,
package_name=package_name,
package_version=package_version,
tool_dependencies_config=config_to_use )
self.remove_file( tmp_filename )
else:
message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \
( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) )
raise Exception( message )
return handled_tool_dependencies
def get_env_shell_file_paths(self, elem):
# Currently only the following tag set is supported.
# <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056">
# <package name="numpy" version="1.7.1" />
# </repository>
env_shell_file_paths = []
toolshed = elem.get('toolshed', None)
repository_name = elem.get('name', None)
repository_owner = elem.get('owner', None)
changeset_revision = elem.get('changeset_revision', None)
if toolshed and repository_name and repository_owner and changeset_revision:
# The protocol is not stored, but the port is if it exists.
toolshed = common_util.remove_protocol_from_tool_shed_url(toolshed)
repository = repository_util.get_repository_for_dependency_relationship(
self.app, toolshed, repository_name, repository_owner,
changeset_revision)
if repository:
for sub_elem in elem:
tool_dependency_type = sub_elem.tag
tool_dependency_name = sub_elem.get('name')
tool_dependency_version = sub_elem.get('version')
if tool_dependency_type and tool_dependency_name and tool_dependency_version:
# Get the tool_dependency so we can get its installation directory.
tool_dependency = None
for tool_dependency in repository.tool_dependencies:
if tool_dependency.type == tool_dependency_type and \
tool_dependency.name == tool_dependency_name and \
tool_dependency.version == tool_dependency_version:
break
if tool_dependency:
installation_directory = tool_dependency.installation_directory(
self.app)
env_shell_file_path = self.get_env_shell_file_path(
installation_directory)
if env_shell_file_path:
env_shell_file_paths.append(
env_shell_file_path)
else:
error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency "
error_message += "type %s, name %s, version %s for repository %s" % \
(str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version), str(repository.name))
log.debug(error_message)
continue
else:
error_message = "Skipping tool dependency definition because unable to locate tool dependency "
error_message += "type %s, name %s, version %s for repository %s" % \
(str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version), str(repository.name))
log.debug(error_message)
continue
else:
error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \
(str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version))
log.debug(error_message)
continue
else:
error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: "
error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \
(str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision))
log.debug(error_message)
else:
error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \
(str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision))
log.debug(error_message)
return env_shell_file_paths
def load_from_element( self, elem, tool_path ):
assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag )
self.declared_id = elem.get( 'id', None )
self.guid = elem.get( 'guid', None )
path = elem.get( 'tool_file', None )
self.version = elem.get( 'version', self.version )
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find( 'tool' )
assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) )
path = tool_elem.get( "file", None )
tool_guid = tool_elem.get( "guid", None )
# need to determine repository info so that dependencies will work correctly
tool_shed_url = tool_elem.find( 'tool_shed' ).text
# Handle protocol changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url )
# The protocol is not stored in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url )
repository_name = tool_elem.find( 'repository_name' ).text
repository_owner = tool_elem.find( 'repository_owner' ).text
installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text
self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision )
tool_shed_repository = \
repository_util.get_installed_repository( self.data_managers.app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision )
if tool_shed_repository is None:
log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' )
# we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path )
except Exception as e:
log.error( 'Error determining tool_path for Data Manager during testing: %s', e )
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id )
# use shed_conf_file to determine tool_path
shed_conf_file = elem.get( "shed_conf_file", None )
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None )
if shed_conf:
tool_path = shed_conf.get( "tool_path", tool_path )
assert path is not None, "A tool file path could not be determined:\n%s" % ( util.xml_to_string( elem ) )
self.load_tool( os.path.join( tool_path, path ),
guid=tool_guid,
data_manager_id=self.id,
tool_shed_repository_id=tool_shed_repository_id )
self.name = elem.get( 'name', self.tool.name )
self.description = elem.get( 'description', self.tool.description )
self.undeclared_tables = util.asbool( elem.get( 'undeclared_tables', self.undeclared_tables ) )
for data_table_elem in elem.findall( 'data_table' ):
data_table_name = data_table_elem.get( "name" )
assert data_table_name is not None, "A name is required for a data table entry"
if data_table_name not in self.data_tables:
self.data_tables[ data_table_name ] = odict()
output_elem = data_table_elem.find( 'output' )
if output_elem is not None:
for column_elem in output_elem.findall( 'column' ):
column_name = column_elem.get( 'name', None )
assert column_name is not None, "Name is required for column entry"
data_table_coumn_name = column_elem.get( 'data_table_name', column_name )
self.data_tables[ data_table_name ][ data_table_coumn_name ] = column_name
output_ref = column_elem.get( 'output_ref', None )
if output_ref is not None:
if data_table_name not in self.output_ref_by_data_table:
self.output_ref_by_data_table[ data_table_name ] = {}
self.output_ref_by_data_table[ data_table_name ][ data_table_coumn_name ] = output_ref
value_translation_elems = column_elem.findall( 'value_translation' )
if value_translation_elems is not None:
for value_translation_elem in value_translation_elems:
value_translation = value_translation_elem.text
if value_translation is not None:
value_translation_type = value_translation_elem.get( 'type', DEFAULT_VALUE_TRANSLATION_TYPE )
if data_table_name not in self.value_translation_by_data_table_column:
self.value_translation_by_data_table_column[ data_table_name ] = {}
if data_table_coumn_name not in self.value_translation_by_data_table_column[ data_table_name ]:
self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = []
if value_translation_type == 'function':
if value_translation in VALUE_TRANSLATION_FUNCTIONS:
value_translation = VALUE_TRANSLATION_FUNCTIONS[ value_translation ]
else:
raise ValueError( "Unsupported value translation function: '%s'" % ( value_translation ) )
else:
assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError( "Unsupported value translation type: '%s'" % ( value_translation_type ) )
self.value_translation_by_data_table_column[ data_table_name ][ data_table_coumn_name ].append( value_translation )
for move_elem in column_elem.findall( 'move' ):
move_type = move_elem.get( 'type', 'directory' )
relativize_symlinks = move_elem.get( 'relativize_symlinks', False ) # TODO: should we instead always relativize links?
source_elem = move_elem.find( 'source' )
if source_elem is None:
source_base = None
source_value = ''
else:
source_base = source_elem.get( 'base', None )
source_value = source_elem.text
target_elem = move_elem.find( 'target' )
if target_elem is None:
target_base = None
target_value = ''
else:
target_base = target_elem.get( 'base', None )
target_value = target_elem.text
if data_table_name not in self.move_by_data_table_column:
self.move_by_data_table_column[ data_table_name ] = {}
self.move_by_data_table_column[ data_table_name ][ data_table_coumn_name ] = \
dict( type=move_type,
source_base=source_base,
source_value=source_value,
target_base=target_base,
target_value=target_value,
relativize_symlinks=relativize_symlinks )
