def tool_shed_is_this_tool_shed(toolshed_base_url):
"""Determine if a tool shed is the current tool shed."""
cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url(
toolshed_base_url)
cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url(
str(url_for('/', qualified=True)))
return cleaned_toolshed_base_url == cleaned_tool_shed
def password_manager_for_url(self, url):
"""
If the tool shed is using external auth, the client to the tool shed must authenticate to that
as well. This provides access to the six.moves.urllib.request.HTTPPasswordMgrWithdefaultRealm() object for the
url passed in.
Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at
some stage...
"""
url_sans_protocol = remove_protocol_from_tool_shed_url(url)
for shed_name, shed_url in self.tool_sheds.items():
shed_url_sans_protocol = remove_protocol_from_tool_shed_url(shed_url)
if url_sans_protocol.startswith(shed_url_sans_protocol):
return self.tool_sheds_auth[shed_name]
log.debug("Invalid url '%s' received by tool shed registry's password_manager_for_url method." % str(url))
return None
def get_or_create_tool_shed_repository(self, tool_shed, name, owner,
changeset_revision):
"""
Return a tool shed repository database record defined by the combination of
tool shed, repository name, repository owner and changeset_revision or
installed_changeset_revision. A new tool shed repository record will be
created if one is not located.
"""
install_model = self.app.install_model
# We store the port in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed)
# This method is used only in Galaxy, not the tool shed.
repository = repository_util.get_repository_for_dependency_relationship(
self.app, tool_shed, name, owner, changeset_revision)
if not repository:
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, tool_shed)
repository_clone_url = os.path.join(tool_shed_url, 'repos', owner,
name)
ctx_rev = get_ctx_rev(self.app, tool_shed_url, name, owner,
changeset_revision)
repository = repository_util.create_or_update_tool_shed_repository(
app=self.app,
name=name,
description=None,
installed_changeset_revision=changeset_revision,
ctx_rev=ctx_rev,
repository_clone_url=repository_clone_url,
status=install_model.ToolShedRepository.installation_status.
NEW,
metadata_dict={},
current_changeset_revision=None,
owner=owner,
dist_to_shed=False)
return repository
def get_installed_repository(app, tool_shed=None, name=None, owner=None, changeset_revision=None, installed_changeset_revision=None, repository_id=None, refresh=False, from_cache=False):
"""
Return a tool shed repository database record defined by the combination of a toolshed, repository name,
repository owner and either current or originally installed changeset_revision.
"""
# We store the port, if one exists, in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed)
if from_cache and hasattr(app, 'tool_shed_repository_cache'):
if refresh:
app.tool_shed_repository_cache.rebuild()
return app.tool_shed_repository_cache.get_installed_repository(tool_shed=tool_shed,
name=name,
owner=owner,
installed_changeset_revision=installed_changeset_revision,
changeset_revision=changeset_revision,
repository_id=repository_id)
query = app.install_model.context.query(app.install_model.ToolShedRepository)
if repository_id:
clause_list = [app.install_model.ToolShedRepository.table.c.id == repository_id]
else:
clause_list = [app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed,
app.install_model.ToolShedRepository.table.c.name == name,
app.install_model.ToolShedRepository.table.c.owner == owner]
if changeset_revision is not None:
clause_list.append(app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision)
if installed_changeset_revision is not None:
clause_list.append(app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision)
return query.filter(and_(*clause_list)).first()
def get_repository_for_dependency_relationship(app, tool_shed, name, owner,
changeset_revision):
"""
Return an installed tool_shed_repository database record that is defined by either the current changeset
revision or the installed_changeset_revision.
"""
# This method is used only in Galaxy, not the Tool Shed. We store the port (if one exists) in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed)
if tool_shed is None or name is None or owner is None or changeset_revision is None:
message = "Unable to retrieve the repository record from the database because one or more of the following "
message += "required parameters is None: tool_shed: %s, name: %s, owner: %s, changeset_revision: %s " % \
(str(tool_shed), str(name), str(owner), str(changeset_revision))
raise Exception(message)
app.tool_shed_repository_cache.rebuild()
repository = get_installed_repository(
app=app,
tool_shed=tool_shed,
name=name,
owner=owner,
installed_changeset_revision=changeset_revision)
if not repository:
repository = get_installed_repository(
app=app,
tool_shed=tool_shed,
name=name,
owner=owner,
changeset_revision=changeset_revision)
if not repository:
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
app, tool_shed)
repository_clone_url = os.path.join(tool_shed_url, 'repos', owner,
name)
repo_info_tuple = (None, repository_clone_url, changeset_revision,
None, owner, None, None)
repository, pcr = repository_was_previously_installed(
app, tool_shed_url, name, repo_info_tuple)
if not repository:
# The received changeset_revision is no longer installable, so get the next changeset_revision
# in the repository's changelog in the tool shed that is associated with repository_metadata.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
app, tool_shed)
params = dict(name=name,
owner=owner,
changeset_revision=changeset_revision)
pathspec = ['repository', 'next_installable_changeset_revision']
text = util.url_get(
tool_shed_url,
auth=app.tool_shed_registry.url_auth(tool_shed_url),
pathspec=pathspec,
params=params)
if text:
repository = get_installed_repository(app=app,
tool_shed=tool_shed,
name=name,
owner=owner,
changeset_revision=text)
return repository
def get_repository_ids_requiring_prior_import_or_install(
app, tsr_ids, repository_dependencies):
"""
This method is used in the Tool Shed when exporting a repository and its dependencies,
and in Galaxy when a repository and its dependencies are being installed. Inspect the
received repository_dependencies and determine if the encoded id of each required
repository is in the received tsr_ids. If so, then determine whether that required
repository should be imported / installed prior to its dependent repository. Return a
list of encoded repository ids, each of which is contained in the received list of tsr_ids,
and whose associated repositories must be imported / installed prior to the dependent
repository associated with the received repository_dependencies.
"""
prior_tsr_ids = []
if repository_dependencies:
for key, rd_tups in repository_dependencies.items():
if key in ['description', 'root_key']:
continue
for rd_tup in rd_tups:
tool_shed, \
name, \
owner, \
changeset_revision, \
prior_installation_required, \
only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple(rd_tup)
# If only_if_compiling_contained_td is False, then the repository dependency
# is not required to be installed prior to the dependent repository even if
# prior_installation_required is True. This is because the only meaningful
# content of the repository dependency is its contained tool dependency, which
# is required in order to compile the dependent repository's tool dependency.
# In the scenario where the repository dependency is not installed prior to the
# dependent repository's tool dependency compilation process, the tool dependency
# compilation framework will install the repository dependency prior to compilation
# of the dependent repository's tool dependency.
if not util.asbool(only_if_compiling_contained_td):
if util.asbool(prior_installation_required):
if is_tool_shed_client(app):
# We store the port, if one exists, in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(
tool_shed)
repository = get_repository_for_dependency_relationship(
app, tool_shed, name, owner,
changeset_revision)
else:
repository = get_repository_by_name_and_owner(
app, name, owner)
if repository:
encoded_repository_id = app.security.encode_id(
repository.id)
if encoded_repository_id in tsr_ids:
prior_tsr_ids.append(encoded_repository_id)
return prior_tsr_ids
def generate_repository_dependencies_key_for_repository(
toolshed_base_url, repository_name, repository_owner,
changeset_revision, prior_installation_required,
only_if_compiling_contained_td):
"""
Assumes tool shed is current tool shed since repository dependencies across tool sheds
is not yet supported.
"""
# The tool_shed portion of the key must be the value that is stored in the tool_shed_repository.tool_shed column
# of the Galaxy database for an installed repository. This value does not include the protocol, but does include
# the port if there is one.
tool_shed = remove_protocol_from_tool_shed_url(toolshed_base_url)
return '%s%s%s%s%s%s%s%s%s%s%s' % (
tool_shed, STRSEP, str(repository_name), STRSEP, str(repository_owner),
STRSEP, str(changeset_revision), STRSEP,
str(prior_installation_required), STRSEP,
str(only_if_compiling_contained_td))
def handle_complex_repository_dependency_for_package(
self,
elem,
package_name,
package_version,
tool_shed_repository,
from_tool_migration_manager=False):
"""
Inspect the repository defined by a complex repository dependency definition and take certain steps to
enable installation of the received package name and version to proceed. The received elem is the
<repository> tag set which defines the complex repository dependency. The received tool_shed_repository
is the installed tool shed repository for which the tool dependency defined by the received package_name
and package_version is being installed.
"""
handled_tool_dependencies = []
tool_shed_url = elem.attrib['toolshed']
required_repository_name = elem.attrib['name']
required_repository_owner = elem.attrib['owner']
default_required_repository_changeset_revision = elem.attrib[
'changeset_revision']
required_repository = get_repository_for_dependency_relationship(
self.app, tool_shed_url, required_repository_name,
required_repository_owner,
default_required_repository_changeset_revision)
tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url)
tmp_filename = None
if required_repository:
required_repository_changeset_revision = required_repository.installed_changeset_revision
# Define the installation directory for the required tool dependency package in the required repository.
required_repository_package_install_dir = \
tool_dependency_util.get_tool_dependency_install_dir(app=self.app,
repository_name=required_repository_name,
repository_owner=required_repository_owner,
repository_changeset_revision=required_repository_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version)
# Define this dependent repository's tool dependency installation directory that will contain
# the env.sh file with a path to the required repository's installed tool dependency package.
dependent_install_dir = \
tool_dependency_util.get_tool_dependency_install_dir(app=self.app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version)
if os.path.exists(dependent_install_dir):
# The install manager handles tool migration stages and the sync_database_with_file_system()
# method handles two scenarios: (1) where a Galaxy file system environment related to installed
# Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy
# database tables associated with these installed items, and (2) the Tool Shed's install and test
# framework which installs repositories in 2 stages, those of type tool_dependency_definition
# followed by those containing valid tools and tool functional test components. Neither of these
# scenarios apply when the install manager is running.
if from_tool_migration_manager:
can_install_tool_dependency = True
else:
# Notice that we'll throw away the following tool_dependency if it can be installed.
tool_dependency, can_install_tool_dependency = self.sync_database_with_file_system(
self.app,
tool_shed_repository,
package_name,
package_version,
dependent_install_dir,
tool_dependency_type='package')
if not can_install_tool_dependency:
log.debug(
"Tool dependency %s version %s cannot be installed (it was probably previously installed), "
"so appending it to the list of handled tool dependencies.",
str(tool_dependency.name),
str(tool_dependency.version))
handled_tool_dependencies.append(tool_dependency)
else:
can_install_tool_dependency = True
if can_install_tool_dependency:
# Set this dependent repository's tool dependency env.sh file with a path to the required repository's
# installed tool dependency package. We can get everything we need from the discovered installed
# required_repository.
if required_repository.is_deactivated_or_installed:
if not os.path.exists(
required_repository_package_install_dir):
log.error(
f'Missing required tool dependency directory {str(required_repository_package_install_dir)}'
)
repo_files_dir = required_repository.repo_files_directory(
self.app)
if not repo_files_dir:
message = "Unable to locate the repository directory for revision %s of installed repository %s owned by %s." % \
(str(required_repository.changeset_revision), str(required_repository.name), str(required_repository.owner))
raise Exception(message)
tool_dependencies_config = get_absolute_path_to_file_in_repository(
repo_files_dir, 'tool_dependencies.xml')
if tool_dependencies_config:
config_to_use = tool_dependencies_config
else:
message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \
(str(required_repository.changeset_revision), str(required_repository.name), str(required_repository.owner))
raise Exception(message)
else:
# Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision
# should be updated if it's not current.
text = get_updated_changeset_revisions_from_tool_shed(
app=self.app,
tool_shed_url=tool_shed,
name=required_repository_name,
owner=required_repository_owner,
changeset_revision=
required_repository_changeset_revision)
if text:
updated_changeset_revisions = listify(text)
# The list of changeset revisions is in reverse order, so the newest will be first.
required_repository_changeset_revision = updated_changeset_revisions[
0]
# Make a call to the tool shed to get the required repository's tool_dependencies.xml file.
tmp_filename = self.create_temporary_tool_dependencies_config(
tool_shed, required_repository_name,
required_repository_owner,
required_repository_changeset_revision)
config_to_use = tmp_filename
handled_tool_dependencies = \
self.create_tool_dependency_with_initialized_env_sh_file(dependent_install_dir=dependent_install_dir,
tool_shed_repository=tool_shed_repository,
required_repository=required_repository,
package_name=package_name,
package_version=package_version,
tool_dependencies_config=config_to_use)
self.remove_file(tmp_filename)
else:
message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \
(str(required_repository_name), str(required_repository_owner), str(default_required_repository_changeset_revision))
raise Exception(message)
return handled_tool_dependencies
def get_env_shell_file_paths(self, elem):
# Currently only the following tag set is supported.
# <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056">
# <package name="numpy" version="1.7.1" />
# </repository>
env_shell_file_paths = []
toolshed = elem.get('toolshed', None)
repository_name = elem.get('name', None)
repository_owner = elem.get('owner', None)
changeset_revision = elem.get('changeset_revision', None)
if toolshed and repository_name and repository_owner and changeset_revision:
# The protocol is not stored, but the port is if it exists.
toolshed = remove_protocol_from_tool_shed_url(toolshed)
repository = get_repository_for_dependency_relationship(self.app,
toolshed,
repository_name,
repository_owner,
changeset_revision)
if repository:
for sub_elem in elem:
tool_dependency_type = sub_elem.tag
tool_dependency_name = sub_elem.get('name')
tool_dependency_version = sub_elem.get('version')
if tool_dependency_type and tool_dependency_name and tool_dependency_version:
# Get the tool_dependency so we can get its installation directory.
tool_dependency = None
for tool_dependency in repository.tool_dependencies:
if tool_dependency.type == tool_dependency_type and \
tool_dependency.name == tool_dependency_name and \
tool_dependency.version == tool_dependency_version:
break
if tool_dependency:
installation_directory = tool_dependency.installation_directory(self.app)
env_shell_file_path = self.get_env_shell_file_path(installation_directory)
if env_shell_file_path:
env_shell_file_paths.append(env_shell_file_path)
else:
error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency "
error_message += "type %s, name %s, version %s for repository %s" % \
(str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version), str(repository.name))
log.debug(error_message)
continue
else:
error_message = "Skipping tool dependency definition because unable to locate tool dependency "
error_message += "type %s, name %s, version %s for repository %s" % \
(str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version), str(repository.name))
log.debug(error_message)
continue
else:
error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \
(str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version))
log.debug(error_message)
continue
else:
error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: "
error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \
(str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision))
log.debug(error_message)
else:
error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \
(str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision))
log.debug(error_message)
return env_shell_file_paths
