def get_repository_for_dependency_relationship(app, tool_shed, name, owner,
changeset_revision):
"""
Return an installed tool_shed_repository database record that is defined by either the current changeset
revision or the installed_changeset_revision.
"""
# This method is used only in Galaxy, not the Tool Shed. We store the port (if one exists) in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed)
if tool_shed is None or name is None or owner is None or changeset_revision is None:
message = "Unable to retrieve the repository record from the database because one or more of the following "
message += "required parameters is None: tool_shed: %s, name: %s, owner: %s, changeset_revision: %s " % \
(str(tool_shed), str(name), str(owner), str(changeset_revision))
raise Exception(message)
app.tool_shed_repository_cache.rebuild()
repository = get_installed_repository(
app=app,
tool_shed=tool_shed,
name=name,
owner=owner,
installed_changeset_revision=changeset_revision)
if not repository:
repository = get_installed_repository(
app=app,
tool_shed=tool_shed,
name=name,
owner=owner,
changeset_revision=changeset_revision)
if not repository:
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
app, tool_shed)
repository_clone_url = os.path.join(tool_shed_url, 'repos', owner,
name)
repo_info_tuple = (None, repository_clone_url, changeset_revision,
None, owner, None, None)
repository, pcr = repository_was_previously_installed(
app, tool_shed_url, name, repo_info_tuple)
if not repository:
# The received changeset_revision is no longer installable, so get the next changeset_revision
# in the repository's changelog in the tool shed that is associated with repository_metadata.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
app, tool_shed)
params = dict(name=name,
owner=owner,
changeset_revision=changeset_revision)
pathspec = ['repository', 'next_installable_changeset_revision']
text = util.url_get(
tool_shed_url,
auth=app.tool_shed_registry.url_auth(tool_shed_url),
pathspec=pathspec,
params=params)
if text:
repository = get_installed_repository(app=app,
tool_shed=tool_shed,
name=name,
owner=owner,
changeset_revision=text)
return repository
def create_temporary_tool_dependencies_config(self, tool_shed_url, name,
owner, changeset_revision):
"""Make a call to the tool shed to get the required repository's tool_dependencies.xml file."""
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(
self.app, tool_shed_url)
if tool_shed_url is None or name is None or owner is None or changeset_revision is None:
message = "Unable to retrieve required tool_dependencies.xml file from the Tool Shed because one or more of the "
message += "following required parameters is None: tool_shed_url: %s, name: %s, owner: %s, changeset_revision: %s " % \
(str(tool_shed_url), str(name), str(owner), str(changeset_revision))
raise Exception(message)
params = dict(name=name,
owner=owner,
changeset_revision=changeset_revision)
pathspec = ['repository', 'get_tool_dependencies_config_contents']
text = url_get(
tool_shed_url,
auth=self.app.tool_shed_registry.url_auth(tool_shed_url),
pathspec=pathspec,
params=params)
if text:
# Write the contents to a temporary file on disk so it can be reloaded and parsed.
fh = tempfile.NamedTemporaryFile('w', prefix="tmp-toolshed-cttdc")
tmp_filename = fh.name
fh.close()
fh = open(tmp_filename, 'w')
fh.write(text)
fh.close()
return tmp_filename
else:
message = "Unable to retrieve required tool_dependencies.xml file from the Tool Shed for revision "
message += f"{str(changeset_revision)} of installed repository {str(name)} owned by {str(owner)}."
raise Exception(message)
def update_repository_record(self, repository, updated_metadata_dict,
updated_changeset_revision, updated_ctx_rev):
"""
Update a tool_shed_repository database record with new information retrieved from the
Tool Shed. This happens when updating an installed repository to a new changeset revision.
"""
repository.metadata = updated_metadata_dict
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(
self.app, repository.tool_shed)
clean_dependency_relationships(self.app, updated_metadata_dict,
repository, tool_shed_url)
# Update the repository.changeset_revision column in the database.
repository.changeset_revision = updated_changeset_revision
repository.ctx_rev = updated_ctx_rev
# Update the repository.tool_shed_status column in the database.
tool_shed_status_dict = get_tool_shed_status_for_installed_repository(
self.app, repository)
if tool_shed_status_dict:
repository.tool_shed_status = tool_shed_status_dict
else:
repository.tool_shed_status = None
self.app.install_model.context.add(repository)
self.app.install_model.context.flush()
self.app.install_model.context.refresh(repository)
return repository
def get_repository_dependencies_for_installed_tool_shed_repository(
self, app, repository):
"""
Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined
for the received repository which is installed into Galaxy. This method is called only from Galaxy.
"""
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
app, str(repository.tool_shed))
params = dict(name=str(repository.name),
owner=str(repository.owner),
changeset_revision=str(repository.changeset_revision))
pathspec = ['repository', 'get_repository_dependencies']
try:
raw_text = url_get(
tool_shed_url,
password_mgr=app.tool_shed_registry.url_auth(tool_shed_url),
pathspec=pathspec,
params=params)
except Exception:
log.exception(
"Error while trying to get URL: %s",
build_url(tool_shed_url, pathspec=pathspec, params=params))
return ''
if len(raw_text) > 2:
encoded_text = json.loads(raw_text)
text = encoding_util.tool_shed_decode(encoded_text)
else:
text = ''
return text
def get_or_create_tool_shed_repository(self, tool_shed, name, owner,
changeset_revision):
"""
Return a tool shed repository database record defined by the combination of
tool shed, repository name, repository owner and changeset_revision or
installed_changeset_revision. A new tool shed repository record will be
created if one is not located.
"""
install_model = self.app.install_model
# We store the port in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed)
# This method is used only in Galaxy, not the tool shed.
repository = repository_util.get_repository_for_dependency_relationship(
self.app, tool_shed, name, owner, changeset_revision)
if not repository:
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, tool_shed)
repository_clone_url = os.path.join(tool_shed_url, 'repos', owner,
name)
ctx_rev = get_ctx_rev(self.app, tool_shed_url, name, owner,
changeset_revision)
repository = repository_util.create_or_update_tool_shed_repository(
app=self.app,
name=name,
description=None,
installed_changeset_revision=changeset_revision,
ctx_rev=ctx_rev,
repository_clone_url=repository_clone_url,
status=install_model.ToolShedRepository.installation_status.
NEW,
metadata_dict={},
current_changeset_revision=None,
owner=owner,
dist_to_shed=False)
return repository
def get_tool_shed_status_for_installed_repository(app, repository):
"""
Send a request to the tool shed to retrieve information about newer installable repository revisions,
current revision updates, whether the repository revision is the latest downloadable revision, and
whether the repository has been deprecated in the tool shed. The received repository is a ToolShedRepository
object from Galaxy.
"""
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed))
params = dict(name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision)
pathspec = ['repository', 'status_for_installed_repository']
try:
encoded_tool_shed_status_dict = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
tool_shed_status_dict = encoding_util.tool_shed_decode(encoded_tool_shed_status_dict)
return tool_shed_status_dict
except HTTPError as e:
# This should handle backward compatility to the Galaxy 12/20/12 release. We used to only handle updates for an installed revision
# using a boolean value.
log.debug("Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" %
(str(repository.name), str(e)))
pathspec = ['repository', 'check_for_updates']
params['from_update_manager'] = True
try:
# The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision.
text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
return dict(revision_update=text)
except Exception:
# The required tool shed may be unavailable, so default the revision_update value to 'false'.
return dict(revision_update='false')
except Exception:
log.exception("Error attempting to get tool shed status for installed repository %s", str(repository.name))
return {}
def repository_was_previously_installed(app,
tool_shed_url,
repository_name,
repo_info_tuple,
from_tip=False):
"""
Find out if a repository is already installed into Galaxy - there are several scenarios where this
is necessary. For example, this method will handle the case where the repository was previously
installed using an older changeset_revsion, but later the repository was updated in the tool shed
and now we're trying to install the latest changeset revision of the same repository instead of
updating the one that was previously installed. We'll look in the database instead of on disk since
the repository may be currently uninstalled.
"""
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
app, tool_shed_url)
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
get_repo_info_tuple_contents(repo_info_tuple)
tool_shed = get_tool_shed_from_clone_url(repository_clone_url)
# See if we can locate the repository using the value of changeset_revision.
tool_shed_repository = get_installed_repository(
app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=changeset_revision)
if tool_shed_repository:
return tool_shed_repository, changeset_revision
# Get all previous changeset revisions from the tool shed for the repository back to, but excluding,
# the previous valid changeset revision to see if it was previously installed using one of them.
params = dict(galaxy_url=web.url_for('/', qualified=True),
name=repository_name,
owner=repository_owner,
changeset_revision=changeset_revision,
from_tip=str(from_tip))
pathspec = ['repository', 'previous_changeset_revisions']
text = util.url_get(tool_shed_url,
auth=app.tool_shed_registry.url_auth(tool_shed_url),
pathspec=pathspec,
params=params)
if text:
changeset_revisions = util.listify(text)
for previous_changeset_revision in changeset_revisions:
tool_shed_repository = get_installed_repository(
app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=previous_changeset_revision)
if tool_shed_repository:
return tool_shed_repository, previous_changeset_revision
return None, None
def get_ctx_rev(app, tool_shed_url, name, owner, changeset_revision):
"""
Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the
combination of a name, owner and changeset revision.
"""
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
app, tool_shed_url)
params = dict(name=name,
owner=owner,
changeset_revision=changeset_revision)
pathspec = ['repository', 'get_ctx_rev']
ctx_rev = util.url_get(tool_shed_url,
auth=app.tool_shed_registry.url_auth(tool_shed_url),
pathspec=pathspec,
params=params)
return ctx_rev
def get_updated_changeset_revisions_from_tool_shed(app, tool_shed_url, name,
owner, changeset_revision):
"""
Get all appropriate newer changeset revisions for the repository defined by
the received tool_shed_url / name / owner combination.
"""
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(
app, tool_shed_url)
if tool_shed_url is None or name is None or owner is None or changeset_revision is None:
message = "Unable to get updated changeset revisions from the Tool Shed because one or more of the following "
message += "required parameters is None: tool_shed_url: %s, name: %s, owner: %s, changeset_revision: %s " % \
(str(tool_shed_url), str(name), str(owner), str(changeset_revision))
raise Exception(message)
params = dict(name=name,
owner=owner,
changeset_revision=changeset_revision)
pathspec = ['repository', 'updated_changeset_revisions']
text = util.url_get(tool_shed_url,
auth=app.tool_shed_registry.url_auth(tool_shed_url),
pathspec=pathspec,
params=params)
return text
def get_update_to_changeset_revision_and_ctx_rev(self, repository):
"""Return the changeset revision hash to which the repository can be updated."""
changeset_revision_dict = {}
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(
self.app, str(repository.tool_shed))
params = dict(name=str(repository.name),
owner=str(repository.owner),
changeset_revision=str(
repository.installed_changeset_revision))
pathspec = ['repository', 'get_changeset_revision_and_ctx_rev']
try:
encoded_update_dict = util.url_get(
tool_shed_url,
password_mgr=self.app.tool_shed_registry.url_auth(
tool_shed_url),
pathspec=pathspec,
params=params)
if encoded_update_dict:
update_dict = tool_shed_decode(encoded_update_dict)
includes_data_managers = update_dict.get(
'includes_data_managers', False)
includes_datatypes = update_dict.get('includes_datatypes',
False)
includes_tools = update_dict.get('includes_tools', False)
includes_tools_for_display_in_tool_panel = update_dict.get(
'includes_tools_for_display_in_tool_panel', False)
includes_tool_dependencies = update_dict.get(
'includes_tool_dependencies', False)
includes_workflows = update_dict.get('includes_workflows',
False)
has_repository_dependencies = update_dict.get(
'has_repository_dependencies', False)
has_repository_dependencies_only_if_compiling_contained_td = update_dict.get(
'has_repository_dependencies_only_if_compiling_contained_td',
False)
changeset_revision = update_dict.get('changeset_revision',
None)
ctx_rev = update_dict.get('ctx_rev', None)
changeset_revision_dict[
'includes_data_managers'] = includes_data_managers
changeset_revision_dict['includes_datatypes'] = includes_datatypes
changeset_revision_dict['includes_tools'] = includes_tools
changeset_revision_dict[
'includes_tools_for_display_in_tool_panel'] = includes_tools_for_display_in_tool_panel
changeset_revision_dict[
'includes_tool_dependencies'] = includes_tool_dependencies
changeset_revision_dict['includes_workflows'] = includes_workflows
changeset_revision_dict[
'has_repository_dependencies'] = has_repository_dependencies
changeset_revision_dict[
'has_repository_dependencies_only_if_compiling_contained_td'] = has_repository_dependencies_only_if_compiling_contained_td
changeset_revision_dict['changeset_revision'] = changeset_revision
changeset_revision_dict['ctx_rev'] = ctx_rev
except Exception as e:
log.debug(
"Error getting change set revision for update from the tool shed for repository '%s': %s"
% (repository.name, str(e)))
changeset_revision_dict['includes_data_managers'] = False
changeset_revision_dict['includes_datatypes'] = False
changeset_revision_dict['includes_tools'] = False
changeset_revision_dict[
'includes_tools_for_display_in_tool_panel'] = False
changeset_revision_dict['includes_tool_dependencies'] = False
changeset_revision_dict['includes_workflows'] = False
changeset_revision_dict['has_repository_dependencies'] = False
changeset_revision_dict[
'has_repository_dependencies_only_if_compiling_contained_td'] = False
changeset_revision_dict['changeset_revision'] = None
changeset_revision_dict['ctx_rev'] = None
return changeset_revision_dict
def __init__(self, app, latest_migration_script_number,
tool_shed_install_config, migrated_tools_config,
install_dependencies):
"""
Check tool settings in tool_shed_install_config and install all repositories
that are not already installed. The tool panel configuration file is the received
migrated_tools_config, which is the reserved file named migrated_tools_conf.xml.
"""
self.app = app
self.toolbox = self.app.toolbox
self.migrated_tools_config = migrated_tools_config
# Initialize the ToolPanelManager.
self.tpm = tool_panel_manager.ToolPanelManager(self.app)
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt
# to install but print informative error message.
if install_dependencies and app.tool_dependency_dir is None:
message = 'You are attempting to install tool dependencies but do not have a value '
message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this '
message += 'location value to the path where you want tool dependencies installed and '
message += 'rerun the migration script.'
raise Exception(message)
# Get the local non-shed related tool panel configs (there can be more than one, and the
# default name is tool_conf.xml).
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems(
latest_migration_script_number)
# Set the location where the repositories will be installed by retrieving the tool_path
# setting from migrated_tools_config.
try:
tree, error_message = parse_xml(migrated_tools_config)
except OSError as exc:
if exc.errno == errno.ENOENT:
with open(migrated_tools_config, 'w') as fh:
fh.write(
MIGRATED_TOOLS_CONF_XML.format(
shed_tools_dir=self.app.config.shed_tools_dir))
tree, error_message = parse_xml(migrated_tools_config)
else:
raise
if tree is None:
log.error(error_message)
else:
root = tree.getroot()
self.tool_path = root.get('tool_path')
log.debug(
"Repositories will be installed into configured tool_path location ",
str(self.tool_path))
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree, error_message = parse_xml(tool_shed_install_config)
if tree is None:
log.error(error_message)
else:
root = tree.getroot()
defined_tool_shed_url = root.get('name')
self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, defined_tool_shed_url)
self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url(
self.tool_shed_url)
self.repository_owner = common_util.REPOSITORY_OWNER
self.shed_config_dict = self.tpm.get_shed_tool_conf_dict(
self.migrated_tools_config)
# Since tool migration scripts can be executed any number of times, we need to
# make sure the appropriate tools are defined in tool_conf.xml. If no tools
# associated with the migration stage are defined, no repositories will be installed
# on disk. The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs(
app)
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = \
common_util.check_for_missing_tools(app,
tool_panel_configs,
latest_migration_script_number)
else:
# It doesn't matter if the tool shed is accessible since there are no migrated
# tools defined in the local Galaxy instance, but we have to set the value of
# tool_shed_accessible to True so that the value of migrate_tools.version can
# be correctly set in the database.
tool_shed_accessible = True
missing_tool_configs_dict = {}
if tool_shed_accessible:
if len(self.proprietary_tool_confs) == 1:
plural = ''
file_names = self.proprietary_tool_confs[0]
else:
plural = 's'
file_names = ', '.join(self.proprietary_tool_confs)
if missing_tool_configs_dict:
for proprietary_tool_conf in self.proprietary_tool_confs:
# Create a backup of the tool configuration in the un-migrated state.
shutil.copy(
proprietary_tool_conf, '%s-pre-stage-%04d' %
(proprietary_tool_conf,
latest_migration_script_number))
for repository_elem in root:
# Make sure we have a valid repository tag.
if self.__is_valid_repository_tag(repository_elem):
# Get all repository dependencies for the repository defined by the
# current repository_elem. Repository dependency definitions contained
# in tool shed repositories with migrated tools must never define a
# relationship to a repository dependency that contains a tool. The
# repository dependency can only contain items that are not loaded into
# the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes,
# etc). This restriction must be followed down the entire dependency hierarchy.
name = repository_elem.get('name')
changeset_revision = repository_elem.get(
'changeset_revision')
tool_shed_accessible, repository_dependencies_dict = \
common_util.get_repository_dependencies(app,
self.tool_shed_url,
name,
self.repository_owner,
changeset_revision)
# Make sure all repository dependency records exist (as tool_shed_repository
# table rows) in the Galaxy database.
created_tool_shed_repositories = \
self.create_or_update_tool_shed_repository_records(name=name,
changeset_revision=changeset_revision,
repository_dependencies_dict=repository_dependencies_dict)
# Order the repositories for proper installation. This process is similar to the
# process used when installing tool shed repositories, but does not handle managing
# tool panel sections and other components since repository dependency definitions
# contained in tool shed repositories with migrated tools must never define a relationship
# to a repository dependency that contains a tool.
ordered_tool_shed_repositories = \
self.order_repositories_for_installation(created_tool_shed_repositories,
repository_dependencies_dict)
for tool_shed_repository in ordered_tool_shed_repositories:
is_repository_dependency = self.__is_repository_dependency(
name, changeset_revision,
tool_shed_repository)
self.install_repository(
repository_elem,
tool_shed_repository,
install_dependencies,
is_repository_dependency=
is_repository_dependency)
else:
message = "\nNo tools associated with migration stage %s are defined in your " % \
str(latest_migration_script_number)
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \
(plural, file_names)
log.info(message)
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \
str(latest_migration_script_number)
message += "Try again later.\n"
log.error(message)
def get_required_repo_info_dicts(self, tool_shed_url, repo_info_dicts):
"""
Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of
them to the list. All repository_dependency entries in each of the received repo_info_dicts includes
all required repositories, so only one pass through this method is required to retrieve all repository
dependencies.
"""
all_required_repo_info_dict = {}
all_repo_info_dicts = []
if repo_info_dicts:
# We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool
# shed to discover repository ids.
required_repository_tups = []
for repo_info_dict in repo_info_dicts:
if repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append(repo_info_dict)
for repository_name, repo_info_tup in repo_info_dict.items():
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
repository_util.get_repo_info_tuple_contents(repo_info_tup)
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in ['root_key', 'description']:
continue
repository_components_tuple = get_components_from_key(
key)
components_list = repository_util.extract_components_from_tuple(
repository_components_tuple)
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository since in this case, the repository dependency is really
# a dependency of the dependent repository's contained tool dependency, and only if that
# tool dependency requires compilation.
# For backward compatibility to the 12/20/12 Galaxy release.
only_if_compiling_contained_td = 'False'
if len(components_list) == 4:
only_if_compiling_contained_td = 'False'
elif len(components_list) == 5:
only_if_compiling_contained_td = 'False'
if not asbool(only_if_compiling_contained_td):
if components_list not in required_repository_tups:
required_repository_tups.append(
components_list)
for components_list in val:
try:
only_if_compiling_contained_td = components_list[
5]
except IndexError:
only_if_compiling_contained_td = 'False'
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository (see above comment).
if not asbool(only_if_compiling_contained_td):
if components_list not in required_repository_tups:
required_repository_tups.append(
components_list)
else:
# We have a single repository with no dependencies.
components_list = [
tool_shed_url, repository_name, repository_owner,
changeset_revision
]
required_repository_tups.append(components_list)
if required_repository_tups:
# The value of required_repository_tups is a list of tuples, so we need to encode it.
encoded_required_repository_tups = []
for required_repository_tup in required_repository_tups:
# Convert every item in required_repository_tup to a string.
required_repository_tup = [
str(item) for item in required_repository_tup
]
encoded_required_repository_tups.append(
encoding_util.encoding_sep.join(
required_repository_tup))
encoded_required_repository_str = encoding_util.encoding_sep2.join(
encoded_required_repository_tups)
encoded_required_repository_str = encoding_util.tool_shed_encode(
encoded_required_repository_str)
if repository_util.is_tool_shed_client(self.app):
# Handle secure / insecure Tool Shed URL protocol changes and port changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, tool_shed_url)
pathspec = ['repository', 'get_required_repo_info_dict']
url = build_url(tool_shed_url, pathspec=pathspec)
# Fix for handling 307 redirect not being handled nicely by urlopen() when the Request() has data provided
try:
url = _urlopen(url).geturl()
except HTTPError as e:
if e.code == 502:
pass
else:
raise
payload = urlencode(
dict(encoded_str=encoded_required_repository_str))
response = _urlopen(url, payload).read()
if response:
try:
required_repo_info_dict = json.loads(
unicodify(response))
except Exception as e:
log.exception(e)
return all_repo_info_dicts
required_repo_info_dicts = []
for k, v in required_repo_info_dict.items():
if k == 'repo_info_dicts':
encoded_dict_strings = required_repo_info_dict[
'repo_info_dicts']
for encoded_dict_str in encoded_dict_strings:
decoded_dict = encoding_util.tool_shed_decode(
encoded_dict_str)
required_repo_info_dicts.append(
decoded_dict)
else:
if k not in all_required_repo_info_dict:
all_required_repo_info_dict[k] = v
else:
if v and not all_required_repo_info_dict[k]:
all_required_repo_info_dict[k] = v
if required_repo_info_dicts:
for required_repo_info_dict in required_repo_info_dicts:
# Each required_repo_info_dict has a single entry, and all_repo_info_dicts is a list
# of dictionaries, each of which has a single entry. We'll check keys here rather than
# the entire dictionary because a dictionary entry in all_repo_info_dicts will include
# lists of discovered repository dependencies, but these lists will be empty in the
# required_repo_info_dict since dependency discovery has not yet been performed for these
# dictionaries.
required_repo_info_dict_key = next(
iter(required_repo_info_dict))
all_repo_info_dicts_keys = [
next(iter(d))
for d in all_repo_info_dicts
]
if required_repo_info_dict_key not in all_repo_info_dicts_keys:
all_repo_info_dicts.append(
required_repo_info_dict)
else:
# required_repo_info_dict_key corresponds to the repo name.
# A single install transaction might require the installation of 2 or more repos
# with the same repo name but different owners or versions.
# Therefore, if required_repo_info_dict_key is already in all_repo_info_dicts,
# check that the tool id is already present. If it is not, we are dealing with the same repo name,
# but a different owner/changeset revision or version and we add the repo to the list of repos to be installed.
tool_id = required_repo_info_dict[
required_repo_info_dict_key][1]
is_present = False
for repo_info_dict in all_repo_info_dicts:
for k, v in repo_info_dict.items():
if required_repo_info_dict_key == k:
if tool_id == v[1]:
is_present = True
break
if not is_present:
all_repo_info_dicts.append(
required_repo_info_dict)
all_required_repo_info_dict[
'all_repo_info_dicts'] = all_repo_info_dicts
return all_required_repo_info_dict
