def get_installed_repositories_from_repository_dependencies(
self, repository_dependencies_dict):
installed_repositories = []
if repository_dependencies_dict and isinstance(
repository_dependencies_dict, dict):
for rd_key, rd_vals in repository_dependencies_dict.items():
if rd_key in ['root_key', 'description']:
continue
# rd_key is something like: 'http://localhost:9009__ESEP__package_rdkit_2012_12__ESEP__test__ESEP__d635ffb9c665__ESEP__True'
# rd_val is something like: [['http://localhost:9009', 'package_numpy_1_7', 'test', 'cddd64ecd985', 'True']]
repository_components_tuple = container_util.get_components_from_key(
rd_key)
components_list = suc.extract_components_from_tuple(
repository_components_tuple)
tool_shed, name, owner, changeset_revision = components_list[
0:4]
installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision(
self.app, tool_shed, name, owner, changeset_revision)
if installed_repository not in installed_repositories:
installed_repositories.append(installed_repository)
for rd_val in rd_vals:
tool_shed, name, owner, changeset_revision = rd_val[0:4]
installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision(
self.app, tool_shed, name, owner, changeset_revision)
if installed_repository not in installed_repositories:
installed_repositories.append(installed_repository)
return installed_repositories
def get_installed_repositories_from_repository_dependencies( self, repository_dependencies_dict ):
installed_repositories = []
if repository_dependencies_dict and isinstance( repository_dependencies_dict, dict ):
for rd_key, rd_vals in repository_dependencies_dict.items():
if rd_key in [ 'root_key', 'description' ]:
continue
# rd_key is something like: 'http://localhost:9009__ESEP__package_rdkit_2012_12__ESEP__test__ESEP__d635ffb9c665__ESEP__True'
# rd_val is something like: [['http://localhost:9009', 'package_numpy_1_7', 'test', 'cddd64ecd985', 'True']]
repository_components_tuple = container_util.get_components_from_key( rd_key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
tool_shed, name, owner, changeset_revision = components_list[ 0:4 ]
installed_repository = suc.get_installed_repository( self.app,
tool_shed=tool_shed,
name=name,
owner=owner,
changeset_revision=changeset_revision )
if ( installed_repository ) and ( installed_repository not in installed_repositories ):
installed_repositories.append( installed_repository )
for rd_val in rd_vals:
tool_shed, name, owner, changeset_revision = rd_val[ 0:4 ]
installed_repository = suc.get_installed_repository( self.app,
tool_shed=tool_shed,
name=name,
owner=owner,
changeset_revision=changeset_revision )
if ( installed_repository ) and ( installed_repository not in installed_repositories ):
installed_repositories.append( installed_repository )
return installed_repositories
def key_is_current_repositorys_key(self, repository_name, repository_owner,
changeset_revision,
prior_installation_required,
only_if_compiling_contained_td, key):
repository_components_tuple = container_util.get_components_from_key(
key)
components_list = repository_util.extract_components_from_tuple(
repository_components_tuple)
toolshed, key_name, key_owner, key_changeset_revision = components_list[
0:4]
# For backward compatibility to the 12/20/12 Galaxy release.
if len(components_list) == 4:
key_prior_installation_required = 'False'
key_only_if_compiling_contained_td = 'False'
elif len(components_list) == 5:
key_prior_installation_required = components_list[4]
key_only_if_compiling_contained_td = 'False'
elif len(components_list) == 6:
key_prior_installation_required = components_list[4]
key_only_if_compiling_contained_td = components_list[5]
if (repository_name == key_name and repository_owner == key_owner
and changeset_revision == key_changeset_revision and
prior_installation_required == key_prior_installation_required
and only_if_compiling_contained_td
== key_only_if_compiling_contained_td):
return True
return False
def key_is_current_repositorys_key( self, repository_name, repository_owner, changeset_revision,
prior_installation_required, only_if_compiling_contained_td, key ):
repository_components_tuple = container_util.get_components_from_key( key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
toolshed, key_name, key_owner, key_changeset_revision = components_list[ 0:4 ]
# For backward compatibility to the 12/20/12 Galaxy release.
if len( components_list ) == 4:
key_prior_installation_required = 'False'
key_only_if_compiling_contained_td = 'False'
elif len( components_list ) == 5:
key_prior_installation_required = components_list[ 4 ]
key_only_if_compiling_contained_td = 'False'
elif len( components_list ) == 6:
key_prior_installation_required = components_list[ 4 ]
key_only_if_compiling_contained_td = components_list[ 5 ]
if ( repository_name == key_name and
repository_owner == key_owner and
changeset_revision == key_changeset_revision and
prior_installation_required == key_prior_installation_required and
only_if_compiling_contained_td == key_only_if_compiling_contained_td ):
return True
return False
def get_required_repo_info_dicts( self, tool_shed_url, repo_info_dicts ):
"""
Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of
them to the list. All repository_dependency entries in each of the received repo_info_dicts includes
all required repositories, so only one pass through this method is required to retrieve all repository
dependencies.
"""
all_required_repo_info_dict = {}
all_repo_info_dicts = []
if repo_info_dicts:
# We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool
# shed to discover repository ids.
required_repository_tups = []
for repo_info_dict in repo_info_dicts:
if repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append( repo_info_dict )
for repository_name, repo_info_tup in repo_info_dict.items():
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tup )
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in [ 'root_key', 'description' ]:
continue
repository_components_tuple = container_util.get_components_from_key( key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository since in this case, the repository dependency is really
# a dependency of the dependent repository's contained tool dependency, and only if that
# tool dependency requires compilation.
# For backward compatibility to the 12/20/12 Galaxy release.
prior_installation_required = 'False'
only_if_compiling_contained_td = 'False'
if len( components_list ) == 4:
prior_installation_required = 'False'
only_if_compiling_contained_td = 'False'
elif len( components_list ) == 5:
prior_installation_required = components_list[ 4 ]
only_if_compiling_contained_td = 'False'
if not asbool( only_if_compiling_contained_td ):
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
for components_list in val:
try:
only_if_compiling_contained_td = components_list[ 5 ]
except:
only_if_compiling_contained_td = 'False'
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository (see above comment).
if not asbool( only_if_compiling_contained_td ):
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
else:
# We have a single repository with no dependencies.
components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision ]
required_repository_tups.append( components_list )
if required_repository_tups:
# The value of required_repository_tups is a list of tuples, so we need to encode it.
encoded_required_repository_tups = []
for required_repository_tup in required_repository_tups:
# Convert every item in required_repository_tup to a string.
required_repository_tup = [ str( item ) for item in required_repository_tup ]
encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
if suc.is_tool_shed_client( self.app ):
# Handle secure / insecure Tool Shed URL protocol changes and port changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
url = common_util.url_join( tool_shed_url, '/repository/get_required_repo_info_dict' )
# Fix for handling 307 redirect not being handled nicely by urllib2.urlopen when the urllib2.Request has data provided
url = urllib2.urlopen( urllib2.Request( url ) ).geturl()
request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
response = urllib2.urlopen( request ).read()
if response:
try:
required_repo_info_dict = json.loads( response )
except Exception, e:
log.exception( e )
return all_repo_info_dicts
required_repo_info_dicts = []
for k, v in required_repo_info_dict.items():
if k == 'repo_info_dicts':
encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
for encoded_dict_str in encoded_dict_strings:
decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
required_repo_info_dicts.append( decoded_dict )
else:
if k not in all_required_repo_info_dict:
all_required_repo_info_dict[ k ] = v
else:
if v and not all_required_repo_info_dict[ k ]:
all_required_repo_info_dict[ k ] = v
if required_repo_info_dicts:
for required_repo_info_dict in required_repo_info_dicts:
# Each required_repo_info_dict has a single entry, and all_repo_info_dicts is a list
# of dictionaries, each of which has a single entry. We'll check keys here rather than
# the entire dictionary because a dictionary entry in all_repo_info_dicts will include
# lists of discovered repository dependencies, but these lists will be empty in the
# required_repo_info_dict since dependency discovery has not yet been performed for these
# dictionaries.
required_repo_info_dict_key = required_repo_info_dict.keys()[ 0 ]
all_repo_info_dicts_keys = [ d.keys()[ 0 ] for d in all_repo_info_dicts ]
if required_repo_info_dict_key not in all_repo_info_dicts_keys:
all_repo_info_dicts.append( required_repo_info_dict )
all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
def build_repository_dependency_relationships( self, repo_info_dicts, tool_shed_repositories ):
"""
Build relationships between installed tool shed repositories and other installed
tool shed repositories upon which they depend. These relationships are defined in
the repository_dependencies entry for each dictionary in the received list of repo_info_dicts.
Each of these dictionaries is associated with a repository in the received tool_shed_repositories
list.
"""
install_model = self.app.install_model
log.debug( "Building repository dependency relationships..." )
for repo_info_dict in repo_info_dicts:
for name, repo_info_tuple in repo_info_dict.items():
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in [ 'root_key', 'description' ]:
continue
d_repository = None
repository_components_tuple = container_util.get_components_from_key( key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
d_toolshed, d_name, d_owner, d_changeset_revision = components_list[ 0:4 ]
for tsr in tool_shed_repositories:
# Get the the tool_shed_repository defined by name, owner and changeset_revision. This is
# the repository that will be dependent upon each of the tool shed repositories contained in
# val. We'll need to check tool_shed_repository.tool_shed as well if/when repository dependencies
# across tool sheds is supported.
if tsr.name == d_name and tsr.owner == d_owner and tsr.changeset_revision == d_changeset_revision:
d_repository = tsr
break
if d_repository is None:
# The dependent repository is not in the received list so look in the database.
d_repository = self.get_or_create_tool_shed_repository( d_toolshed,
d_name,
d_owner,
d_changeset_revision )
# Process each repository_dependency defined for the current dependent repository.
for repository_dependency_components_list in val:
required_repository = None
rd_toolshed, \
rd_name, \
rd_owner, \
rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( repository_dependency_components_list )
# Get the the tool_shed_repository defined by rd_name, rd_owner and rd_changeset_revision. This
# is the repository that will be required by the current d_repository.
# TODO: Check tool_shed_repository.tool_shed as well when repository dependencies across tool sheds is supported.
for tsr in tool_shed_repositories:
if tsr.name == rd_name and tsr.owner == rd_owner and tsr.changeset_revision == rd_changeset_revision:
required_repository = tsr
break
if required_repository is None:
# The required repository is not in the received list so look in the database.
required_repository = self.get_or_create_tool_shed_repository( rd_toolshed,
rd_name,
rd_owner,
rd_changeset_revision )
# Ensure there is a repository_dependency relationship between d_repository and required_repository.
rrda = None
for rd in d_repository.repository_dependencies:
if rd.id == required_repository.id:
rrda = rd
break
if not rrda:
# Make sure required_repository is in the repository_dependency table.
repository_dependency = self.get_repository_dependency_by_repository_id( install_model,
required_repository.id )
if not repository_dependency:
log.debug( 'Creating new repository_dependency record for installed revision %s of repository: %s owned by %s.' % \
( str( required_repository.installed_changeset_revision ),
str( required_repository.name ),
str( required_repository.owner ) ) )
repository_dependency = install_model.RepositoryDependency( tool_shed_repository_id=required_repository.id )
install_model.context.add( repository_dependency )
install_model.context.flush()
# Build the relationship between the d_repository and the required_repository.
rrda = install_model.RepositoryRepositoryDependencyAssociation( tool_shed_repository_id=d_repository.id,
repository_dependency_id=repository_dependency.id )
install_model.context.add( rrda )
install_model.context.flush()
def handle_repository_dependencies_container_entry( self, repository_dependencies_folder, rd_key, rd_value, folder_id,
repository_dependency_id, folder_keys ):
repository_components_tuple = container_util.get_components_from_key( rd_key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
toolshed, repository_name, repository_owner, changeset_revision = components_list[ 0:4 ]
# For backward compatibility to the 12/20/12 Galaxy release.
if len( components_list ) == 4:
prior_installation_required = 'False'
only_if_compiling_contained_td = 'False'
elif len( components_list ) == 5:
prior_installation_required = components_list[ 4 ]
only_if_compiling_contained_td = 'False'
elif len( components_list ) == 6:
prior_installation_required = components_list[ 4 ]
only_if_compiling_contained_td = components_list[ 5 ]
folder = self.get_folder( repository_dependencies_folder, rd_key )
label = self.generate_repository_dependencies_folder_label_from_key( repository_name,
repository_owner,
changeset_revision,
prior_installation_required,
only_if_compiling_contained_td,
repository_dependencies_folder.key )
if folder:
if rd_key not in folder_keys:
folder_id += 1
sub_folder = Folder( id=folder_id, key=rd_key, label=label, parent=folder )
folder.folders.append( sub_folder )
else:
sub_folder = folder
else:
folder_id += 1
sub_folder = Folder( id=folder_id, key=rd_key, label=label, parent=repository_dependencies_folder )
repository_dependencies_folder.folders.append( sub_folder )
if self.app.name == 'galaxy':
# Insert a header row.
repository_dependency_id += 1
repository_dependency = RepositoryDependency( id=repository_dependency_id,
repository_name='Name',
changeset_revision='Revision',
repository_owner='Owner',
installation_status='Installation status' )
# Insert the header row into the folder.
sub_folder.repository_dependencies.append( repository_dependency )
for repository_dependency in rd_value:
if self.app.name == 'galaxy':
tool_shed_repository_id, installation_status, repository_dependency = \
self.get_components_from_repository_dependency_for_installed_repository( repository_dependency )
else:
tool_shed_repository_id = None
installation_status = None
can_create_dependency = not self.is_subfolder_of( sub_folder, repository_dependency )
if can_create_dependency:
toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( repository_dependency )
repository_dependency_id += 1
repository_dependency = RepositoryDependency( id=repository_dependency_id,
toolshed=toolshed,
repository_name=repository_name,
repository_owner=repository_owner,
changeset_revision=changeset_revision,
prior_installation_required=util.asbool( prior_installation_required ),
only_if_compiling_contained_td=util.asbool( only_if_compiling_contained_td ),
installation_status=installation_status,
tool_shed_repository_id=tool_shed_repository_id )
# Insert the repository_dependency into the folder.
sub_folder.repository_dependencies.append( repository_dependency )
return repository_dependencies_folder, folder_id, repository_dependency_id
def get_updated_changeset_revisions_for_repository_dependencies(self, key_rd_dicts):
updated_key_rd_dicts = []
for key_rd_dict in key_rd_dicts:
key = next(iter(key_rd_dict))
repository_dependency = key_rd_dict[key]
rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple(repository_dependency)
if suc.tool_shed_is_this_tool_shed(rd_toolshed):
repository = tool_shed.util.repository_util.get_repository_by_name_and_owner(self.app, rd_name, rd_owner)
if repository:
repository_id = self.app.security.encode_id(repository.id)
repository_metadata = \
metadata_util.get_repository_metadata_by_repository_id_changeset_revision(self.app,
repository_id,
rd_changeset_revision)
if repository_metadata:
# The repository changeset_revision is installable, so no updates are available.
new_key_rd_dict = {}
new_key_rd_dict[key] = repository_dependency
updated_key_rd_dicts.append(key_rd_dict)
else:
# The repository changeset_revision is no longer installable, so see if there's been an update.
changeset_revision = metadata_util.get_next_downloadable_changeset_revision(self.app, repository, rd_changeset_revision)
if changeset_revision != rd_changeset_revision:
repository_metadata = \
metadata_util.get_repository_metadata_by_repository_id_changeset_revision(self.app,
repository_id,
changeset_revision)
if repository_metadata:
new_key_rd_dict = {}
new_key_rd_dict[key] = \
[rd_toolshed,
rd_name,
rd_owner,
repository_metadata.changeset_revision,
rd_prior_installation_required,
rd_only_if_compiling_contained_td]
# We have the updated changeset revision.
updated_key_rd_dicts.append(new_key_rd_dict)
else:
repository_components_tuple = container_util.get_components_from_key(key)
components_list = tool_shed.util.repository_util.extract_components_from_tuple(repository_components_tuple)
toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[0:4]
# For backward compatibility to the 12/20/12 Galaxy release.
if len(components_list) in (4, 5):
rd_only_if_compiling_contained_td = 'False'
message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \
(str(rd_changeset_revision), str(rd_name), str(rd_owner))
message += "dependencies defined for repository %s will be ignored." % str(repository_name)
log.debug(message)
else:
repository_components_tuple = container_util.get_components_from_key(key)
components_list = tool_shed.util.repository_util.extract_components_from_tuple(repository_components_tuple)
toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[0:4]
message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \
(str(rd_changeset_revision), str(rd_name), str(rd_owner))
message += "dependencies defined for repository %s will be ignored." % str(repository_name)
log.debug(message)
return updated_key_rd_dicts
def get_required_repo_info_dicts(self, tool_shed_url, repo_info_dicts):
"""
Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of
them to the list. All repository_dependency entries in each of the received repo_info_dicts includes
all required repositories, so only one pass through this method is required to retrieve all repository
dependencies.
"""
all_required_repo_info_dict = {}
all_repo_info_dicts = []
if repo_info_dicts:
# We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool
# shed to discover repository ids.
required_repository_tups = []
for repo_info_dict in repo_info_dicts:
if repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append(repo_info_dict)
for repository_name, repo_info_tup in repo_info_dict.items():
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
repository_util.get_repo_info_tuple_contents(repo_info_tup)
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in ['root_key', 'description']:
continue
repository_components_tuple = container_util.get_components_from_key(
key)
components_list = repository_util.extract_components_from_tuple(
repository_components_tuple)
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository since in this case, the repository dependency is really
# a dependency of the dependent repository's contained tool dependency, and only if that
# tool dependency requires compilation.
# For backward compatibility to the 12/20/12 Galaxy release.
only_if_compiling_contained_td = 'False'
if len(components_list) == 4:
only_if_compiling_contained_td = 'False'
elif len(components_list) == 5:
only_if_compiling_contained_td = 'False'
if not asbool(only_if_compiling_contained_td):
if components_list not in required_repository_tups:
required_repository_tups.append(
components_list)
for components_list in val:
try:
only_if_compiling_contained_td = components_list[
5]
except IndexError:
only_if_compiling_contained_td = 'False'
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository (see above comment).
if not asbool(only_if_compiling_contained_td):
if components_list not in required_repository_tups:
required_repository_tups.append(
components_list)
else:
# We have a single repository with no dependencies.
components_list = [
tool_shed_url, repository_name, repository_owner,
changeset_revision
]
required_repository_tups.append(components_list)
if required_repository_tups:
# The value of required_repository_tups is a list of tuples, so we need to encode it.
encoded_required_repository_tups = []
for required_repository_tup in required_repository_tups:
# Convert every item in required_repository_tup to a string.
required_repository_tup = [
str(item) for item in required_repository_tup
]
encoded_required_repository_tups.append(
encoding_util.encoding_sep.join(
required_repository_tup))
encoded_required_repository_str = encoding_util.encoding_sep2.join(
encoded_required_repository_tups)
encoded_required_repository_str = encoding_util.tool_shed_encode(
encoded_required_repository_str)
if repository_util.is_tool_shed_client(self.app):
# Handle secure / insecure Tool Shed URL protocol changes and port changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
self.app, tool_shed_url)
pathspec = ['repository', 'get_required_repo_info_dict']
url = build_url(tool_shed_url, pathspec=pathspec)
# Fix for handling 307 redirect not being handled nicely by urlopen() when the Request() has data provided
try:
url = _urlopen(url).geturl()
except HTTPError as e:
if e.code == 502:
pass
else:
raise
payload = urlencode(
dict(encoded_str=encoded_required_repository_str))
response = _urlopen(url, payload).read()
if response:
try:
required_repo_info_dict = json.loads(
unicodify(response))
except Exception as e:
log.exception(e)
return all_repo_info_dicts
required_repo_info_dicts = []
for k, v in required_repo_info_dict.items():
if k == 'repo_info_dicts':
encoded_dict_strings = required_repo_info_dict[
'repo_info_dicts']
for encoded_dict_str in encoded_dict_strings:
decoded_dict = encoding_util.tool_shed_decode(
encoded_dict_str)
required_repo_info_dicts.append(
decoded_dict)
else:
if k not in all_required_repo_info_dict:
all_required_repo_info_dict[k] = v
else:
if v and not all_required_repo_info_dict[k]:
all_required_repo_info_dict[k] = v
if required_repo_info_dicts:
for required_repo_info_dict in required_repo_info_dicts:
# Each required_repo_info_dict has a single entry, and all_repo_info_dicts is a list
# of dictionaries, each of which has a single entry. We'll check keys here rather than
# the entire dictionary because a dictionary entry in all_repo_info_dicts will include
# lists of discovered repository dependencies, but these lists will be empty in the
# required_repo_info_dict since dependency discovery has not yet been performed for these
# dictionaries.
required_repo_info_dict_key = next(
iter(required_repo_info_dict))
all_repo_info_dicts_keys = [
next(iter(d))
for d in all_repo_info_dicts
]
if required_repo_info_dict_key not in all_repo_info_dicts_keys:
all_repo_info_dicts.append(
required_repo_info_dict)
else:
# required_repo_info_dict_key corresponds to the repo name.
# A single install transaction might require the installation of 2 or more repos
# with the same repo name but different owners or versions.
# Therefore, if required_repo_info_dict_key is already in all_repo_info_dicts,
# check that the tool id is already present. If it is not, we are dealing with the same repo name,
# but a different owner/changeset revision or version and we add the repo to the list of repos to be installed.
tool_id = required_repo_info_dict[
required_repo_info_dict_key][1]
is_present = False
for repo_info_dict in all_repo_info_dicts:
for k, v in repo_info_dict.items():
if required_repo_info_dict_key == k:
if tool_id == v[1]:
is_present = True
break
if not is_present:
all_repo_info_dicts.append(
required_repo_info_dict)
all_required_repo_info_dict[
'all_repo_info_dicts'] = all_repo_info_dicts
return all_required_repo_info_dict
def merge_containers_dicts_for_new_install(self, containers_dicts):
"""
When installing one or more tool shed repositories for the first time, the received list of
containers_dicts contains a containers_dict for each repository being installed. Since the
repositories are being installed for the first time, all entries are None except the repository
dependencies and tool dependencies. The entries for missing dependencies are all None since
they have previously been merged into the installed dependencies. This method will merge the
dependencies entries into a single container and return it for display.
"""
new_containers_dict = dict(readme_files=None,
datatypes=None,
missing_repository_dependencies=None,
repository_dependencies=None,
missing_tool_dependencies=None,
tool_dependencies=None,
invalid_tools=None,
valid_tools=None,
workflows=None)
if containers_dicts:
lock = threading.Lock()
lock.acquire(True)
try:
repository_dependencies_root_folder = None
tool_dependencies_root_folder = None
# Use a unique folder id (hopefully the following is).
folder_id = 867
for old_container_dict in containers_dicts:
# Merge repository_dependencies.
old_container_repository_dependencies_root = old_container_dict[
'repository_dependencies']
if old_container_repository_dependencies_root:
if repository_dependencies_root_folder is None:
repository_dependencies_root_folder = utility_container_manager.Folder(
id=folder_id,
key='root',
label='root',
parent=None)
folder_id += 1
repository_dependencies_folder = utility_container_manager.Folder(
id=folder_id,
key='merged',
label='Repository dependencies',
parent=repository_dependencies_root_folder)
folder_id += 1
# The old_container_repository_dependencies_root will be a root folder containing a single sub_folder.
old_container_repository_dependencies_folder = old_container_repository_dependencies_root.folders[
0]
# Change the folder id so it won't confict with others being merged.
old_container_repository_dependencies_folder.id = folder_id
folder_id += 1
repository_components_tuple = \
container_util.get_components_from_key( old_container_repository_dependencies_folder.key )
components_list = suc.extract_components_from_tuple(
repository_components_tuple)
name = components_list[1]
# Generate the label by retrieving the repository name.
old_container_repository_dependencies_folder.label = str(
name)
repository_dependencies_folder.folders.append(
old_container_repository_dependencies_folder)
# Merge tool_dependencies.
old_container_tool_dependencies_root = old_container_dict[
'tool_dependencies']
if old_container_tool_dependencies_root:
if tool_dependencies_root_folder is None:
tool_dependencies_root_folder = utility_container_manager.Folder(
id=folder_id,
key='root',
label='root',
parent=None)
folder_id += 1
tool_dependencies_folder = utility_container_manager.Folder(
id=folder_id,
key='merged',
label='Tool dependencies',
parent=tool_dependencies_root_folder)
folder_id += 1
else:
td_list = [
td.listify for td in
tool_dependencies_folder.tool_dependencies
]
# The old_container_tool_dependencies_root will be a root folder containing a single sub_folder.
old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[
0]
for td in old_container_tool_dependencies_folder.tool_dependencies:
if td.listify not in td_list:
tool_dependencies_folder.tool_dependencies.append(
td)
if repository_dependencies_root_folder:
repository_dependencies_root_folder.folders.append(
repository_dependencies_folder)
new_containers_dict[
'repository_dependencies'] = repository_dependencies_root_folder
if tool_dependencies_root_folder:
tool_dependencies_root_folder.folders.append(
tool_dependencies_folder)
new_containers_dict[
'tool_dependencies'] = tool_dependencies_root_folder
except Exception, e:
log.debug(
"Exception in merge_containers_dicts_for_new_install: %s" %
str(e))
finally:
def get_required_repo_info_dicts( self, tool_shed_url, repo_info_dicts ):
"""
Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of
them to the list. All repository_dependency entries in each of the received repo_info_dicts includes
all required repositories, so only one pass through this method is required to retrieve all repository
dependencies.
"""
all_required_repo_info_dict = {}
all_repo_info_dicts = []
if repo_info_dicts:
# We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool
# shed to discover repository ids.
required_repository_tups = []
for repo_info_dict in repo_info_dicts:
if repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append( repo_info_dict )
for repository_name, repo_info_tup in repo_info_dict.items():
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tup )
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in [ 'root_key', 'description' ]:
continue
repository_components_tuple = container_util.get_components_from_key( key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository since in this case, the repository dependency is really
# a dependency of the dependent repository's contained tool dependency, and only if that
# tool dependency requires compilation.
# For backward compatibility to the 12/20/12 Galaxy release.
only_if_compiling_contained_td = 'False'
if len( components_list ) == 4:
only_if_compiling_contained_td = 'False'
elif len( components_list ) == 5:
only_if_compiling_contained_td = 'False'
if not asbool( only_if_compiling_contained_td ):
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
for components_list in val:
try:
only_if_compiling_contained_td = components_list[ 5 ]
except:
only_if_compiling_contained_td = 'False'
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository (see above comment).
if not asbool( only_if_compiling_contained_td ):
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
else:
# We have a single repository with no dependencies.
components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision ]
required_repository_tups.append( components_list )
if required_repository_tups:
# The value of required_repository_tups is a list of tuples, so we need to encode it.
encoded_required_repository_tups = []
for required_repository_tup in required_repository_tups:
# Convert every item in required_repository_tup to a string.
required_repository_tup = [ str( item ) for item in required_repository_tup ]
encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
if suc.is_tool_shed_client( self.app ):
# Handle secure / insecure Tool Shed URL protocol changes and port changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
pathspec = [ 'repository', 'get_required_repo_info_dict' ]
url = common_util.url_join( tool_shed_url, pathspec=pathspec )
# Fix for handling 307 redirect not being handled nicely by urllib2.urlopen when the urllib2.Request has data provided
url = urllib2.urlopen( urllib2.Request( url ) ).geturl()
request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
response = urllib2.urlopen( request ).read()
if response:
try:
required_repo_info_dict = json.loads( response )
except Exception, e:
log.exception( e )
return all_repo_info_dicts
required_repo_info_dicts = []
for k, v in required_repo_info_dict.items():
if k == 'repo_info_dicts':
encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
for encoded_dict_str in encoded_dict_strings:
decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
required_repo_info_dicts.append( decoded_dict )
else:
if k not in all_required_repo_info_dict:
all_required_repo_info_dict[ k ] = v
else:
if v and not all_required_repo_info_dict[ k ]:
all_required_repo_info_dict[ k ] = v
if required_repo_info_dicts:
for required_repo_info_dict in required_repo_info_dicts:
# Each required_repo_info_dict has a single entry, and all_repo_info_dicts is a list
# of dictionaries, each of which has a single entry. We'll check keys here rather than
# the entire dictionary because a dictionary entry in all_repo_info_dicts will include
# lists of discovered repository dependencies, but these lists will be empty in the
# required_repo_info_dict since dependency discovery has not yet been performed for these
# dictionaries.
required_repo_info_dict_key = required_repo_info_dict.keys()[ 0 ]
all_repo_info_dicts_keys = [ d.keys()[ 0 ] for d in all_repo_info_dicts ]
if required_repo_info_dict_key not in all_repo_info_dicts_keys:
all_repo_info_dicts.append( required_repo_info_dict )
all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
def build_repository_dependency_relationships( self, repo_info_dicts, tool_shed_repositories ):
"""
Build relationships between installed tool shed repositories and other installed
tool shed repositories upon which they depend. These relationships are defined in
the repository_dependencies entry for each dictionary in the received list of repo_info_dicts.
Each of these dictionaries is associated with a repository in the received tool_shed_repositories
list.
"""
install_model = self.app.install_model
log.debug( "Building repository dependency relationships..." )
for repo_info_dict in repo_info_dicts:
for name, repo_info_tuple in repo_info_dict.items():
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in [ 'root_key', 'description' ]:
continue
d_repository = None
repository_components_tuple = container_util.get_components_from_key( key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
d_toolshed, d_name, d_owner, d_changeset_revision = components_list[ 0:4 ]
for tsr in tool_shed_repositories:
# Get the the tool_shed_repository defined by name, owner and changeset_revision. This is
# the repository that will be dependent upon each of the tool shed repositories contained in
# val. We'll need to check tool_shed_repository.tool_shed as well if/when repository dependencies
# across tool sheds is supported.
if tsr.name == d_name and tsr.owner == d_owner and tsr.changeset_revision == d_changeset_revision:
d_repository = tsr
break
if d_repository is None:
# The dependent repository is not in the received list so look in the database.
d_repository = self.get_or_create_tool_shed_repository( d_toolshed,
d_name,
d_owner,
d_changeset_revision )
# Process each repository_dependency defined for the current dependent repository.
for repository_dependency_components_list in val:
required_repository = None
rd_toolshed, \
rd_name, \
rd_owner, \
rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( repository_dependency_components_list )
# Get the the tool_shed_repository defined by rd_name, rd_owner and rd_changeset_revision. This
# is the repository that will be required by the current d_repository.
# TODO: Check tool_shed_repository.tool_shed as well when repository dependencies across tool sheds is supported.
for tsr in tool_shed_repositories:
if tsr.name == rd_name and tsr.owner == rd_owner and tsr.changeset_revision == rd_changeset_revision:
required_repository = tsr
break
if required_repository is None:
# The required repository is not in the received list so look in the database.
required_repository = self.get_or_create_tool_shed_repository( rd_toolshed,
rd_name,
rd_owner,
rd_changeset_revision )
# Ensure there is a repository_dependency relationship between d_repository and required_repository.
rrda = None
for rd in d_repository.repository_dependencies:
if rd.id == required_repository.id:
rrda = rd
break
if not rrda:
# Make sure required_repository is in the repository_dependency table.
repository_dependency = self.get_repository_dependency_by_repository_id( install_model,
required_repository.id )
if not repository_dependency:
log.debug( 'Creating new repository_dependency record for installed revision %s of repository: %s owned by %s.' %
( str( required_repository.installed_changeset_revision ),
str( required_repository.name ),
str( required_repository.owner ) ) )
repository_dependency = install_model.RepositoryDependency( tool_shed_repository_id=required_repository.id )
install_model.context.add( repository_dependency )
install_model.context.flush()
# Build the relationship between the d_repository and the required_repository.
rrda = install_model.RepositoryRepositoryDependencyAssociation( tool_shed_repository_id=d_repository.id,
repository_dependency_id=repository_dependency.id )
install_model.context.add( rrda )
install_model.context.flush()
def get_updated_changeset_revisions_for_repository_dependencies( self, key_rd_dicts ):
updated_key_rd_dicts = []
for key_rd_dict in key_rd_dicts:
key = next(iter(key_rd_dict))
repository_dependency = key_rd_dict[ key ]
rd_toolshed, rd_name, rd_owner, rd_changeset_revision, \
rd_prior_installation_required, \
rd_only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( repository_dependency )
if suc.tool_shed_is_this_tool_shed( rd_toolshed ):
repository = tool_shed.util.repository_util.get_repository_by_name_and_owner( self.app, rd_name, rd_owner )
if repository:
repository_id = self.app.security.encode_id( repository.id )
repository_metadata = \
metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app,
repository_id,
rd_changeset_revision )
if repository_metadata:
# The repository changeset_revision is installable, so no updates are available.
new_key_rd_dict = {}
new_key_rd_dict[ key ] = repository_dependency
updated_key_rd_dicts.append( key_rd_dict )
else:
# The repository changeset_revision is no longer installable, so see if there's been an update.
repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False )
changeset_revision = metadata_util.get_next_downloadable_changeset_revision( repository, repo, rd_changeset_revision )
if changeset_revision != rd_changeset_revision:
repository_metadata = \
metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app,
repository_id,
changeset_revision )
if repository_metadata:
new_key_rd_dict = {}
new_key_rd_dict[ key ] = \
[ rd_toolshed,
rd_name,
rd_owner,
repository_metadata.changeset_revision,
rd_prior_installation_required,
rd_only_if_compiling_contained_td ]
# We have the updated changeset revision.
updated_key_rd_dicts.append( new_key_rd_dict )
else:
repository_components_tuple = container_util.get_components_from_key( key )
components_list = tool_shed.util.repository_util.extract_components_from_tuple( repository_components_tuple )
toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ]
# For backward compatibility to the 12/20/12 Galaxy release.
if len( components_list ) in (4, 5):
rd_only_if_compiling_contained_td = 'False'
message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \
( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ) )
message += "dependencies defined for repository %s will be ignored." % str( repository_name )
log.debug( message )
else:
repository_components_tuple = container_util.get_components_from_key( key )
components_list = tool_shed.util.repository_util.extract_components_from_tuple( repository_components_tuple )
toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ]
message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \
( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ) )
message += "dependencies defined for repository %s will be ignored." % str( repository_name )
log.debug( message )
return updated_key_rd_dicts
def merge_containers_dicts_for_new_install( self, containers_dicts ):
"""
When installing one or more tool shed repositories for the first time, the received list of
containers_dicts contains a containers_dict for each repository being installed. Since the
repositories are being installed for the first time, all entries are None except the repository
dependencies and tool dependencies. The entries for missing dependencies are all None since
they have previously been merged into the installed dependencies. This method will merge the
dependencies entries into a single container and return it for display.
"""
new_containers_dict = dict( readme_files=None,
datatypes=None,
missing_repository_dependencies=None,
repository_dependencies=None,
missing_tool_dependencies=None,
tool_dependencies=None,
invalid_tools=None,
valid_tools=None,
workflows=None )
if containers_dicts:
lock = threading.Lock()
lock.acquire( True )
try:
repository_dependencies_root_folder = None
tool_dependencies_root_folder = None
# Use a unique folder id (hopefully the following is).
folder_id = 867
for old_container_dict in containers_dicts:
# Merge repository_dependencies.
old_container_repository_dependencies_root = old_container_dict[ 'repository_dependencies' ]
if old_container_repository_dependencies_root:
if repository_dependencies_root_folder is None:
repository_dependencies_root_folder = utility_container_manager.Folder( id=folder_id,
key='root',
label='root',
parent=None )
folder_id += 1
repository_dependencies_folder = utility_container_manager.Folder( id=folder_id,
key='merged',
label='Repository dependencies',
parent=repository_dependencies_root_folder )
folder_id += 1
# The old_container_repository_dependencies_root will be a root folder containing a single sub_folder.
old_container_repository_dependencies_folder = old_container_repository_dependencies_root.folders[ 0 ]
# Change the folder id so it won't confict with others being merged.
old_container_repository_dependencies_folder.id = folder_id
folder_id += 1
repository_components_tuple = \
container_util.get_components_from_key( old_container_repository_dependencies_folder.key )
components_list = repository_util.extract_components_from_tuple( repository_components_tuple )
name = components_list[ 1 ]
# Generate the label by retrieving the repository name.
old_container_repository_dependencies_folder.label = str( name )
repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder )
# Merge tool_dependencies.
old_container_tool_dependencies_root = old_container_dict[ 'tool_dependencies' ]
if old_container_tool_dependencies_root:
if tool_dependencies_root_folder is None:
tool_dependencies_root_folder = utility_container_manager.Folder( id=folder_id,
key='root',
label='root',
parent=None )
folder_id += 1
tool_dependencies_folder = utility_container_manager.Folder( id=folder_id,
key='merged',
label='Tool dependencies',
parent=tool_dependencies_root_folder )
folder_id += 1
else:
td_list = [ td.listify for td in tool_dependencies_folder.tool_dependencies ]
# The old_container_tool_dependencies_root will be a root folder containing a single sub_folder.
old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[ 0 ]
for td in old_container_tool_dependencies_folder.tool_dependencies:
if td.listify not in td_list:
tool_dependencies_folder.tool_dependencies.append( td )
if repository_dependencies_root_folder:
repository_dependencies_root_folder.folders.append( repository_dependencies_folder )
new_containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder
if tool_dependencies_root_folder:
tool_dependencies_root_folder.folders.append( tool_dependencies_folder )
new_containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder
except Exception as e:
log.debug( "Exception in merge_containers_dicts_for_new_install: %s" % str( e ) )
finally:
lock.release()
return new_containers_dict
def handle_repository_dependencies_container_entry(
self, repository_dependencies_folder, rd_key, rd_value, folder_id,
repository_dependency_id, folder_keys):
repository_components_tuple = container_util.get_components_from_key(
rd_key)
components_list = repository_util.extract_components_from_tuple(
repository_components_tuple)
toolshed, repository_name, repository_owner, changeset_revision = components_list[
0:4]
# For backward compatibility to the 12/20/12 Galaxy release.
if len(components_list) == 4:
prior_installation_required = 'False'
only_if_compiling_contained_td = 'False'
elif len(components_list) == 5:
prior_installation_required = components_list[4]
only_if_compiling_contained_td = 'False'
elif len(components_list) == 6:
prior_installation_required = components_list[4]
only_if_compiling_contained_td = components_list[5]
folder = self.get_folder(repository_dependencies_folder, rd_key)
label = self.generate_repository_dependencies_folder_label_from_key(
repository_name, repository_owner, changeset_revision,
prior_installation_required, only_if_compiling_contained_td,
repository_dependencies_folder.key)
if folder:
if rd_key not in folder_keys:
folder_id += 1
sub_folder = Folder(id=folder_id,
key=rd_key,
label=label,
parent=folder)
folder.folders.append(sub_folder)
else:
sub_folder = folder
else:
folder_id += 1
sub_folder = Folder(id=folder_id,
key=rd_key,
label=label,
parent=repository_dependencies_folder)
repository_dependencies_folder.folders.append(sub_folder)
if self.app.name == 'galaxy':
# Insert a header row.
repository_dependency_id += 1
repository_dependency = RepositoryDependency(
id=repository_dependency_id,
repository_name='Name',
changeset_revision='Revision',
repository_owner='Owner',
installation_status='Installation status')
# Insert the header row into the folder.
sub_folder.repository_dependencies.append(repository_dependency)
for repository_dependency in rd_value:
if self.app.name == 'galaxy':
tool_shed_repository_id, installation_status, repository_dependency = \
self.get_components_from_repository_dependency_for_installed_repository( repository_dependency )
else:
tool_shed_repository_id = None
installation_status = None
can_create_dependency = not self.is_subfolder_of(
sub_folder, repository_dependency)
if can_create_dependency:
toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
common_util.parse_repository_dependency_tuple( repository_dependency )
repository_dependency_id += 1
repository_dependency = RepositoryDependency(
id=repository_dependency_id,
toolshed=toolshed,
repository_name=repository_name,
repository_owner=repository_owner,
changeset_revision=changeset_revision,
prior_installation_required=util.asbool(
prior_installation_required),
only_if_compiling_contained_td=util.asbool(
only_if_compiling_contained_td),
installation_status=installation_status,
tool_shed_repository_id=tool_shed_repository_id)
# Insert the repository_dependency into the folder.
sub_folder.repository_dependencies.append(
repository_dependency)
return repository_dependencies_folder, folder_id, repository_dependency_id
def get_required_repo_info_dicts(self, tool_shed_url, repo_info_dicts):
"""
Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of
them to the list. All repository_dependency entries in each of the received repo_info_dicts includes
all required repositories, so only one pass through this method is required to retrieve all repository
dependencies.
"""
all_required_repo_info_dict = {}
all_repo_info_dicts = []
if repo_info_dicts:
# We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool
# shed to discover repository ids.
required_repository_tups = []
for repo_info_dict in repo_info_dicts:
if repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append(repo_info_dict)
for repository_name, repo_info_tup in repo_info_dict.items():
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
repository_util.get_repo_info_tuple_contents(repo_info_tup)
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in ['root_key', 'description']:
continue
repository_components_tuple = container_util.get_components_from_key(key)
components_list = repository_util.extract_components_from_tuple(repository_components_tuple)
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository since in this case, the repository dependency is really
# a dependency of the dependent repository's contained tool dependency, and only if that
# tool dependency requires compilation.
# For backward compatibility to the 12/20/12 Galaxy release.
only_if_compiling_contained_td = 'False'
if len(components_list) == 4:
only_if_compiling_contained_td = 'False'
elif len(components_list) == 5:
only_if_compiling_contained_td = 'False'
if not asbool(only_if_compiling_contained_td):
if components_list not in required_repository_tups:
required_repository_tups.append(components_list)
for components_list in val:
try:
only_if_compiling_contained_td = components_list[5]
except:
only_if_compiling_contained_td = 'False'
# Skip listing a repository dependency if it is required only to compile a tool dependency
# defined for the dependent repository (see above comment).
if not asbool(only_if_compiling_contained_td):
if components_list not in required_repository_tups:
required_repository_tups.append(components_list)
else:
# We have a single repository with no dependencies.
components_list = [tool_shed_url, repository_name, repository_owner, changeset_revision]
required_repository_tups.append(components_list)
if required_repository_tups:
# The value of required_repository_tups is a list of tuples, so we need to encode it.
encoded_required_repository_tups = []
for required_repository_tup in required_repository_tups:
# Convert every item in required_repository_tup to a string.
required_repository_tup = [str(item) for item in required_repository_tup]
encoded_required_repository_tups.append(encoding_util.encoding_sep.join(required_repository_tup))
encoded_required_repository_str = encoding_util.encoding_sep2.join(encoded_required_repository_tups)
encoded_required_repository_str = encoding_util.tool_shed_encode(encoded_required_repository_str)
if repository_util.is_tool_shed_client(self.app):
# Handle secure / insecure Tool Shed URL protocol changes and port changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.app, tool_shed_url)
pathspec = ['repository', 'get_required_repo_info_dict']
url = build_url(tool_shed_url, pathspec=pathspec)
# Fix for handling 307 redirect not being handled nicely by urlopen() when the Request() has data provided
url = _urlopen(url).geturl()
response = _urlopen(url, urlencode(dict(encoded_str=encoded_required_repository_str))).read()
if response:
try:
required_repo_info_dict = json.loads(response)
except Exception as e:
log.exception(e)
return all_repo_info_dicts
required_repo_info_dicts = []
for k, v in required_repo_info_dict.items():
if k == 'repo_info_dicts':
encoded_dict_strings = required_repo_info_dict['repo_info_dicts']
for encoded_dict_str in encoded_dict_strings:
decoded_dict = encoding_util.tool_shed_decode(encoded_dict_str)
required_repo_info_dicts.append(decoded_dict)
else:
if k not in all_required_repo_info_dict:
all_required_repo_info_dict[k] = v
else:
if v and not all_required_repo_info_dict[k]:
all_required_repo_info_dict[k] = v
if required_repo_info_dicts:
for required_repo_info_dict in required_repo_info_dicts:
# Each required_repo_info_dict has a single entry, and all_repo_info_dicts is a list
# of dictionaries, each of which has a single entry. We'll check keys here rather than
# the entire dictionary because a dictionary entry in all_repo_info_dicts will include
# lists of discovered repository dependencies, but these lists will be empty in the
# required_repo_info_dict since dependency discovery has not yet been performed for these
# dictionaries.
required_repo_info_dict_key = next(iter(required_repo_info_dict))
all_repo_info_dicts_keys = [next(iter(d)) for d in all_repo_info_dicts]
if required_repo_info_dict_key not in all_repo_info_dicts_keys:
all_repo_info_dicts.append(required_repo_info_dict)
else:
# required_repo_info_dict_key corresponds to the repo name.
# A single install transaction might require the installation of 2 or more repos
# with the same repo name but different owners or versions.
# Therefore, if required_repo_info_dict_key is already in all_repo_info_dicts,
# check that the tool id is already present. If it is not, we are dealing with the same repo name,
# but a different owner/changeset revision or version and we add the repo to the list of repos to be installed.
tool_id = required_repo_info_dict[required_repo_info_dict_key][1]
is_present = False
for repo_info_dict in all_repo_info_dicts:
for k, v in repo_info_dict.items():
if required_repo_info_dict_key == k:
if tool_id == v[1]:
is_present = True
break
if not is_present:
all_repo_info_dicts.append(required_repo_info_dict)
all_required_repo_info_dict['all_repo_info_dicts'] = all_repo_info_dicts
return all_required_repo_info_dict
def get_required_repo_info_dicts( trans, tool_shed_url, repo_info_dicts ):
"""
Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of them to the list. All
repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through
this method is required to retrieve all repository dependencies.
"""
all_required_repo_info_dict = {}
all_repo_info_dicts = []
if repo_info_dicts:
# We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids.
required_repository_tups = []
for repo_info_dict in repo_info_dicts:
if repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append( repo_info_dict )
for repository_name, repo_info_tup in repo_info_dict.items():
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tup )
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in [ 'root_key', 'description' ]:
continue
repository_components_tuple = container_util.get_components_from_key( key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
# Skip listing a repository dependency if it is required only to compile a tool dependency defined for the dependent repository since
# in this case, the repository dependency is really a dependency of the dependent repository's contained tool dependency, and only if
# that tool dependency requires compilation.
# For backward compatibility to the 12/20/12 Galaxy release.
prior_installation_required = 'False'
only_if_compiling_contained_td = 'False'
if len( components_list ) == 4:
prior_installation_required = 'False'
only_if_compiling_contained_td = 'False'
elif len( components_list ) == 5:
prior_installation_required = components_list[ 4 ]
only_if_compiling_contained_td = 'False'
if not util.asbool( only_if_compiling_contained_td ):
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
for components_list in val:
try:
only_if_compiling_contained_td = components_list[ 5 ]
except:
only_if_compiling_contained_td = 'False'
# Skip listing a repository dependency if it is required only to compile a tool dependency defined for the dependent repository
# (see above comment).
if not util.asbool( only_if_compiling_contained_td ):
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
else:
# We have a single repository with no dependencies.
components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision ]
required_repository_tups.append( components_list )
if required_repository_tups:
# The value of required_repository_tups is a list of tuples, so we need to encode it.
encoded_required_repository_tups = []
for required_repository_tup in required_repository_tups:
# Convert every item in required_repository_tup to a string.
required_repository_tup = [ str( item ) for item in required_repository_tup ]
encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict' )
request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
response = urllib2.urlopen( request ).read()
if response:
try:
required_repo_info_dict = json.from_json_string( response )
except Exception, e:
log.exception( e )
return all_repo_info_dicts
required_repo_info_dicts = []
for k, v in required_repo_info_dict.items():
if k == 'repo_info_dicts':
encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
for encoded_dict_str in encoded_dict_strings:
decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
required_repo_info_dicts.append( decoded_dict )
else:
if k not in all_required_repo_info_dict:
all_required_repo_info_dict[ k ] = v
else:
if v and not all_required_repo_info_dict[ k ]:
all_required_repo_info_dict[ k ] = v
if required_repo_info_dicts:
for required_repo_info_dict in required_repo_info_dicts:
if required_repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append( required_repo_info_dict )
all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
