def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, owner, changeset_revision ):
"""
Get all appropriate newer changeset revisions for the repository defined by
the received tool_shed_url / name / owner combination.
"""
url = suc.url_join( tool_shed_url,
'repository/updated_changeset_revisions?name=%s&owner=%s&changeset_revision=%s' % ( name, owner, changeset_revision ) )
text = cu.tool_shed_get( app, tool_shed_url, url )
return text
def get_api_url( base, parts=[], params=None ):
if 'api' in parts and parts.index( 'api' ) != 0:
parts.pop( parts.index( 'api' ) )
parts.insert( 0, 'api' )
elif 'api' not in parts:
parts.insert( 0, 'api' )
url = suc.url_join( base, *parts )
if params:
url += '?%s' % params
return url
def get_updated_changeset_revisions_from_tool_shed(app, tool_shed_url, name,
owner, changeset_revision):
"""
Get all appropriate newer changeset revisions for the repository defined by
the received tool_shed_url / name / owner combination.
"""
url = suc.url_join(
tool_shed_url,
'repository/updated_changeset_revisions?name=%s&owner=%s&changeset_revision=%s'
% (name, owner, changeset_revision))
text = cu.tool_shed_get(app, tool_shed_url, url)
return text
def get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ):
"""
Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
selection page.
"""
name = repo_info_dict.keys()[ 0 ]
repo_info_tuple = repo_info_dict[ name ]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
# Handle README files.
url = suc.url_join( tool_shed_url,
'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % ( name, repository_owner, changeset_revision ) )
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url )
readme_files_dict = json.from_json_string( raw_text )
return readme_files_dict
def get_api_url( base, parts=[], params=None ):
"""Compose and return a URL for the Tool Shed API."""
if 'api' in parts and parts.index( 'api' ) != 0:
parts.pop( parts.index( 'api' ) )
parts.insert( 0, 'api' )
elif 'api' not in parts:
parts.insert( 0, 'api' )
url = suc.url_join( base, *parts )
if params is not None:
try:
query_string = urllib.urlencode( params )
except Exception, e:
# The value of params must be a string.
query_string = params
url += '?%s' % query_string
def get_readme_files_dict_for_display(trans, tool_shed_url, repo_info_dict):
"""
Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
selection page.
"""
name = repo_info_dict.keys()[0]
repo_info_tuple = repo_info_dict[name]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
# Handle README files.
url = suc.url_join(
tool_shed_url,
'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' %
(name, repository_owner, changeset_revision))
raw_text = common_util.tool_shed_get(trans.app, tool_shed_url, url)
readme_files_dict = json.from_json_string(raw_text)
return readme_files_dict
def build_citable_url( host, repository ):
return url_join( host, 'view', repository.user.username, repository.name )
def install_repository_revision( self, trans, payload, **kwd ):
"""
POST /api/tool_shed_repositories/install_repository_revision
Install a specified repository revision from a specified tool shed into Galaxy.
:param key: the current Galaxy admin user's API key
The following parameters are included in the payload.
:param tool_shed_url (required): the base URL of the Tool Shed from which to install the Repository
:param name (required): the name of the Repository
:param owner (required): the owner of the Repository
:param changset_revision (required): the changset_revision of the RepositoryMetadata object associated with the Repository
:param new_tool_panel_section_label (optional): label of a new section to be added to the Galaxy tool panel in which to load
tools contained in the Repository. Either this parameter must be an empty string or
the tool_panel_section_id parameter must be an empty string or both must be an empty
string (both cannot be used simultaneously).
:param tool_panel_section_id (optional): id of the Galaxy tool panel section in which to load tools contained in the Repository.
If this parameter is an empty string and the above new_tool_panel_section_label parameter is an
empty string, tools will be loaded outside of any sections in the tool panel. Either this
parameter must be an empty string or the tool_panel_section_id parameter must be an empty string
of both must be an empty string (both cannot be used simultaneously).
:param install_repository_dependencies (optional): Set to True if you want to install repository dependencies defined for the specified
repository being installed. The default setting is False.
:param install_tool_dependencies (optional): Set to True if you want to install tool dependencies defined for the specified repository being
installed. The default setting is False.
:param shed_tool_conf (optional): The shed-related tool panel configuration file configured in the "tool_config_file" setting in the Galaxy config file
(e.g., universe_wsgi.ini). At least one shed-related tool panel config file is required to be configured. Setting
this parameter to a specific file enables you to choose where the specified repository will be installed because
the tool_path attribute of the <toolbox> from the specified file is used as the installation location
(e.g., <toolbox tool_path="../shed_tools">). If this parameter is not set, a shed-related tool panel configuration
file will be selected automatically.
"""
# Get the information about the repository to be installed from the payload.
tool_shed_url = payload.get( 'tool_shed_url', '' )
if not tool_shed_url:
raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." )
name = payload.get( 'name', '' )
if not name:
raise HTTPBadRequest( detail="Missing required parameter 'name'." )
owner = payload.get( 'owner', '' )
if not owner:
raise HTTPBadRequest( detail="Missing required parameter 'owner'." )
changeset_revision = payload.get( 'changeset_revision', '' )
if not changeset_revision:
raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." )
# Make sure this Galaxy instance is configured with a shed-related tool panel configuration file.
if not suc.have_shed_tool_conf_for_install( trans ):
message = get_message_for_no_shed_tool_config()
log.error( message, exc_info=True )
trans.response.status = 500
return dict( status='error', error=message )
# Make sure the current user's API key proves he is an admin user in this Galaxy instance.
if not trans.user_is_admin():
raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' )
# Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed.
installed_tool_shed_repositories = []
# Get all of the information necessary for installing the repository from the specified tool shed.
url = suc.url_join( tool_shed_url,
'api/repositories/get_repository_revision_install_info?name=%s&owner=%s&changeset_revision=%s' % \
( name, owner, changeset_revision ) )
try:
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url )
except Exception, e:
message = "Error attempting to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \
( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) )
log.error( message, exc_info=True )
trans.response.status = 500
return dict( status='error', error=message )
tmp_export_info = xml_util.create_and_write_tmp_file( export_elem, use_indent=True )
repositories_archive.add( tmp_export_info, arcname='export_info.xml' )
# Write the manifest, which must preserve the order in which the repositories should be imported.
exported_repository_root = xml_util.create_element( 'repositories' )
for exported_repository_elem in exported_repository_registry.exported_repository_elems:
exported_repository_root.append( exported_repository_elem )
tmp_manifest = xml_util.create_and_write_tmp_file( exported_repository_root, use_indent=True )
repositories_archive.add( tmp_manifest, arcname='manifest.xml' )
except Exception, e:
log.exception( str( e ) )
finally:
lock.release()
repositories_archive.close()
if api:
encoded_repositories_archive_name = encoding_util.tool_shed_encode( repositories_archive_filename )
download_url = suc.url_join( web.url_for( '/', qualified=True ),
'repository/export_via_api?encoded_repositories_archive_name=%s' % encoded_repositories_archive_name )
return dict( download_url=download_url, error_messages=error_messages )
return repositories_archive, error_messages
def generate_repository_archive( trans, work_dir, tool_shed_url, repository, changeset_revision, file_type ):
file_type_str = suc.get_file_type_str( changeset_revision, file_type )
file_name = '%s-%s' % ( repository.name, file_type_str )
return_code, error_message = archive_repository_revision( trans, ui, repository, work_dir, changeset_revision )
if return_code:
return None, error_message
repository_archive_name = os.path.join( work_dir, file_name )
# Create a compressed tar archive that will contain only valid files and possibly altered dependency definition files.
repository_archive = tarfile.open( repository_archive_name, "w:%s" % file_type )
for root, dirs, files in os.walk( work_dir ):
if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0:
for dir in dirs:
def get_required_repo_info_dicts( trans, tool_shed_url, repo_info_dicts ):
"""
Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of them to the list. All
repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through
this method is required to retrieve all repository dependencies.
"""
all_required_repo_info_dict = {}
all_repo_info_dicts = []
if repo_info_dicts:
# We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids.
required_repository_tups = []
for repo_info_dict in repo_info_dicts:
if repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append( repo_info_dict )
for repository_name, repo_info_tup in repo_info_dict.items():
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tup )
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in [ 'root_key', 'description' ]:
continue
try:
toolshed, name, owner, changeset_revision, prior_installation_required = container_util.get_components_from_key( key )
components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required ]
except ValueError:
# For backward compatibility to the 12/20/12 Galaxy release, default prior_installation_required to False in the caller.
toolshed, name, owner, changeset_revision = container_util.get_components_from_key( key )
components_list = [ toolshed, name, owner, changeset_revision ]
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
for components_list in val:
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
else:
# We have a single repository with no dependencies.
components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision, 'False' ]
required_repository_tups.append( components_list )
if required_repository_tups:
# The value of required_repository_tups is a list of tuples, so we need to encode it.
encoded_required_repository_tups = []
for required_repository_tup in required_repository_tups:
# Convert every item in required_repository_tup to a string.
required_repository_tup = [ str( item ) for item in required_repository_tup ]
encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict' )
request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
response = urllib2.urlopen( request ).read()
if response:
try:
required_repo_info_dict = json.from_json_string( response )
except Exception, e:
log.exception( e )
return all_repo_info_dicts
required_repo_info_dicts = []
for k, v in required_repo_info_dict.items():
if k == 'repo_info_dicts':
encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
for encoded_dict_str in encoded_dict_strings:
decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
required_repo_info_dicts.append( decoded_dict )
else:
if k not in all_required_repo_info_dict:
all_required_repo_info_dict[ k ] = v
else:
if v and not all_required_repo_info_dict[ k ]:
all_required_repo_info_dict[ k ] = v
if required_repo_info_dicts:
for required_repo_info_dict in required_repo_info_dicts:
if required_repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append( required_repo_info_dict )
all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
def build_citable_url( host, repository ):
return url_join( host, 'view', repository.user.username, repository.name )
def get_required_repo_info_dicts( trans, tool_shed_url, repo_info_dicts ):
"""
Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of them to the list. All
repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through
this method is required to retrieve all repository dependencies.
"""
all_required_repo_info_dict = {}
all_repo_info_dicts = []
if repo_info_dicts:
# We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids.
required_repository_tups = []
for repo_info_dict in repo_info_dicts:
if repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append( repo_info_dict )
for repository_name, repo_info_tup in repo_info_dict.items():
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tup )
if repository_dependencies:
for key, val in repository_dependencies.items():
if key in [ 'root_key', 'description' ]:
continue
repository_components_tuple = container_util.get_components_from_key( key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
# Skip listing a repository dependency if it is required only to compile a tool dependency defined for the dependent repository since
# in this case, the repository dependency is really a dependency of the dependent repository's contained tool dependency, and only if
# that tool dependency requires compilation.
# For backward compatibility to the 12/20/12 Galaxy release.
prior_installation_required = 'False'
only_if_compiling_contained_td = 'False'
if len( components_list ) == 4:
prior_installation_required = 'False'
only_if_compiling_contained_td = 'False'
elif len( components_list ) == 5:
prior_installation_required = components_list[ 4 ]
only_if_compiling_contained_td = 'False'
if not util.asbool( only_if_compiling_contained_td ):
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
for components_list in val:
try:
only_if_compiling_contained_td = components_list[ 5 ]
except:
only_if_compiling_contained_td = 'False'
# Skip listing a repository dependency if it is required only to compile a tool dependency defined for the dependent repository
# (see above comment).
if not util.asbool( only_if_compiling_contained_td ):
if components_list not in required_repository_tups:
required_repository_tups.append( components_list )
else:
# We have a single repository with no dependencies.
components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision ]
required_repository_tups.append( components_list )
if required_repository_tups:
# The value of required_repository_tups is a list of tuples, so we need to encode it.
encoded_required_repository_tups = []
for required_repository_tup in required_repository_tups:
# Convert every item in required_repository_tup to a string.
required_repository_tup = [ str( item ) for item in required_repository_tup ]
encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict' )
request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
response = urllib2.urlopen( request ).read()
if response:
try:
required_repo_info_dict = json.from_json_string( response )
except Exception, e:
log.exception( e )
return all_repo_info_dicts
required_repo_info_dicts = []
for k, v in required_repo_info_dict.items():
if k == 'repo_info_dicts':
encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
for encoded_dict_str in encoded_dict_strings:
decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
required_repo_info_dicts.append( decoded_dict )
else:
if k not in all_required_repo_info_dict:
all_required_repo_info_dict[ k ] = v
else:
if v and not all_required_repo_info_dict[ k ]:
all_required_repo_info_dict[ k ] = v
if required_repo_info_dicts:
for required_repo_info_dict in required_repo_info_dicts:
if required_repo_info_dict not in all_repo_info_dicts:
all_repo_info_dicts.append( required_repo_info_dict )
all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
