def get_version_lineage_for_tool(self, repository_id, repository_metadata, guid):
"""
Return the tool version lineage chain in descendant order for the received
guid contained in the received repsitory_metadata.tool_versions. This function
is called only from the Tool Shed.
"""
repository = repository_util.get_repository_by_id(self.app, repository_id)
repo = hg_util.get_repo_for_repository(self.app, repository=repository)
# Initialize the tool lineage
version_lineage = [guid]
# Get all ancestor guids of the received guid.
current_child_guid = guid
for changeset in hg_util.reversed_upper_bounded_changelog(repo, repository_metadata.changeset_revision):
ctx = repo.changectx(changeset)
rm = metadata_util.get_repository_metadata_by_changeset_revision(self.app, repository_id, str(ctx))
if rm:
parent_guid = rm.tool_versions.get(current_child_guid, None)
if parent_guid:
version_lineage.append(parent_guid)
current_child_guid = parent_guid
# Get all descendant guids of the received guid.
current_parent_guid = guid
for changeset in hg_util.reversed_lower_upper_bounded_changelog(repo,
repository_metadata.changeset_revision,
repository.tip(self.app)):
ctx = repo.changectx(changeset)
rm = metadata_util.get_repository_metadata_by_changeset_revision(self.app, repository_id, str(ctx))
if rm:
tool_versions = rm.tool_versions
for child_guid, parent_guid in tool_versions.items():
if parent_guid == current_parent_guid:
version_lineage.insert(0, child_guid)
current_parent_guid = child_guid
break
return version_lineage
def get_latest_downloadable_repository_metadata(trans, repository):
"""
Return the latest downloadable repository_metadata record for the received repository. This will
return repositories of type unrestricted as well as types repository_suite_definition and
tool_dependency_definition.
"""
encoded_repository_id = trans.security.encode_id(repository.id)
repo = hg_util.get_repo_for_repository(trans.app,
repository=repository,
repo_path=None,
create=False)
tip_ctx = str(repo.changectx(repo.changelog.tip()))
repository_metadata = None
try:
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
trans.app, encoded_repository_id, tip_ctx)
if repository_metadata is not None and repository_metadata.downloadable:
return repository_metadata
return None
except Exception:
latest_downloadable_revision = metadata_util.get_previous_metadata_changeset_revision(
repository, repo, tip_ctx, downloadable=True)
if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH:
return None
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
trans.app, encoded_repository_id, latest_downloadable_revision)
if repository_metadata is not None and repository_metadata.downloadable:
return repository_metadata
return None
def get_latest_downloadable_repository_metadata( trans, repository ):
"""
Return the latest downloadable repository_metadata record for the received repository. This will
return repositories of type unrestricted as well as types repository_suite_definition and
tool_dependency_definition.
"""
encoded_repository_id = trans.security.encode_id( repository.id )
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
tip_ctx = str( repo.changectx( repo.changelog.tip() ) )
repository_metadata = None
try:
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx )
if repository_metadata is not None and repository_metadata.downloadable:
return repository_metadata
return None
except:
latest_downloadable_revision = metadata_util.get_previous_metadata_changeset_revision( repository,
repo,
tip_ctx,
downloadable=True )
if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH:
return None
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app,
encoded_repository_id,
latest_downloadable_revision )
if repository_metadata is not None and repository_metadata.downloadable:
return repository_metadata
return None
def get_repo_info_dict(app, user, repository_id, changeset_revision):
repository = get_repository_in_tool_shed(app, repository_id)
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(
user, repository)
repository_metadata = get_repository_metadata_by_changeset_revision(
app, repository_id, changeset_revision)
if not repository_metadata:
# The received changeset_revision is no longer installable, so get the next changeset_revision
# in the repository's changelog. This generally occurs only with repositories of type
# repository_suite_definition or tool_dependency_definition.
next_downloadable_changeset_revision = \
get_next_downloadable_changeset_revision(app, repository, changeset_revision)
if next_downloadable_changeset_revision and next_downloadable_changeset_revision != changeset_revision:
repository_metadata = get_repository_metadata_by_changeset_revision(
app, repository_id, next_downloadable_changeset_revision)
if repository_metadata:
# For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption
# is not valid we'll have to enhance the callers to handle repository_metadata values of None in the
# returned repo_info_dict.
metadata = repository_metadata.metadata
if 'tools' in metadata:
includes_tools = True
else:
includes_tools = False
includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel
repository_dependencies_dict = metadata.get('repository_dependencies',
{})
repository_dependencies = repository_dependencies_dict.get(
'repository_dependencies', [])
has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \
get_repository_dependency_types(repository_dependencies)
if 'tool_dependencies' in metadata:
includes_tool_dependencies = True
else:
includes_tool_dependencies = False
else:
# Here's where we may have to handle enhancements to the callers. See above comment.
includes_tools = False
has_repository_dependencies = False
has_repository_dependencies_only_if_compiling_contained_td = False
includes_tool_dependencies = False
includes_tools_for_display_in_tool_panel = False
repo_path = repository.repo_path(app)
ctx_rev = str(changeset2rev(repo_path, changeset_revision))
repo_info_dict = create_repo_info_dict(
app=app,
repository_clone_url=repository_clone_url,
changeset_revision=changeset_revision,
ctx_rev=ctx_rev,
repository_owner=repository.user.username,
repository_name=repository.name,
repository=repository,
repository_metadata=repository_metadata,
tool_dependencies=None,
repository_dependencies=None)
return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \
has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td
def get_certified_level_one_tuple(self, repository):
"""
Return True if the latest installable changeset_revision of the received repository is level one certified.
"""
if repository is None:
return (None, False)
if repository.deleted or repository.deprecated:
return (None, False)
repo = hg_util.get_repo_for_repository(self.app,
repository=repository,
repo_path=None,
create=False)
# Get the latest installable changeset revision since that is all that is currently configured for testing.
latest_installable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision(
self.app, repository, repo)
if latest_installable_changeset_revision not in [
None, hg_util.INITIAL_CHANGELOG_HASH
]:
encoded_repository_id = self.app.security.encode_id(repository.id)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
self.app, encoded_repository_id,
latest_installable_changeset_revision)
if repository_metadata:
# No repository_metadata.
return (latest_installable_changeset_revision, True)
else:
# No installable changeset_revision.
return (None, False)
def manage_repository_reviews(self, trans, mine=False, **kwd):
# The value of the received id is the encoded repository id.
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
repository_id = kwd.get('id', None)
if repository_id:
repository = suc.get_repository_in_tool_shed(
trans.app, repository_id)
repo = hg_util.get_repo_for_repository(trans.app,
repository=repository)
metadata_revision_hashes = [
metadata_revision.changeset_revision
for metadata_revision in repository.metadata_revisions
]
reviewed_revision_hashes = [
review.changeset_revision for review in repository.reviews
]
reviews_dict = OrderedDict()
for changeset in hg_util.get_reversed_changelog_changesets(repo):
changeset_revision = str(repo[changeset])
if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes:
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision(
repo, changeset_revision)
if changeset_revision in reviewed_revision_hashes:
# Find the review for this changeset_revision
repository_reviews = \
review_util.get_reviews_by_repository_id_changeset_revision(trans.app,
repository_id,
changeset_revision)
# Determine if the current user can add a review to this revision.
can_add_review = trans.user not in [
repository_review.user
for repository_review in repository_reviews
]
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
trans.app, repository_id, changeset_revision)
if repository_metadata:
repository_metadata_reviews = util.listify(
repository_metadata.reviews)
else:
repository_metadata_reviews = []
else:
repository_reviews = []
repository_metadata_reviews = []
can_add_review = True
installable = changeset_revision in metadata_revision_hashes
revision_dict = dict(
changeset_revision_label=changeset_revision_label,
repository_reviews=repository_reviews,
repository_metadata_reviews=repository_metadata_reviews,
installable=installable,
can_add_review=can_add_review)
reviews_dict[changeset_revision] = revision_dict
return trans.fill_template(
'/webapps/tool_shed/repository_review/reviews_of_repository.mako',
repository=repository,
reviews_dict=reviews_dict,
mine=mine,
message=message,
status=status)
def get_repositories_by_category(app, category_id, installable=False, sort_order='asc', sort_key='name', page=None, per_page=25):
sa_session = app.model.context.current
query = sa_session.query(app.model.Repository) \
.join(app.model.RepositoryCategoryAssociation, app.model.Repository.id == app.model.RepositoryCategoryAssociation.repository_id) \
.join(app.model.User, app.model.User.id == app.model.Repository.user_id) \
.filter(app.model.RepositoryCategoryAssociation.category_id == category_id)
if installable:
subquery = select([app.model.RepositoryMetadata.table.c.repository_id])
query = query.filter(app.model.Repository.id.in_(subquery))
if sort_key == 'owner':
query = query.order_by(app.model.User.username) if sort_order == 'asc' else query.order_by(app.model.User.username.desc())
else:
query = query.order_by(app.model.Repository.name) if sort_order == 'asc' else query.order_by(app.model.Repository.name.desc())
if page is not None:
page = int(page)
query = query.limit(per_page)
if page > 1:
query = query.offset((page - 1) * per_page)
resultset = query.all()
repositories = []
for repository in resultset:
default_value_mapper = {'id': app.security.encode_id, 'user_id': app.security.encode_id, 'repository_id': app.security.encode_id}
repository_dict = repository.to_dict(value_mapper=default_value_mapper)
repository_dict['metadata'] = {}
for changeset, changehash in repository.installable_revisions(app):
encoded_id = app.security.encode_id(repository.id)
metadata = get_repository_metadata_by_changeset_revision(app, encoded_id, changehash)
repository_dict['metadata']['%s:%s' % (changeset, changehash)] = metadata.to_dict(value_mapper=default_value_mapper)
if installable:
if len(repository.installable_revisions(app)):
repositories.append(repository_dict)
else:
repositories.append(repository_dict)
return repositories
def load_viewable_repositories_and_suites_by_category(self):
# Clear all dictionaries just in case they were previously loaded.
self.certified_level_one_viewable_repositories_and_suites_by_category = {}
self.certified_level_one_viewable_suites_by_category = {}
self.certified_level_two_viewable_repositories_and_suites_by_category = {}
self.certified_level_two_viewable_suites_by_category = {}
self.viewable_repositories_and_suites_by_category = {}
self.viewable_suites_by_category = {}
self.viewable_valid_repositories_and_suites_by_category = {}
self.viewable_valid_suites_by_category = {}
for category in self.sa_session.query(model.Category):
category_name = str(category.name)
if category not in self.certified_level_one_viewable_repositories_and_suites_by_category:
self.certified_level_one_viewable_repositories_and_suites_by_category[
category_name] = 0
if category not in self.certified_level_one_viewable_suites_by_category:
self.certified_level_one_viewable_suites_by_category[
category_name] = 0
if category not in self.viewable_repositories_and_suites_by_category:
self.viewable_repositories_and_suites_by_category[
category_name] = 0
if category not in self.viewable_suites_by_category:
self.viewable_suites_by_category[category_name] = 0
if category not in self.viewable_valid_repositories_and_suites_by_category:
self.viewable_valid_repositories_and_suites_by_category[
category_name] = 0
if category not in self.viewable_valid_suites_by_category:
self.viewable_valid_suites_by_category[category_name] = 0
for rca in category.repositories:
repository = rca.repository
if not repository.deleted and not repository.deprecated:
is_valid = self.is_valid(repository)
encoded_repository_id = self.app.security.encode_id(
repository.id)
tip_changeset_hash = repository.tip(self.app)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
self.app, encoded_repository_id, tip_changeset_hash)
self.viewable_repositories_and_suites_by_category[
category_name] += 1
if is_valid:
self.viewable_valid_repositories_and_suites_by_category[
category_name] += 1
if repository.type in [
rt_util.REPOSITORY_SUITE_DEFINITION
]:
self.viewable_suites_by_category[category_name] += 1
if is_valid:
self.viewable_valid_suites_by_category[
category_name] += 1
if self.is_level_one_certified(repository_metadata):
self.certified_level_one_viewable_repositories_and_suites_by_category[
category_name] += 1
if repository.type in [
rt_util.REPOSITORY_SUITE_DEFINITION
]:
self.certified_level_one_viewable_suites_by_category[
category_name] += 1
def get_latest_repository_metadata(trans, repository):
"""
Return the latest repository_metadata record for the received repository if it exists. This will
return repositories of type unrestricted as well as types repository_suite_definition and
tool_dependency_definition.
"""
encoded_repository_id = trans.security.encode_id(repository.id)
repo = repository.hg_repo
tip_ctx = str(repo[repo.changelog.tip()])
try:
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
trans.app, encoded_repository_id, tip_ctx)
return repository_metadata
except Exception:
latest_downloadable_revision = metadata_util.get_previous_metadata_changeset_revision(
trans.app, repository, tip_ctx, downloadable=False)
if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH:
return None
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
trans.app, encoded_repository_id, latest_downloadable_revision)
return repository_metadata
def manage_repository_reviews( self, trans, mine=False, **kwd ):
# The value of the received id is the encoded repository id.
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
if repository_id:
repository = suc.get_repository_in_tool_shed( trans.app, repository_id )
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
metadata_revision_hashes = [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ]
reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ]
reviews_dict = odict()
for changeset in hg_util.get_reversed_changelog_changesets( repo ):
ctx = repo.changectx( changeset )
changeset_revision = str( ctx )
if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes:
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision )
if changeset_revision in reviewed_revision_hashes:
# Find the review for this changeset_revision
repository_reviews = \
review_util.get_reviews_by_repository_id_changeset_revision( trans.app,
repository_id,
changeset_revision )
# Determine if the current user can add a review to this revision.
can_add_review = trans.user not in [ repository_review.user for repository_review in repository_reviews ]
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision )
if repository_metadata:
repository_metadata_reviews = util.listify( repository_metadata.reviews )
else:
repository_metadata_reviews = []
else:
repository_reviews = []
repository_metadata_reviews = []
can_add_review = True
installable = changeset_revision in metadata_revision_hashes
revision_dict = dict( changeset_revision_label=changeset_revision_label,
repository_reviews=repository_reviews,
repository_metadata_reviews=repository_metadata_reviews,
installable=installable,
can_add_review=can_add_review )
reviews_dict[ changeset_revision ] = revision_dict
return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_repository.mako',
repository=repository,
reviews_dict=reviews_dict,
mine=mine,
message=message,
status=status )
def load_viewable_repositories_and_suites_by_category(self):
# Clear all dictionaries just in case they were previously loaded.
self.certified_level_one_viewable_repositories_and_suites_by_category = {}
self.certified_level_one_viewable_suites_by_category = {}
self.certified_level_two_viewable_repositories_and_suites_by_category = {}
self.certified_level_two_viewable_suites_by_category = {}
self.viewable_repositories_and_suites_by_category = {}
self.viewable_suites_by_category = {}
self.viewable_valid_repositories_and_suites_by_category = {}
self.viewable_valid_suites_by_category = {}
for category in self.sa_session.query(model.Category):
category_name = str(category.name)
if category not in self.certified_level_one_viewable_repositories_and_suites_by_category:
self.certified_level_one_viewable_repositories_and_suites_by_category[category_name] = 0
if category not in self.certified_level_one_viewable_suites_by_category:
self.certified_level_one_viewable_suites_by_category[category_name] = 0
if category not in self.viewable_repositories_and_suites_by_category:
self.viewable_repositories_and_suites_by_category[category_name] = 0
if category not in self.viewable_suites_by_category:
self.viewable_suites_by_category[category_name] = 0
if category not in self.viewable_valid_repositories_and_suites_by_category:
self.viewable_valid_repositories_and_suites_by_category[category_name] = 0
if category not in self.viewable_valid_suites_by_category:
self.viewable_valid_suites_by_category[category_name] = 0
for rca in category.repositories:
repository = rca.repository
if not repository.deleted and not repository.deprecated:
is_valid = self.is_valid(repository)
encoded_repository_id = self.app.security.encode_id(repository.id)
tip_changeset_hash = repository.tip(self.app)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(self.app,
encoded_repository_id,
tip_changeset_hash)
self.viewable_repositories_and_suites_by_category[category_name] += 1
if is_valid:
self.viewable_valid_repositories_and_suites_by_category[category_name] += 1
if repository.type in [rt_util.REPOSITORY_SUITE_DEFINITION]:
self.viewable_suites_by_category[category_name] += 1
if is_valid:
self.viewable_valid_suites_by_category[category_name] += 1
if self.is_level_one_certified(repository_metadata):
self.certified_level_one_viewable_repositories_and_suites_by_category[category_name] += 1
if repository.type in [rt_util.REPOSITORY_SUITE_DEFINITION]:
self.certified_level_one_viewable_suites_by_category[category_name] += 1
def get_certified_level_one_tuple(self, repository):
"""
Return True if the latest installable changeset_revision of the received repository is level one certified.
"""
if repository is None:
return (None, False)
if repository.deleted or repository.deprecated:
return (None, False)
# Get the latest installable changeset revision since that is all that is currently configured for testing.
latest_installable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision(self.app, repository)
if latest_installable_changeset_revision not in [None, hg_util.INITIAL_CHANGELOG_HASH]:
encoded_repository_id = self.app.security.encode_id(repository.id)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(self.app,
encoded_repository_id,
latest_installable_changeset_revision)
if repository_metadata:
# No repository_metadata.
return (latest_installable_changeset_revision, True)
else:
# No installable changeset_revision.
return (None, False)
def show_tools(self, trans, id, changeset, **kwd):
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(self.app, id, changeset)
if repository_metadata is not None:
encoded_repository_metadata_id = trans.security.encode_id(repository_metadata.id)
repository_metadata_dict = repository_metadata.to_dict(
view="collection", value_mapper=self.__get_value_mapper(trans)
)
repository_metadata_dict["url"] = web.url_for(
controller="repository_revisions", action="show", id=encoded_repository_metadata_id
)
if "tools" in repository_metadata.metadata:
repository_metadata_dict["valid_tools"] = repository_metadata.metadata["tools"]
# Get the repo_info_dict for installing the repository.
repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = repository_util.get_repo_info_dict(
self.app, trans.user, id, changeset
)
return repository_metadata_dict
else:
log.debug(
"Unable to locate repository_metadata record for repository id %s and changeset_revision %s"
% (str(id), str(changeset))
)
return {}
def create_review( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
changeset_revision = kwd.get( 'changeset_revision', None )
previous_review_id = kwd.get( 'previous_review_id', None )
create_without_copying = 'create_without_copying' in kwd
if repository_id:
if changeset_revision:
# Make sure there is not already a review of the revision by the user.
repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id )
if review_util.get_review_by_repository_id_changeset_revision_user_id( app=trans.app,
repository_id=repository_id,
changeset_revision=changeset_revision,
user_id=trans.security.encode_id( trans.user.id ) ):
message = "You have already created a review for revision <b>%s</b> of repository <b>%s</b>." % ( changeset_revision, escape( repository.name ) )
status = "error"
else:
# See if there are any reviews for previous changeset revisions that the user can copy.
if not create_without_copying and \
not previous_review_id and \
review_util.has_previous_repository_reviews( trans.app, repository, changeset_revision ):
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='select_previous_review',
**kwd ) )
# A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated
# with the received changeset_revision.
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision )
if repository_metadata:
metadata = repository_metadata.metadata
if metadata:
review = trans.app.model.RepositoryReview( repository_id=repository_metadata.repository_id,
changeset_revision=changeset_revision,
user_id=trans.user.id,
rating=None,
deleted=False )
trans.sa_session.add( review )
trans.sa_session.flush()
if previous_review_id:
review_to_copy = review_util.get_review( trans.app, previous_review_id )
self.copy_review( trans, review_to_copy, review )
review_id = trans.security.encode_id( review.id )
message = "Begin your review of revision <b>%s</b> of repository <b>%s</b>." \
% ( changeset_revision, repository.name )
status = 'done'
trans.response.send_redirect( web.url_for( controller='repository_review',
action='edit_review',
id=review_id,
message=message,
status=status ) )
else:
message = "A new review cannot be created for revision <b>%s</b> of repository <b>%s</b>. Select a valid revision and try again." \
% ( changeset_revision, escape( repository.name ) )
kwd[ 'message' ] = message
kwd[ 'status' ] = 'error'
else:
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='manage_repository_reviews',
**kwd ) )
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='view_or_manage_repository',
**kwd ) )
def json(self, trans, **kwd):
"""
GET /api/tools/json
Get the tool form JSON for a tool in a repository.
:param guid: the GUID of the tool
:param guid: str
:param tsr_id: the ID of the repository
:param tsr_id: str
:param changeset: the changeset at which to load the tool json
:param changeset: str
"""
guid = kwd.get('guid', None)
tsr_id = kwd.get('tsr_id', None)
changeset = kwd.get('changeset', None)
if None in [changeset, tsr_id, guid]:
message = 'Changeset, repository ID, and tool GUID are all required parameters.'
trans.response.status = 400
return {'status': 'error', 'message': message}
tsucm = ToolShedUtilityContainerManager(trans.app)
repository = repository_util.get_repository_in_tool_shed(
self.app, tsr_id)
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(
repository.user, repository)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
trans.app, tsr_id, changeset)
toolshed_base_url = str(web.url_for('/', qualified=True)).rstrip('/')
rb = relation_builder.RelationBuilder(trans.app, repository,
repository_metadata,
toolshed_base_url)
repository_dependencies = rb.get_repository_dependencies_for_changeset_revision(
)
containers_dict = tsucm.build_repository_containers(
repository, changeset, repository_dependencies,
repository_metadata)
found_tool = None
for folder in containers_dict['valid_tools'].folders:
if hasattr(folder, 'valid_tools'):
for tool in folder.valid_tools:
tool.id = tool.tool_id
tool_guid = suc.generate_tool_guid(repository_clone_url,
tool)
if tool_guid == guid:
found_tool = tool
break
if found_tool is None:
message = f'Unable to find tool with guid {guid} in repository {repository.name}.'
trans.response.status = 404
return {'status': 'error', 'message': message}
with ValidationContext.from_app(trans.app) as validation_context:
tv = tool_validator.ToolValidator(validation_context)
repository, tool, valid, message = tv.load_tool_from_changeset_revision(
tsr_id, changeset, found_tool.tool_config)
if message or not valid:
status = 'error'
return dict(message=message, status=status)
tool_help = ''
if tool.help:
tool_help = tool.help.render(static_path=web.url_for('/static'),
host_url=web.url_for('/',
qualified=True))
tool_help = util.unicodify(tool_help, 'utf-8')
tool_dict = tool.to_dict(trans)
tool_dict['inputs'] = {}
tool.populate_model(trans, tool.inputs, {}, tool_dict['inputs'])
tool_dict.update({
'help':
tool_help,
'citations':
bool(tool.citations),
'requirements': [{
'name': r.name,
'version': r.version
} for r in tool.requirements],
'state_inputs':
params_to_strings(tool.inputs, {}, trans.app),
'display':
tool.display_interface,
'action':
web.url_for(tool.action),
'method':
tool.method,
'enctype':
tool.enctype
})
return json.dumps(tool_dict)
def create_review(self, trans, **kwd):
# The value of the received id is the encoded repository id.
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
repository_id = kwd.get('id', None)
changeset_revision = kwd.get('changeset_revision', None)
previous_review_id = kwd.get('previous_review_id', None)
create_without_copying = 'create_without_copying' in kwd
if repository_id:
if changeset_revision:
# Make sure there is not already a review of the revision by the user.
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
if review_util.get_review_by_repository_id_changeset_revision_user_id(app=trans.app,
repository_id=repository_id,
changeset_revision=changeset_revision,
user_id=trans.security.encode_id(trans.user.id)):
message = "You have already created a review for revision <b>%s</b> of repository <b>%s</b>." % (changeset_revision, escape(repository.name))
status = "error"
else:
# See if there are any reviews for previous changeset revisions that the user can copy.
if not create_without_copying and \
not previous_review_id and \
review_util.has_previous_repository_reviews(trans.app, repository, changeset_revision):
return trans.response.send_redirect(web.url_for(controller='repository_review',
action='select_previous_review',
**kwd))
# A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated
# with the received changeset_revision.
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, repository_id, changeset_revision)
if repository_metadata:
metadata = repository_metadata.metadata
if metadata:
review = trans.app.model.RepositoryReview(repository_id=repository_metadata.repository_id,
changeset_revision=changeset_revision,
user_id=trans.user.id,
rating=None,
deleted=False)
trans.sa_session.add(review)
trans.sa_session.flush()
if previous_review_id:
review_to_copy = review_util.get_review(trans.app, previous_review_id)
self.copy_review(trans, review_to_copy, review)
review_id = trans.security.encode_id(review.id)
message = "Begin your review of revision <b>%s</b> of repository <b>%s</b>." \
% (changeset_revision, repository.name)
status = 'done'
trans.response.send_redirect(web.url_for(controller='repository_review',
action='edit_review',
id=review_id,
message=message,
status=status))
else:
message = "A new review cannot be created for revision <b>%s</b> of repository <b>%s</b>. Select a valid revision and try again." \
% (changeset_revision, escape(repository.name))
kwd['message'] = message
kwd['status'] = 'error'
else:
return trans.response.send_redirect(web.url_for(controller='repository_review',
action='manage_repository_reviews',
**kwd))
return trans.response.send_redirect(web.url_for(controller='repository_review',
action='view_or_manage_repository',
**kwd))
def repository_dependencies(self, trans, id, **kwd):
"""
GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies
Returns a list of dictionaries that each define a specific downloadable revision of a
repository in the Tool Shed. This method returns dictionaries with more information in
them than other methods in this controller. The information about repository_metdata is
enhanced to include information about the repository (e.g., name, owner, etc) associated
with the repository_metadata record.
:param id: the encoded id of the `RepositoryMetadata` object
"""
# Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb125606ff9ea620
repository_dependencies_dicts = []
repository_metadata = metadata_util.get_repository_metadata_by_id(
trans.app, id)
if repository_metadata is None:
log.debug('Invalid repository_metadata id received: %s' % str(id))
return repository_dependencies_dicts
metadata = repository_metadata.metadata
if metadata is None:
log.debug(
'The repository_metadata record with id %s has no metadata.' %
str(id))
return repository_dependencies_dicts
if 'repository_dependencies' in metadata:
rd_tups = metadata['repository_dependencies'][
'repository_dependencies']
for rd_tup in rd_tups:
tool_shed, name, owner, changeset_revision = rd_tup[0:4]
repository_dependency = repository_util.get_repository_by_name_and_owner(
trans.app, name, owner)
if repository_dependency is None:
log.dbug(
'Cannot locate repository dependency %s owned by %s.' %
(name, owner))
continue
repository_dependency_id = trans.security.encode_id(
repository_dependency.id)
repository_dependency_repository_metadata = \
metadata_util.get_repository_metadata_by_changeset_revision(trans.app, repository_dependency_id, changeset_revision)
if repository_dependency_repository_metadata is None:
# The changeset_revision column in the repository_metadata table has been updated with a new
# value value, so find the changeset_revision to which we need to update.
new_changeset_revision = metadata_util.get_next_downloadable_changeset_revision(
trans.app, repository_dependency, changeset_revision)
if new_changeset_revision != changeset_revision:
repository_dependency_repository_metadata = \
metadata_util.get_repository_metadata_by_changeset_revision(trans.app,
repository_dependency_id,
new_changeset_revision)
changeset_revision = new_changeset_revision
else:
decoded_repository_dependency_id = trans.security.decode_id(
repository_dependency_id)
debug_msg = 'Cannot locate repository_metadata with id %d for repository dependency %s owned by %s ' % \
(decoded_repository_dependency_id, str(name), str(owner))
debug_msg += 'using either of these changeset_revisions: %s, %s.' % \
(str(changeset_revision), str(new_changeset_revision))
log.debug(debug_msg)
continue
repository_dependency_metadata_dict = \
repository_dependency_repository_metadata.to_dict(view='element',
value_mapper=self.__get_value_mapper(trans))
repository_dependency_dict = repository_dependency.to_dict(
view='element',
value_mapper=self.__get_value_mapper(trans))
# We need to be careful with the entries in our repository_dependency_dict here since this Tool Shed API
# controller is working with repository_metadata records. The above to_dict() method returns a dictionary
# with an id entry for the repository record. However, all of the other methods in this controller have
# the id entry associated with a repository_metadata record id. To avoid confusion, we'll update the
# repository_dependency_metadata_dict with entries from the repository_dependency_dict without using the
# Python dictionary update() method because we do not want to overwrite existing entries.
for k, v in repository_dependency_dict.items():
if k not in repository_dependency_metadata_dict:
repository_dependency_metadata_dict[k] = v
repository_dependency_metadata_dict['url'] = web.url_for(
controller='repositories',
action='show',
id=repository_dependency_id)
repository_dependencies_dicts.append(
repository_dependency_metadata_dict)
return repository_dependencies_dicts
def json(self, trans, **kwd):
"""
GET /api/tools/json
Get the tool form JSON for a tool in a repository.
:param guid: the GUID of the tool
:param guid: str
:param tsr_id: the ID of the repository
:param tsr_id: str
:param changeset: the changeset at which to load the tool json
:param changeset: str
"""
guid = kwd.get('guid', None)
tsr_id = kwd.get('tsr_id', None)
changeset = kwd.get('changeset', None)
if None in [changeset, tsr_id, guid]:
message = 'Changeset, repository ID, and tool GUID are all required parameters.'
trans.response.status = 400
return {'status': 'error', 'message': message}
tsucm = ToolShedUtilityContainerManager(trans.app)
repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id)
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(repository.user, repository)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, tsr_id, changeset)
toolshed_base_url = str(web.url_for('/', qualified=True)).rstrip('/')
rb = relation_builder.RelationBuilder(trans.app, repository, repository_metadata, toolshed_base_url)
repository_dependencies = rb.get_repository_dependencies_for_changeset_revision()
containers_dict = tsucm.build_repository_containers(repository,
changeset,
repository_dependencies,
repository_metadata)
found_tool = None
for folder in containers_dict['valid_tools'].folders:
if hasattr(folder, 'valid_tools'):
for tool in folder.valid_tools:
tool.id = tool.tool_id
tool_guid = suc.generate_tool_guid(repository_clone_url, tool)
if tool_guid == guid:
found_tool = tool
break
if found_tool is None:
message = 'Unable to find tool with guid %s in repository %s.' % (guid, repository.name)
trans.response.status = 404
return {'status': 'error', 'message': message}
tv = tool_validator.ToolValidator(trans.app)
repository, tool, message = tv.load_tool_from_changeset_revision(tsr_id,
changeset,
found_tool.tool_config)
if message:
status = 'error'
return dict(message=message, status=status)
tool_help = ''
if tool.help:
tool_help = tool.help.render(static_path=web.url_for('/static'), host_url=web.url_for('/', qualified=True))
tool_help = util.unicodify(tool_help, 'utf-8')
tool_dict = tool.to_dict(trans)
tool_dict['inputs'] = {}
tool.populate_model(trans, tool.inputs, {}, tool_dict['inputs'])
tool_dict.update({
'help' : tool_help,
'citations' : bool(tool.citations),
'biostar_url' : trans.app.config.biostar_url,
'requirements' : [{'name' : r.name, 'version' : r.version} for r in tool.requirements],
'state_inputs' : params_to_strings(tool.inputs, {}, trans.app),
'display' : tool.display_interface,
'action' : web.url_for(tool.action),
'method' : tool.method,
'enctype' : tool.enctype
})
return json.dumps(tool_dict)
def create_repo_info_dict(app,
repository_clone_url,
changeset_revision,
ctx_rev,
repository_owner,
repository_name=None,
repository=None,
repository_metadata=None,
tool_dependencies=None,
repository_dependencies=None):
"""
Return a dictionary that includes all of the information needed to install a repository into a local
Galaxy instance. The dictionary will also contain the recursive list of repository dependencies defined
for the repository, as well as the defined tool dependencies.
This method is called from Galaxy under four scenarios:
1. During the tool shed repository installation process via the tool shed's get_repository_information()
method. In this case both the received repository and repository_metadata will be objects, but
tool_dependencies and repository_dependencies will be None.
2. When getting updates for an installed repository where the updates include newly defined repository
dependency definitions. This scenario is similar to 1. above. The tool shed's get_repository_information()
method is the caller, and both the received repository and repository_metadata will be objects, but
tool_dependencies and repository_dependencies will be None.
3. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with no
updates available. In this case, both repository and repository_metadata will be None, but tool_dependencies
and repository_dependencies will be objects previously retrieved from the tool shed if the repository includes
definitions for them.
4. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with updates
available. In this case, this method is reached via the tool shed's get_updated_repository_information()
method, and both repository and repository_metadata will be objects but tool_dependencies and
repository_dependencies will be None.
"""
repo_info_dict = {}
repository = get_repository_by_name_and_owner(app, repository_name,
repository_owner)
if app.name == 'tool_shed':
# We're in the tool shed.
repository_metadata = get_repository_metadata_by_changeset_revision(
app, app.security.encode_id(repository.id), changeset_revision)
if repository_metadata:
metadata = repository_metadata.metadata
if metadata:
tool_shed_url = web.url_for('/', qualified=True).rstrip('/')
rb = tool_shed.dependencies.repository.relation_builder.RelationBuilder(
app, repository, repository_metadata, tool_shed_url)
# Get a dictionary of all repositories upon which the contents of the received repository depends.
repository_dependencies = rb.get_repository_dependencies_for_changeset_revision(
)
tool_dependencies = metadata.get('tool_dependencies', {})
if tool_dependencies:
new_tool_dependencies = {}
for dependency_key, requirements_dict in tool_dependencies.items():
if dependency_key in ['set_environment']:
new_set_environment_dict_list = []
for set_environment_dict in requirements_dict:
set_environment_dict['repository_name'] = repository_name
set_environment_dict['repository_owner'] = repository_owner
set_environment_dict[
'changeset_revision'] = changeset_revision
new_set_environment_dict_list.append(set_environment_dict)
new_tool_dependencies[
dependency_key] = new_set_environment_dict_list
else:
requirements_dict['repository_name'] = repository_name
requirements_dict['repository_owner'] = repository_owner
requirements_dict['changeset_revision'] = changeset_revision
new_tool_dependencies[dependency_key] = requirements_dict
tool_dependencies = new_tool_dependencies
repo_info_dict[repository.name] = (repository.description,
repository_clone_url,
changeset_revision, ctx_rev,
repository_owner,
repository_dependencies,
tool_dependencies)
return repo_info_dict
def repository_dependencies(self, trans, id, **kwd):
"""
GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies
Returns a list of dictionaries that each define a specific downloadable revision of a
repository in the Tool Shed. This method returns dictionaries with more information in
them than other methods in this controller. The information about repository_metdata is
enhanced to include information about the repository (e.g., name, owner, etc) associated
with the repository_metadata record.
:param id: the encoded id of the `RepositoryMetadata` object
"""
# Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb125606ff9ea620
repository_dependencies_dicts = []
repository_metadata = metadata_util.get_repository_metadata_by_id(trans.app, id)
if repository_metadata is None:
log.debug('Invalid repository_metadata id received: %s' % str(id))
return repository_dependencies_dicts
metadata = repository_metadata.metadata
if metadata is None:
log.debug('The repository_metadata record with id %s has no metadata.' % str(id))
return repository_dependencies_dicts
if 'repository_dependencies' in metadata:
rd_tups = metadata['repository_dependencies']['repository_dependencies']
for rd_tup in rd_tups:
tool_shed, name, owner, changeset_revision = rd_tup[0:4]
repository_dependency = repository_util.get_repository_by_name_and_owner(trans.app, name, owner)
if repository_dependency is None:
log.dbug('Cannot locate repository dependency %s owned by %s.' % (name, owner))
continue
repository_dependency_id = trans.security.encode_id(repository_dependency.id)
repository_dependency_repository_metadata = \
metadata_util.get_repository_metadata_by_changeset_revision(trans.app, repository_dependency_id, changeset_revision)
if repository_dependency_repository_metadata is None:
# The changeset_revision column in the repository_metadata table has been updated with a new
# value value, so find the changeset_revision to which we need to update.
repo = hg_util.get_repo_for_repository(trans.app,
repository=repository_dependency,
repo_path=None,
create=False)
new_changeset_revision = metadata_util.get_next_downloadable_changeset_revision(repository_dependency,
repo,
changeset_revision)
if new_changeset_revision != changeset_revision:
repository_dependency_repository_metadata = \
metadata_util.get_repository_metadata_by_changeset_revision(trans.app,
repository_dependency_id,
new_changeset_revision)
changeset_revision = new_changeset_revision
else:
decoded_repository_dependency_id = trans.security.decode_id(repository_dependency_id)
debug_msg = 'Cannot locate repository_metadata with id %d for repository dependency %s owned by %s ' % \
(decoded_repository_dependency_id, str(name), str(owner))
debug_msg += 'using either of these changeset_revisions: %s, %s.' % \
(str(changeset_revision), str(new_changeset_revision))
log.debug(debug_msg)
continue
repository_dependency_metadata_dict = \
repository_dependency_repository_metadata.to_dict(view='element',
value_mapper=self.__get_value_mapper(trans))
repository_dependency_dict = repository_dependency.to_dict(view='element',
value_mapper=self.__get_value_mapper(trans))
# We need to be careful with the entries in our repository_dependency_dict here since this Tool Shed API
# controller is working with repository_metadata records. The above to_dict() method returns a dictionary
# with an id entry for the repository record. However, all of the other methods in this controller have
# the id entry associated with a repository_metadata record id. To avoid confusion, we'll update the
# repository_dependency_metadata_dict with entries from the repository_dependency_dict without using the
# Python dictionary update() method because we do not want to overwrite existing entries.
for k, v in repository_dependency_dict.items():
if k not in repository_dependency_metadata_dict:
repository_dependency_metadata_dict[k] = v
repository_dependency_metadata_dict['url'] = web.url_for(controller='repositories',
action='show',
id=repository_dependency_id)
repository_dependencies_dicts.append(repository_dependency_metadata_dict)
return repository_dependencies_dicts
def get_repository_revision_install_info(self, trans, name, owner, changeset_revision, **kwd):
"""
GET /api/repositories/get_repository_revision_install_info
:param name: the name of the Repository
:param owner: the owner of the Repository
:param changeset_revision: the changeset_revision of the RepositoryMetadata object associated with the Repository
Returns a list of the following dictionaries::
- a dictionary defining the Repository. For example:
{
"deleted": false,
"deprecated": false,
"description": "add_column hello",
"id": "f9cad7b01a472135",
"long_description": "add_column hello",
"name": "add_column",
"owner": "test",
"private": false,
"times_downloaded": 6,
"url": "/api/repositories/f9cad7b01a472135",
"user_id": "f9cad7b01a472135"
}
- a dictionary defining the Repository revision (RepositoryMetadata). For example:
{
"changeset_revision": "3a08cc21466f",
"downloadable": true,
"has_repository_dependencies": false,
"has_repository_dependencies_only_if_compiling_contained_td": false,
"id": "f9cad7b01a472135",
"includes_datatypes": false,
"includes_tool_dependencies": false,
"includes_tools": true,
"includes_tools_for_display_in_tool_panel": true,
"includes_workflows": false,
"malicious": false,
"repository_id": "f9cad7b01a472135",
"url": "/api/repository_revisions/f9cad7b01a472135",
"valid_tools": [{u'add_to_tool_panel': True,
u'description': u'data on any column using simple expressions',
u'guid': u'localhost:9009/repos/enis/sample_repo_1/Filter1/2.2.0',
u'id': u'Filter1',
u'name': u'Filter',
u'requirements': [],
u'tests': [{u'inputs': [[u'input', u'1.bed'], [u'cond', u"c1=='chr22'"]],
u'name': u'Test-1',
u'outputs': [[u'out_file1', u'filter1_test1.bed']],
u'required_files': [u'1.bed', u'filter1_test1.bed']}],
u'tool_config': u'database/community_files/000/repo_1/filtering.xml',
u'tool_type': u'default',
u'version': u'2.2.0',
u'version_string_cmd': None}]
}
- a dictionary including the additional information required to install the repository. For example:
{
"add_column": [
"add_column hello",
"http://test@localhost:9009/repos/test/add_column",
"3a08cc21466f",
"1",
"test",
{},
{}
]
}
"""
# Example URL:
# http://<xyz>/api/repositories/get_repository_revision_install_info?name=<n>&owner=<o>&changeset_revision=<cr>
if name and owner and changeset_revision:
# Get the repository information.
repository = repository_util.get_repository_by_name_and_owner(self.app, name, owner)
if repository is None:
log.debug("Cannot locate repository %s owned by %s" % (str(name), str(owner)))
return {}, {}, {}
encoded_repository_id = trans.security.encode_id(repository.id)
repository_dict = repository.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
repository_dict["url"] = web.url_for(controller="repositories", action="show", id=encoded_repository_id)
# Get the repository_metadata information.
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
self.app, encoded_repository_id, changeset_revision
)
if repository_metadata is None:
# The changeset_revision column in the repository_metadata table has been updated with a new
# value value, so find the changeset_revision to which we need to update.
repo = hg_util.get_repo_for_repository(self.app, repository=repository, repo_path=None, create=False)
new_changeset_revision = metadata_util.get_next_downloadable_changeset_revision(
repository, repo, changeset_revision
)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
self.app, encoded_repository_id, new_changeset_revision
)
changeset_revision = new_changeset_revision
if repository_metadata is not None:
encoded_repository_metadata_id = trans.security.encode_id(repository_metadata.id)
repository_metadata_dict = repository_metadata.to_dict(
view="collection", value_mapper=self.__get_value_mapper(trans)
)
repository_metadata_dict["url"] = web.url_for(
controller="repository_revisions", action="show", id=encoded_repository_metadata_id
)
if "tools" in repository_metadata.metadata:
repository_metadata_dict["valid_tools"] = repository_metadata.metadata["tools"]
# Get the repo_info_dict for installing the repository.
repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = repository_util.get_repo_info_dict(
self.app, trans.user, encoded_repository_id, changeset_revision
)
return repository_dict, repository_metadata_dict, repo_info_dict
else:
log.debug(
"Unable to locate repository_metadata record for repository id %s and changeset_revision %s"
% (str(repository.id), str(changeset_revision))
)
return repository_dict, {}, {}
else:
debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: "
debug_msg += "Invalid name %s or owner %s or changeset_revision %s received." % (
str(name),
str(owner),
str(changeset_revision),
)
log.debug(debug_msg)
return {}, {}, {}
