def manage_repository_reviews_of_revision(self, trans, **kwd):
# The value of the received id is the encoded repository id.
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
repository_id = kwd.get('id', None)
changeset_revision = kwd.get('changeset_revision', None)
repository = repository_util.get_repository_in_tool_shed(
trans.app, repository_id)
repo = hg_util.get_repo_for_repository(trans.app,
repository=repository)
installable = changeset_revision in [
metadata_revision.changeset_revision
for metadata_revision in repository.metadata_revisions
]
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision(
repo, changeset_revision)
reviews = review_util.get_reviews_by_repository_id_changeset_revision(
trans.app, repository_id, changeset_revision)
return trans.fill_template(
'/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako',
repository=repository,
changeset_revision=changeset_revision,
changeset_revision_label=changeset_revision_label,
reviews=reviews,
installable=installable,
message=message,
status=status)
def get_ordered_installable_revisions(self, trans, name=None, owner=None, **kwd):
"""
GET /api/repositories/get_ordered_installable_revisions
:param name: the name of the Repository
:param owner: the owner of the Repository
Returns the ordered list of changeset revision hash strings that are associated with installable revisions.
As in the changelog, the list is ordered oldest to newest.
"""
# Example URL: http://localhost:9009/api/repositories/get_ordered_installable_revisions?name=add_column&owner=test
if name is None:
name = kwd.get("name", None)
if owner is None:
owner = kwd.get("owner", None)
tsr_id = kwd.get("tsr_id", None)
if None not in [name, owner]:
# Get the repository information.
repository = repository_util.get_repository_by_name_and_owner(self.app, name, owner)
if repository is None:
trans.response.status = 404
return {"status": "error", "message": "No repository named %s found with owner %s" % (name, owner)}
elif tsr_id is not None:
repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id)
else:
error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: "
error_message += "invalid parameters received."
log.debug(error_message)
return []
return [revision[1] for revision in repository.installable_revisions(self.app, sort_revisions=True)]
def load_tool_from_changeset_revision(self, repository_id,
changeset_revision,
tool_config_filename):
"""
Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value
of tool_config_filename. The value of changeset_revision is a valid (downloadable)
changeset revision. The tool config will be located in the repository manifest between
the received valid changeset revision and the first changeset revision in the repository,
searching backwards.
"""
original_tool_data_path = self.app.config.tool_data_path
repository = repository_util.get_repository_in_tool_shed(
self.app, repository_id)
repo_files_dir = repository.repo_path(self.app)
repo = hg_util.get_repo_for_repository(self.app,
repository=None,
repo_path=repo_files_dir,
create=False)
message = ''
tool = None
can_use_disk_file = False
tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository(
repo_files_dir, tool_config_filename)
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr")
can_use_disk_file = self.can_use_tool_config_disk_file(
repository, repo, tool_config_filepath, changeset_revision)
if can_use_disk_file:
self.app.config.tool_data_path = work_dir
tool, valid, message, sample_files = \
self.handle_sample_files_and_load_tool_from_disk( repo_files_dir,
repository_id,
tool_config_filepath,
work_dir )
if tool is not None:
invalid_files_and_errors_tups = \
self.check_tool_input_params( repo_files_dir,
tool_config_filename,
tool,
sample_files )
if invalid_files_and_errors_tups:
message2 = tool_util.generate_message_for_invalid_tools(
self.app,
invalid_files_and_errors_tups,
repository,
metadata_dict=None,
as_html=True,
displaying_invalid_tool=True)
message = self.concat_messages(message, message2)
else:
tool, message, sample_files = \
self.handle_sample_files_and_load_tool_from_tmp_config( repo,
repository_id,
changeset_revision,
tool_config_filename,
work_dir )
basic_util.remove_dir(work_dir)
self.app.config.tool_data_path = original_tool_data_path
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
self.tdtm.reset_tool_data_tables()
return repository, tool, message
def show(self, trans, id, **kwd):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
:param id: the encoded id of the Repository object
:type id: encoded str
:returns: detailed repository information
:rtype: dict
:raises: ObjectNotFound, MalformedId
"""
try:
trans.security.decode_id(id)
except Exception:
raise MalformedId("The given id is invalid.")
repository = repository_util.get_repository_in_tool_shed(self.app, id)
if repository is None:
raise ObjectNotFound("Unable to locate repository for the given id.")
repository_dict = repository.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
# TODO the following property would be better suited in the to_dict method
repository_dict["category_ids"] = [trans.security.encode_id(x.category.id) for x in repository.categories]
return repository_dict
def reset_metadata_on_repository(self, trans, payload, **kwd):
"""
PUT /api/repositories/reset_metadata_on_repository
Resets all metadata on a specified repository in the Tool Shed.
:param key: the API key of the Tool Shed user.
The following parameters must be included in the payload.
:param repository_id: the encoded id of the repository on which metadata is to be reset.
"""
def handle_repository(trans, start_time, repository):
results = dict(start_time=start_time, repository_status=[])
try:
rmm = repository_metadata_manager.RepositoryMetadataManager(
app=self.app,
user=trans.user,
repository=repository,
resetting_all_metadata_on_repository=True,
updating_installed_repository=False,
persist=False,
)
rmm.reset_all_metadata_on_repository_in_tool_shed()
rmm_invalid_file_tups = rmm.get_invalid_file_tups()
if rmm_invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools(
self.app, rmm_invalid_file_tups, repository, None, as_html=False
)
results["status"] = "warning"
else:
message = "Successfully reset metadata on repository %s owned by %s" % (
str(repository.name),
str(repository.user.username),
)
results["status"] = "ok"
except Exception as e:
message = "Error resetting metadata on repository %s owned by %s: %s" % (
str(repository.name),
str(repository.user.username),
str(e),
)
results["status"] = "error"
status = "%s : %s" % (str(repository.name), message)
results["repository_status"].append(status)
return results
repository_id = payload.get("repository_id", None)
if repository_id is not None:
repository = repository_util.get_repository_in_tool_shed(self.app, repository_id)
start_time = strftime("%Y-%m-%d %H:%M:%S")
log.debug("%s...resetting metadata on repository %s" % (start_time, str(repository.name)))
results = handle_repository(trans, start_time, repository)
stop_time = strftime("%Y-%m-%d %H:%M:%S")
results["stop_time"] = stop_time
return results
def view_or_manage_repository( self, trans, **kwd ):
repository = repository_util.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] )
if trans.user_is_admin() or repository.user == trans.user:
return trans.response.send_redirect( web.url_for( controller='repository',
action='manage_repository',
**kwd ) )
else:
return trans.response.send_redirect( web.url_for( controller='repository',
action='view_repository',
**kwd ) )
def view_or_manage_repository(self, trans, **kwd):
repository = repository_util.get_repository_in_tool_shed(trans.app, kwd['id'])
if trans.user_is_admin or repository.user == trans.user:
return trans.response.send_redirect(web.url_for(controller='repository',
action='manage_repository',
**kwd))
else:
return trans.response.send_redirect(web.url_for(controller='repository',
action='view_repository',
**kwd))
def load_tool_from_changeset_revision( self, repository_id, changeset_revision, tool_config_filename ):
"""
Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value
of tool_config_filename. The value of changeset_revision is a valid (downloadable)
changeset revision. The tool config will be located in the repository manifest between
the received valid changeset revision and the first changeset revision in the repository,
searching backwards.
"""
original_tool_data_path = self.app.config.tool_data_path
repository = repository_util.get_repository_in_tool_shed( self.app, repository_id )
repo_files_dir = repository.repo_path( self.app )
repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_files_dir, create=False )
message = ''
tool = None
can_use_disk_file = False
tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename )
work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ltfcr" )
can_use_disk_file = self.can_use_tool_config_disk_file( repository,
repo,
tool_config_filepath,
changeset_revision )
if can_use_disk_file:
self.app.config.tool_data_path = work_dir
tool, valid, message, sample_files = \
self.handle_sample_files_and_load_tool_from_disk( repo_files_dir,
repository_id,
tool_config_filepath,
work_dir )
if tool is not None:
invalid_files_and_errors_tups = \
self.check_tool_input_params( repo_files_dir,
tool_config_filename,
tool,
sample_files )
if invalid_files_and_errors_tups:
message2 = tool_util.generate_message_for_invalid_tools( self.app,
invalid_files_and_errors_tups,
repository,
metadata_dict=None,
as_html=True,
displaying_invalid_tool=True )
message = self.concat_messages( message, message2 )
else:
tool, message, sample_files = \
self.handle_sample_files_and_load_tool_from_tmp_config( repo,
repository_id,
changeset_revision,
tool_config_filename,
work_dir )
basic_util.remove_dir( work_dir )
self.app.config.tool_data_path = original_tool_data_path
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
self.tdtm.reset_tool_data_tables()
return repository, tool, message
def get_repository_from_refresh_on_change(app, **kwd):
# The changeset_revision_select_field in several grids performs a refresh_on_change which sends in request parameters like
# changeset_revison_1, changeset_revision_2, etc. One of the many select fields on the grid performed the refresh_on_change,
# so we loop through all of the received values to see which value is not the repository tip. If we find it, we know the
# refresh_on_change occurred and we have the necessary repository id and change set revision to pass on.
repository_id = None
v = None
for k, v in kwd.items():
changeset_revision_str = 'changeset_revision_'
if k.startswith(changeset_revision_str):
repository_id = app.security.encode_id(int(k.lstrip(changeset_revision_str)))
repository = repository_util.get_repository_in_tool_shed(app, repository_id)
if repository.tip(app) != v:
return v, repository
# This should never be reached - raise an exception?
return v, None
def get_repository_from_refresh_on_change(app, **kwd):
# The changeset_revision_select_field in several grids performs a refresh_on_change which sends in request parameters like
# changeset_revison_1, changeset_revision_2, etc. One of the many select fields on the grid performed the refresh_on_change,
# so we loop through all of the received values to see which value is not the repository tip. If we find it, we know the
# refresh_on_change occurred and we have the necessary repository id and change set revision to pass on.
repository_id = None
v = None
for k, v in kwd.items():
changeset_revision_str = 'changeset_revision_'
if k.startswith(changeset_revision_str):
repository_id = app.security.encode_id(int(k.lstrip(changeset_revision_str)))
repository = repository_util.get_repository_in_tool_shed(app, repository_id)
if repository.tip(app) != v:
return v, repository
# This should never be reached - raise an exception?
return v, None
def undelete_repository(self, trans, **kwd):
message = escape(kwd.get('message', ''))
id = kwd.get('id', None)
if id:
# Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify(id)
count = 0
undeleted_repositories = ""
for repository_id in ids:
repository = repository_util.get_repository_in_tool_shed(
trans.app, repository_id)
if repository:
if repository.deleted:
# Inspect all repository_metadata records to determine those that are installable, and mark
# them accordingly.
for repository_metadata in repository.metadata_revisions:
metadata = repository_metadata.metadata
if metadata:
if metadata_util.is_downloadable(metadata):
repository_metadata.downloadable = True
trans.sa_session.add(repository_metadata)
# Mark the repository admin role as not deleted.
repository_admin_role = repository.admin_role
if repository_admin_role is not None:
repository_admin_role.deleted = False
trans.sa_session.add(repository_admin_role)
repository.deleted = False
trans.sa_session.add(repository)
trans.sa_session.flush()
if not repository.deprecated:
# Update the repository registry.
trans.app.repository_registry.add_entry(repository)
count += 1
undeleted_repositories += " %s" % repository.name
if count:
message = "Undeleted %d %s: %s" % (
count, inflector.cond_plural(
count, "repository"), undeleted_repositories)
else:
message = "No selected repositories were marked deleted, so they could not be undeleted."
else:
message = "No repository ids received for undeleting."
trans.response.send_redirect(
web.url_for(controller='admin',
action='browse_repositories',
message=util.sanitize_text(message),
status='done'))
def metadata(self, trans, id, **kwd):
"""
GET /api/repositories/{encoded_repository_id}/metadata
Returns information about a repository in the Tool Shed.
Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135/metadata
:param id: the encoded id of the Repository object
:returns: A dictionary containing the specified repository's metadata, by changeset,
recursively including dependencies and their metadata.
:not found: Empty dictionary.
"""
try:
trans.security.decode_id(id)
except Exception:
raise MalformedId("The given id is invalid.")
recursive = util.asbool(kwd.get("recursive", "True"))
all_metadata = {}
repository = repository_util.get_repository_in_tool_shed(self.app, id)
for changeset, changehash in repository.installable_revisions(self.app):
metadata = metadata_util.get_current_repository_metadata_for_changeset_revision(
self.app, repository, changehash
)
if metadata is None:
continue
metadata_dict = metadata.to_dict(
value_mapper={"id": self.app.security.encode_id, "repository_id": self.app.security.encode_id}
)
metadata_dict["repository"] = repository.to_dict(value_mapper={"id": self.app.security.encode_id})
if metadata.has_repository_dependencies and recursive:
metadata_dict["repository_dependencies"] = metadata_util.get_all_dependencies(
self.app, metadata, processed_dependency_links=[]
)
else:
metadata_dict["repository_dependencies"] = []
if metadata.includes_tool_dependencies and recursive:
metadata_dict["tool_dependencies"] = repository.get_tool_dependencies(self.app, changehash)
else:
metadata_dict["tool_dependencies"] = {}
if metadata.includes_tools:
metadata_dict["tools"] = metadata.metadata["tools"]
all_metadata["%s:%s" % (int(changeset), changehash)] = metadata_dict
return all_metadata
def get_installable_revisions(self, trans, **kwd):
"""
GET /api/repositories/get_installable_revisions
:param tsr_id: the encoded toolshed ID of the repository
Returns a list of lists of changesets, in the format [ [ 0, fbb391dc803c ], [ 1, 9d9ec4d9c03e ], [ 2, 9b5b20673b89 ], [ 3, e8c99ce51292 ] ].
"""
# Example URL: http://localhost:9009/api/repositories/get_installable_revisions?tsr_id=9d37e53072ff9fa4
tsr_id = kwd.get("tsr_id", None)
if tsr_id is not None:
repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id)
else:
error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: "
error_message += "missing or invalid parameter received."
log.debug(error_message)
return []
return repository.installable_revisions(self.app)
def select_previous_review(self, trans, **kwd):
# The value of the received id is the encoded repository id.
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
repository = repository_util.get_repository_in_tool_shed(trans.app, kwd['id'])
changeset_revision = kwd.get('changeset_revision', None)
repo = hg_util.get_repo_for_repository(trans.app, repository=repository)
previous_reviews_dict = review_util.get_previous_repository_reviews(trans.app,
repository,
changeset_revision)
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision(repo, changeset_revision)
return trans.fill_template('/webapps/tool_shed/repository_review/select_previous_review.mako',
repository=repository,
changeset_revision=changeset_revision,
changeset_revision_label=changeset_revision_label,
previous_reviews_dict=previous_reviews_dict,
message=message,
status=status)
def select_previous_review( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
repository = repository_util.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] )
changeset_revision = kwd.get( 'changeset_revision', None )
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
previous_reviews_dict = review_util.get_previous_repository_reviews( trans.app,
repository,
changeset_revision )
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision )
return trans.fill_template( '/webapps/tool_shed/repository_review/select_previous_review.mako',
repository=repository,
changeset_revision=changeset_revision,
changeset_revision_label=changeset_revision_label,
previous_reviews_dict=previous_reviews_dict,
message=message,
status=status )
def delete_repository(self, trans, **kwd):
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
id = kwd.get('id', None)
if id:
# Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify(id)
count = 0
deleted_repositories = ""
for repository_id in ids:
repository = repository_util.get_repository_in_tool_shed(
trans.app, repository_id)
if repository:
if not repository.deleted:
# Mark all installable repository_metadata records as not installable.
for repository_metadata in repository.downloadable_revisions:
repository_metadata.downloadable = False
trans.sa_session.add(repository_metadata)
# Mark the repository admin role as deleted.
repository_admin_role = repository.admin_role
if repository_admin_role is not None:
repository_admin_role.deleted = True
trans.sa_session.add(repository_admin_role)
repository.deleted = True
trans.sa_session.add(repository)
trans.sa_session.flush()
# Update the repository registry.
trans.app.repository_registry.remove_entry(repository)
count += 1
deleted_repositories += " %s " % repository.name
if count:
message = "Deleted %d %s: %s" % (
count, inflector.cond_plural(
len(ids), "repository"), escape(deleted_repositories))
else:
message = "All selected repositories were already marked deleted."
else:
message = "No repository ids received for deleting."
status = 'error'
trans.response.send_redirect(
web.url_for(controller='admin',
action='browse_repositories',
message=util.sanitize_text(message),
status=status))
def undelete_repository(self, trans, **kwd):
message = escape(kwd.get('message', ''))
id = kwd.get('id', None)
if id:
# Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify(id)
count = 0
undeleted_repositories = ""
for repository_id in ids:
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
if repository:
if repository.deleted:
# Inspect all repository_metadata records to determine those that are installable, and mark
# them accordingly.
for repository_metadata in repository.metadata_revisions:
metadata = repository_metadata.metadata
if metadata:
if metadata_util.is_downloadable(metadata):
repository_metadata.downloadable = True
trans.sa_session.add(repository_metadata)
# Mark the repository admin role as not deleted.
repository_admin_role = repository.admin_role
if repository_admin_role is not None:
repository_admin_role.deleted = False
trans.sa_session.add(repository_admin_role)
repository.deleted = False
trans.sa_session.add(repository)
trans.sa_session.flush()
if not repository.deprecated:
# Update the repository registry.
trans.app.repository_registry.add_entry(repository)
count += 1
undeleted_repositories += " %s" % repository.name
if count:
message = "Undeleted %d %s: %s" % (count, inflector.cond_plural(count, "repository"), undeleted_repositories)
else:
message = "No selected repositories were marked deleted, so they could not be undeleted."
else:
message = "No repository ids received for undeleting."
trans.response.send_redirect(web.url_for(controller='admin',
action='browse_repositories',
message=util.sanitize_text(message),
status='done'))
def manage_repository_reviews_of_revision( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
changeset_revision = kwd.get( 'changeset_revision', None )
repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id )
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ]
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision )
reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans.app,
repository_id,
changeset_revision )
return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako',
repository=repository,
changeset_revision=changeset_revision,
changeset_revision_label=changeset_revision_label,
reviews=reviews,
installable=installable,
message=message,
status=status )
def delete_repository(self, trans, **kwd):
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
id = kwd.get('id', None)
if id:
# Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify(id)
count = 0
deleted_repositories = ""
for repository_id in ids:
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
if repository:
if not repository.deleted:
# Mark all installable repository_metadata records as not installable.
for repository_metadata in repository.downloadable_revisions:
repository_metadata.downloadable = False
trans.sa_session.add(repository_metadata)
# Mark the repository admin role as deleted.
repository_admin_role = repository.admin_role
if repository_admin_role is not None:
repository_admin_role.deleted = True
trans.sa_session.add(repository_admin_role)
repository.deleted = True
trans.sa_session.add(repository)
trans.sa_session.flush()
# Update the repository registry.
trans.app.repository_registry.remove_entry(repository)
count += 1
deleted_repositories += " %s " % repository.name
if count:
message = "Deleted %d %s: %s" % (count, inflector.cond_plural(len(ids), "repository"), escape(deleted_repositories))
else:
message = "All selected repositories were already marked deleted."
else:
message = "No repository ids received for deleting."
status = 'error'
trans.response.send_redirect(web.url_for(controller='admin',
action='browse_repositories',
message=util.sanitize_text(message),
status=status))
def browse_repositories(self, trans, **kwd):
# We add parameters to the keyword dict in this method in order to rename the param
# with an "f-" prefix, simulating filtering by clicking a search link. We have
# to take this approach because the "-" character is illegal in HTTP requests.
if 'operation' in kwd:
operation = kwd['operation'].lower()
if operation == "view_or_manage_repository":
return trans.response.send_redirect(
web.url_for(controller='repository',
action='browse_repositories',
**kwd))
elif operation == "edit_repository":
return trans.response.send_redirect(
web.url_for(controller='repository',
action='edit_repository',
**kwd))
elif operation == "repositories_by_user":
# Eliminate the current filters if any exist.
for k, v in list(kwd.items()):
if k.startswith('f-'):
del kwd[k]
if 'user_id' in kwd:
user = suc.get_user(trans.app, kwd['user_id'])
kwd['f-email'] = user.email
del kwd['user_id']
else:
# The received id is the repository id, so we need to get the id of the user
# that uploaded the repository.
repository_id = kwd.get('id', None)
repository = repository_util.get_repository_in_tool_shed(
trans.app, repository_id)
kwd['f-email'] = repository.user.email
elif operation == "repositories_by_category":
# Eliminate the current filters if any exist.
for k, v in list(kwd.items()):
if k.startswith('f-'):
del kwd[k]
category_id = kwd.get('id', None)
category = suc.get_category(trans.app, category_id)
kwd['f-Category.name'] = category.name
elif operation == "receive email alerts":
if kwd['id']:
kwd['caller'] = 'browse_repositories'
return trans.response.send_redirect(
web.url_for(controller='repository',
action='set_email_alerts',
**kwd))
else:
del kwd['operation']
elif operation == 'delete':
return self.delete_repository(trans, **kwd)
elif operation == "undelete":
return self.undelete_repository(trans, **kwd)
# The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change
# which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One
# of the many select fields on the grid performed the refresh_on_change, so we loop through
# all of the received values to see which value is not the repository tip. If we find it, we
# know the refresh_on_change occurred, and we have the necessary repository id and change set
# revision to pass on.
for k, v in kwd.items():
changeset_revision_str = 'changeset_revision_'
if k.startswith(changeset_revision_str):
repository_id = trans.security.encode_id(
int(k.lstrip(changeset_revision_str)))
repository = repository_util.get_repository_in_tool_shed(
trans.app, repository_id)
if repository.tip() != v:
return trans.response.send_redirect(
web.url_for(controller='repository',
action='browse_repositories',
operation='view_or_manage_repository',
id=trans.security.encode_id(repository.id),
changeset_revision=v))
# Render the list view
return self.repository_grid(trans, **kwd)
def json(self, trans, **kwd):
"""
GET /api/tools/json
Get the tool form JSON for a tool in a repository.
:param guid: the GUID of the tool
:param guid: str
:param tsr_id: the ID of the repository
:param tsr_id: str
:param changeset: the changeset at which to load the tool json
:param changeset: str
"""
guid = kwd.get('guid', None)
tsr_id = kwd.get('tsr_id', None)
changeset = kwd.get('changeset', None)
if None in [changeset, tsr_id, guid]:
message = 'Changeset, repository ID, and tool GUID are all required parameters.'
trans.response.status = 400
return {'status': 'error', 'message': message}
tsucm = ToolShedUtilityContainerManager(trans.app)
repository = repository_util.get_repository_in_tool_shed(
self.app, tsr_id)
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(
repository.user, repository)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
trans.app, tsr_id, changeset)
toolshed_base_url = str(web.url_for('/', qualified=True)).rstrip('/')
rb = relation_builder.RelationBuilder(trans.app, repository,
repository_metadata,
toolshed_base_url)
repository_dependencies = rb.get_repository_dependencies_for_changeset_revision(
)
containers_dict = tsucm.build_repository_containers(
repository, changeset, repository_dependencies,
repository_metadata)
found_tool = None
for folder in containers_dict['valid_tools'].folders:
if hasattr(folder, 'valid_tools'):
for tool in folder.valid_tools:
tool.id = tool.tool_id
tool_guid = suc.generate_tool_guid(repository_clone_url,
tool)
if tool_guid == guid:
found_tool = tool
break
if found_tool is None:
message = f'Unable to find tool with guid {guid} in repository {repository.name}.'
trans.response.status = 404
return {'status': 'error', 'message': message}
with ValidationContext.from_app(trans.app) as validation_context:
tv = tool_validator.ToolValidator(validation_context)
repository, tool, valid, message = tv.load_tool_from_changeset_revision(
tsr_id, changeset, found_tool.tool_config)
if message or not valid:
status = 'error'
return dict(message=message, status=status)
tool_help = ''
if tool.help:
tool_help = tool.help.render(static_path=web.url_for('/static'),
host_url=web.url_for('/',
qualified=True))
tool_help = util.unicodify(tool_help, 'utf-8')
tool_dict = tool.to_dict(trans)
tool_dict['inputs'] = {}
tool.populate_model(trans, tool.inputs, {}, tool_dict['inputs'])
tool_dict.update({
'help':
tool_help,
'citations':
bool(tool.citations),
'requirements': [{
'name': r.name,
'version': r.version
} for r in tool.requirements],
'state_inputs':
params_to_strings(tool.inputs, {}, trans.app),
'display':
tool.display_interface,
'action':
web.url_for(tool.action),
'method':
tool.method,
'enctype':
tool.enctype
})
return json.dumps(tool_dict)
def upload(self, trans, **kwd):
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
commit_message = escape(kwd.get('commit_message', 'Uploaded'))
repository_id = kwd.get('repository_id', '')
repository = repository_util.get_repository_in_tool_shed(
trans.app, repository_id)
repo_dir = repository.repo_path(trans.app)
uncompress_file = util.string_as_bool(
kwd.get('uncompress_file', 'true'))
remove_repo_files_not_in_tar = util.string_as_bool(
kwd.get('remove_repo_files_not_in_tar', 'true'))
uploaded_file = None
upload_point = commit_util.get_upload_point(repository, **kwd)
tip = repository.tip()
file_data = kwd.get('file_data', '')
url = kwd.get('url', '')
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new()
uploaded_directory = None
if kwd.get('upload_button', False):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith('hg'):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[len('hg'):]
cloned_ok, error_message = hg_util.clone_repository(
repo_url, uploaded_directory)
if not cloned_ok:
message = 'Error uploading via mercurial clone: %s' % error_message
status = 'error'
basic_util.remove_dir(uploaded_directory)
uploaded_directory = None
elif url:
valid_url = True
try:
stream = requests.get(url, stream=True)
except Exception as e:
valid_url = False
message = 'Error uploading file via http: %s' % util.unicodify(
e)
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open(uploaded_file_name, 'wb')
for chunk in stream.iter_content(
chunk_size=util.CHUNK_SIZE):
if chunk:
uploaded_file.write(chunk)
uploaded_file.flush()
uploaded_file_filename = url.split('/')[-1]
isempty = os.path.getsize(
os.path.abspath(uploaded_file_name)) == 0
elif file_data not in ('', None):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = os.path.split(file_data.filename)[-1]
isempty = os.path.getsize(
os.path.abspath(uploaded_file_name)) == 0
if uploaded_file or uploaded_directory:
rdah = attribute_handlers.RepositoryDependencyAttributeHandler(
trans.app, unpopulate=False)
tdah = attribute_handlers.ToolDependencyAttributeHandler(
trans.app, unpopulate=False)
stdtm = ShedToolDataTableManager(trans.app)
ok = True
isgzip = False
isbz2 = False
if uploaded_file:
if uncompress_file:
isgzip = checkers.is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = checkers.is_bz2(uploaded_file_name)
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if (isgzip or isbz2) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, 'r:*')
else:
tar = tarfile.open(uploaded_file_name)
istar = True
except tarfile.ReadError:
tar = None
istar = False
else:
# Uploaded directory
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
repository_content_util.upload_tar(
trans,
rdah,
tdah,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert
)
elif uploaded_directory:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_directory(trans,
rdah,
tdah,
repository,
uploaded_directory,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert)
else:
if (isgzip or isbz2) and uncompress_file:
uploaded_file_filename = commit_util.uncompress(
repository,
uploaded_file_name,
uploaded_file_filename,
isgzip=isgzip,
isbz2=isbz2)
if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \
uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named '
message += '<b>repository_dependencies.xml</b>.'
elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \
uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named '
message += '<b>tool_dependencies.xml</b>.'
if ok:
if upload_point is not None:
full_path = os.path.abspath(
os.path.join(repo_dir, upload_point,
uploaded_file_filename))
else:
full_path = os.path.abspath(
os.path.join(repo_dir, uploaded_file_filename))
# Move some version of the uploaded file to the load_point within the repository hierarchy.
if uploaded_file_filename in [
rt_util.
REPOSITORY_DEPENDENCY_DEFINITION_FILENAME
]:
# Inspect the contents of the file to see if toolshed or changeset_revision attributes
# are missing and if so, set them appropriately.
altered, root_elem, error_message = rdah.handle_tag_attributes(
uploaded_file_name)
if error_message:
ok = False
message = error_message
status = 'error'
elif altered:
tmp_filename = xml_util.create_and_write_tmp_file(
root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
elif uploaded_file_filename in [
rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME
]:
# Inspect the contents of the file to see if changeset_revision values are
# missing and if so, set them appropriately.
altered, root_elem, error_message = tdah.handle_tag_attributes(
uploaded_file_name)
if error_message:
ok = False
message = error_message
status = 'error'
if ok:
if altered:
tmp_filename = xml_util.create_and_write_tmp_file(
root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
else:
shutil.move(uploaded_file_name, full_path)
if ok:
# See if any admin users have chosen to receive email alerts when a repository is updated.
# If so, check every uploaded file to ensure content is appropriate.
check_contents = commit_util.check_file_contents_for_email_alerts(
trans.app)
if check_contents and os.path.isfile(full_path):
content_alert_str = commit_util.check_file_content_for_html_and_images(
full_path)
else:
content_alert_str = ''
hg_util.add_changeset(repo_dir, full_path)
hg_util.commit_changeset(
repo_dir,
full_path_to_changeset=full_path,
username=trans.user.username,
message=commit_message)
if full_path.endswith(
'tool_data_table_conf.xml.sample'):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
# by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables
# dictionary.
error, error_message = stdtm.handle_sample_tool_data_table_conf_file(
full_path, persist=False)
if error:
message = '%s<br/>%s' % (message,
error_message)
# See if the content of the change set was valid.
admin_only = len(
repository.downloadable_revisions) != 1
suc.handle_email_alerts(
trans.app,
trans.request.host,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only)
if ok:
# Update the repository files for browsing.
hg_util.update_repository(repo_dir)
# Get the new repository tip.
if tip == repository.tip():
message = 'No changes to repository. '
status = 'warning'
else:
if (isgzip or isbz2) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
if uploaded_directory:
source_type = "repository"
source = url
else:
source_type = "file"
source = uploaded_file_filename
message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \
(source_type, escape(source), uncompress_str)
if istar and (undesirable_dirs_removed
or undesirable_files_removed):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed
message += "were removed from the archive. "
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \
(len(files_to_remove), upload_point)
else:
message += " %d files were removed from the repository root. " % len(
files_to_remove)
rmm = repository_metadata_manager.RepositoryMetadataManager(
app=trans.app,
user=trans.user,
repository=repository)
status, error_message = \
rmm.set_repository_metadata_due_to_new_tip(trans.request.host,
content_alert_str=content_alert_str,
**kwd)
if error_message:
message = error_message
kwd['message'] = message
if repository.metadata_revisions:
# A repository's metadata revisions are order descending by update_time, so the zeroth revision
# will be the tip just after an upload.
metadata_dict = repository.metadata_revisions[
0].metadata
else:
metadata_dict = {}
dd = dependency_display.DependencyDisplayer(trans.app)
if str(repository.type) not in [
rt_util.REPOSITORY_SUITE_DEFINITION,
rt_util.TOOL_DEPENDENCY_DEFINITION
]:
change_repository_type_message = rt_util.generate_message_for_repository_type_change(
trans.app, repository)
if change_repository_type_message:
message += change_repository_type_message
status = 'warning'
else:
# Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies
# weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency
# definitions can define orphan tool dependencies (no relationship to any tools contained in the
# repository), so warning messages are important because orphans are always valid. The repository
# owner must be warned in case they did not intend to define an orphan dependency, but simply
# provided incorrect information (tool shed, name owner, changeset_revision) for the definition.
orphan_message = dd.generate_message_for_orphan_tool_dependencies(
repository, metadata_dict)
if orphan_message:
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies(
metadata_dict)
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
# Handle messaging for invalid repository dependencies.
invalid_repository_dependencies_message = \
dd.generate_message_for_invalid_repository_dependencies(metadata_dict,
error_from_tuple=True)
if invalid_repository_dependencies_message:
message += invalid_repository_dependencies_message
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
stdtm.reset_tool_data_tables()
if uploaded_directory:
basic_util.remove_dir(uploaded_directory)
trans.response.send_redirect(
web.url_for(controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
message=message,
status=status))
else:
if uploaded_directory:
basic_util.remove_dir(uploaded_directory)
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
stdtm.reset_tool_data_tables()
return trans.fill_template(
'/webapps/tool_shed/repository/upload.mako',
repository=repository,
changeset_revision=tip,
url=url,
commit_message=commit_message,
uncompress_file=uncompress_file,
remove_repo_files_not_in_tar=remove_repo_files_not_in_tar,
message=message,
status=status)
def json(self, trans, **kwd):
"""
GET /api/tools/json
Get the tool form JSON for a tool in a repository.
:param guid: the GUID of the tool
:param guid: str
:param tsr_id: the ID of the repository
:param tsr_id: str
:param changeset: the changeset at which to load the tool json
:param changeset: str
"""
guid = kwd.get('guid', None)
tsr_id = kwd.get('tsr_id', None)
changeset = kwd.get('changeset', None)
if None in [changeset, tsr_id, guid]:
message = 'Changeset, repository ID, and tool GUID are all required parameters.'
trans.response.status = 400
return {'status': 'error', 'message': message}
tsucm = ToolShedUtilityContainerManager(trans.app)
repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id)
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(repository.user, repository)
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, tsr_id, changeset)
toolshed_base_url = str(web.url_for('/', qualified=True)).rstrip('/')
rb = relation_builder.RelationBuilder(trans.app, repository, repository_metadata, toolshed_base_url)
repository_dependencies = rb.get_repository_dependencies_for_changeset_revision()
containers_dict = tsucm.build_repository_containers(repository,
changeset,
repository_dependencies,
repository_metadata)
found_tool = None
for folder in containers_dict['valid_tools'].folders:
if hasattr(folder, 'valid_tools'):
for tool in folder.valid_tools:
tool.id = tool.tool_id
tool_guid = suc.generate_tool_guid(repository_clone_url, tool)
if tool_guid == guid:
found_tool = tool
break
if found_tool is None:
message = 'Unable to find tool with guid %s in repository %s.' % (guid, repository.name)
trans.response.status = 404
return {'status': 'error', 'message': message}
tv = tool_validator.ToolValidator(trans.app)
repository, tool, message = tv.load_tool_from_changeset_revision(tsr_id,
changeset,
found_tool.tool_config)
if message:
status = 'error'
return dict(message=message, status=status)
tool_help = ''
if tool.help:
tool_help = tool.help.render(static_path=web.url_for('/static'), host_url=web.url_for('/', qualified=True))
tool_help = util.unicodify(tool_help, 'utf-8')
tool_dict = tool.to_dict(trans)
tool_dict['inputs'] = {}
tool.populate_model(trans, tool.inputs, {}, tool_dict['inputs'])
tool_dict.update({
'help' : tool_help,
'citations' : bool(tool.citations),
'biostar_url' : trans.app.config.biostar_url,
'requirements' : [{'name' : r.name, 'version' : r.version} for r in tool.requirements],
'state_inputs' : params_to_strings(tool.inputs, {}, trans.app),
'display' : tool.display_interface,
'action' : web.url_for(tool.action),
'method' : tool.method,
'enctype' : tool.enctype
})
return json.dumps(tool_dict)
def create_changeset_revision(self, trans, id, payload, **kwd):
"""
POST /api/repositories/{encoded_repository_id}/changeset_revision
Create a new tool shed repository commit - leaving PUT on parent
resource open for updating meta-attributes of the repository (and
Galaxy doesn't allow PUT multipart data anyway
https://trello.com/c/CQwmCeG6).
:param id: the encoded id of the Repository object
The following parameters may be included in the payload.
:param commit_message: hg commit message for update.
"""
# Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
rdah = attribute_handlers.RepositoryDependencyAttributeHandler(self.app, unpopulate=False)
tdah = attribute_handlers.ToolDependencyAttributeHandler(self.app, unpopulate=False)
repository = repository_util.get_repository_in_tool_shed(self.app, id)
if not (
trans.user_is_admin()
or self.app.security_agent.user_can_administer_repository(trans.user, repository)
or self.app.security_agent.can_push(self.app, trans.user, repository)
):
trans.response.status = 400
return {"err_msg": "You do not have permission to update this repository."}
repo_dir = repository.repo_path(self.app)
repo = hg_util.get_repo_for_repository(self.app, repository=None, repo_path=repo_dir, create=False)
upload_point = commit_util.get_upload_point(repository, **kwd)
tip = repository.tip(self.app)
file_data = payload.get("file")
# Code stolen from gx's upload_common.py
if isinstance(file_data, FieldStorage):
assert not isinstance(file_data.file, StringIO.StringIO)
assert file_data.file.name != "<fdopen>"
local_filename = util.mkstemp_ln(file_data.file.name, "upload_file_data_")
file_data.file.close()
file_data = dict(filename=file_data.filename, local_filename=local_filename)
elif type(file_data) == dict and "local_filename" not in file_data:
raise Exception("Uploaded file was encoded in a way not understood.")
commit_message = kwd.get("commit_message", "Uploaded")
uploaded_file = open(file_data["local_filename"], "rb")
uploaded_file_name = file_data["local_filename"]
isgzip = False
isbz2 = False
isgzip = checkers.is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = checkers.is_bz2(uploaded_file_name)
if isgzip or isbz2:
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, "r:*")
else:
tar = tarfile.open(uploaded_file_name)
new_repo_alert = False
remove_repo_files_not_in_tar = True
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = repository_content_util.upload_tar(
trans,
rdah,
tdah,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert,
)
if ok:
# Update the repository files for browsing.
hg_util.update_repository(repo)
# Get the new repository tip.
if tip == repository.tip(self.app):
trans.response.status = 400
message = "No changes to repository."
ok = False
else:
rmm = repository_metadata_manager.RepositoryMetadataManager(
app=self.app, user=trans.user, repository=repository
)
status, error_message = rmm.set_repository_metadata_due_to_new_tip(
trans.request.host, content_alert_str=content_alert_str, **kwd
)
if error_message:
ok = False
trans.response.status = 500
message = error_message
else:
trans.response.status = 500
if os.path.exists(uploaded_file_name):
os.remove(uploaded_file_name)
if not ok:
return {"err_msg": message, "content_alert": content_alert_str}
else:
return {"message": message, "content_alert": content_alert_str}
def create_review(self, trans, **kwd):
# The value of the received id is the encoded repository id.
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
repository_id = kwd.get('id', None)
changeset_revision = kwd.get('changeset_revision', None)
previous_review_id = kwd.get('previous_review_id', None)
create_without_copying = 'create_without_copying' in kwd
if repository_id:
if changeset_revision:
# Make sure there is not already a review of the revision by the user.
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
if review_util.get_review_by_repository_id_changeset_revision_user_id(app=trans.app,
repository_id=repository_id,
changeset_revision=changeset_revision,
user_id=trans.security.encode_id(trans.user.id)):
message = "You have already created a review for revision <b>%s</b> of repository <b>%s</b>." % (changeset_revision, escape(repository.name))
status = "error"
else:
# See if there are any reviews for previous changeset revisions that the user can copy.
if not create_without_copying and \
not previous_review_id and \
review_util.has_previous_repository_reviews(trans.app, repository, changeset_revision):
return trans.response.send_redirect(web.url_for(controller='repository_review',
action='select_previous_review',
**kwd))
# A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated
# with the received changeset_revision.
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, repository_id, changeset_revision)
if repository_metadata:
metadata = repository_metadata.metadata
if metadata:
review = trans.app.model.RepositoryReview(repository_id=repository_metadata.repository_id,
changeset_revision=changeset_revision,
user_id=trans.user.id,
rating=None,
deleted=False)
trans.sa_session.add(review)
trans.sa_session.flush()
if previous_review_id:
review_to_copy = review_util.get_review(trans.app, previous_review_id)
self.copy_review(trans, review_to_copy, review)
review_id = trans.security.encode_id(review.id)
message = "Begin your review of revision <b>%s</b> of repository <b>%s</b>." \
% (changeset_revision, repository.name)
status = 'done'
trans.response.send_redirect(web.url_for(controller='repository_review',
action='edit_review',
id=review_id,
message=message,
status=status))
else:
message = "A new review cannot be created for revision <b>%s</b> of repository <b>%s</b>. Select a valid revision and try again." \
% (changeset_revision, escape(repository.name))
kwd['message'] = message
kwd['status'] = 'error'
else:
return trans.response.send_redirect(web.url_for(controller='repository_review',
action='manage_repository_reviews',
**kwd))
return trans.response.send_redirect(web.url_for(controller='repository_review',
action='view_or_manage_repository',
**kwd))
def create_review( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
changeset_revision = kwd.get( 'changeset_revision', None )
previous_review_id = kwd.get( 'previous_review_id', None )
create_without_copying = 'create_without_copying' in kwd
if repository_id:
if changeset_revision:
# Make sure there is not already a review of the revision by the user.
repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id )
if review_util.get_review_by_repository_id_changeset_revision_user_id( app=trans.app,
repository_id=repository_id,
changeset_revision=changeset_revision,
user_id=trans.security.encode_id( trans.user.id ) ):
message = "You have already created a review for revision <b>%s</b> of repository <b>%s</b>." % ( changeset_revision, escape( repository.name ) )
status = "error"
else:
# See if there are any reviews for previous changeset revisions that the user can copy.
if not create_without_copying and \
not previous_review_id and \
review_util.has_previous_repository_reviews( trans.app, repository, changeset_revision ):
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='select_previous_review',
**kwd ) )
# A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated
# with the received changeset_revision.
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision )
if repository_metadata:
metadata = repository_metadata.metadata
if metadata:
review = trans.app.model.RepositoryReview( repository_id=repository_metadata.repository_id,
changeset_revision=changeset_revision,
user_id=trans.user.id,
rating=None,
deleted=False )
trans.sa_session.add( review )
trans.sa_session.flush()
if previous_review_id:
review_to_copy = review_util.get_review( trans.app, previous_review_id )
self.copy_review( trans, review_to_copy, review )
review_id = trans.security.encode_id( review.id )
message = "Begin your review of revision <b>%s</b> of repository <b>%s</b>." \
% ( changeset_revision, repository.name )
status = 'done'
trans.response.send_redirect( web.url_for( controller='repository_review',
action='edit_review',
id=review_id,
message=message,
status=status ) )
else:
message = "A new review cannot be created for revision <b>%s</b> of repository <b>%s</b>. Select a valid revision and try again." \
% ( changeset_revision, escape( repository.name ) )
kwd[ 'message' ] = message
kwd[ 'status' ] = 'error'
else:
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='manage_repository_reviews',
**kwd ) )
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='view_or_manage_repository',
**kwd ) )
def browse_repositories(self, trans, **kwd):
# We add parameters to the keyword dict in this method in order to rename the param
# with an "f-" prefix, simulating filtering by clicking a search link. We have
# to take this approach because the "-" character is illegal in HTTP requests.
if 'operation' in kwd:
operation = kwd['operation'].lower()
if operation == "view_or_manage_repository":
return trans.response.send_redirect(web.url_for(controller='repository',
action='browse_repositories',
**kwd))
elif operation == "edit_repository":
return trans.response.send_redirect(web.url_for(controller='repository',
action='edit_repository',
**kwd))
elif operation == "repositories_by_user":
# Eliminate the current filters if any exist.
for k, v in list(kwd.items()):
if k.startswith('f-'):
del kwd[k]
if 'user_id' in kwd:
user = suc.get_user(trans.app, kwd['user_id'])
kwd['f-email'] = user.email
del kwd['user_id']
else:
# The received id is the repository id, so we need to get the id of the user
# that uploaded the repository.
repository_id = kwd.get('id', None)
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
kwd['f-email'] = repository.user.email
elif operation == "repositories_by_category":
# Eliminate the current filters if any exist.
for k, v in list(kwd.items()):
if k.startswith('f-'):
del kwd[k]
category_id = kwd.get('id', None)
category = suc.get_category(trans.app, category_id)
kwd['f-Category.name'] = category.name
elif operation == "receive email alerts":
if kwd['id']:
kwd['caller'] = 'browse_repositories'
return trans.response.send_redirect(web.url_for(controller='repository',
action='set_email_alerts',
**kwd))
else:
del kwd['operation']
elif operation == 'delete':
return self.delete_repository(trans, **kwd)
elif operation == "undelete":
return self.undelete_repository(trans, **kwd)
# The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change
# which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One
# of the many select fields on the grid performed the refresh_on_change, so we loop through
# all of the received values to see which value is not the repository tip. If we find it, we
# know the refresh_on_change occurred, and we have the necessary repository id and change set
# revision to pass on.
for k, v in kwd.items():
changeset_revision_str = 'changeset_revision_'
if k.startswith(changeset_revision_str):
repository_id = trans.security.encode_id(int(k.lstrip(changeset_revision_str)))
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
if repository.tip(trans.app) != v:
return trans.response.send_redirect(web.url_for(controller='repository',
action='browse_repositories',
operation='view_or_manage_repository',
id=trans.security.encode_id(repository.id),
changeset_revision=v))
# Render the list view
return self.repository_grid(trans, **kwd)
def upload(self, trans, **kwd):
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
commit_message = escape(kwd.get('commit_message', 'Uploaded'))
repository_id = kwd.get('repository_id', '')
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
repo_dir = repository.repo_path(trans.app)
repo = hg_util.get_repo_for_repository(trans.app, repository=None, repo_path=repo_dir, create=False)
uncompress_file = util.string_as_bool(kwd.get('uncompress_file', 'true'))
remove_repo_files_not_in_tar = util.string_as_bool(kwd.get('remove_repo_files_not_in_tar', 'true'))
uploaded_file = None
upload_point = commit_util.get_upload_point(repository, **kwd)
tip = repository.tip(trans.app)
file_data = kwd.get('file_data', '')
url = kwd.get('url', '')
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new(trans.app)
uploaded_directory = None
if kwd.get('upload_button', False):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith('hg'):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[len('hg'):]
repo_url = repo_url.encode('ascii', 'replace')
try:
commands.clone(hg_util.get_configured_ui(), repo_url, uploaded_directory)
except Exception as e:
message = 'Error uploading via mercurial clone: %s' % basic_util.to_html_string(str(e))
status = 'error'
basic_util.remove_dir(uploaded_directory)
uploaded_directory = None
elif url:
valid_url = True
try:
stream = requests.get(url, stream=True)
except Exception as e:
valid_url = False
message = 'Error uploading file via http: %s' % str(e)
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open(uploaded_file_name, 'wb')
for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE):
if chunk:
uploaded_file.write(chunk)
uploaded_file.flush()
uploaded_file_filename = url.split('/')[-1]
isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0
elif file_data not in ('', None):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = os.path.split(file_data.filename)[-1]
isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0
if uploaded_file or uploaded_directory:
rdah = attribute_handlers.RepositoryDependencyAttributeHandler(trans.app, unpopulate=False)
tdah = attribute_handlers.ToolDependencyAttributeHandler(trans.app, unpopulate=False)
stdtm = ShedToolDataTableManager(trans.app)
ok = True
isgzip = False
isbz2 = False
if uploaded_file:
if uncompress_file:
isgzip = checkers.is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = checkers.is_bz2(uploaded_file_name)
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if (isgzip or isbz2) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, 'r:*')
else:
tar = tarfile.open(uploaded_file_name)
istar = True
except tarfile.ReadError as e:
tar = None
istar = False
else:
# Uploaded directory
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
repository_content_util.upload_tar(
trans,
rdah,
tdah,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert
)
elif uploaded_directory:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_directory(trans,
rdah,
tdah,
repository,
uploaded_directory,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert)
else:
if (isgzip or isbz2) and uncompress_file:
uploaded_file_filename = commit_util.uncompress(repository,
uploaded_file_name,
uploaded_file_filename,
isgzip=isgzip,
isbz2=isbz2)
if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \
uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named '
message += '<b>repository_dependencies.xml</b>.'
elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \
uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named '
message += '<b>tool_dependencies.xml</b>.'
if ok:
if upload_point is not None:
full_path = os.path.abspath(os.path.join(repo_dir, upload_point, uploaded_file_filename))
else:
full_path = os.path.abspath(os.path.join(repo_dir, uploaded_file_filename))
# Move some version of the uploaded file to the load_point within the repository hierarchy.
if uploaded_file_filename in [rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME]:
# Inspect the contents of the file to see if toolshed or changeset_revision attributes
# are missing and if so, set them appropriately.
altered, root_elem, error_message = rdah.handle_tag_attributes(uploaded_file_name)
if error_message:
ok = False
message = error_message
status = 'error'
elif altered:
tmp_filename = xml_util.create_and_write_tmp_file(root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
elif uploaded_file_filename in [rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME]:
# Inspect the contents of the file to see if changeset_revision values are
# missing and if so, set them appropriately.
altered, root_elem, error_message = tdah.handle_tag_attributes(uploaded_file_name)
if error_message:
ok = False
message = error_message
status = 'error'
if ok:
if altered:
tmp_filename = xml_util.create_and_write_tmp_file(root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
else:
shutil.move(uploaded_file_name, full_path)
if ok:
# See if any admin users have chosen to receive email alerts when a repository is updated.
# If so, check every uploaded file to ensure content is appropriate.
check_contents = commit_util.check_file_contents_for_email_alerts(trans.app)
if check_contents and os.path.isfile(full_path):
content_alert_str = commit_util.check_file_content_for_html_and_images(full_path)
else:
content_alert_str = ''
hg_util.add_changeset(repo.ui, repo, full_path)
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode('ascii', 'replace')
hg_util.commit_changeset(repo.ui,
repo,
full_path_to_changeset=full_path,
username=trans.user.username,
message=commit_message)
if full_path.endswith('tool_data_table_conf.xml.sample'):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
# by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables
# dictionary.
error, error_message = stdtm.handle_sample_tool_data_table_conf_file(full_path, persist=False)
if error:
message = '%s<br/>%s' % (message, error_message)
# See if the content of the change set was valid.
admin_only = len(repository.downloadable_revisions) != 1
suc.handle_email_alerts(trans.app,
trans.request.host,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only)
if ok:
# Update the repository files for browsing.
hg_util.update_repository(repo)
# Get the new repository tip.
if tip == repository.tip(trans.app):
message = 'No changes to repository. '
status = 'warning'
else:
if (isgzip or isbz2) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
if uploaded_directory:
source_type = "repository"
source = url
else:
source_type = "file"
source = uploaded_file_filename
message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \
(source_type, escape(source), uncompress_str)
if istar and (undesirable_dirs_removed or undesirable_files_removed):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed
message += "were removed from the archive. "
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \
(len(files_to_remove), upload_point)
else:
message += " %d files were removed from the repository root. " % len(files_to_remove)
rmm = repository_metadata_manager.RepositoryMetadataManager(app=trans.app,
user=trans.user,
repository=repository)
status, error_message = \
rmm.set_repository_metadata_due_to_new_tip(trans.request.host,
content_alert_str=content_alert_str,
**kwd)
if error_message:
message = error_message
kwd['message'] = message
if repository.metadata_revisions:
# A repository's metadata revisions are order descending by update_time, so the zeroth revision
# will be the tip just after an upload.
metadata_dict = repository.metadata_revisions[0].metadata
else:
metadata_dict = {}
dd = dependency_display.DependencyDisplayer(trans.app)
if str(repository.type) not in [rt_util.REPOSITORY_SUITE_DEFINITION,
rt_util.TOOL_DEPENDENCY_DEFINITION]:
change_repository_type_message = rt_util.generate_message_for_repository_type_change(trans.app,
repository)
if change_repository_type_message:
message += change_repository_type_message
status = 'warning'
else:
# Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies
# weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency
# definitions can define orphan tool dependencies (no relationship to any tools contained in the
# repository), so warning messages are important because orphans are always valid. The repository
# owner must be warned in case they did not intend to define an orphan dependency, but simply
# provided incorrect information (tool shed, name owner, changeset_revision) for the definition.
orphan_message = dd.generate_message_for_orphan_tool_dependencies(repository, metadata_dict)
if orphan_message:
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies(metadata_dict)
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
# Handle messaging for invalid repository dependencies.
invalid_repository_dependencies_message = \
dd.generate_message_for_invalid_repository_dependencies(metadata_dict,
error_from_tuple=True)
if invalid_repository_dependencies_message:
message += invalid_repository_dependencies_message
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
stdtm.reset_tool_data_tables()
if uploaded_directory:
basic_util.remove_dir(uploaded_directory)
trans.response.send_redirect(web.url_for(controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
message=message,
status=status))
else:
if uploaded_directory:
basic_util.remove_dir(uploaded_directory)
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
stdtm.reset_tool_data_tables()
return trans.fill_template('/webapps/tool_shed/repository/upload.mako',
repository=repository,
changeset_revision=tip,
url=url,
commit_message=commit_message,
uncompress_file=uncompress_file,
remove_repo_files_not_in_tar=remove_repo_files_not_in_tar,
message=message,
status=status)
