def manage_repository_reviews_of_revision(self, trans, **kwd): # The value of the received id is the encoded repository id. message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository_id = kwd.get('id', None) changeset_revision = kwd.get('changeset_revision', None) repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id) repo = hg_util.get_repo_for_repository(trans.app, repository=repository) installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision) reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans.app, repository_id, changeset_revision) return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako', repository=repository, changeset_revision=changeset_revision, changeset_revision_label=changeset_revision_label, reviews=reviews, installable=installable, message=message, status=status)
def get_ordered_installable_revisions(self, trans, name=None, owner=None, **kwd): """ GET /api/repositories/get_ordered_installable_revisions :param name: the name of the Repository :param owner: the owner of the Repository Returns the ordered list of changeset revision hash strings that are associated with installable revisions. As in the changelog, the list is ordered oldest to newest. """ # Example URL: http://localhost:9009/api/repositories/get_ordered_installable_revisions?name=add_column&owner=test if name is None: name = kwd.get("name", None) if owner is None: owner = kwd.get("owner", None) tsr_id = kwd.get("tsr_id", None) if None not in [name, owner]: # Get the repository information. repository = repository_util.get_repository_by_name_and_owner(self.app, name, owner) if repository is None: trans.response.status = 404 return {"status": "error", "message": "No repository named %s found with owner %s" % (name, owner)} elif tsr_id is not None: repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id) else: error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " error_message += "invalid parameters received." log.debug(error_message) return [] return [revision[1] for revision in repository.installable_revisions(self.app, sort_revisions=True)]
def load_tool_from_changeset_revision(self, repository_id, changeset_revision, tool_config_filename): """ Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision is a valid (downloadable) changeset revision. The tool config will be located in the repository manifest between the received valid changeset revision and the first changeset revision in the repository, searching backwards. """ original_tool_data_path = self.app.config.tool_data_path repository = repository_util.get_repository_in_tool_shed( self.app, repository_id) repo_files_dir = repository.repo_path(self.app) repo = hg_util.get_repo_for_repository(self.app, repository=None, repo_path=repo_files_dir, create=False) message = '' tool = None can_use_disk_file = False tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename) work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr") can_use_disk_file = self.can_use_tool_config_disk_file( repository, repo, tool_config_filepath, changeset_revision) if can_use_disk_file: self.app.config.tool_data_path = work_dir tool, valid, message, sample_files = \ self.handle_sample_files_and_load_tool_from_disk( repo_files_dir, repository_id, tool_config_filepath, work_dir ) if tool is not None: invalid_files_and_errors_tups = \ self.check_tool_input_params( repo_files_dir, tool_config_filename, tool, sample_files ) if invalid_files_and_errors_tups: message2 = tool_util.generate_message_for_invalid_tools( self.app, invalid_files_and_errors_tups, repository, metadata_dict=None, as_html=True, displaying_invalid_tool=True) message = self.concat_messages(message, message2) else: tool, message, sample_files = \ self.handle_sample_files_and_load_tool_from_tmp_config( repo, repository_id, changeset_revision, tool_config_filename, work_dir ) basic_util.remove_dir(work_dir) self.app.config.tool_data_path = original_tool_data_path # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.tdtm.reset_tool_data_tables() return repository, tool, message
def show(self, trans, id, **kwd): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 :param id: the encoded id of the Repository object :type id: encoded str :returns: detailed repository information :rtype: dict :raises: ObjectNotFound, MalformedId """ try: trans.security.decode_id(id) except Exception: raise MalformedId("The given id is invalid.") repository = repository_util.get_repository_in_tool_shed(self.app, id) if repository is None: raise ObjectNotFound("Unable to locate repository for the given id.") repository_dict = repository.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) # TODO the following property would be better suited in the to_dict method repository_dict["category_ids"] = [trans.security.encode_id(x.category.id) for x in repository.categories] return repository_dict
def reset_metadata_on_repository(self, trans, payload, **kwd): """ PUT /api/repositories/reset_metadata_on_repository Resets all metadata on a specified repository in the Tool Shed. :param key: the API key of the Tool Shed user. The following parameters must be included in the payload. :param repository_id: the encoded id of the repository on which metadata is to be reset. """ def handle_repository(trans, start_time, repository): results = dict(start_time=start_time, repository_status=[]) try: rmm = repository_metadata_manager.RepositoryMetadataManager( app=self.app, user=trans.user, repository=repository, resetting_all_metadata_on_repository=True, updating_installed_repository=False, persist=False, ) rmm.reset_all_metadata_on_repository_in_tool_shed() rmm_invalid_file_tups = rmm.get_invalid_file_tups() if rmm_invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( self.app, rmm_invalid_file_tups, repository, None, as_html=False ) results["status"] = "warning" else: message = "Successfully reset metadata on repository %s owned by %s" % ( str(repository.name), str(repository.user.username), ) results["status"] = "ok" except Exception as e: message = "Error resetting metadata on repository %s owned by %s: %s" % ( str(repository.name), str(repository.user.username), str(e), ) results["status"] = "error" status = "%s : %s" % (str(repository.name), message) results["repository_status"].append(status) return results repository_id = payload.get("repository_id", None) if repository_id is not None: repository = repository_util.get_repository_in_tool_shed(self.app, repository_id) start_time = strftime("%Y-%m-%d %H:%M:%S") log.debug("%s...resetting metadata on repository %s" % (start_time, str(repository.name))) results = handle_repository(trans, start_time, repository) stop_time = strftime("%Y-%m-%d %H:%M:%S") results["stop_time"] = stop_time return results
def view_or_manage_repository( self, trans, **kwd ): repository = repository_util.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] ) if trans.user_is_admin() or repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', action='manage_repository', **kwd ) ) else: return trans.response.send_redirect( web.url_for( controller='repository', action='view_repository', **kwd ) )
def view_or_manage_repository(self, trans, **kwd): repository = repository_util.get_repository_in_tool_shed(trans.app, kwd['id']) if trans.user_is_admin or repository.user == trans.user: return trans.response.send_redirect(web.url_for(controller='repository', action='manage_repository', **kwd)) else: return trans.response.send_redirect(web.url_for(controller='repository', action='view_repository', **kwd))
def load_tool_from_changeset_revision( self, repository_id, changeset_revision, tool_config_filename ): """ Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision is a valid (downloadable) changeset revision. The tool config will be located in the repository manifest between the received valid changeset revision and the first changeset revision in the repository, searching backwards. """ original_tool_data_path = self.app.config.tool_data_path repository = repository_util.get_repository_in_tool_shed( self.app, repository_id ) repo_files_dir = repository.repo_path( self.app ) repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_files_dir, create=False ) message = '' tool = None can_use_disk_file = False tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename ) work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ltfcr" ) can_use_disk_file = self.can_use_tool_config_disk_file( repository, repo, tool_config_filepath, changeset_revision ) if can_use_disk_file: self.app.config.tool_data_path = work_dir tool, valid, message, sample_files = \ self.handle_sample_files_and_load_tool_from_disk( repo_files_dir, repository_id, tool_config_filepath, work_dir ) if tool is not None: invalid_files_and_errors_tups = \ self.check_tool_input_params( repo_files_dir, tool_config_filename, tool, sample_files ) if invalid_files_and_errors_tups: message2 = tool_util.generate_message_for_invalid_tools( self.app, invalid_files_and_errors_tups, repository, metadata_dict=None, as_html=True, displaying_invalid_tool=True ) message = self.concat_messages( message, message2 ) else: tool, message, sample_files = \ self.handle_sample_files_and_load_tool_from_tmp_config( repo, repository_id, changeset_revision, tool_config_filename, work_dir ) basic_util.remove_dir( work_dir ) self.app.config.tool_data_path = original_tool_data_path # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.tdtm.reset_tool_data_tables() return repository, tool, message
def get_repository_from_refresh_on_change(app, **kwd): # The changeset_revision_select_field in several grids performs a refresh_on_change which sends in request parameters like # changeset_revison_1, changeset_revision_2, etc. One of the many select fields on the grid performed the refresh_on_change, # so we loop through all of the received values to see which value is not the repository tip. If we find it, we know the # refresh_on_change occurred and we have the necessary repository id and change set revision to pass on. repository_id = None v = None for k, v in kwd.items(): changeset_revision_str = 'changeset_revision_' if k.startswith(changeset_revision_str): repository_id = app.security.encode_id(int(k.lstrip(changeset_revision_str))) repository = repository_util.get_repository_in_tool_shed(app, repository_id) if repository.tip(app) != v: return v, repository # This should never be reached - raise an exception? return v, None
def get_repository_from_refresh_on_change(app, **kwd): # The changeset_revision_select_field in several grids performs a refresh_on_change which sends in request parameters like # changeset_revison_1, changeset_revision_2, etc. One of the many select fields on the grid performed the refresh_on_change, # so we loop through all of the received values to see which value is not the repository tip. If we find it, we know the # refresh_on_change occurred and we have the necessary repository id and change set revision to pass on. repository_id = None v = None for k, v in kwd.items(): changeset_revision_str = 'changeset_revision_' if k.startswith(changeset_revision_str): repository_id = app.security.encode_id(int(k.lstrip(changeset_revision_str))) repository = repository_util.get_repository_in_tool_shed(app, repository_id) if repository.tip(app) != v: return v, repository # This should never be reached - raise an exception? return v, None
def undelete_repository(self, trans, **kwd): message = escape(kwd.get('message', '')) id = kwd.get('id', None) if id: # Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify(id) count = 0 undeleted_repositories = "" for repository_id in ids: repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id) if repository: if repository.deleted: # Inspect all repository_metadata records to determine those that are installable, and mark # them accordingly. for repository_metadata in repository.metadata_revisions: metadata = repository_metadata.metadata if metadata: if metadata_util.is_downloadable(metadata): repository_metadata.downloadable = True trans.sa_session.add(repository_metadata) # Mark the repository admin role as not deleted. repository_admin_role = repository.admin_role if repository_admin_role is not None: repository_admin_role.deleted = False trans.sa_session.add(repository_admin_role) repository.deleted = False trans.sa_session.add(repository) trans.sa_session.flush() if not repository.deprecated: # Update the repository registry. trans.app.repository_registry.add_entry(repository) count += 1 undeleted_repositories += " %s" % repository.name if count: message = "Undeleted %d %s: %s" % ( count, inflector.cond_plural( count, "repository"), undeleted_repositories) else: message = "No selected repositories were marked deleted, so they could not be undeleted." else: message = "No repository ids received for undeleting." trans.response.send_redirect( web.url_for(controller='admin', action='browse_repositories', message=util.sanitize_text(message), status='done'))
def metadata(self, trans, id, **kwd): """ GET /api/repositories/{encoded_repository_id}/metadata Returns information about a repository in the Tool Shed. Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135/metadata :param id: the encoded id of the Repository object :returns: A dictionary containing the specified repository's metadata, by changeset, recursively including dependencies and their metadata. :not found: Empty dictionary. """ try: trans.security.decode_id(id) except Exception: raise MalformedId("The given id is invalid.") recursive = util.asbool(kwd.get("recursive", "True")) all_metadata = {} repository = repository_util.get_repository_in_tool_shed(self.app, id) for changeset, changehash in repository.installable_revisions(self.app): metadata = metadata_util.get_current_repository_metadata_for_changeset_revision( self.app, repository, changehash ) if metadata is None: continue metadata_dict = metadata.to_dict( value_mapper={"id": self.app.security.encode_id, "repository_id": self.app.security.encode_id} ) metadata_dict["repository"] = repository.to_dict(value_mapper={"id": self.app.security.encode_id}) if metadata.has_repository_dependencies and recursive: metadata_dict["repository_dependencies"] = metadata_util.get_all_dependencies( self.app, metadata, processed_dependency_links=[] ) else: metadata_dict["repository_dependencies"] = [] if metadata.includes_tool_dependencies and recursive: metadata_dict["tool_dependencies"] = repository.get_tool_dependencies(self.app, changehash) else: metadata_dict["tool_dependencies"] = {} if metadata.includes_tools: metadata_dict["tools"] = metadata.metadata["tools"] all_metadata["%s:%s" % (int(changeset), changehash)] = metadata_dict return all_metadata
def get_installable_revisions(self, trans, **kwd): """ GET /api/repositories/get_installable_revisions :param tsr_id: the encoded toolshed ID of the repository Returns a list of lists of changesets, in the format [ [ 0, fbb391dc803c ], [ 1, 9d9ec4d9c03e ], [ 2, 9b5b20673b89 ], [ 3, e8c99ce51292 ] ]. """ # Example URL: http://localhost:9009/api/repositories/get_installable_revisions?tsr_id=9d37e53072ff9fa4 tsr_id = kwd.get("tsr_id", None) if tsr_id is not None: repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id) else: error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " error_message += "missing or invalid parameter received." log.debug(error_message) return [] return repository.installable_revisions(self.app)
def select_previous_review(self, trans, **kwd): # The value of the received id is the encoded repository id. message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository = repository_util.get_repository_in_tool_shed(trans.app, kwd['id']) changeset_revision = kwd.get('changeset_revision', None) repo = hg_util.get_repo_for_repository(trans.app, repository=repository) previous_reviews_dict = review_util.get_previous_repository_reviews(trans.app, repository, changeset_revision) rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision(repo, changeset_revision) return trans.fill_template('/webapps/tool_shed/repository_review/select_previous_review.mako', repository=repository, changeset_revision=changeset_revision, changeset_revision_label=changeset_revision_label, previous_reviews_dict=previous_reviews_dict, message=message, status=status)
def select_previous_review( self, trans, **kwd ): # The value of the received id is the encoded repository id. message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) repository = repository_util.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] ) changeset_revision = kwd.get( 'changeset_revision', None ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) previous_reviews_dict = review_util.get_previous_repository_reviews( trans.app, repository, changeset_revision ) rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository_review/select_previous_review.mako', repository=repository, changeset_revision=changeset_revision, changeset_revision_label=changeset_revision_label, previous_reviews_dict=previous_reviews_dict, message=message, status=status )
def delete_repository(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') id = kwd.get('id', None) if id: # Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify(id) count = 0 deleted_repositories = "" for repository_id in ids: repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id) if repository: if not repository.deleted: # Mark all installable repository_metadata records as not installable. for repository_metadata in repository.downloadable_revisions: repository_metadata.downloadable = False trans.sa_session.add(repository_metadata) # Mark the repository admin role as deleted. repository_admin_role = repository.admin_role if repository_admin_role is not None: repository_admin_role.deleted = True trans.sa_session.add(repository_admin_role) repository.deleted = True trans.sa_session.add(repository) trans.sa_session.flush() # Update the repository registry. trans.app.repository_registry.remove_entry(repository) count += 1 deleted_repositories += " %s " % repository.name if count: message = "Deleted %d %s: %s" % ( count, inflector.cond_plural( len(ids), "repository"), escape(deleted_repositories)) else: message = "All selected repositories were already marked deleted." else: message = "No repository ids received for deleting." status = 'error' trans.response.send_redirect( web.url_for(controller='admin', action='browse_repositories', message=util.sanitize_text(message), status=status))
def undelete_repository(self, trans, **kwd): message = escape(kwd.get('message', '')) id = kwd.get('id', None) if id: # Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify(id) count = 0 undeleted_repositories = "" for repository_id in ids: repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) if repository: if repository.deleted: # Inspect all repository_metadata records to determine those that are installable, and mark # them accordingly. for repository_metadata in repository.metadata_revisions: metadata = repository_metadata.metadata if metadata: if metadata_util.is_downloadable(metadata): repository_metadata.downloadable = True trans.sa_session.add(repository_metadata) # Mark the repository admin role as not deleted. repository_admin_role = repository.admin_role if repository_admin_role is not None: repository_admin_role.deleted = False trans.sa_session.add(repository_admin_role) repository.deleted = False trans.sa_session.add(repository) trans.sa_session.flush() if not repository.deprecated: # Update the repository registry. trans.app.repository_registry.add_entry(repository) count += 1 undeleted_repositories += " %s" % repository.name if count: message = "Undeleted %d %s: %s" % (count, inflector.cond_plural(count, "repository"), undeleted_repositories) else: message = "No selected repositories were marked deleted, so they could not be undeleted." else: message = "No repository ids received for undeleting." trans.response.send_redirect(web.url_for(controller='admin', action='browse_repositories', message=util.sanitize_text(message), status='done'))
def manage_repository_reviews_of_revision( self, trans, **kwd ): # The value of the received id is the encoded repository id. message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision ) reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans.app, repository_id, changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako', repository=repository, changeset_revision=changeset_revision, changeset_revision_label=changeset_revision_label, reviews=reviews, installable=installable, message=message, status=status )
def delete_repository(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') id = kwd.get('id', None) if id: # Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify(id) count = 0 deleted_repositories = "" for repository_id in ids: repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) if repository: if not repository.deleted: # Mark all installable repository_metadata records as not installable. for repository_metadata in repository.downloadable_revisions: repository_metadata.downloadable = False trans.sa_session.add(repository_metadata) # Mark the repository admin role as deleted. repository_admin_role = repository.admin_role if repository_admin_role is not None: repository_admin_role.deleted = True trans.sa_session.add(repository_admin_role) repository.deleted = True trans.sa_session.add(repository) trans.sa_session.flush() # Update the repository registry. trans.app.repository_registry.remove_entry(repository) count += 1 deleted_repositories += " %s " % repository.name if count: message = "Deleted %d %s: %s" % (count, inflector.cond_plural(len(ids), "repository"), escape(deleted_repositories)) else: message = "All selected repositories were already marked deleted." else: message = "No repository ids received for deleting." status = 'error' trans.response.send_redirect(web.url_for(controller='admin', action='browse_repositories', message=util.sanitize_text(message), status=status))
def browse_repositories(self, trans, **kwd): # We add parameters to the keyword dict in this method in order to rename the param # with an "f-" prefix, simulating filtering by clicking a search link. We have # to take this approach because the "-" character is illegal in HTTP requests. if 'operation' in kwd: operation = kwd['operation'].lower() if operation == "view_or_manage_repository": return trans.response.send_redirect( web.url_for(controller='repository', action='browse_repositories', **kwd)) elif operation == "edit_repository": return trans.response.send_redirect( web.url_for(controller='repository', action='edit_repository', **kwd)) elif operation == "repositories_by_user": # Eliminate the current filters if any exist. for k, v in list(kwd.items()): if k.startswith('f-'): del kwd[k] if 'user_id' in kwd: user = suc.get_user(trans.app, kwd['user_id']) kwd['f-email'] = user.email del kwd['user_id'] else: # The received id is the repository id, so we need to get the id of the user # that uploaded the repository. repository_id = kwd.get('id', None) repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id) kwd['f-email'] = repository.user.email elif operation == "repositories_by_category": # Eliminate the current filters if any exist. for k, v in list(kwd.items()): if k.startswith('f-'): del kwd[k] category_id = kwd.get('id', None) category = suc.get_category(trans.app, category_id) kwd['f-Category.name'] = category.name elif operation == "receive email alerts": if kwd['id']: kwd['caller'] = 'browse_repositories' return trans.response.send_redirect( web.url_for(controller='repository', action='set_email_alerts', **kwd)) else: del kwd['operation'] elif operation == 'delete': return self.delete_repository(trans, **kwd) elif operation == "undelete": return self.undelete_repository(trans, **kwd) # The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One # of the many select fields on the grid performed the refresh_on_change, so we loop through # all of the received values to see which value is not the repository tip. If we find it, we # know the refresh_on_change occurred, and we have the necessary repository id and change set # revision to pass on. for k, v in kwd.items(): changeset_revision_str = 'changeset_revision_' if k.startswith(changeset_revision_str): repository_id = trans.security.encode_id( int(k.lstrip(changeset_revision_str))) repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id) if repository.tip() != v: return trans.response.send_redirect( web.url_for(controller='repository', action='browse_repositories', operation='view_or_manage_repository', id=trans.security.encode_id(repository.id), changeset_revision=v)) # Render the list view return self.repository_grid(trans, **kwd)
def json(self, trans, **kwd): """ GET /api/tools/json Get the tool form JSON for a tool in a repository. :param guid: the GUID of the tool :param guid: str :param tsr_id: the ID of the repository :param tsr_id: str :param changeset: the changeset at which to load the tool json :param changeset: str """ guid = kwd.get('guid', None) tsr_id = kwd.get('tsr_id', None) changeset = kwd.get('changeset', None) if None in [changeset, tsr_id, guid]: message = 'Changeset, repository ID, and tool GUID are all required parameters.' trans.response.status = 400 return {'status': 'error', 'message': message} tsucm = ToolShedUtilityContainerManager(trans.app) repository = repository_util.get_repository_in_tool_shed( self.app, tsr_id) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( repository.user, repository) repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, tsr_id, changeset) toolshed_base_url = str(web.url_for('/', qualified=True)).rstrip('/') rb = relation_builder.RelationBuilder(trans.app, repository, repository_metadata, toolshed_base_url) repository_dependencies = rb.get_repository_dependencies_for_changeset_revision( ) containers_dict = tsucm.build_repository_containers( repository, changeset, repository_dependencies, repository_metadata) found_tool = None for folder in containers_dict['valid_tools'].folders: if hasattr(folder, 'valid_tools'): for tool in folder.valid_tools: tool.id = tool.tool_id tool_guid = suc.generate_tool_guid(repository_clone_url, tool) if tool_guid == guid: found_tool = tool break if found_tool is None: message = f'Unable to find tool with guid {guid} in repository {repository.name}.' trans.response.status = 404 return {'status': 'error', 'message': message} with ValidationContext.from_app(trans.app) as validation_context: tv = tool_validator.ToolValidator(validation_context) repository, tool, valid, message = tv.load_tool_from_changeset_revision( tsr_id, changeset, found_tool.tool_config) if message or not valid: status = 'error' return dict(message=message, status=status) tool_help = '' if tool.help: tool_help = tool.help.render(static_path=web.url_for('/static'), host_url=web.url_for('/', qualified=True)) tool_help = util.unicodify(tool_help, 'utf-8') tool_dict = tool.to_dict(trans) tool_dict['inputs'] = {} tool.populate_model(trans, tool.inputs, {}, tool_dict['inputs']) tool_dict.update({ 'help': tool_help, 'citations': bool(tool.citations), 'requirements': [{ 'name': r.name, 'version': r.version } for r in tool.requirements], 'state_inputs': params_to_strings(tool.inputs, {}, trans.app), 'display': tool.display_interface, 'action': web.url_for(tool.action), 'method': tool.method, 'enctype': tool.enctype }) return json.dumps(tool_dict)
def upload(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') commit_message = escape(kwd.get('commit_message', 'Uploaded')) repository_id = kwd.get('repository_id', '') repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id) repo_dir = repository.repo_path(trans.app) uncompress_file = util.string_as_bool( kwd.get('uncompress_file', 'true')) remove_repo_files_not_in_tar = util.string_as_bool( kwd.get('remove_repo_files_not_in_tar', 'true')) uploaded_file = None upload_point = commit_util.get_upload_point(repository, **kwd) tip = repository.tip() file_data = kwd.get('file_data', '') url = kwd.get('url', '') # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new() uploaded_directory = None if kwd.get('upload_button', False): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None elif url and url.startswith('hg'): # Use mercurial clone to fetch repository, contents will then be copied over. uploaded_directory = tempfile.mkdtemp() repo_url = 'http%s' % url[len('hg'):] cloned_ok, error_message = hg_util.clone_repository( repo_url, uploaded_directory) if not cloned_ok: message = 'Error uploading via mercurial clone: %s' % error_message status = 'error' basic_util.remove_dir(uploaded_directory) uploaded_directory = None elif url: valid_url = True try: stream = requests.get(url, stream=True) except Exception as e: valid_url = False message = 'Error uploading file via http: %s' % util.unicodify( e) status = 'error' uploaded_file = None if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open(uploaded_file_name, 'wb') for chunk in stream.iter_content( chunk_size=util.CHUNK_SIZE): if chunk: uploaded_file.write(chunk) uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize( os.path.abspath(uploaded_file_name)) == 0 elif file_data not in ('', None): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name uploaded_file_filename = os.path.split(file_data.filename)[-1] isempty = os.path.getsize( os.path.abspath(uploaded_file_name)) == 0 if uploaded_file or uploaded_directory: rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False) tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False) stdtm = ShedToolDataTableManager(trans.app) ok = True isgzip = False isbz2 = False if uploaded_file: if uncompress_file: isgzip = checkers.is_gzip(uploaded_file_name) if not isgzip: isbz2 = checkers.is_bz2(uploaded_file_name) if isempty: tar = None istar = False else: # Determine what we have - a single file or an archive try: if (isgzip or isbz2) and uncompress_file: # Open for reading with transparent compression. tar = tarfile.open(uploaded_file_name, 'r:*') else: tar = tarfile.open(uploaded_file_name) istar = True except tarfile.ReadError: tar = None istar = False else: # Uploaded directory istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ repository_content_util.upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_directory(trans, rdah, tdah, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert) else: if (isgzip or isbz2) and uncompress_file: uploaded_file_filename = commit_util.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip=isgzip, isbz2=isbz2) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named ' message += '<b>repository_dependencies.xml</b>.' elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \ uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named ' message += '<b>tool_dependencies.xml</b>.' if ok: if upload_point is not None: full_path = os.path.abspath( os.path.join(repo_dir, upload_point, uploaded_file_filename)) else: full_path = os.path.abspath( os.path.join(repo_dir, uploaded_file_filename)) # Move some version of the uploaded file to the load_point within the repository hierarchy. if uploaded_file_filename in [ rt_util. REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if toolshed or changeset_revision attributes # are missing and if so, set them appropriately. altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name) if error_message: ok = False message = error_message status = 'error' elif altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) elif uploaded_file_filename in [ rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if changeset_revision values are # missing and if so, set them appropriately. altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name) if error_message: ok = False message = error_message status = 'error' if ok: if altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) else: shutil.move(uploaded_file_name, full_path) if ok: # See if any admin users have chosen to receive email alerts when a repository is updated. # If so, check every uploaded file to ensure content is appropriate. check_contents = commit_util.check_file_contents_for_email_alerts( trans.app) if check_contents and os.path.isfile(full_path): content_alert_str = commit_util.check_file_content_for_html_and_images( full_path) else: content_alert_str = '' hg_util.add_changeset(repo_dir, full_path) hg_util.commit_changeset( repo_dir, full_path_to_changeset=full_path, username=trans.user.username, message=commit_message) if full_path.endswith( 'tool_data_table_conf.xml.sample'): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded # by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables # dictionary. error, error_message = stdtm.handle_sample_tool_data_table_conf_file( full_path, persist=False) if error: message = '%s<br/>%s' % (message, error_message) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions) != 1 suc.handle_email_alerts( trans.app, trans.request.host, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only) if ok: # Update the repository files for browsing. hg_util.update_repository(repo_dir) # Get the new repository tip. if tip == repository.tip(): message = 'No changes to repository. ' status = 'warning' else: if (isgzip or isbz2) and uncompress_file: uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' if uploaded_directory: source_type = "repository" source = url else: source_type = "file" source = uploaded_file_filename message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \ (source_type, escape(source), uncompress_str) if istar and (undesirable_dirs_removed or undesirable_files_removed): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed message += "were removed from the archive. " if istar and remove_repo_files_not_in_tar and files_to_remove: if upload_point is not None: message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \ (len(files_to_remove), upload_point) else: message += " %d files were removed from the repository root. " % len( files_to_remove) rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app, user=trans.user, repository=repository) status, error_message = \ rmm.set_repository_metadata_due_to_new_tip(trans.request.host, content_alert_str=content_alert_str, **kwd) if error_message: message = error_message kwd['message'] = message if repository.metadata_revisions: # A repository's metadata revisions are order descending by update_time, so the zeroth revision # will be the tip just after an upload. metadata_dict = repository.metadata_revisions[ 0].metadata else: metadata_dict = {} dd = dependency_display.DependencyDisplayer(trans.app) if str(repository.type) not in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app, repository) if change_repository_type_message: message += change_repository_type_message status = 'warning' else: # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies # weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency # definitions can define orphan tool dependencies (no relationship to any tools contained in the # repository), so warning messages are important because orphans are always valid. The repository # owner must be warned in case they did not intend to define an orphan dependency, but simply # provided incorrect information (tool shed, name owner, changeset_revision) for the definition. orphan_message = dd.generate_message_for_orphan_tool_dependencies( repository, metadata_dict) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies( metadata_dict) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = \ dd.generate_message_for_invalid_repository_dependencies(metadata_dict, error_from_tuple=True) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. stdtm.reset_tool_data_tables() if uploaded_directory: basic_util.remove_dir(uploaded_directory) trans.response.send_redirect( web.url_for(controller='repository', action='browse_repository', id=repository_id, commit_message='Deleted selected files', message=message, status=status)) else: if uploaded_directory: basic_util.remove_dir(uploaded_directory) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. stdtm.reset_tool_data_tables() return trans.fill_template( '/webapps/tool_shed/repository/upload.mako', repository=repository, changeset_revision=tip, url=url, commit_message=commit_message, uncompress_file=uncompress_file, remove_repo_files_not_in_tar=remove_repo_files_not_in_tar, message=message, status=status)
def json(self, trans, **kwd): """ GET /api/tools/json Get the tool form JSON for a tool in a repository. :param guid: the GUID of the tool :param guid: str :param tsr_id: the ID of the repository :param tsr_id: str :param changeset: the changeset at which to load the tool json :param changeset: str """ guid = kwd.get('guid', None) tsr_id = kwd.get('tsr_id', None) changeset = kwd.get('changeset', None) if None in [changeset, tsr_id, guid]: message = 'Changeset, repository ID, and tool GUID are all required parameters.' trans.response.status = 400 return {'status': 'error', 'message': message} tsucm = ToolShedUtilityContainerManager(trans.app) repository = repository_util.get_repository_in_tool_shed(self.app, tsr_id) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(repository.user, repository) repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, tsr_id, changeset) toolshed_base_url = str(web.url_for('/', qualified=True)).rstrip('/') rb = relation_builder.RelationBuilder(trans.app, repository, repository_metadata, toolshed_base_url) repository_dependencies = rb.get_repository_dependencies_for_changeset_revision() containers_dict = tsucm.build_repository_containers(repository, changeset, repository_dependencies, repository_metadata) found_tool = None for folder in containers_dict['valid_tools'].folders: if hasattr(folder, 'valid_tools'): for tool in folder.valid_tools: tool.id = tool.tool_id tool_guid = suc.generate_tool_guid(repository_clone_url, tool) if tool_guid == guid: found_tool = tool break if found_tool is None: message = 'Unable to find tool with guid %s in repository %s.' % (guid, repository.name) trans.response.status = 404 return {'status': 'error', 'message': message} tv = tool_validator.ToolValidator(trans.app) repository, tool, message = tv.load_tool_from_changeset_revision(tsr_id, changeset, found_tool.tool_config) if message: status = 'error' return dict(message=message, status=status) tool_help = '' if tool.help: tool_help = tool.help.render(static_path=web.url_for('/static'), host_url=web.url_for('/', qualified=True)) tool_help = util.unicodify(tool_help, 'utf-8') tool_dict = tool.to_dict(trans) tool_dict['inputs'] = {} tool.populate_model(trans, tool.inputs, {}, tool_dict['inputs']) tool_dict.update({ 'help' : tool_help, 'citations' : bool(tool.citations), 'biostar_url' : trans.app.config.biostar_url, 'requirements' : [{'name' : r.name, 'version' : r.version} for r in tool.requirements], 'state_inputs' : params_to_strings(tool.inputs, {}, trans.app), 'display' : tool.display_interface, 'action' : web.url_for(tool.action), 'method' : tool.method, 'enctype' : tool.enctype }) return json.dumps(tool_dict)
def create_changeset_revision(self, trans, id, payload, **kwd): """ POST /api/repositories/{encoded_repository_id}/changeset_revision Create a new tool shed repository commit - leaving PUT on parent resource open for updating meta-attributes of the repository (and Galaxy doesn't allow PUT multipart data anyway https://trello.com/c/CQwmCeG6). :param id: the encoded id of the Repository object The following parameters may be included in the payload. :param commit_message: hg commit message for update. """ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 rdah = attribute_handlers.RepositoryDependencyAttributeHandler(self.app, unpopulate=False) tdah = attribute_handlers.ToolDependencyAttributeHandler(self.app, unpopulate=False) repository = repository_util.get_repository_in_tool_shed(self.app, id) if not ( trans.user_is_admin() or self.app.security_agent.user_can_administer_repository(trans.user, repository) or self.app.security_agent.can_push(self.app, trans.user, repository) ): trans.response.status = 400 return {"err_msg": "You do not have permission to update this repository."} repo_dir = repository.repo_path(self.app) repo = hg_util.get_repo_for_repository(self.app, repository=None, repo_path=repo_dir, create=False) upload_point = commit_util.get_upload_point(repository, **kwd) tip = repository.tip(self.app) file_data = payload.get("file") # Code stolen from gx's upload_common.py if isinstance(file_data, FieldStorage): assert not isinstance(file_data.file, StringIO.StringIO) assert file_data.file.name != "<fdopen>" local_filename = util.mkstemp_ln(file_data.file.name, "upload_file_data_") file_data.file.close() file_data = dict(filename=file_data.filename, local_filename=local_filename) elif type(file_data) == dict and "local_filename" not in file_data: raise Exception("Uploaded file was encoded in a way not understood.") commit_message = kwd.get("commit_message", "Uploaded") uploaded_file = open(file_data["local_filename"], "rb") uploaded_file_name = file_data["local_filename"] isgzip = False isbz2 = False isgzip = checkers.is_gzip(uploaded_file_name) if not isgzip: isbz2 = checkers.is_bz2(uploaded_file_name) if isgzip or isbz2: # Open for reading with transparent compression. tar = tarfile.open(uploaded_file_name, "r:*") else: tar = tarfile.open(uploaded_file_name) new_repo_alert = False remove_repo_files_not_in_tar = True ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = repository_content_util.upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert, ) if ok: # Update the repository files for browsing. hg_util.update_repository(repo) # Get the new repository tip. if tip == repository.tip(self.app): trans.response.status = 400 message = "No changes to repository." ok = False else: rmm = repository_metadata_manager.RepositoryMetadataManager( app=self.app, user=trans.user, repository=repository ) status, error_message = rmm.set_repository_metadata_due_to_new_tip( trans.request.host, content_alert_str=content_alert_str, **kwd ) if error_message: ok = False trans.response.status = 500 message = error_message else: trans.response.status = 500 if os.path.exists(uploaded_file_name): os.remove(uploaded_file_name) if not ok: return {"err_msg": message, "content_alert": content_alert_str} else: return {"message": message, "content_alert": content_alert_str}
def create_review(self, trans, **kwd): # The value of the received id is the encoded repository id. message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository_id = kwd.get('id', None) changeset_revision = kwd.get('changeset_revision', None) previous_review_id = kwd.get('previous_review_id', None) create_without_copying = 'create_without_copying' in kwd if repository_id: if changeset_revision: # Make sure there is not already a review of the revision by the user. repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) if review_util.get_review_by_repository_id_changeset_revision_user_id(app=trans.app, repository_id=repository_id, changeset_revision=changeset_revision, user_id=trans.security.encode_id(trans.user.id)): message = "You have already created a review for revision <b>%s</b> of repository <b>%s</b>." % (changeset_revision, escape(repository.name)) status = "error" else: # See if there are any reviews for previous changeset revisions that the user can copy. if not create_without_copying and \ not previous_review_id and \ review_util.has_previous_repository_reviews(trans.app, repository, changeset_revision): return trans.response.send_redirect(web.url_for(controller='repository_review', action='select_previous_review', **kwd)) # A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated # with the received changeset_revision. repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, repository_id, changeset_revision) if repository_metadata: metadata = repository_metadata.metadata if metadata: review = trans.app.model.RepositoryReview(repository_id=repository_metadata.repository_id, changeset_revision=changeset_revision, user_id=trans.user.id, rating=None, deleted=False) trans.sa_session.add(review) trans.sa_session.flush() if previous_review_id: review_to_copy = review_util.get_review(trans.app, previous_review_id) self.copy_review(trans, review_to_copy, review) review_id = trans.security.encode_id(review.id) message = "Begin your review of revision <b>%s</b> of repository <b>%s</b>." \ % (changeset_revision, repository.name) status = 'done' trans.response.send_redirect(web.url_for(controller='repository_review', action='edit_review', id=review_id, message=message, status=status)) else: message = "A new review cannot be created for revision <b>%s</b> of repository <b>%s</b>. Select a valid revision and try again." \ % (changeset_revision, escape(repository.name)) kwd['message'] = message kwd['status'] = 'error' else: return trans.response.send_redirect(web.url_for(controller='repository_review', action='manage_repository_reviews', **kwd)) return trans.response.send_redirect(web.url_for(controller='repository_review', action='view_or_manage_repository', **kwd))
def create_review( self, trans, **kwd ): # The value of the received id is the encoded repository id. message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) changeset_revision = kwd.get( 'changeset_revision', None ) previous_review_id = kwd.get( 'previous_review_id', None ) create_without_copying = 'create_without_copying' in kwd if repository_id: if changeset_revision: # Make sure there is not already a review of the revision by the user. repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) if review_util.get_review_by_repository_id_changeset_revision_user_id( app=trans.app, repository_id=repository_id, changeset_revision=changeset_revision, user_id=trans.security.encode_id( trans.user.id ) ): message = "You have already created a review for revision <b>%s</b> of repository <b>%s</b>." % ( changeset_revision, escape( repository.name ) ) status = "error" else: # See if there are any reviews for previous changeset revisions that the user can copy. if not create_without_copying and \ not previous_review_id and \ review_util.has_previous_repository_reviews( trans.app, repository, changeset_revision ): return trans.response.send_redirect( web.url_for( controller='repository_review', action='select_previous_review', **kwd ) ) # A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated # with the received changeset_revision. repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: review = trans.app.model.RepositoryReview( repository_id=repository_metadata.repository_id, changeset_revision=changeset_revision, user_id=trans.user.id, rating=None, deleted=False ) trans.sa_session.add( review ) trans.sa_session.flush() if previous_review_id: review_to_copy = review_util.get_review( trans.app, previous_review_id ) self.copy_review( trans, review_to_copy, review ) review_id = trans.security.encode_id( review.id ) message = "Begin your review of revision <b>%s</b> of repository <b>%s</b>." \ % ( changeset_revision, repository.name ) status = 'done' trans.response.send_redirect( web.url_for( controller='repository_review', action='edit_review', id=review_id, message=message, status=status ) ) else: message = "A new review cannot be created for revision <b>%s</b> of repository <b>%s</b>. Select a valid revision and try again." \ % ( changeset_revision, escape( repository.name ) ) kwd[ 'message' ] = message kwd[ 'status' ] = 'error' else: return trans.response.send_redirect( web.url_for( controller='repository_review', action='manage_repository_reviews', **kwd ) ) return trans.response.send_redirect( web.url_for( controller='repository_review', action='view_or_manage_repository', **kwd ) )
def browse_repositories(self, trans, **kwd): # We add parameters to the keyword dict in this method in order to rename the param # with an "f-" prefix, simulating filtering by clicking a search link. We have # to take this approach because the "-" character is illegal in HTTP requests. if 'operation' in kwd: operation = kwd['operation'].lower() if operation == "view_or_manage_repository": return trans.response.send_redirect(web.url_for(controller='repository', action='browse_repositories', **kwd)) elif operation == "edit_repository": return trans.response.send_redirect(web.url_for(controller='repository', action='edit_repository', **kwd)) elif operation == "repositories_by_user": # Eliminate the current filters if any exist. for k, v in list(kwd.items()): if k.startswith('f-'): del kwd[k] if 'user_id' in kwd: user = suc.get_user(trans.app, kwd['user_id']) kwd['f-email'] = user.email del kwd['user_id'] else: # The received id is the repository id, so we need to get the id of the user # that uploaded the repository. repository_id = kwd.get('id', None) repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) kwd['f-email'] = repository.user.email elif operation == "repositories_by_category": # Eliminate the current filters if any exist. for k, v in list(kwd.items()): if k.startswith('f-'): del kwd[k] category_id = kwd.get('id', None) category = suc.get_category(trans.app, category_id) kwd['f-Category.name'] = category.name elif operation == "receive email alerts": if kwd['id']: kwd['caller'] = 'browse_repositories' return trans.response.send_redirect(web.url_for(controller='repository', action='set_email_alerts', **kwd)) else: del kwd['operation'] elif operation == 'delete': return self.delete_repository(trans, **kwd) elif operation == "undelete": return self.undelete_repository(trans, **kwd) # The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One # of the many select fields on the grid performed the refresh_on_change, so we loop through # all of the received values to see which value is not the repository tip. If we find it, we # know the refresh_on_change occurred, and we have the necessary repository id and change set # revision to pass on. for k, v in kwd.items(): changeset_revision_str = 'changeset_revision_' if k.startswith(changeset_revision_str): repository_id = trans.security.encode_id(int(k.lstrip(changeset_revision_str))) repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) if repository.tip(trans.app) != v: return trans.response.send_redirect(web.url_for(controller='repository', action='browse_repositories', operation='view_or_manage_repository', id=trans.security.encode_id(repository.id), changeset_revision=v)) # Render the list view return self.repository_grid(trans, **kwd)
def upload(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') commit_message = escape(kwd.get('commit_message', 'Uploaded')) repository_id = kwd.get('repository_id', '') repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) repo_dir = repository.repo_path(trans.app) repo = hg_util.get_repo_for_repository(trans.app, repository=None, repo_path=repo_dir, create=False) uncompress_file = util.string_as_bool(kwd.get('uncompress_file', 'true')) remove_repo_files_not_in_tar = util.string_as_bool(kwd.get('remove_repo_files_not_in_tar', 'true')) uploaded_file = None upload_point = commit_util.get_upload_point(repository, **kwd) tip = repository.tip(trans.app) file_data = kwd.get('file_data', '') url = kwd.get('url', '') # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new(trans.app) uploaded_directory = None if kwd.get('upload_button', False): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None elif url and url.startswith('hg'): # Use mercurial clone to fetch repository, contents will then be copied over. uploaded_directory = tempfile.mkdtemp() repo_url = 'http%s' % url[len('hg'):] repo_url = repo_url.encode('ascii', 'replace') try: commands.clone(hg_util.get_configured_ui(), repo_url, uploaded_directory) except Exception as e: message = 'Error uploading via mercurial clone: %s' % basic_util.to_html_string(str(e)) status = 'error' basic_util.remove_dir(uploaded_directory) uploaded_directory = None elif url: valid_url = True try: stream = requests.get(url, stream=True) except Exception as e: valid_url = False message = 'Error uploading file via http: %s' % str(e) status = 'error' uploaded_file = None if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open(uploaded_file_name, 'wb') for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE): if chunk: uploaded_file.write(chunk) uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0 elif file_data not in ('', None): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name uploaded_file_filename = os.path.split(file_data.filename)[-1] isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0 if uploaded_file or uploaded_directory: rdah = attribute_handlers.RepositoryDependencyAttributeHandler(trans.app, unpopulate=False) tdah = attribute_handlers.ToolDependencyAttributeHandler(trans.app, unpopulate=False) stdtm = ShedToolDataTableManager(trans.app) ok = True isgzip = False isbz2 = False if uploaded_file: if uncompress_file: isgzip = checkers.is_gzip(uploaded_file_name) if not isgzip: isbz2 = checkers.is_bz2(uploaded_file_name) if isempty: tar = None istar = False else: # Determine what we have - a single file or an archive try: if (isgzip or isbz2) and uncompress_file: # Open for reading with transparent compression. tar = tarfile.open(uploaded_file_name, 'r:*') else: tar = tarfile.open(uploaded_file_name) istar = True except tarfile.ReadError as e: tar = None istar = False else: # Uploaded directory istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ repository_content_util.upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_directory(trans, rdah, tdah, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert) else: if (isgzip or isbz2) and uncompress_file: uploaded_file_filename = commit_util.uncompress(repository, uploaded_file_name, uploaded_file_filename, isgzip=isgzip, isbz2=isbz2) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named ' message += '<b>repository_dependencies.xml</b>.' elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \ uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named ' message += '<b>tool_dependencies.xml</b>.' if ok: if upload_point is not None: full_path = os.path.abspath(os.path.join(repo_dir, upload_point, uploaded_file_filename)) else: full_path = os.path.abspath(os.path.join(repo_dir, uploaded_file_filename)) # Move some version of the uploaded file to the load_point within the repository hierarchy. if uploaded_file_filename in [rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME]: # Inspect the contents of the file to see if toolshed or changeset_revision attributes # are missing and if so, set them appropriately. altered, root_elem, error_message = rdah.handle_tag_attributes(uploaded_file_name) if error_message: ok = False message = error_message status = 'error' elif altered: tmp_filename = xml_util.create_and_write_tmp_file(root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) elif uploaded_file_filename in [rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME]: # Inspect the contents of the file to see if changeset_revision values are # missing and if so, set them appropriately. altered, root_elem, error_message = tdah.handle_tag_attributes(uploaded_file_name) if error_message: ok = False message = error_message status = 'error' if ok: if altered: tmp_filename = xml_util.create_and_write_tmp_file(root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) else: shutil.move(uploaded_file_name, full_path) if ok: # See if any admin users have chosen to receive email alerts when a repository is updated. # If so, check every uploaded file to ensure content is appropriate. check_contents = commit_util.check_file_contents_for_email_alerts(trans.app) if check_contents and os.path.isfile(full_path): content_alert_str = commit_util.check_file_content_for_html_and_images(full_path) else: content_alert_str = '' hg_util.add_changeset(repo.ui, repo, full_path) # Convert from unicode to prevent "TypeError: array item must be char" full_path = full_path.encode('ascii', 'replace') hg_util.commit_changeset(repo.ui, repo, full_path_to_changeset=full_path, username=trans.user.username, message=commit_message) if full_path.endswith('tool_data_table_conf.xml.sample'): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded # by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables # dictionary. error, error_message = stdtm.handle_sample_tool_data_table_conf_file(full_path, persist=False) if error: message = '%s<br/>%s' % (message, error_message) # See if the content of the change set was valid. admin_only = len(repository.downloadable_revisions) != 1 suc.handle_email_alerts(trans.app, trans.request.host, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only) if ok: # Update the repository files for browsing. hg_util.update_repository(repo) # Get the new repository tip. if tip == repository.tip(trans.app): message = 'No changes to repository. ' status = 'warning' else: if (isgzip or isbz2) and uncompress_file: uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' if uploaded_directory: source_type = "repository" source = url else: source_type = "file" source = uploaded_file_filename message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \ (source_type, escape(source), uncompress_str) if istar and (undesirable_dirs_removed or undesirable_files_removed): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed message += "were removed from the archive. " if istar and remove_repo_files_not_in_tar and files_to_remove: if upload_point is not None: message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \ (len(files_to_remove), upload_point) else: message += " %d files were removed from the repository root. " % len(files_to_remove) rmm = repository_metadata_manager.RepositoryMetadataManager(app=trans.app, user=trans.user, repository=repository) status, error_message = \ rmm.set_repository_metadata_due_to_new_tip(trans.request.host, content_alert_str=content_alert_str, **kwd) if error_message: message = error_message kwd['message'] = message if repository.metadata_revisions: # A repository's metadata revisions are order descending by update_time, so the zeroth revision # will be the tip just after an upload. metadata_dict = repository.metadata_revisions[0].metadata else: metadata_dict = {} dd = dependency_display.DependencyDisplayer(trans.app) if str(repository.type) not in [rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION]: change_repository_type_message = rt_util.generate_message_for_repository_type_change(trans.app, repository) if change_repository_type_message: message += change_repository_type_message status = 'warning' else: # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies # weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency # definitions can define orphan tool dependencies (no relationship to any tools contained in the # repository), so warning messages are important because orphans are always valid. The repository # owner must be warned in case they did not intend to define an orphan dependency, but simply # provided incorrect information (tool shed, name owner, changeset_revision) for the definition. orphan_message = dd.generate_message_for_orphan_tool_dependencies(repository, metadata_dict) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies(metadata_dict) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = \ dd.generate_message_for_invalid_repository_dependencies(metadata_dict, error_from_tuple=True) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. stdtm.reset_tool_data_tables() if uploaded_directory: basic_util.remove_dir(uploaded_directory) trans.response.send_redirect(web.url_for(controller='repository', action='browse_repository', id=repository_id, commit_message='Deleted selected files', message=message, status=status)) else: if uploaded_directory: basic_util.remove_dir(uploaded_directory) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. stdtm.reset_tool_data_tables() return trans.fill_template('/webapps/tool_shed/repository/upload.mako', repository=repository, changeset_revision=tip, url=url, commit_message=commit_message, uncompress_file=uncompress_file, remove_repo_files_not_in_tar=remove_repo_files_not_in_tar, message=message, status=status)