def show( self, trans, id, **kwd ): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 :param id: the encoded id of the Repository object :type id: encoded str :returns: detailed repository information :rtype: dict :raises: ObjectNotFound, MalformedId """ try: trans.security.decode_id( id ) except Exception: raise MalformedId( 'The given id is invalid.' ) repository = suc.get_repository_in_tool_shed( trans.app, id ) if repository is None: raise ObjectNotFound( 'Unable to locate repository for the given id.' ) repository_dict = repository.to_dict( view='element', value_mapper=self.__get_value_mapper( trans ) ) # TODO the following property would be better suited in the to_dict method repository_dict[ 'category_ids' ] = \ [ trans.security.encode_id( x.category.id ) for x in repository.categories ] return repository_dict
def manage_repository_reviews(self, trans, mine=False, **kwd): # The value of the received id is the encoded repository id. message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository_id = kwd.get('id', None) if repository_id: repository = suc.get_repository_in_tool_shed( trans.app, repository_id) repo = hg_util.get_repo_for_repository(trans.app, repository=repository) metadata_revision_hashes = [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] reviews_dict = OrderedDict() for changeset in hg_util.get_reversed_changelog_changesets(repo): changeset_revision = str(repo[changeset]) if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes: rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision) if changeset_revision in reviewed_revision_hashes: # Find the review for this changeset_revision repository_reviews = \ review_util.get_reviews_by_repository_id_changeset_revision(trans.app, repository_id, changeset_revision) # Determine if the current user can add a review to this revision. can_add_review = trans.user not in [ repository_review.user for repository_review in repository_reviews ] repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision) if repository_metadata: repository_metadata_reviews = util.listify( repository_metadata.reviews) else: repository_metadata_reviews = [] else: repository_reviews = [] repository_metadata_reviews = [] can_add_review = True installable = changeset_revision in metadata_revision_hashes revision_dict = dict( changeset_revision_label=changeset_revision_label, repository_reviews=repository_reviews, repository_metadata_reviews=repository_metadata_reviews, installable=installable, can_add_review=can_add_review) reviews_dict[changeset_revision] = revision_dict return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_repository.mako', repository=repository, reviews_dict=reviews_dict, mine=mine, message=message, status=status)
def delete_repository( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) id = kwd.get( 'id', None ) if id: # Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify( id ) count = 0 deleted_repositories = "" for repository_id in ids: repository = suc.get_repository_in_tool_shed( trans, repository_id ) if repository: if not repository.deleted: # Mark all installable repository_metadata records as not installable. for repository_metadata in repository.downloadable_revisions: repository_metadata.downloadable = False trans.sa_session.add( repository_metadata ) repository.deleted = True trans.sa_session.add( repository ) trans.sa_session.flush() count += 1 deleted_repositories += " %s " % repository.name if count: message = "Deleted %d %s: %s" % ( count, inflector.cond_plural( len( ids ), "repository" ), deleted_repositories ) else: message = "All selected repositories were already marked deleted." else: message = "No repository ids received for deleting." status = 'error' trans.response.send_redirect( web.url_for( controller='admin', action='browse_repositories', message=util.sanitize_text( message ), status=status ) )
def get_repo_info_dict(app, user, repository_id, changeset_revision): repository = suc.get_repository_in_tool_shed(app, repository_id) repo = hg_util.get_repo_for_repository(app, repository=repository, repo_path=None, create=False) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository) repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, repository_id, changeset_revision) if not repository_metadata: # The received changeset_revision is no longer installable, so get the next changeset_revision # in the repository's changelog. This generally occurs only with repositories of type # repository_suite_definition or tool_dependency_definition. next_downloadable_changeset_revision = \ suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if next_downloadable_changeset_revision: repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, repository_id, next_downloadable_changeset_revision) if repository_metadata: # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the # returned repo_info_dict. metadata = repository_metadata.metadata if 'tools' in metadata: includes_tools = True else: includes_tools = False includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel repository_dependencies_dict = metadata.get('repository_dependencies', {}) repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', []) has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ suc.get_repository_dependency_types( repository_dependencies ) if 'tool_dependencies' in metadata: includes_tool_dependencies = True else: includes_tool_dependencies = False else: # Here's where we may have to handle enhancements to the callers. See above comment. includes_tools = False has_repository_dependencies = False has_repository_dependencies_only_if_compiling_contained_td = False includes_tool_dependencies = False includes_tools_for_display_in_tool_panel = False ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) repo_info_dict = create_repo_info_dict( app=app, repository_clone_url=repository_clone_url, changeset_revision=changeset_revision, ctx_rev=str(ctx.rev()), repository_owner=repository.user.username, repository_name=repository.name, repository=repository, repository_metadata=repository_metadata, tool_dependencies=None, repository_dependencies=None) return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td
def show(self, trans, id, **kwd): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. :param id: the encoded id of the Repository object """ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 try: repository = suc.get_repository_in_tool_shed(trans, id) repository_dict = repository.get_api_value( view='element', value_mapper=default_repository_value_mapper( trans, repository)) repository_dict['url'] = web.url_for(controller='repositories', action='show', id=trans.security.encode_id( repository.id)) return repository_dict except Exception, e: message = "Error in the Tool Shed repositories API in show: %s" % str( e) log.error(message, exc_info=True) trans.response.status = 500 return message
def get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( trans, repository_id ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. repository_dependencies = \ repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans, repository=repository, repository_metadata=repository_metadata, toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, handled_key_rd_dicts=None ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = {} # Cast unicode to string. repo_info_dict[ str( repository.name ) ] = ( str( repository.description ), suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ), str( changeset_revision ), str( ctx.rev() ), str( repository.user.username ), repository_dependencies, None ) all_required_repo_info_dict = common_install_util.get_required_repo_info_dicts( trans, tool_shed_url, [ repo_info_dict ] ) all_repo_info_dicts = all_required_repo_info_dict.get( 'all_repo_info_dicts', [] ) return all_repo_info_dicts
def manage_repository_reviews_of_revision(self, trans, **kwd): # The value of the received id is the encoded repository id. message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') repository_id = kwd.get('id', None) changeset_revision = kwd.get('changeset_revision', None) repository = suc.get_repository_in_tool_shed(trans.app, repository_id) repo = hg_util.get_repo_for_repository(trans.app, repository=repository, repo_path=None, create=False) installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision) reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans.app, repository_id, changeset_revision) return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako', repository=repository, changeset_revision=changeset_revision, changeset_revision_label=changeset_revision_label, reviews=reviews, installable=installable, message=message, status=status)
def load_tool_from_changeset_revision(self, repository_id, changeset_revision, tool_config_filename): """ Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision is a valid (downloadable) changeset revision. The tool config will be located in the repository manifest between the received valid changeset revision and the first changeset revision in the repository, searching backwards. """ original_tool_data_path = self.app.config.tool_data_path repository = suc.get_repository_in_tool_shed(self.app, repository_id) repo_files_dir = repository.repo_path(self.app) repo = hg_util.get_repo_for_repository(self.app, repository=None, repo_path=repo_files_dir, create=False) message = '' tool = None can_use_disk_file = False tool_config_filepath = suc.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename) work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr") can_use_disk_file = self.can_use_tool_config_disk_file( repository, repo, tool_config_filepath, changeset_revision) if can_use_disk_file: self.app.config.tool_data_path = work_dir tool, valid, message, sample_files = \ self.handle_sample_files_and_load_tool_from_disk( repo_files_dir, repository_id, tool_config_filepath, work_dir ) if tool is not None: invalid_files_and_errors_tups = \ self.check_tool_input_params( repo_files_dir, tool_config_filename, tool, sample_files ) if invalid_files_and_errors_tups: message2 = tool_util.generate_message_for_invalid_tools( self.app, invalid_files_and_errors_tups, repository, metadata_dict=None, as_html=True, displaying_invalid_tool=True) message = self.concat_messages(message, message2) else: tool, message, sample_files = \ self.handle_sample_files_and_load_tool_from_tmp_config( repo, repository_id, changeset_revision, tool_config_filename, work_dir ) basic_util.remove_dir(work_dir) self.app.config.tool_data_path = original_tool_data_path # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.tdtm.reset_tool_data_tables() return repository, tool, message
def upload( self, trans, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) commit_message = escape( kwd.get( 'commit_message', 'Uploaded' ) ) category_ids = util.listify( kwd.get( 'category_id', '' ) ) categories = suc.get_categories( trans.app ) repository_id = kwd.get( 'repository_id', '' ) repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) ) remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) ) uploaded_file = None upload_point = commit_util.get_upload_point( repository, **kwd ) tip = repository.tip( trans.app ) file_data = kwd.get( 'file_data', '' ) url = kwd.get( 'url', '' ) # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new( trans.app ) uploaded_directory = None if kwd.get( 'upload_button', False ): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None elif url and url.startswith( 'hg' ): # Use mercurial clone to fetch repository, contents will then be copied over. uploaded_directory = tempfile.mkdtemp() repo_url = 'http%s' % url[ len( 'hg' ): ] repo_url = repo_url.encode( 'ascii', 'replace' ) try: commands.clone( hg_util.get_configured_ui(), repo_url, uploaded_directory ) except Exception, e: message = 'Error uploading via mercurial clone: %s' % basic_util.to_html_string( str( e ) ) status = 'error' basic_util.remove_dir( uploaded_directory ) uploaded_directory = None elif url: valid_url = True try: stream = urllib.urlopen( url ) except Exception, e: valid_url = False message = 'Error uploading file via http: %s' % str( e ) status = 'error' uploaded_file = None if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open( uploaded_file_name, 'wb' ) while 1: chunk = stream.read( util.CHUNK_SIZE ) if not chunk: break uploaded_file.write( chunk ) uploaded_file.flush() uploaded_file_filename = url.split( '/' )[ -1 ] isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
def upload( self, trans, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) commit_message = escape( kwd.get( 'commit_message', 'Uploaded' ) ) repository_id = kwd.get( 'repository_id', '' ) repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) ) remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) ) uploaded_file = None upload_point = commit_util.get_upload_point( repository, **kwd ) tip = repository.tip( trans.app ) file_data = kwd.get( 'file_data', '' ) url = kwd.get( 'url', '' ) # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new( trans.app ) uploaded_directory = None if kwd.get( 'upload_button', False ): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None elif url and url.startswith( 'hg' ): # Use mercurial clone to fetch repository, contents will then be copied over. uploaded_directory = tempfile.mkdtemp() repo_url = 'http%s' % url[ len( 'hg' ): ] repo_url = repo_url.encode( 'ascii', 'replace' ) try: commands.clone( hg_util.get_configured_ui(), repo_url, uploaded_directory ) except Exception, e: message = 'Error uploading via mercurial clone: %s' % basic_util.to_html_string( str( e ) ) status = 'error' basic_util.remove_dir( uploaded_directory ) uploaded_directory = None elif url: valid_url = True try: stream = urllib.urlopen( url ) except Exception, e: valid_url = False message = 'Error uploading file via http: %s' % str( e ) status = 'error' uploaded_file = None if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open( uploaded_file_name, 'wb' ) while 1: chunk = stream.read( util.CHUNK_SIZE ) if not chunk: break uploaded_file.write( chunk ) uploaded_file.flush() uploaded_file_filename = url.split( '/' )[ -1 ] isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
def view_or_manage_repository( self, trans, **kwd ): repository = suc.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] ) if trans.user_is_admin() or repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', action='manage_repository', **kwd ) ) else: return trans.response.send_redirect( web.url_for( controller='repository', action='view_repository', **kwd ) )
def get_repo_info_dict( app, user, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( app, repository_id ) repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, repository_id, changeset_revision ) if not repository_metadata: # The received changeset_revision is no longer installable, so get the next changeset_revision # in the repository's changelog. This generally occurs only with repositories of type # repository_suite_definition or tool_dependency_definition. next_downloadable_changeset_revision = \ suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if next_downloadable_changeset_revision: repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, repository_id, next_downloadable_changeset_revision ) if repository_metadata: # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the # returned repo_info_dict. metadata = repository_metadata.metadata if 'tools' in metadata: includes_tools = True else: includes_tools = False includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ suc.get_repository_dependency_types( repository_dependencies ) if 'tool_dependencies' in metadata: includes_tool_dependencies = True else: includes_tool_dependencies = False else: # Here's where we may have to handle enhancements to the callers. See above comment. includes_tools = False has_repository_dependencies = False has_repository_dependencies_only_if_compiling_contained_td = False includes_tool_dependencies = False includes_tools_for_display_in_tool_panel = False ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = create_repo_info_dict( app=app, repository_clone_url=repository_clone_url, changeset_revision=changeset_revision, ctx_rev=str( ctx.rev() ), repository_owner=repository.user.username, repository_name=repository.name, repository=repository, repository_metadata=repository_metadata, tool_dependencies=None, repository_dependencies=None ) return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td
def load_tool_from_changeset_revision( self, repository_id, changeset_revision, tool_config_filename ): """ Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision is a valid (downloadable) changeset revision. The tool config will be located in the repository manifest between the received valid changeset revision and the first changeset revision in the repository, searching backwards. """ original_tool_data_path = self.app.config.tool_data_path repository = suc.get_repository_in_tool_shed( self.app, repository_id ) repo_files_dir = repository.repo_path( self.app ) repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_files_dir, create=False ) message = '' tool = None can_use_disk_file = False tool_config_filepath = suc.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename ) work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ltfcr" ) can_use_disk_file = self.can_use_tool_config_disk_file( repository, repo, tool_config_filepath, changeset_revision ) if can_use_disk_file: self.app.config.tool_data_path = work_dir tool, valid, message, sample_files = \ self.handle_sample_files_and_load_tool_from_disk( repo_files_dir, repository_id, tool_config_filepath, work_dir ) if tool is not None: invalid_files_and_errors_tups = \ self.check_tool_input_params( repo_files_dir, tool_config_filename, tool, sample_files ) if invalid_files_and_errors_tups: message2 = tool_util.generate_message_for_invalid_tools( self.app, invalid_files_and_errors_tups, repository, metadata_dict=None, as_html=True, displaying_invalid_tool=True ) message = self.concat_messages( message, message2 ) else: tool, message, sample_files = \ self.handle_sample_files_and_load_tool_from_tmp_config( repo, repository_id, changeset_revision, tool_config_filename, work_dir ) basic_util.remove_dir( work_dir ) self.app.config.tool_data_path = original_tool_data_path # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.tdtm.reset_tool_data_tables() return repository, tool, message
def undelete_repository(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') id = kwd.get('id', None) if id: # Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify(id) count = 0 undeleted_repositories = "" for repository_id in ids: repository = suc.get_repository_in_tool_shed( trans.app, repository_id) if repository: if repository.deleted: # Inspect all repository_metadata records to determine those that are installable, and mark # them accordingly. for repository_metadata in repository.metadata_revisions: metadata = repository_metadata.metadata if metadata: if metadata_util.is_downloadable(metadata): repository_metadata.downloadable = True trans.sa_session.add(repository_metadata) # Mark the repository admin role as not deleted. repository_admin_role = repository.admin_role if repository_admin_role is not None: repository_admin_role.deleted = False trans.sa_session.add(repository_admin_role) repository.deleted = False trans.sa_session.add(repository) trans.sa_session.flush() if not repository.deprecated: # Update the repository registry. trans.app.repository_registry.add_entry(repository) count += 1 undeleted_repositories += " %s" % repository.name if count: message = "Undeleted %d %s: %s" % ( count, inflector.cond_plural( count, "repository"), undeleted_repositories) else: message = "No selected repositories were marked deleted, so they could not be undeleted." else: message = "No repository ids received for undeleting." status = 'error' trans.response.send_redirect( web.url_for(controller='admin', action='browse_repositories', message=util.sanitize_text(message), status='done'))
def undelete_repository(self, trans, **kwd): message = escape(kwd.get("message", "")) id = kwd.get("id", None) if id: # Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify(id) count = 0 undeleted_repositories = "" for repository_id in ids: repository = suc.get_repository_in_tool_shed(trans.app, repository_id) if repository: if repository.deleted: # Inspect all repository_metadata records to determine those that are installable, and mark # them accordingly. for repository_metadata in repository.metadata_revisions: metadata = repository_metadata.metadata if metadata: if metadata_util.is_downloadable(metadata): repository_metadata.downloadable = True trans.sa_session.add(repository_metadata) # Mark the repository admin role as not deleted. repository_admin_role = repository.admin_role if repository_admin_role is not None: repository_admin_role.deleted = False trans.sa_session.add(repository_admin_role) repository.deleted = False trans.sa_session.add(repository) trans.sa_session.flush() if not repository.deprecated: # Update the repository registry. trans.app.repository_registry.add_entry(repository) count += 1 undeleted_repositories += " %s" % repository.name if count: message = "Undeleted %d %s: %s" % ( count, inflector.cond_plural(count, "repository"), undeleted_repositories, ) else: message = "No selected repositories were marked deleted, so they could not be undeleted." else: message = "No repository ids received for undeleting." trans.response.send_redirect( web.url_for( controller="admin", action="browse_repositories", message=util.sanitize_text(message), status="done" ) )
def select_previous_review( self, trans, **kwd ): # The value of the received id is the encoded repository id. message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, kwd[ 'id' ] ) changeset_revision = kwd.get( 'changeset_revision', None ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) previous_reviews_dict = review_util.get_previous_repository_reviews( trans, repository, changeset_revision ) rev, changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository_review/select_previous_review.mako', repository=repository, changeset_revision=changeset_revision, changeset_revision_label=changeset_revision_label, previous_reviews_dict=previous_reviews_dict, message=message, status=status )
def manage_repository_reviews( self, trans, mine=False, **kwd ): # The value of the received id is the encoded repository id. message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) if repository_id: repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) metadata_revision_hashes = [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] reviews_dict = odict() for changeset in hg_util.get_reversed_changelog_changesets( repo ): ctx = repo.changectx( changeset ) changeset_revision = str( ctx ) if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes: rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision ) if changeset_revision in reviewed_revision_hashes: # Find the review for this changeset_revision repository_reviews = \ review_util.get_reviews_by_repository_id_changeset_revision( trans.app, repository_id, changeset_revision ) # Determine if the current user can add a review to this revision. can_add_review = trans.user not in [ repository_review.user for repository_review in repository_reviews ] repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_reviews = util.listify( repository_metadata.reviews ) else: repository_metadata_reviews = [] else: repository_reviews = [] repository_metadata_reviews = [] can_add_review = True installable = changeset_revision in metadata_revision_hashes revision_dict = dict( changeset_revision_label=changeset_revision_label, repository_reviews=repository_reviews, repository_metadata_reviews=repository_metadata_reviews, installable=installable, can_add_review=can_add_review ) reviews_dict[ changeset_revision ] = revision_dict return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_repository.mako', repository=repository, reviews_dict=reviews_dict, mine=mine, message=message, status=status )
def show(self, trans, id, **kwd): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. :param id: the encoded id of the Repository object """ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 repository = suc.get_repository_in_tool_shed(trans.app, id) if repository is None: log.debug("Unable to locate repository record for id %s." % (str(id))) return {} repository_dict = repository.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) repository_dict["url"] = web.url_for( controller="repositories", action="show", id=trans.security.encode_id(repository.id) ) return repository_dict
def delete_repository(self, trans, **kwd): message = escape(kwd.get("message", "")) status = kwd.get("status", "done") id = kwd.get("id", None) if id: # Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify(id) count = 0 deleted_repositories = "" for repository_id in ids: repository = suc.get_repository_in_tool_shed(trans.app, repository_id) if repository: if not repository.deleted: # Mark all installable repository_metadata records as not installable. for repository_metadata in repository.downloadable_revisions: repository_metadata.downloadable = False trans.sa_session.add(repository_metadata) # Mark the repository admin role as deleted. repository_admin_role = repository.admin_role if repository_admin_role is not None: repository_admin_role.deleted = True trans.sa_session.add(repository_admin_role) repository.deleted = True trans.sa_session.add(repository) trans.sa_session.flush() # Update the repository registry. trans.app.repository_registry.remove_entry(repository) count += 1 deleted_repositories += " %s " % repository.name if count: message = "Deleted %d %s: %s" % ( count, inflector.cond_plural(len(ids), "repository"), escape(deleted_repositories), ) else: message = "All selected repositories were already marked deleted." else: message = "No repository ids received for deleting." status = "error" trans.response.send_redirect( web.url_for( controller="admin", action="browse_repositories", message=util.sanitize_text(message), status=status ) )
def show( self, trans, id, **kwd ): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. :param id: the encoded id of the Repository object """ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 try: repository = suc.get_repository_in_tool_shed( trans, id ) repository_dict = repository.get_api_value( view='element', value_mapper=default_repository_value_mapper( trans, repository ) ) repository_dict[ 'url' ] = web.url_for( controller='repositories', action='show', id=trans.security.encode_id( repository.id ) ) return repository_dict except Exception, e: message = "Error in the Tool Shed repositories API in show: %s" % str( e ) log.error( message, exc_info=True ) trans.response.status = 500 return message
def show(self, trans, id, **kwd): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. :param id: the encoded id of the Repository object """ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 repository = suc.get_repository_in_tool_shed(trans.app, id) if repository is None: log.debug("Unable to locate repository record for id %s." % (str(id))) return {} repository_dict = repository.to_dict( view='element', value_mapper=self.__get_value_mapper(trans)) repository_dict['url'] = web.url_for(controller='repositories', action='show', id=trans.security.encode_id( repository.id)) return repository_dict
def manage_repository_reviews_of_revision( self, trans, **kwd ): # The value of the received id is the encoded repository id. message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] rev, changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, changeset_revision ) reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako', repository=repository, changeset_revision=changeset_revision, changeset_revision_label=changeset_revision_label, reviews=reviews, installable=installable, message=message, status=status )
def undelete_repository( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) id = kwd.get( 'id', None ) if id: # Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify( id ) count = 0 undeleted_repositories = "" for repository_id in ids: repository = suc.get_repository_in_tool_shed( trans, repository_id ) if repository: if repository.deleted: # Inspect all repository_metadata records to determine those that are installable, and mark them accordingly. for repository_metadata in repository.metadata_revisions: metadata = repository_metadata.metadata if metadata: if metadata_util.is_downloadable( metadata ): repository_metadata.downloadable = True trans.sa_session.add( repository_metadata ) repository.deleted = False trans.sa_session.add( repository ) trans.sa_session.flush() count += 1 undeleted_repositories += " %s" % repository.name if count: message = "Undeleted %d %s: %s" % ( count, inflector.cond_plural( count, "repository" ), undeleted_repositories ) else: message = "No selected repositories were marked deleted, so they could not be undeleted." else: message = "No repository ids received for undeleting." status = 'error' trans.response.send_redirect( web.url_for( controller='admin', action='browse_repositories', message=util.sanitize_text( message ), status='done' ) )
def browse_repositories( self, trans, **kwd ): # We add parameters to the keyword dict in this method in order to rename the param # with an "f-" prefix, simulating filtering by clicking a search link. We have # to take this approach because the "-" character is illegal in HTTP requests. if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if operation == "view_or_manage_repository": return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', **kwd ) ) elif operation == "edit_repository": return trans.response.send_redirect( web.url_for( controller='repository', action='edit_repository', **kwd ) ) elif operation == "repositories_by_user": # Eliminate the current filters if any exist. for k, v in kwd.items(): if k.startswith( 'f-' ): del kwd[ k ] if 'user_id' in kwd: user = suc.get_user( trans.app, kwd[ 'user_id' ] ) kwd[ 'f-email' ] = user.email del kwd[ 'user_id' ] else: # The received id is the repository id, so we need to get the id of the user # that uploaded the repository. repository_id = kwd.get( 'id', None ) repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) kwd[ 'f-email' ] = repository.user.email elif operation == "repositories_by_category": # Eliminate the current filters if any exist. for k, v in kwd.items(): if k.startswith( 'f-' ): del kwd[ k ] category_id = kwd.get( 'id', None ) category = suc.get_category( trans.app, category_id ) kwd[ 'f-Category.name' ] = category.name elif operation == "receive email alerts": if kwd[ 'id' ]: kwd[ 'caller' ] = 'browse_repositories' return trans.response.send_redirect( web.url_for( controller='repository', action='set_email_alerts', **kwd ) ) else: del kwd[ 'operation' ] elif operation == 'delete': return self.delete_repository( trans, **kwd ) elif operation == "undelete": return self.undelete_repository( trans, **kwd ) # The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One # of the many select fields on the grid performed the refresh_on_change, so we loop through # all of the received values to see which value is not the repository tip. If we find it, we # know the refresh_on_change occurred, and we have the necessary repository id and change set # revision to pass on. for k, v in kwd.items(): changeset_revision_str = 'changeset_revision_' if k.startswith( changeset_revision_str ): repository_id = trans.security.encode_id( int( k.lstrip( changeset_revision_str ) ) ) repository = suc.get_repository_in_tool_shed( tran.apps, repository_id ) if repository.tip( trans.app ) != v: return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', operation='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=v ) ) # Render the list view return self.repository_grid( trans, **kwd )
def browse_repositories( self, trans, **kwd ): # We add params to the keyword dict in this method in order to rename the param # with an "f-" prefix, simulating filtering by clicking a search link. We have # to take this approach because the "-" character is illegal in HTTP requests. if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if operation == "view_or_manage_repository": return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', **kwd ) ) elif operation == "edit_repository": return trans.response.send_redirect( web.url_for( controller='repository', action='edit_repository', **kwd ) ) elif operation == "repositories_by_user": # Eliminate the current filters if any exist. for k, v in kwd.items(): if k.startswith( 'f-' ): del kwd[ k ] if 'user_id' in kwd: user = suc.get_user( trans, kwd[ 'user_id' ] ) kwd[ 'f-email' ] = user.email del kwd[ 'user_id' ] else: # The received id is the repository id, so we need to get the id of the user # that uploaded the repository. repository_id = kwd.get( 'id', None ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) kwd[ 'f-email' ] = repository.user.email elif operation == "repositories_by_category": # Eliminate the current filters if any exist. for k, v in kwd.items(): if k.startswith( 'f-' ): del kwd[ k ] category_id = kwd.get( 'id', None ) category = suc.get_category( trans, category_id ) kwd[ 'f-Category.name' ] = category.name elif operation == "receive email alerts": if kwd[ 'id' ]: kwd[ 'caller' ] = 'browse_repositories' return trans.response.send_redirect( web.url_for( controller='repository', action='set_email_alerts', **kwd ) ) else: del kwd[ 'operation' ] elif operation == 'delete': return self.delete_repository( trans, **kwd ) elif operation == "undelete": return self.undelete_repository( trans, **kwd ) # The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One # of the many select fields on the grid performed the refresh_on_change, so we loop through # all of the received values to see which value is not the repository tip. If we find it, we # know the refresh_on_change occurred, and we have the necessary repository id and change set # revision to pass on. for k, v in kwd.items(): changeset_revision_str = 'changeset_revision_' if k.startswith( changeset_revision_str ): repository_id = trans.security.encode_id( int( k.lstrip( changeset_revision_str ) ) ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) if repository.tip( trans.app ) != v: return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', operation='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=v ) ) # Render the list view return self.repository_grid( trans, **kwd )
def upload( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) commit_message = kwd.get( 'commit_message', 'Uploaded' ) category_ids = util.listify( kwd.get( 'category_id', '' ) ) categories = suc.get_categories( trans ) repository_id = kwd.get( 'repository_id', '' ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) ) remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) ) uploaded_file = None upload_point = commit_util.get_upload_point( repository, **kwd ) tip = repository.tip( trans.app ) file_data = kwd.get( 'file_data', '' ) url = kwd.get( 'url', '' ) # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new( trans.app ) uploaded_directory = None if kwd.get( 'upload_button', False ): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None elif url and url.startswith( 'hg' ): # Use mercurial clone to fetch repository, contents will then be copied over. uploaded_directory = tempfile.mkdtemp() repo_url = 'http%s' % url[ len( 'hg' ): ] repo_url = repo_url.encode( 'ascii', 'replace' ) commands.clone( suc.get_configured_ui(), repo_url, uploaded_directory ) elif url: valid_url = True try: stream = urllib.urlopen( url ) except Exception, e: valid_url = False message = 'Error uploading file via http: %s' % str( e ) status = 'error' uploaded_file = None if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open( uploaded_file_name, 'wb' ) while 1: chunk = stream.read( util.CHUNK_SIZE ) if not chunk: break uploaded_file.write( chunk ) uploaded_file.flush() uploaded_file_filename = url.split( '/' )[ -1 ] isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0 elif file_data not in ( '', None ): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name uploaded_file_filename = os.path.split( file_data.filename )[ -1 ] isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0 if uploaded_file or uploaded_directory: ok = True isgzip = False isbz2 = False if uploaded_file: if uncompress_file: isgzip = checkers.is_gzip( uploaded_file_name ) if not isgzip: isbz2 = checkers.is_bz2( uploaded_file_name ) if isempty: tar = None istar = False else: # Determine what we have - a single file or an archive try: if ( isgzip or isbz2 ) and uncompress_file: # Open for reading with transparent compression. tar = tarfile.open( uploaded_file_name, 'r:*' ) else: tar = tarfile.open( uploaded_file_name ) istar = True except tarfile.ReadError, e: tar = None istar = False else: # Uploaded directory istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_tar( trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_directory( trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) else: if ( isgzip or isbz2 ) and uncompress_file: uploaded_file_filename = commit_util.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip=isgzip, isbz2=isbz2 ) if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named <b>tool_dependencies.xml</b>.' if ok: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) ) else: full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) # Move some version of the uploaded file to the load_point within the repository hierarchy. if uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name, unpopulate=False ) if error_message: ok = False message = error_message status = 'error' elif altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, full_path ) else: shutil.move( uploaded_file_name, full_path ) elif uploaded_file_filename in [ suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if it defines a complex repository dependency definition whose changeset_revision values # are missing and if so, set them appropriately. altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) if error_message: ok = False message = error_message status = 'error' if ok: if altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, full_path ) else: shutil.move( uploaded_file_name, full_path ) else: shutil.move( uploaded_file_name, full_path ) if ok: # See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure # content is appropriate. check_contents = commit_util.check_file_contents_for_email_alerts( trans ) if check_contents and os.path.isfile( full_path ): content_alert_str = commit_util.check_file_content_for_html_and_images( full_path ) else: content_alert_str = '' commands.add( repo.ui, repo, full_path ) # Convert from unicode to prevent "TypeError: array item must be char" full_path = full_path.encode( 'ascii', 'replace' ) commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) if full_path.endswith( 'tool_data_table_conf.xml.sample' ): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries # to the in-memory trans.app.tool_data_tables dictionary. error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path ) if error: message = '%s<br/>%s' % ( message, error_message ) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions ) != 1 suc.handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) if ok: # Update the repository files for browsing. suc.update_repository( repo ) # Get the new repository tip. if tip == repository.tip( trans.app ): message = 'No changes to repository. ' status = 'warning' else: if ( isgzip or isbz2 ) and uncompress_file: uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' if uploaded_directory: source_type = "repository" source = url else: source_type = "file" source = uploaded_file_filename message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str ) if istar and ( undesirable_dirs_removed or undesirable_files_removed ): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed if istar and remove_repo_files_not_in_tar and files_to_remove: if upload_point is not None: message += " %d files were removed from the repository relative to the selected upload point '%s'. " % ( len( files_to_remove ), upload_point ) else: message += " %d files were removed from the repository root. " % len( files_to_remove ) kwd[ 'message' ] = message metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd ) if repository.metadata_revisions: # A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload. metadata_dict = repository.metadata_revisions[0].metadata else: metadata_dict = {} if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: change_repository_type_message = tool_dependency_util.generate_message_for_repository_type_change( trans, repository ) if change_repository_type_message: message += change_repository_type_message status = 'warning' else: # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies weren't loaded due to a requirement tag mismatch # or some other problem. Tool dependency definitions can define orphan tool dependencies (no relationship to any tools contained in the repository), # so warning messages are important because orphans are always valid. The repository owner must be warned in case they did not intend to define an # orphan dependency, but simply provided incorrect information (tool shed, name owner, changeset_revision) for the definition. orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata_dict ) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. invalid_tool_dependencies_message = tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict ) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict ) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. tool_util.reset_tool_data_tables( trans.app ) if uploaded_directory: suc.remove_dir( uploaded_directory ) trans.response.send_redirect( web.url_for( controller='repository', action='browse_repository', id=repository_id, commit_message='Deleted selected files', message=message, status=status ) ) else: if uploaded_directory: suc.remove_dir( uploaded_directory ) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. tool_util.reset_tool_data_tables( trans.app )
def create_review( self, trans, **kwd ): # The value of the received id is the encoded repository id. message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) changeset_revision = kwd.get( 'changeset_revision', None ) previous_review_id = kwd.get( 'previous_review_id', None ) create_without_copying = 'create_without_copying' in kwd if repository_id: if changeset_revision: # Make sure there is not already a review of the revision by the user. repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) if review_util.get_review_by_repository_id_changeset_revision_user_id( app=trans.app, repository_id=repository_id, changeset_revision=changeset_revision, user_id=trans.security.encode_id( trans.user.id ) ): message = "You have already created a review for revision <b>%s</b> of repository <b>%s</b>." % ( changeset_revision, escape( repository.name ) ) status = "error" else: # See if there are any reviews for previous changeset revisions that the user can copy. if not create_without_copying and \ not previous_review_id and \ review_util.has_previous_repository_reviews( trans.app, repository, changeset_revision ): return trans.response.send_redirect( web.url_for( controller='repository_review', action='select_previous_review', **kwd ) ) # A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated # with the received changeset_revision. repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: review = trans.app.model.RepositoryReview( repository_id=repository_metadata.repository_id, changeset_revision=changeset_revision, user_id=trans.user.id, rating=None, deleted=False ) trans.sa_session.add( review ) trans.sa_session.flush() if previous_review_id: review_to_copy = review_util.get_review( trans.app, previous_review_id ) self.copy_review( trans, review_to_copy, review ) review_id = trans.security.encode_id( review.id ) message = "Begin your review of revision <b>%s</b> of repository <b>%s</b>." \ % ( changeset_revision, repository.name ) status = 'done' trans.response.send_redirect( web.url_for( controller='repository_review', action='edit_review', id=review_id, message=message, status=status ) ) else: message = "A new review cannot be created for revision <b>%s</b> of repository <b>%s</b>. Select a valid revision and try again." \ % ( changeset_revision, escape( repository.name ) ) kwd[ 'message' ] = message kwd[ 'status' ] = 'error' else: return trans.response.send_redirect( web.url_for( controller='repository_review', action='manage_repository_reviews', **kwd ) ) return trans.response.send_redirect( web.url_for( controller='repository_review', action='view_or_manage_repository', **kwd ) )
def upload(self, trans, **kwd): message = kwd.get('message', '') status = kwd.get('status', 'done') commit_message = kwd.get('commit_message', 'Uploaded') category_ids = util.listify(kwd.get('category_id', '')) categories = suc.get_categories(trans) repository_id = kwd.get('repository_id', '') repository = suc.get_repository_in_tool_shed(trans, repository_id) repo_dir = repository.repo_path(trans.app) repo = hg.repository(suc.get_configured_ui(), repo_dir) uncompress_file = util.string_as_bool( kwd.get('uncompress_file', 'true')) remove_repo_files_not_in_tar = util.string_as_bool( kwd.get('remove_repo_files_not_in_tar', 'true')) uploaded_file = None upload_point = commit_util.get_upload_point(repository, **kwd) tip = repository.tip(trans.app) file_data = kwd.get('file_data', '') url = kwd.get('url', '') # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new(trans.app) uploaded_directory = None if kwd.get('upload_button', False): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None elif url and url.startswith('hg'): # Use mercurial clone to fetch repository, contents will then be copied over. uploaded_directory = tempfile.mkdtemp() repo_url = 'http%s' % url[len('hg'):] repo_url = repo_url.encode('ascii', 'replace') commands.clone(suc.get_configured_ui(), repo_url, uploaded_directory) elif url: valid_url = True try: stream = urllib.urlopen(url) except Exception, e: valid_url = False message = 'Error uploading file via http: %s' % str(e) status = 'error' uploaded_file = None if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open(uploaded_file_name, 'wb') while 1: chunk = stream.read(util.CHUNK_SIZE) if not chunk: break uploaded_file.write(chunk) uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize( os.path.abspath(uploaded_file_name)) == 0 elif file_data not in ('', None): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name uploaded_file_filename = os.path.split(file_data.filename)[-1] isempty = os.path.getsize( os.path.abspath(uploaded_file_name)) == 0 if uploaded_file or uploaded_directory: ok = True isgzip = False isbz2 = False if uploaded_file: if uncompress_file: isgzip = checkers.is_gzip(uploaded_file_name) if not isgzip: isbz2 = checkers.is_bz2(uploaded_file_name) if isempty: tar = None istar = False else: # Determine what we have - a single file or an archive try: if (isgzip or isbz2) and uncompress_file: # Open for reading with transparent compression. tar = tarfile.open(uploaded_file_name, 'r:*') else: tar = tarfile.open(uploaded_file_name) istar = True except tarfile.ReadError, e: tar = None istar = False else: # Uploaded directory istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_tar( trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_directory( trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) else: if (isgzip or isbz2) and uncompress_file: uploaded_file_filename = commit_util.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2) if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named <b>tool_dependencies.xml</b>.' if ok: if upload_point is not None: full_path = os.path.abspath( os.path.join(repo_dir, upload_point, uploaded_file_filename)) else: full_path = os.path.abspath( os.path.join(repo_dir, uploaded_file_filename)) # Move some version of the uploaded file to the load_point within the repository hierarchy. if uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name) if altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) elif uploaded_file_filename in [ suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if it defines a complex repository dependency definition whose changeset_revision values # are missing and if so, set them appropriately. altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name) if altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) else: shutil.move(uploaded_file_name, full_path) # See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure # content is appropriate. check_contents = commit_util.check_file_contents_for_email_alerts( trans) if check_contents and os.path.isfile(full_path): content_alert_str = commit_util.check_file_content_for_html_and_images( full_path) else: content_alert_str = '' commands.add(repo.ui, repo, full_path) # Convert from unicode to prevent "TypeError: array item must be char" full_path = full_path.encode('ascii', 'replace') commands.commit(repo.ui, repo, full_path, user=trans.user.username, message=commit_message) if full_path.endswith( 'tool_data_table_conf.xml.sample'): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries # to the in-memory trans.app.tool_data_tables dictionary. error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path) if error: message = '%s<br/>%s' % (message, error_message) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions) != 1 suc.handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only) if ok: # Update the repository files for browsing. suc.update_repository(repo) # Get the new repository tip. if tip == repository.tip(trans.app): message = 'No changes to repository. ' status = 'warning' else: if (isgzip or isbz2) and uncompress_file: uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' if uploaded_directory: source_type = "repository" source = url else: source_type = "file" source = uploaded_file_filename message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str) if istar and (undesirable_dirs_removed or undesirable_files_removed): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed if istar and remove_repo_files_not_in_tar and files_to_remove: if upload_point is not None: message += " %d files were removed from the repository relative to the selected upload point '%s'. " % ( len(files_to_remove), upload_point) else: message += " %d files were removed from the repository root. " % len( files_to_remove) kwd['message'] = message metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd) if repository.metadata_revisions: # A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload. metadata_dict = repository.metadata_revisions[ 0].metadata else: metadata_dict = {} if str(repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies weren't loaded due to a requirement tag mismatch # or some other problem. Tool dependency definitions can define orphan tool dependencies (no relationship to any tools contained in the repository), # so warning messages are important because orphans are always valid. The repository owner must be warned in case they did not intend to define an # orphan dependency, but simply provided incorrect information (tool shed, name owner, changeset_revision) for the definition. # Handle messaging for orphan tool dependencies. orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata_dict) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. invalid_tool_dependencies_message = tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. tool_util.reset_tool_data_tables(trans.app) trans.response.send_redirect( web.url_for(controller='repository', action='browse_repository', id=repository_id, commit_message='Deleted selected files', message=message, status=status)) else: status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. tool_util.reset_tool_data_tables(trans.app)
def create_changeset_revision( self, trans, id, payload, **kwd ): """ POST /api/repositories/{encoded_repository_id}/changeset_revision Create a new tool shed repository commit - leaving PUT on parent resource open for updating meta-attributes of the repository (and Galaxy doesn't allow PUT multipart data anyway https://trello.com/c/CQwmCeG6). :param id: the encoded id of the Repository object The following parameters may be included in the payload. :param commit_message: hg commit message for update. """ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False ) tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False ) repository = suc.get_repository_in_tool_shed( trans.app, id ) repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) upload_point = commit_util.get_upload_point( repository, **kwd ) tip = repository.tip( trans.app ) file_data = payload.get('file') # Code stolen from gx's upload_common.py if isinstance( file_data, FieldStorage ): assert not isinstance( file_data.file, StringIO.StringIO ) assert file_data.file.name != '<fdopen>' local_filename = util.mkstemp_ln( file_data.file.name, 'upload_file_data_' ) file_data.file.close() file_data = dict( filename=file_data.filename, local_filename=local_filename ) elif type( file_data ) == dict and 'local_filename' not in file_data: raise Exception( 'Uploaded file was encoded in a way not understood.' ) commit_message = kwd.get( 'commit_message', 'Uploaded' ) uploaded_file = open(file_data['local_filename'], 'rb') uploaded_file_name = file_data['local_filename'] isgzip = False isbz2 = False isgzip = checkers.is_gzip( uploaded_file_name ) if not isgzip: isbz2 = checkers.is_bz2( uploaded_file_name ) if ( isgzip or isbz2 ): # Open for reading with transparent compression. tar = tarfile.open( uploaded_file_name, 'r:*' ) else: tar = tarfile.open( uploaded_file_name ) new_repo_alert = False remove_repo_files_not_in_tar = True ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ repository_content_util.upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) if ok: # Update the repository files for browsing. hg_util.update_repository( repo ) # Get the new repository tip. if tip == repository.tip( trans.app ): trans.response.status = 400 message = 'No changes to repository.' ok = False else: rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app, user=trans.user, repository=repository ) status, error_message = \ rmm.set_repository_metadata_due_to_new_tip( trans.request.host, content_alert_str=content_alert_str, **kwd ) if error_message: ok = False trans.response.status = 500 message = error_message else: trans.response.status = 500 if not ok: return { "err_msg": message, "content_alert": content_alert_str } else: return { "message": message, "content_alert": content_alert_str }
repository, None, as_html=False ) else: message = "Successfully reset metadata on repository %s owned by %s" % \ ( str( repository.name ), str( repository.user.username ) ) except Exception, e: message = "Error resetting metadata on repository %s owned by %s: %s" % \ ( str( repository.name ), str( repository.user.username ), str( e ) ) status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results repository_id = payload.get( 'repository_id', None ) if repository_id is not None: repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) start_time = strftime( "%Y-%m-%d %H:%M:%S" ) log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) ) results = handle_repository( trans, start_time, repository ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time return json.dumps( results, sort_keys=True, indent=4 ) @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. :param id: the encoded id of the Repository object """
repository, None, as_html=False) else: message = "Successfully reset metadata on repository %s owned by %s" % \ ( str( repository.name ), str( repository.user.username ) ) except Exception, e: message = "Error resetting metadata on repository %s owned by %s: %s" % \ ( str( repository.name ), str( repository.user.username ), str( e ) ) status = '%s : %s' % (str(repository.name), message) results['repository_status'].append(status) return results repository_id = payload.get('repository_id', None) if repository_id is not None: repository = suc.get_repository_in_tool_shed( trans.app, repository_id) start_time = strftime("%Y-%m-%d %H:%M:%S") log.debug("%s...resetting metadata on repository %s" % (start_time, str(repository.name))) results = handle_repository(trans, start_time, repository) stop_time = strftime("%Y-%m-%d %H:%M:%S") results['stop_time'] = stop_time return json.dumps(results, sort_keys=True, indent=4) @web.expose_api_anonymous def show(self, trans, id, **kwd): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. :param id: the encoded id of the Repository object
def create_changeset_revision(self, trans, id, payload, **kwd): """ POST /api/repositories/{encoded_repository_id}/changeset_revision Create a new tool shed repository commit - leaving PUT on parent resource open for updating meta-attributes of the repository (and Galaxy doesn't allow PUT multipart data anyway https://trello.com/c/CQwmCeG6). :param id: the encoded id of the Repository object The following parameters may be included in the payload. :param commit_message: hg commit message for update. """ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False) tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False) repository = suc.get_repository_in_tool_shed(trans.app, id) repo_dir = repository.repo_path(trans.app) repo = hg_util.get_repo_for_repository(trans.app, repository=None, repo_path=repo_dir, create=False) upload_point = commit_util.get_upload_point(repository, **kwd) tip = repository.tip(trans.app) file_data = payload.get('file') # Code stolen from gx's upload_common.py if isinstance(file_data, FieldStorage): assert not isinstance(file_data.file, StringIO.StringIO) assert file_data.file.name != '<fdopen>' local_filename = util.mkstemp_ln(file_data.file.name, 'upload_file_data_') file_data.file.close() file_data = dict(filename=file_data.filename, local_filename=local_filename) elif type(file_data) == dict and 'local_filename' not in file_data: raise Exception( 'Uploaded file was encoded in a way not understood.') commit_message = kwd.get('commit_message', 'Uploaded') uploaded_file = open(file_data['local_filename'], 'rb') uploaded_file_name = file_data['local_filename'] isgzip = False isbz2 = False isgzip = checkers.is_gzip(uploaded_file_name) if not isgzip: isbz2 = checkers.is_bz2(uploaded_file_name) if (isgzip or isbz2): # Open for reading with transparent compression. tar = tarfile.open(uploaded_file_name, 'r:*') else: tar = tarfile.open(uploaded_file_name) new_repo_alert = False remove_repo_files_not_in_tar = True ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ repository_content_util.upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) if ok: # Update the repository files for browsing. hg_util.update_repository(repo) # Get the new repository tip. if tip == repository.tip(trans.app): trans.response.status = 400 message = 'No changes to repository.' ok = False else: rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app, user=trans.user, repository=repository) status, error_message = \ rmm.set_repository_metadata_due_to_new_tip( trans.request.host, content_alert_str=content_alert_str, **kwd ) if error_message: ok = False trans.response.status = 500 message = error_message else: trans.response.status = 500 if not ok: return {"err_msg": message, "content_alert": content_alert_str} else: return {"message": message, "content_alert": content_alert_str}
def export_repository( trans, tool_shed_url, repository_id, repository_name, changeset_revision, file_type, export_repository_dependencies, api=False ): repository = suc.get_repository_in_tool_shed( trans, repository_id ) repositories_archive_filename = generate_repository_archive_filename( tool_shed_url, str( repository.name ), str( repository.user.username ), changeset_revision, file_type, export_repository_dependencies=export_repository_dependencies, use_tmp_archive_dir=True ) if export_repository_dependencies: repo_info_dicts = get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision ) repository_ids = get_repository_ids( trans, repo_info_dicts ) ordered_repository_ids, ordered_repositories, ordered_changeset_revisions = order_components_for_import( trans, repository_ids, repo_info_dicts ) else: ordered_repository_ids = [] ordered_repositories = [] ordered_changeset_revisions = [] if repository: repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision ) if repository_metadata: ordered_repository_ids = [ repository_id ] ordered_repositories = [ repository ] ordered_changeset_revisions = [ repository_metadata.changeset_revision ] repositories_archive = None error_messages = '' lock = threading.Lock() lock.acquire( True ) try: repositories_archive = tarfile.open( repositories_archive_filename, "w:%s" % file_type ) exported_repository_registry = ExportedRepositoryRegistry() for index, repository_id in enumerate( ordered_repository_ids ): work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-export-er" ) ordered_repository = ordered_repositories[ index ] ordered_changeset_revision = ordered_changeset_revisions[ index ] repository_archive, error_message = generate_repository_archive( trans, work_dir, tool_shed_url, ordered_repository, ordered_changeset_revision, file_type ) if error_message: error_messages = '%s %s' % ( error_messages, error_message ) else: archive_name = str( os.path.basename( repository_archive.name ) ) repositories_archive.add( repository_archive.name, arcname=archive_name ) attributes, sub_elements = get_repository_attributes_and_sub_elements( ordered_repository, archive_name ) elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements ) exported_repository_registry.exported_repository_elems.append( elem ) suc.remove_dir( work_dir ) # Keep information about the export in a file name export_info.xml in the archive. sub_elements = generate_export_elem( tool_shed_url, repository, changeset_revision, export_repository_dependencies, api ) export_elem = xml_util.create_element( 'export_info', attributes=None, sub_elements=sub_elements ) tmp_export_info = xml_util.create_and_write_tmp_file( export_elem, use_indent=True ) repositories_archive.add( tmp_export_info, arcname='export_info.xml' ) # Write the manifest, which must preserve the order in which the repositories should be imported. exported_repository_root = xml_util.create_element( 'repositories' ) for exported_repository_elem in exported_repository_registry.exported_repository_elems: exported_repository_root.append( exported_repository_elem ) tmp_manifest = xml_util.create_and_write_tmp_file( exported_repository_root, use_indent=True ) repositories_archive.add( tmp_manifest, arcname='manifest.xml' ) except Exception, e: log.exception( str( e ) )