def get_repo_info_dict(app, user, repository_id, changeset_revision): repository = suc.get_repository_in_tool_shed(app, repository_id) repo = hg_util.get_repo_for_repository(app, repository=repository, repo_path=None, create=False) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository) repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, repository_id, changeset_revision) if not repository_metadata: # The received changeset_revision is no longer installable, so get the next changeset_revision # in the repository's changelog. This generally occurs only with repositories of type # repository_suite_definition or tool_dependency_definition. next_downloadable_changeset_revision = \ suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if next_downloadable_changeset_revision: repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, repository_id, next_downloadable_changeset_revision) if repository_metadata: # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the # returned repo_info_dict. metadata = repository_metadata.metadata if 'tools' in metadata: includes_tools = True else: includes_tools = False includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel repository_dependencies_dict = metadata.get('repository_dependencies', {}) repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', []) has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ suc.get_repository_dependency_types( repository_dependencies ) if 'tool_dependencies' in metadata: includes_tool_dependencies = True else: includes_tool_dependencies = False else: # Here's where we may have to handle enhancements to the callers. See above comment. includes_tools = False has_repository_dependencies = False has_repository_dependencies_only_if_compiling_contained_td = False includes_tool_dependencies = False includes_tools_for_display_in_tool_panel = False ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) repo_info_dict = create_repo_info_dict( app=app, repository_clone_url=repository_clone_url, changeset_revision=changeset_revision, ctx_rev=str(ctx.rev()), repository_owner=repository.user.username, repository_name=repository.name, repository=repository, repository_metadata=repository_metadata, tool_dependencies=None, repository_dependencies=None) return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td
def handle_sample_files_and_load_tool_from_tmp_config( self, repo, repository_id, changeset_revision, tool_config_filename, work_dir): tool = None message = '' ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) # We're not currently doing anything with the returned list of deleted_sample_files here. It is # intended to help handle sample files that are in the manifest, but have been deleted from disk. sample_files, deleted_sample_files = self.get_list_of_copied_sample_files( repo, ctx, dir=work_dir) if sample_files: self.app.config.tool_data_path = work_dir if 'tool_data_table_conf.xml.sample' in sample_files: # Load entries into the tool_data_tables if the tool requires them. tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml') if tool_data_table_config: error, message = self.tdtm.handle_sample_tool_data_table_conf_file( tool_data_table_config, persist=False) if error: log.debug(message) manifest_ctx, ctx_file = hg_util.get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision) if manifest_ctx and ctx_file: tool, message2 = self.load_tool_from_tmp_config( repo, repository_id, manifest_ctx, ctx_file, work_dir) message = self.concat_messages(message, message2) return tool, message, sample_files
def handle_sample_files_and_load_tool_from_tmp_config( self, repo, repository_id, changeset_revision, tool_config_filename, work_dir ): tool = None message = "" ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) # We're not currently doing anything with the returned list of deleted_sample_files here. It is # intended to help handle sample files that are in the manifest, but have been deleted from disk. sample_files, deleted_sample_files = self.get_list_of_copied_sample_files(repo, ctx, dir=work_dir) if sample_files: self.app.config.tool_data_path = work_dir if "tool_data_table_conf.xml.sample" in sample_files: # Load entries into the tool_data_tables if the tool requires them. tool_data_table_config = os.path.join(work_dir, "tool_data_table_conf.xml") if tool_data_table_config: error, message = self.tdtm.handle_sample_tool_data_table_conf_file( tool_data_table_config, persist=False ) if error: log.debug(message) manifest_ctx, ctx_file = hg_util.get_ctx_file_path_from_manifest(tool_config_filename, repo, changeset_revision) if manifest_ctx and ctx_file: tool, message2 = self.load_tool_from_tmp_config(repo, repository_id, manifest_ctx, ctx_file, work_dir) message = self.concat_messages(message, message2) return tool, message, sample_files
def get_repo_info_dict( app, user, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( app, repository_id ) repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, repository_id, changeset_revision ) if not repository_metadata: # The received changeset_revision is no longer installable, so get the next changeset_revision # in the repository's changelog. This generally occurs only with repositories of type # repository_suite_definition or tool_dependency_definition. next_downloadable_changeset_revision = \ suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if next_downloadable_changeset_revision: repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, repository_id, next_downloadable_changeset_revision ) if repository_metadata: # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the # returned repo_info_dict. metadata = repository_metadata.metadata if 'tools' in metadata: includes_tools = True else: includes_tools = False includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ suc.get_repository_dependency_types( repository_dependencies ) if 'tool_dependencies' in metadata: includes_tool_dependencies = True else: includes_tool_dependencies = False else: # Here's where we may have to handle enhancements to the callers. See above comment. includes_tools = False has_repository_dependencies = False has_repository_dependencies_only_if_compiling_contained_td = False includes_tool_dependencies = False includes_tools_for_display_in_tool_panel = False ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = create_repo_info_dict( app=app, repository_clone_url=repository_clone_url, changeset_revision=changeset_revision, ctx_rev=str( ctx.rev() ), repository_owner=repository.user.username, repository_name=repository.name, repository=repository, repository_metadata=repository_metadata, tool_dependencies=None, repository_dependencies=None ) return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td
def get_metadata_changeset_revisions( repository, repo ): """ Return an unordered list of changeset_revisions and changeset numbers that are defined as installable. """ changeset_tups = [] for repository_metadata in repository.downloadable_revisions: ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) if ctx: rev = ctx.rev() else: rev = -1 changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) return sorted( changeset_tups )
def get_metadata_changeset_revisions(repository, repo): """ Return an unordered list of changeset_revisions and changeset numbers that are defined as installable. """ changeset_tups = [] for repository_metadata in repository.downloadable_revisions: ctx = hg_util.get_changectx_for_changeset(repo, repository_metadata.changeset_revision) if ctx: rev = ctx.rev() else: rev = -1 changeset_tups.append((rev, repository_metadata.changeset_revision)) return sorted(changeset_tups)
def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False, downloadable=True ): """ Return a list of changesets for the provided repository. """ if downloadable: metadata_revisions = repository.downloadable_revisions else: metadata_revisions = repository.metadata_revisions changeset_tups = [] for repository_metadata in metadata_revisions: ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) if ctx: rev = '%04d' % ctx.rev() else: rev = -1 changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) if sort_revisions: changeset_tups.sort( key=itemgetter( 0 ), reverse=reverse ) return changeset_tups
def get_metadata_revisions(repository, repo, sort_revisions=True, reverse=False, downloadable=True): """ Return a list of changesets for the provided repository. """ if downloadable: metadata_revisions = repository.downloadable_revisions else: metadata_revisions = repository.metadata_revisions changeset_tups = [] for repository_metadata in metadata_revisions: ctx = hg_util.get_changectx_for_changeset(repo, repository_metadata.changeset_revision) if ctx: rev = '%04d' % ctx.rev() else: rev = -1 changeset_tups.append((rev, repository_metadata.changeset_revision)) if sort_revisions: changeset_tups.sort(key=itemgetter(0), reverse=reverse) return changeset_tups
def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_path=None): """ Return a dictionary of valid readme file name <-> readme file content pairs for all readme files defined in the received metadata. Since the received changeset_revision (which is associated with the received metadata) may not be the latest installable changeset revision, the README file contents may not be available on disk. This method is used by both Galaxy and the Tool Shed. """ if app.name == 'galaxy': can_use_disk_files = True else: latest_downloadable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision(app, repository) can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision readme_files_dict = {} if metadata: if 'readme_files' in metadata: for relative_path_to_readme_file in metadata['readme_files']: readme_file_name = os.path.split(relative_path_to_readme_file)[1] if can_use_disk_files: if tool_path: full_path_to_readme_file = os.path.abspath(os.path.join(tool_path, relative_path_to_readme_file)) else: full_path_to_readme_file = os.path.abspath(relative_path_to_readme_file) text = None try: with open(full_path_to_readme_file, encoding='utf-8') as f: text = f.read() except Exception: log.exception("Error reading README file '%s' from disk", relative_path_to_readme_file) text = None if text: text_of_reasonable_length = basic_util.size_string(text) if text_of_reasonable_length.find('.. image:: ') >= 0: # Handle image display for README files that are contained in repositories in the tool shed or installed into Galaxy. try: text_of_reasonable_length = suc.set_image_paths(app, text_of_reasonable_length, encoded_repository_id=app.security.encode_id(repository.id)) except Exception: log.exception("Exception in build_readme_files_dict, so images may not be properly displayed") if readme_file_name.endswith('.rst'): text_of_reasonable_length = Template(rst_to_html(text_of_reasonable_length), input_encoding='utf-8', default_filters=['decode.utf8'], encoding_errors='replace') text_of_reasonable_length = text_of_reasonable_length.render(static_path=web.url_for('/static'), host_url=web.url_for('/', qualified=True)) text_of_reasonable_length = unicodify(text_of_reasonable_length) else: text_of_reasonable_length = basic_util.to_html_string(text_of_reasonable_length) readme_files_dict[readme_file_name] = text_of_reasonable_length else: # We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest. repo = repository.hg_repo ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) if ctx: fctx = hg_util.get_file_context_from_ctx(ctx, readme_file_name) if fctx and fctx not in ['DELETED']: try: text = unicodify(fctx.data()) readme_files_dict[readme_file_name] = basic_util.size_string(text) except Exception: log.exception("Error reading README file '%s' from repository manifest", relative_path_to_readme_file) return readme_files_dict
def build_readme_files_dict( app, repository, changeset_revision, metadata, tool_path=None ): """ Return a dictionary of valid readme file name <-> readme file content pairs for all readme files defined in the received metadata. Since the received changeset_revision (which is associated with the received metadata) may not be the latest installable changeset revision, the README file contents may not be available on disk. This method is used by both Galaxy and the Tool Shed. """ if app.name == 'galaxy': can_use_disk_files = True else: repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) latest_downloadable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision( app, repository, repo ) can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision readme_files_dict = {} if metadata: if 'readme_files' in metadata: for relative_path_to_readme_file in metadata[ 'readme_files' ]: readme_file_name = os.path.split( relative_path_to_readme_file )[ 1 ] if can_use_disk_files: if tool_path: full_path_to_readme_file = os.path.abspath( os.path.join( tool_path, relative_path_to_readme_file ) ) else: full_path_to_readme_file = os.path.abspath( relative_path_to_readme_file ) text = None try: f = open( full_path_to_readme_file, 'r' ) text = unicodify( f.read() ) f.close() except Exception as e: log.exception( "Error reading README file '%s' from disk", str( relative_path_to_readme_file ) ) text = None if text: text_of_reasonable_length = basic_util.size_string( text ) if text_of_reasonable_length.find( '.. image:: ' ) >= 0: # Handle image display for README files that are contained in repositories in the tool shed or installed into Galaxy. lock = threading.Lock() lock.acquire( True ) try: text_of_reasonable_length = suc.set_image_paths( app, app.security.encode_id( repository.id ), text_of_reasonable_length ) except Exception as e: log.exception( "Exception in build_readme_files_dict, so images may not be properly displayed" ) finally: lock.release() if readme_file_name.endswith( '.rst' ): text_of_reasonable_length = Template( rst_to_html( text_of_reasonable_length ), input_encoding='utf-8', output_encoding='utf-8', default_filters=[ 'decode.utf8' ], encoding_errors='replace' ) text_of_reasonable_length = text_of_reasonable_length.render( static_path=web.url_for( '/static' ), host_url=web.url_for( '/', qualified=True ) ) text_of_reasonable_length = unicodify( text_of_reasonable_length ) else: text_of_reasonable_length = basic_util.to_html_string( text_of_reasonable_length ) readme_files_dict[ readme_file_name ] = text_of_reasonable_length else: # We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest. ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) if ctx: fctx = hg_util.get_file_context_from_ctx( ctx, readme_file_name ) if fctx and fctx not in [ 'DELETED' ]: try: text = unicodify( fctx.data() ) readme_files_dict[ readme_file_name ] = basic_util.size_string( text ) except Exception as e: log.exception( "Error reading README file '%s' from repository manifest: %s" % ( str( relative_path_to_readme_file ), str( e ) ) ) return readme_files_dict
lock.release() if readme_file_name.endswith( '.rst' ): text_of_reasonable_length = Template( rst_to_html( text_of_reasonable_length ), input_encoding='utf-8', output_encoding='utf-8', default_filters=[ 'decode.utf8' ], encoding_errors='replace' ) text_of_reasonable_length = text_of_reasonable_length.render( static_path=web.url_for( '/static' ), host_url=web.url_for( '/', qualified=True ) ) text_of_reasonable_length = unicodify( text_of_reasonable_length ) else: text_of_reasonable_length = basic_util.to_html_string( text_of_reasonable_length ) readme_files_dict[ readme_file_name ] = text_of_reasonable_length else: # We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest. ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) if ctx: fctx = hg_util.get_file_context_from_ctx( ctx, readme_file_name ) if fctx and fctx not in [ 'DELETED' ]: try: text = unicodify( fctx.data() ) readme_files_dict[ readme_file_name ] = basic_util.size_string( text ) except Exception, e: log.exception( "Error reading README file '%s' from repository manifest: %s" % ( str( relative_path_to_readme_file ), str( e ) ) ) return readme_files_dict def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ): """ Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
output_encoding='utf-8', default_filters=['decode.utf8'], encoding_errors='replace') text_of_reasonable_length = text_of_reasonable_length.render( static_path=web.url_for('/static'), host_url=web.url_for('/', qualified=True)) text_of_reasonable_length = unicodify( text_of_reasonable_length) else: text_of_reasonable_length = basic_util.to_html_string( text_of_reasonable_length) readme_files_dict[ readme_file_name] = text_of_reasonable_length else: # We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest. ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision) if ctx: fctx = hg_util.get_file_context_from_ctx( ctx, readme_file_name) if fctx and fctx not in ['DELETED']: try: text = unicodify(fctx.data()) readme_files_dict[ readme_file_name] = basic_util.size_string( text) except Exception, e: log.exception( "Error reading README file '%s' from repository manifest: %s" % \ ( str( relative_path_to_readme_file ), str( e ) ) ) return readme_files_dict