def upload(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') commit_message = escape(kwd.get('commit_message', 'Uploaded')) repository_id = kwd.get('repository_id', '') repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id) repo_dir = repository.repo_path(trans.app) uncompress_file = util.string_as_bool( kwd.get('uncompress_file', 'true')) remove_repo_files_not_in_tar = util.string_as_bool( kwd.get('remove_repo_files_not_in_tar', 'true')) uploaded_file = None upload_point = commit_util.get_upload_point(repository, **kwd) tip = repository.tip() file_data = kwd.get('file_data', '') url = kwd.get('url', '') # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new() uploaded_directory = None if kwd.get('upload_button', False): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None elif url and url.startswith('hg'): # Use mercurial clone to fetch repository, contents will then be copied over. uploaded_directory = tempfile.mkdtemp() repo_url = 'http%s' % url[len('hg'):] cloned_ok, error_message = hg_util.clone_repository( repo_url, uploaded_directory) if not cloned_ok: message = 'Error uploading via mercurial clone: %s' % error_message status = 'error' basic_util.remove_dir(uploaded_directory) uploaded_directory = None elif url: valid_url = True try: stream = requests.get(url, stream=True) except Exception as e: valid_url = False message = 'Error uploading file via http: %s' % util.unicodify( e) status = 'error' uploaded_file = None if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open(uploaded_file_name, 'wb') for chunk in stream.iter_content( chunk_size=util.CHUNK_SIZE): if chunk: uploaded_file.write(chunk) uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize( os.path.abspath(uploaded_file_name)) == 0 elif file_data not in ('', None): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name uploaded_file_filename = os.path.split(file_data.filename)[-1] isempty = os.path.getsize( os.path.abspath(uploaded_file_name)) == 0 if uploaded_file or uploaded_directory: rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False) tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False) stdtm = ShedToolDataTableManager(trans.app) ok = True isgzip = False isbz2 = False if uploaded_file: if uncompress_file: isgzip = checkers.is_gzip(uploaded_file_name) if not isgzip: isbz2 = checkers.is_bz2(uploaded_file_name) if isempty: tar = None istar = False else: # Determine what we have - a single file or an archive try: if (isgzip or isbz2) and uncompress_file: # Open for reading with transparent compression. tar = tarfile.open(uploaded_file_name, 'r:*') else: tar = tarfile.open(uploaded_file_name) istar = True except tarfile.ReadError: tar = None istar = False else: # Uploaded directory istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ repository_content_util.upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_directory(trans, rdah, tdah, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert) else: if (isgzip or isbz2) and uncompress_file: uploaded_file_filename = commit_util.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip=isgzip, isbz2=isbz2) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named ' message += '<b>repository_dependencies.xml</b>.' elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \ uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named ' message += '<b>tool_dependencies.xml</b>.' if ok: if upload_point is not None: full_path = os.path.abspath( os.path.join(repo_dir, upload_point, uploaded_file_filename)) else: full_path = os.path.abspath( os.path.join(repo_dir, uploaded_file_filename)) # Move some version of the uploaded file to the load_point within the repository hierarchy. if uploaded_file_filename in [ rt_util. REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if toolshed or changeset_revision attributes # are missing and if so, set them appropriately. altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name) if error_message: ok = False message = error_message status = 'error' elif altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) elif uploaded_file_filename in [ rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if changeset_revision values are # missing and if so, set them appropriately. altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name) if error_message: ok = False message = error_message status = 'error' if ok: if altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) else: shutil.move(uploaded_file_name, full_path) if ok: # See if any admin users have chosen to receive email alerts when a repository is updated. # If so, check every uploaded file to ensure content is appropriate. check_contents = commit_util.check_file_contents_for_email_alerts( trans.app) if check_contents and os.path.isfile(full_path): content_alert_str = commit_util.check_file_content_for_html_and_images( full_path) else: content_alert_str = '' hg_util.add_changeset(repo_dir, full_path) hg_util.commit_changeset( repo_dir, full_path_to_changeset=full_path, username=trans.user.username, message=commit_message) if full_path.endswith( 'tool_data_table_conf.xml.sample'): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded # by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables # dictionary. error, error_message = stdtm.handle_sample_tool_data_table_conf_file( full_path, persist=False) if error: message = '%s<br/>%s' % (message, error_message) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions) != 1 suc.handle_email_alerts( trans.app, trans.request.host, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only) if ok: # Update the repository files for browsing. hg_util.update_repository(repo_dir) # Get the new repository tip. if tip == repository.tip(): message = 'No changes to repository. ' status = 'warning' else: if (isgzip or isbz2) and uncompress_file: uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' if uploaded_directory: source_type = "repository" source = url else: source_type = "file" source = uploaded_file_filename message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \ (source_type, escape(source), uncompress_str) if istar and (undesirable_dirs_removed or undesirable_files_removed): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed message += "were removed from the archive. " if istar and remove_repo_files_not_in_tar and files_to_remove: if upload_point is not None: message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \ (len(files_to_remove), upload_point) else: message += " %d files were removed from the repository root. " % len( files_to_remove) rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app, user=trans.user, repository=repository) status, error_message = \ rmm.set_repository_metadata_due_to_new_tip(trans.request.host, content_alert_str=content_alert_str, **kwd) if error_message: message = error_message kwd['message'] = message if repository.metadata_revisions: # A repository's metadata revisions are order descending by update_time, so the zeroth revision # will be the tip just after an upload. metadata_dict = repository.metadata_revisions[ 0].metadata else: metadata_dict = {} dd = dependency_display.DependencyDisplayer(trans.app) if str(repository.type) not in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app, repository) if change_repository_type_message: message += change_repository_type_message status = 'warning' else: # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies # weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency # definitions can define orphan tool dependencies (no relationship to any tools contained in the # repository), so warning messages are important because orphans are always valid. The repository # owner must be warned in case they did not intend to define an orphan dependency, but simply # provided incorrect information (tool shed, name owner, changeset_revision) for the definition. orphan_message = dd.generate_message_for_orphan_tool_dependencies( repository, metadata_dict) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies( metadata_dict) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = \ dd.generate_message_for_invalid_repository_dependencies(metadata_dict, error_from_tuple=True) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. stdtm.reset_tool_data_tables() if uploaded_directory: basic_util.remove_dir(uploaded_directory) trans.response.send_redirect( web.url_for(controller='repository', action='browse_repository', id=repository_id, commit_message='Deleted selected files', message=message, status=status)) else: if uploaded_directory: basic_util.remove_dir(uploaded_directory) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. stdtm.reset_tool_data_tables() return trans.fill_template( '/webapps/tool_shed/repository/upload.mako', repository=repository, changeset_revision=tip, url=url, commit_message=commit_message, uncompress_file=uncompress_file, remove_repo_files_not_in_tar=remove_repo_files_not_in_tar, message=message, status=status)
def handle_directory_changes(app, host, username, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed): repo = hg_util.get_repo_for_repository(app, repository=repository, repo_path=None, create=False) content_alert_str = '' files_to_remove = [] filenames_in_archive = [ os.path.join(full_path, name) for name in filenames_in_archive ] if remove_repo_files_not_in_tar and not repository.is_new(app): # We have a repository that is not new (it contains files), so discover those files that are in the # repository, but not in the uploaded archive. for root, dirs, files in os.walk(full_path): if root.find('.hg') < 0 and root.find('hgrc') < 0: for undesirable_dir in UNDESIRABLE_DIRS: if undesirable_dir in dirs: dirs.remove(undesirable_dir) undesirable_dirs_removed += 1 for undesirable_file in UNDESIRABLE_FILES: if undesirable_file in files: files.remove(undesirable_file) undesirable_files_removed += 1 for name in files: full_name = os.path.join(root, name) if full_name not in filenames_in_archive: files_to_remove.append(full_name) for repo_file in files_to_remove: # Remove files in the repository (relative to the upload point) that are not in # the uploaded archive. try: hg_util.remove_file(repo.ui, repo, repo_file, force=True) except Exception as e: log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str(e)) relative_selected_file = repo_file.split( 'repo_%d' % repository.id)[1].lstrip('/') repo.dirstate.remove(relative_selected_file) repo.dirstate.write() absolute_selected_file = os.path.abspath(repo_file) if os.path.isdir(absolute_selected_file): try: os.rmdir(absolute_selected_file) except OSError as e: # The directory is not empty. pass elif os.path.isfile(absolute_selected_file): os.remove(absolute_selected_file) dir = os.path.split(absolute_selected_file)[0] try: os.rmdir(dir) except OSError as e: # The directory is not empty. pass # See if any admin users have chosen to receive email alerts when a repository is updated. # If so, check every uploaded file to ensure content is appropriate. check_contents = check_file_contents_for_email_alerts(app) for filename_in_archive in filenames_in_archive: # Check file content to ensure it is appropriate. if check_contents and os.path.isfile(filename_in_archive): content_alert_str += check_file_content_for_html_and_images( filename_in_archive) hg_util.add_changeset(repo.ui, repo, filename_in_archive) if filename_in_archive.endswith('tool_data_table_conf.xml.sample'): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded # by parsing the file and adding new entries to the in-memory app.tool_data_tables # dictionary. stdtm = ShedToolDataTableManager(app) error, message = stdtm.handle_sample_tool_data_table_conf_file( filename_in_archive, persist=False) if error: return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed hg_util.commit_changeset(repo.ui, repo, full_path_to_changeset=full_path, username=username, message=commit_message) admin_only = len(repository.downloadable_revisions) != 1 suc.handle_email_alerts(app, host, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only) return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
class ToolValidator(object): def __init__(self, app): self.app = app self.stdtm = ShedToolDataTableManager(self.app) def can_use_tool_config_disk_file(self, repository, repo, file_path, changeset_revision): """ Determine if repository's tool config file on disk can be used. This method is restricted to tool config files since, with the exception of tool config files, multiple files with the same name will likely be in various directories in the repository and we're comparing file names only (not relative paths). """ if not file_path or not os.path.exists(file_path): # The file no longer exists on disk, so it must have been deleted at some previous # point in the change log. return False if changeset_revision == repository.tip(self.app): return True file_name = basic_util.strip_path(file_path) latest_version_of_file = \ self.get_latest_tool_config_revision_from_repository_manifest(repo, file_name, changeset_revision) can_use_disk_file = filecmp.cmp(file_path, latest_version_of_file) try: os.unlink(latest_version_of_file) except Exception: pass return can_use_disk_file def check_tool_input_params(self, repo_dir, tool_config_name, tool, sample_files): """ Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make sure the files exist. """ invalid_files_and_errors_tups = [] for input_param in tool.input_params: if isinstance(input_param, parameters.basic.SelectToolParameter) and input_param.is_dynamic: # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, # make sure all requirements exist. options = input_param.dynamic_options or input_param.options if options and isinstance(options, dynamic_options.DynamicOptions): if options.tool_data_table or options.missing_tool_data_table_name: # Make sure the repository contains a tool_data_table_conf.xml.sample file. sample_tool_data_table_conf = hg_util.get_config_from_disk('tool_data_table_conf.xml.sample', repo_dir) if sample_tool_data_table_conf: error, correction_msg = \ self.stdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf, persist=False) if error: invalid_files_and_errors_tups.append(('tool_data_table_conf.xml.sample', correction_msg)) else: correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. " correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository " correction_msg += "that includes the required entry to correct this error.<br/>" invalid_tup = (tool_config_name, correction_msg) if invalid_tup not in invalid_files_and_errors_tups: invalid_files_and_errors_tups.append(invalid_tup) if options.index_file or options.tool_data_table and options.tool_data_table.missing_index_file: # Make sure the repository contains the required xxx.loc.sample file. index_file = options.index_file or options.tool_data_table.missing_index_file index_file_name = basic_util.strip_path(index_file) sample_found = False for sample_file in sample_files: sample_file_name = basic_util.strip_path(sample_file) if sample_file_name == '%s.sample' % index_file_name: options.index_file = index_file_name if options.tool_data_table: options.tool_data_table.missing_index_file = None sample_found = True break if not sample_found: correction_msg = "This file refers to a file named <b>%s</b>. " % str(index_file_name) correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % \ str(index_file_name) invalid_files_and_errors_tups.append((tool_config_name, correction_msg)) return invalid_files_and_errors_tups def concat_messages(self, msg1, msg2): if msg1: if msg2: message = '%s %s' % (msg1, msg2) else: message = msg1 elif msg2: message = msg2 else: message = '' return message def copy_disk_sample_files_to_dir(self, repo_files_dir, dest_path): """ Copy all files currently on disk that end with the .sample extension to the directory to which dest_path refers. """ sample_files = [] for root, dirs, files in os.walk(repo_files_dir): if root.find('.hg') < 0: for name in files: if name.endswith('.sample'): relative_path = os.path.join(root, name) tool_util.copy_sample_file(self.app, relative_path, dest_path=dest_path) sample_files.append(name) return sample_files def get_latest_tool_config_revision_from_repository_manifest(self, repo, filename, changeset_revision): """ Get the latest revision of a tool config file named filename from the repository manifest up to the value of changeset_revision. This method is restricted to tool_config files rather than any file since it is likely that, with the exception of tool config files, multiple files will have the same name in various directories within the repository. """ stripped_filename = basic_util.strip_path(filename) for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision): manifest_ctx = repo.changectx(changeset) for ctx_file in manifest_ctx.files(): ctx_file_name = basic_util.strip_path(ctx_file) if ctx_file_name == stripped_filename: try: fctx = manifest_ctx[ctx_file] except LookupError: # The ctx_file may have been moved in the change set. For example, # 'ncbi_blastp_wrapper.xml' was moved to 'tools/ncbi_blast_plus/ncbi_blastp_wrapper.xml', # so keep looking for the file until we find the new location. continue fh = tempfile.NamedTemporaryFile('wb', prefix="tmp-toolshed-gltcrfrm") tmp_filename = fh.name fh.close() fh = open(tmp_filename, 'wb') fh.write(fctx.data()) fh.close() return tmp_filename return None def get_list_of_copied_sample_files(self, repo, changeset_revision, dir): """ Find all sample files (files in the repository with the special .sample extension) in the reversed repository manifest up to changeset_revision. Copy each discovered file to dir and return the list of filenames. If a .sample file was added in a changeset and then deleted in a later changeset, it will be returned in the deleted_sample_files list. The caller will set the value of app.config.tool_data_path to dir in order to load the tools and generate metadata for them. """ deleted_sample_files = [] sample_files = [] for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision): changeset_ctx = repo.changectx(changeset) for ctx_file in changeset_ctx.files(): ctx_file_name = basic_util.strip_path(ctx_file) # If we decide in the future that files deleted later in the changelog should # not be used, we can use the following if statement. if ctx_file_name.endswith( '.sample' ) # and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files: if ctx_file_name.endswith('.sample') and ctx_file_name not in sample_files: fctx = hg_util.get_file_context_from_ctx(changeset_ctx, ctx_file) if fctx in ['DELETED']: # Since the possibly future used if statement above is commented out, the # same file that was initially added will be discovered in an earlier changeset # in the change log and fall through to the else block below. In other words, # if a file named blast2go.loc.sample was added in change set 0 and then deleted # in changeset 3, the deleted file in changeset 3 will be handled here, but the # later discovered file in changeset 0 will be handled in the else block below. # In this way, the file contents will always be found for future tools even though # the file was deleted. if ctx_file_name not in deleted_sample_files: deleted_sample_files.append(ctx_file_name) else: sample_files.append(ctx_file_name) tmp_ctx_file_name = os.path.join(dir, ctx_file_name.replace('.sample', '')) fh = open(tmp_ctx_file_name, 'wb') fh.write(fctx.data()) fh.close() return sample_files, deleted_sample_files def handle_sample_files_and_load_tool_from_disk(self, repo_files_dir, repository_id, tool_config_filepath, work_dir): """ Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. """ message = '' sample_files = self.copy_disk_sample_files_to_dir(repo_files_dir, work_dir) if sample_files: if 'tool_data_table_conf.xml.sample' in sample_files: # Load entries into the tool_data_tables if the tool requires them. tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml') error, message = self.stdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config, persist=False) tool, valid, message2 = self.load_tool_from_config(repository_id, tool_config_filepath) message = self.concat_messages(message, message2) return tool, valid, message, sample_files def handle_sample_files_and_load_tool_from_tmp_config(self, repo, repository_id, changeset_revision, tool_config_filename, work_dir): tool = None valid = False message = '' # We're not currently doing anything with the returned list of deleted_sample_files here. It is # intended to help handle sample files that are in the manifest, but have been deleted from disk. sample_files, deleted_sample_files = self.get_list_of_copied_sample_files(repo, changeset_revision, dir=work_dir) if sample_files: if 'tool_data_table_conf.xml.sample' in sample_files: # Load entries into the tool_data_tables if the tool requires them. tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml') error, message = self.stdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config, persist=False) manifest_ctx, ctx_file = hg_util.get_ctx_file_path_from_manifest(tool_config_filename, repo, changeset_revision) if manifest_ctx and ctx_file: tool, valid, message2 = self.load_tool_from_tmp_config(repo, repository_id, manifest_ctx, ctx_file, work_dir) message = self.concat_messages(message, message2) return tool, valid, message, sample_files def load_tool_from_changeset_revision(self, repository_id, changeset_revision, tool_config_filename): """ Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision is a valid (downloadable) changeset revision. The tool config will be located in the repository manifest between the received valid changeset revision and the first changeset revision in the repository, searching backwards. """ repository = repository_util.get_repository_in_tool_shed(self.app, repository_id) repo_files_dir = repository.repo_path(self.app) repo = hg_util.get_repo_for_repository(self.app, repo_path=repo_files_dir) tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository(repo_files_dir, tool_config_filename) work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr") can_use_disk_file = self.can_use_tool_config_disk_file(repository, repo, tool_config_filepath, changeset_revision) if can_use_disk_file: tool, valid, message, sample_files = \ self.handle_sample_files_and_load_tool_from_disk(repo_files_dir, repository_id, tool_config_filepath, work_dir) if tool is not None: invalid_files_and_errors_tups = \ self.check_tool_input_params(repo_files_dir, tool_config_filename, tool, sample_files) if invalid_files_and_errors_tups: message2 = tool_util.generate_message_for_invalid_tools(self.app, invalid_files_and_errors_tups, repository, metadata_dict=None, as_html=True, displaying_invalid_tool=True) message = self.concat_messages(message, message2) else: tool, valid, message, sample_files = \ self.handle_sample_files_and_load_tool_from_tmp_config(repo, repository_id, changeset_revision, tool_config_filename, work_dir) basic_util.remove_dir(work_dir) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.stdtm.reset_tool_data_tables() return repository, tool, valid, message def load_tool_from_config(self, repository_id, full_path): tool_source = get_tool_source( full_path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False), tool_location_fetcher=ToolLocationFetcher(), ) try: tool = create_tool_from_source(config_file=full_path, app=self.app, tool_source=tool_source, repository_id=repository_id, allow_code_files=False) valid = True error_message = None except KeyError as e: tool = None valid = False error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str(e) error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' error_message += 'this error. ' except Exception as e: tool = None valid = False error_message = str(e) return tool, valid, error_message def load_tool_from_tmp_config(self, repo, repository_id, ctx, ctx_file, work_dir): tool = None valid = False message = '' tmp_tool_config = hg_util.get_named_tmpfile_from_ctx(ctx, ctx_file, work_dir) if tmp_tool_config: tool_element, error_message = xml_util.parse_xml(tmp_tool_config) if tool_element is None: return tool, message # Look for external files required by the tool config. tmp_code_files = [] external_paths = Tool.get_externally_referenced_paths(tmp_tool_config) changeset_revision = str(ctx) for path in external_paths: tmp_code_file_name = hg_util.copy_file_from_manifest(repo, changeset_revision, path, work_dir) if tmp_code_file_name: tmp_code_files.append(tmp_code_file_name) tool, valid, message = self.load_tool_from_config(repository_id, tmp_tool_config) for tmp_code_file in tmp_code_files: try: os.unlink(tmp_code_file) except Exception: pass try: os.unlink(tmp_tool_config) except Exception: pass return tool, valid, message
def upload(self, trans, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') commit_message = escape(kwd.get('commit_message', 'Uploaded')) repository_id = kwd.get('repository_id', '') repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id) repo_dir = repository.repo_path(trans.app) repo = hg_util.get_repo_for_repository(trans.app, repository=None, repo_path=repo_dir, create=False) uncompress_file = util.string_as_bool(kwd.get('uncompress_file', 'true')) remove_repo_files_not_in_tar = util.string_as_bool(kwd.get('remove_repo_files_not_in_tar', 'true')) uploaded_file = None upload_point = commit_util.get_upload_point(repository, **kwd) tip = repository.tip(trans.app) file_data = kwd.get('file_data', '') url = kwd.get('url', '') # Part of the upload process is sending email notification to those that have registered to # receive them. One scenario occurs when the first change set is produced for the repository. # See the suc.handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new(trans.app) uploaded_directory = None if kwd.get('upload_button', False): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None elif url and url.startswith('hg'): # Use mercurial clone to fetch repository, contents will then be copied over. uploaded_directory = tempfile.mkdtemp() repo_url = 'http%s' % url[len('hg'):] repo_url = repo_url.encode('ascii', 'replace') try: commands.clone(hg_util.get_configured_ui(), repo_url, uploaded_directory) except Exception as e: message = 'Error uploading via mercurial clone: %s' % basic_util.to_html_string(str(e)) status = 'error' basic_util.remove_dir(uploaded_directory) uploaded_directory = None elif url: valid_url = True try: stream = requests.get(url, stream=True) except Exception as e: valid_url = False message = 'Error uploading file via http: %s' % str(e) status = 'error' uploaded_file = None if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open(uploaded_file_name, 'wb') for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE): if chunk: uploaded_file.write(chunk) uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0 elif file_data not in ('', None): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name uploaded_file_filename = os.path.split(file_data.filename)[-1] isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0 if uploaded_file or uploaded_directory: rdah = attribute_handlers.RepositoryDependencyAttributeHandler(trans.app, unpopulate=False) tdah = attribute_handlers.ToolDependencyAttributeHandler(trans.app, unpopulate=False) stdtm = ShedToolDataTableManager(trans.app) ok = True isgzip = False isbz2 = False if uploaded_file: if uncompress_file: isgzip = checkers.is_gzip(uploaded_file_name) if not isgzip: isbz2 = checkers.is_bz2(uploaded_file_name) if isempty: tar = None istar = False else: # Determine what we have - a single file or an archive try: if (isgzip or isbz2) and uncompress_file: # Open for reading with transparent compression. tar = tarfile.open(uploaded_file_name, 'r:*') else: tar = tarfile.open(uploaded_file_name) istar = True except tarfile.ReadError as e: tar = None istar = False else: # Uploaded directory istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ repository_content_util.upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_directory(trans, rdah, tdah, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert) else: if (isgzip or isbz2) and uncompress_file: uploaded_file_filename = commit_util.uncompress(repository, uploaded_file_name, uploaded_file_filename, isgzip=isgzip, isbz2=isbz2) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named ' message += '<b>repository_dependencies.xml</b>.' elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \ uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named ' message += '<b>tool_dependencies.xml</b>.' if ok: if upload_point is not None: full_path = os.path.abspath(os.path.join(repo_dir, upload_point, uploaded_file_filename)) else: full_path = os.path.abspath(os.path.join(repo_dir, uploaded_file_filename)) # Move some version of the uploaded file to the load_point within the repository hierarchy. if uploaded_file_filename in [rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME]: # Inspect the contents of the file to see if toolshed or changeset_revision attributes # are missing and if so, set them appropriately. altered, root_elem, error_message = rdah.handle_tag_attributes(uploaded_file_name) if error_message: ok = False message = error_message status = 'error' elif altered: tmp_filename = xml_util.create_and_write_tmp_file(root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) elif uploaded_file_filename in [rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME]: # Inspect the contents of the file to see if changeset_revision values are # missing and if so, set them appropriately. altered, root_elem, error_message = tdah.handle_tag_attributes(uploaded_file_name) if error_message: ok = False message = error_message status = 'error' if ok: if altered: tmp_filename = xml_util.create_and_write_tmp_file(root_elem) shutil.move(tmp_filename, full_path) else: shutil.move(uploaded_file_name, full_path) else: shutil.move(uploaded_file_name, full_path) if ok: # See if any admin users have chosen to receive email alerts when a repository is updated. # If so, check every uploaded file to ensure content is appropriate. check_contents = commit_util.check_file_contents_for_email_alerts(trans.app) if check_contents and os.path.isfile(full_path): content_alert_str = commit_util.check_file_content_for_html_and_images(full_path) else: content_alert_str = '' hg_util.add_changeset(repo.ui, repo, full_path) # Convert from unicode to prevent "TypeError: array item must be char" full_path = full_path.encode('ascii', 'replace') hg_util.commit_changeset(repo.ui, repo, full_path_to_changeset=full_path, username=trans.user.username, message=commit_message) if full_path.endswith('tool_data_table_conf.xml.sample'): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded # by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables # dictionary. error, error_message = stdtm.handle_sample_tool_data_table_conf_file(full_path, persist=False) if error: message = '%s<br/>%s' % (message, error_message) # See if the content of the change set was valid. admin_only = len(repository.downloadable_revisions) != 1 suc.handle_email_alerts(trans.app, trans.request.host, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only) if ok: # Update the repository files for browsing. hg_util.update_repository(repo) # Get the new repository tip. if tip == repository.tip(trans.app): message = 'No changes to repository. ' status = 'warning' else: if (isgzip or isbz2) and uncompress_file: uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' if uploaded_directory: source_type = "repository" source = url else: source_type = "file" source = uploaded_file_filename message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \ (source_type, escape(source), uncompress_str) if istar and (undesirable_dirs_removed or undesirable_files_removed): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed message += "were removed from the archive. " if istar and remove_repo_files_not_in_tar and files_to_remove: if upload_point is not None: message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \ (len(files_to_remove), upload_point) else: message += " %d files were removed from the repository root. " % len(files_to_remove) rmm = repository_metadata_manager.RepositoryMetadataManager(app=trans.app, user=trans.user, repository=repository) status, error_message = \ rmm.set_repository_metadata_due_to_new_tip(trans.request.host, content_alert_str=content_alert_str, **kwd) if error_message: message = error_message kwd['message'] = message if repository.metadata_revisions: # A repository's metadata revisions are order descending by update_time, so the zeroth revision # will be the tip just after an upload. metadata_dict = repository.metadata_revisions[0].metadata else: metadata_dict = {} dd = dependency_display.DependencyDisplayer(trans.app) if str(repository.type) not in [rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION]: change_repository_type_message = rt_util.generate_message_for_repository_type_change(trans.app, repository) if change_repository_type_message: message += change_repository_type_message status = 'warning' else: # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies # weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency # definitions can define orphan tool dependencies (no relationship to any tools contained in the # repository), so warning messages are important because orphans are always valid. The repository # owner must be warned in case they did not intend to define an orphan dependency, but simply # provided incorrect information (tool shed, name owner, changeset_revision) for the definition. orphan_message = dd.generate_message_for_orphan_tool_dependencies(repository, metadata_dict) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies(metadata_dict) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' # Handle messaging for invalid repository dependencies. invalid_repository_dependencies_message = \ dd.generate_message_for_invalid_repository_dependencies(metadata_dict, error_from_tuple=True) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. stdtm.reset_tool_data_tables() if uploaded_directory: basic_util.remove_dir(uploaded_directory) trans.response.send_redirect(web.url_for(controller='repository', action='browse_repository', id=repository_id, commit_message='Deleted selected files', message=message, status=status)) else: if uploaded_directory: basic_util.remove_dir(uploaded_directory) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. stdtm.reset_tool_data_tables() return trans.fill_template('/webapps/tool_shed/repository/upload.mako', repository=repository, changeset_revision=tip, url=url, commit_message=commit_message, uncompress_file=uncompress_file, remove_repo_files_not_in_tar=remove_repo_files_not_in_tar, message=message, status=status)
def handle_directory_changes(app, host, username, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed): repo_path = repository.repo_path(app) repo = hg_util.get_repo_for_repository(app, repo_path=repo_path) content_alert_str = '' files_to_remove = [] filenames_in_archive = [os.path.join(full_path, name) for name in filenames_in_archive] if remove_repo_files_not_in_tar and not repository.is_new(app): # We have a repository that is not new (it contains files), so discover those files that are in the # repository, but not in the uploaded archive. for root, dirs, files in os.walk(full_path): if root.find('.hg') < 0 and root.find('hgrc') < 0: for undesirable_dir in UNDESIRABLE_DIRS: if undesirable_dir in dirs: dirs.remove(undesirable_dir) undesirable_dirs_removed += 1 for undesirable_file in UNDESIRABLE_FILES: if undesirable_file in files: files.remove(undesirable_file) undesirable_files_removed += 1 for name in files: full_name = os.path.join(root, name) if full_name not in filenames_in_archive: files_to_remove.append(full_name) for repo_file in files_to_remove: # Remove files in the repository (relative to the upload point) that are not in # the uploaded archive. try: hg_util.remove_file(repo_path, repo_file, force=True) except Exception as e: log.debug("Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str(e)) relative_selected_file = repo_file.split('repo_%d' % repository.id)[1].lstrip('/') repo.dirstate.remove(relative_selected_file) repo.dirstate.write() absolute_selected_file = os.path.abspath(repo_file) if os.path.isdir(absolute_selected_file): try: os.rmdir(absolute_selected_file) except OSError as e: # The directory is not empty. pass elif os.path.isfile(absolute_selected_file): os.remove(absolute_selected_file) dir = os.path.split(absolute_selected_file)[0] try: os.rmdir(dir) except OSError as e: # The directory is not empty. pass # See if any admin users have chosen to receive email alerts when a repository is updated. # If so, check every uploaded file to ensure content is appropriate. check_contents = check_file_contents_for_email_alerts(app) for filename_in_archive in filenames_in_archive: # Check file content to ensure it is appropriate. if check_contents and os.path.isfile(filename_in_archive): content_alert_str += check_file_content_for_html_and_images(filename_in_archive) hg_util.add_changeset(repo_path, filename_in_archive) if filename_in_archive.endswith('tool_data_table_conf.xml.sample'): # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded # by parsing the file and adding new entries to the in-memory app.tool_data_tables # dictionary. stdtm = ShedToolDataTableManager(app) error, message = stdtm.handle_sample_tool_data_table_conf_file(filename_in_archive, persist=False) if error: return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed hg_util.commit_changeset(repo_path, full_path_to_changeset=full_path, username=username, message=commit_message) admin_only = len(repository.downloadable_revisions) != 1 suc.handle_email_alerts(app, host, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only) return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
class ToolValidator(object): def __init__(self, app): self.app = app self.stdtm = ShedToolDataTableManager(self.app) def can_use_tool_config_disk_file(self, repository, repo, file_path, changeset_revision): """ Determine if repository's tool config file on disk can be used. This method is restricted to tool config files since, with the exception of tool config files, multiple files with the same name will likely be in various directories in the repository and we're comparing file names only (not relative paths). """ if not file_path or not os.path.exists(file_path): # The file no longer exists on disk, so it must have been deleted at some previous # point in the change log. return False if changeset_revision == repository.tip(self.app): return True file_name = basic_util.strip_path(file_path) latest_version_of_file = \ self.get_latest_tool_config_revision_from_repository_manifest(repo, file_name, changeset_revision) can_use_disk_file = filecmp.cmp(file_path, latest_version_of_file) try: os.unlink(latest_version_of_file) except Exception: pass return can_use_disk_file def check_tool_input_params(self, repo_dir, tool_config_name, tool, sample_files): """ Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make sure the files exist. """ invalid_files_and_errors_tups = [] for input_param in tool.input_params: if isinstance(input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, # make sure all requirements exist. options = input_param.dynamic_options or input_param.options if options and isinstance(options, dynamic_options.DynamicOptions): if options.tool_data_table or options.missing_tool_data_table_name: # Make sure the repository contains a tool_data_table_conf.xml.sample file. sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir) if sample_tool_data_table_conf: error, correction_msg = \ self.stdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf, persist=False) if error: invalid_files_and_errors_tups.append( ('tool_data_table_conf.xml.sample', correction_msg)) else: correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. " correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository " correction_msg += "that includes the required entry to correct this error.<br/>" invalid_tup = (tool_config_name, correction_msg) if invalid_tup not in invalid_files_and_errors_tups: invalid_files_and_errors_tups.append( invalid_tup) if options.index_file or options.tool_data_table and options.tool_data_table.missing_index_file: # Make sure the repository contains the required xxx.loc.sample file. index_file = options.index_file or options.tool_data_table.missing_index_file index_file_name = basic_util.strip_path(index_file) sample_found = False for sample_file in sample_files: sample_file_name = basic_util.strip_path( sample_file) if sample_file_name == '%s.sample' % index_file_name: options.index_file = index_file_name if options.tool_data_table: options.tool_data_table.missing_index_file = None sample_found = True break if not sample_found: correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file_name) correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % \ str(index_file_name) invalid_files_and_errors_tups.append( (tool_config_name, correction_msg)) return invalid_files_and_errors_tups def concat_messages(self, msg1, msg2): if msg1: if msg2: message = '%s %s' % (msg1, msg2) else: message = msg1 elif msg2: message = msg2 else: message = '' return message def copy_disk_sample_files_to_dir(self, repo_files_dir, dest_path): """ Copy all files currently on disk that end with the .sample extension to the directory to which dest_path refers. """ sample_files = [] for root, dirs, files in os.walk(repo_files_dir): if root.find('.hg') < 0: for name in files: if name.endswith('.sample'): relative_path = os.path.join(root, name) tool_util.copy_sample_file(self.app, relative_path, dest_path=dest_path) sample_files.append(name) return sample_files def get_latest_tool_config_revision_from_repository_manifest( self, repo, filename, changeset_revision): """ Get the latest revision of a tool config file named filename from the repository manifest up to the value of changeset_revision. This method is restricted to tool_config files rather than any file since it is likely that, with the exception of tool config files, multiple files will have the same name in various directories within the repository. """ stripped_filename = basic_util.strip_path(filename) for changeset in hg_util.reversed_upper_bounded_changelog( repo, changeset_revision): manifest_ctx = repo.changectx(changeset) for ctx_file in manifest_ctx.files(): ctx_file_name = basic_util.strip_path(ctx_file) if ctx_file_name == stripped_filename: try: fctx = manifest_ctx[ctx_file] except LookupError: # The ctx_file may have been moved in the change set. For example, # 'ncbi_blastp_wrapper.xml' was moved to 'tools/ncbi_blast_plus/ncbi_blastp_wrapper.xml', # so keep looking for the file until we find the new location. continue fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-gltcrfrm") tmp_filename = fh.name fh.close() fh = open(tmp_filename, 'wb') fh.write(fctx.data()) fh.close() return tmp_filename return None def get_list_of_copied_sample_files(self, repo, changeset_revision, dir): """ Find all sample files (files in the repository with the special .sample extension) in the reversed repository manifest up to changeset_revision. Copy each discovered file to dir and return the list of filenames. If a .sample file was added in a changeset and then deleted in a later changeset, it will be returned in the deleted_sample_files list. The caller will set the value of app.config.tool_data_path to dir in order to load the tools and generate metadata for them. """ deleted_sample_files = [] sample_files = [] for changeset in hg_util.reversed_upper_bounded_changelog( repo, changeset_revision): changeset_ctx = repo.changectx(changeset) for ctx_file in changeset_ctx.files(): ctx_file_name = basic_util.strip_path(ctx_file) # If we decide in the future that files deleted later in the changelog should # not be used, we can use the following if statement. if ctx_file_name.endswith( '.sample' ) # and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files: if ctx_file_name.endswith( '.sample') and ctx_file_name not in sample_files: fctx = hg_util.get_file_context_from_ctx( changeset_ctx, ctx_file) if fctx in ['DELETED']: # Since the possibly future used if statement above is commented out, the # same file that was initially added will be discovered in an earlier changeset # in the change log and fall through to the else block below. In other words, # if a file named blast2go.loc.sample was added in change set 0 and then deleted # in changeset 3, the deleted file in changeset 3 will be handled here, but the # later discovered file in changeset 0 will be handled in the else block below. # In this way, the file contents will always be found for future tools even though # the file was deleted. if ctx_file_name not in deleted_sample_files: deleted_sample_files.append(ctx_file_name) else: sample_files.append(ctx_file_name) tmp_ctx_file_name = os.path.join( dir, ctx_file_name.replace('.sample', '')) fh = open(tmp_ctx_file_name, 'wb') fh.write(fctx.data()) fh.close() return sample_files, deleted_sample_files def handle_sample_files_and_load_tool_from_disk(self, repo_files_dir, repository_id, tool_config_filepath, work_dir): """ Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. """ message = '' sample_files = self.copy_disk_sample_files_to_dir( repo_files_dir, work_dir) if sample_files: if 'tool_data_table_conf.xml.sample' in sample_files: # Load entries into the tool_data_tables if the tool requires them. tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml') error, message = self.stdtm.handle_sample_tool_data_table_conf_file( tool_data_table_config, persist=False) tool, valid, message2 = self.load_tool_from_config( repository_id, tool_config_filepath) message = self.concat_messages(message, message2) return tool, valid, message, sample_files def handle_sample_files_and_load_tool_from_tmp_config( self, repo, repository_id, changeset_revision, tool_config_filename, work_dir): tool = None valid = False message = '' # We're not currently doing anything with the returned list of deleted_sample_files here. It is # intended to help handle sample files that are in the manifest, but have been deleted from disk. sample_files, deleted_sample_files = self.get_list_of_copied_sample_files( repo, changeset_revision, dir=work_dir) if sample_files: if 'tool_data_table_conf.xml.sample' in sample_files: # Load entries into the tool_data_tables if the tool requires them. tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml') error, message = self.stdtm.handle_sample_tool_data_table_conf_file( tool_data_table_config, persist=False) manifest_ctx, ctx_file = hg_util.get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision) if manifest_ctx and ctx_file: tool, valid, message2 = self.load_tool_from_tmp_config( repo, repository_id, manifest_ctx, ctx_file, work_dir) message = self.concat_messages(message, message2) return tool, valid, message, sample_files def load_tool_from_changeset_revision(self, repository_id, changeset_revision, tool_config_filename): """ Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision is a valid (downloadable) changeset revision. The tool config will be located in the repository manifest between the received valid changeset revision and the first changeset revision in the repository, searching backwards. """ repository = repository_util.get_repository_in_tool_shed( self.app, repository_id) repo_files_dir = repository.repo_path(self.app) repo = hg_util.get_repo_for_repository(self.app, repo_path=repo_files_dir) tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename) work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr") can_use_disk_file = self.can_use_tool_config_disk_file( repository, repo, tool_config_filepath, changeset_revision) if can_use_disk_file: tool, valid, message, sample_files = \ self.handle_sample_files_and_load_tool_from_disk(repo_files_dir, repository_id, tool_config_filepath, work_dir) if tool is not None: invalid_files_and_errors_tups = \ self.check_tool_input_params(repo_files_dir, tool_config_filename, tool, sample_files) if invalid_files_and_errors_tups: message2 = tool_util.generate_message_for_invalid_tools( self.app, invalid_files_and_errors_tups, repository, metadata_dict=None, as_html=True, displaying_invalid_tool=True) message = self.concat_messages(message, message2) else: tool, valid, message, sample_files = \ self.handle_sample_files_and_load_tool_from_tmp_config(repo, repository_id, changeset_revision, tool_config_filename, work_dir) basic_util.remove_dir(work_dir) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.stdtm.reset_tool_data_tables() return repository, tool, valid, message def load_tool_from_config(self, repository_id, full_path): tool_source = get_tool_source( full_path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False), tool_location_fetcher=ToolLocationFetcher(), ) try: tool = create_tool_from_source(config_file=full_path, app=self.app, tool_source=tool_source, repository_id=repository_id, allow_code_files=False) valid = True error_message = None except KeyError as e: tool = None valid = False error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e) error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' error_message += 'this error. ' log.exception(error_message) except Exception as e: tool = None valid = False error_message = str(e) log.exception('Caught exception loading tool from %s:', full_path) return tool, valid, error_message def load_tool_from_tmp_config(self, repo, repository_id, ctx, ctx_file, work_dir): tool = None valid = False message = '' tmp_tool_config = hg_util.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir) if tmp_tool_config: tool_element, error_message = xml_util.parse_xml(tmp_tool_config) if tool_element is None: return tool, message # Look for external files required by the tool config. tmp_code_files = [] external_paths = Tool.get_externally_referenced_paths( tmp_tool_config) changeset_revision = str(ctx) for path in external_paths: tmp_code_file_name = hg_util.copy_file_from_manifest( repo, changeset_revision, path, work_dir) if tmp_code_file_name: tmp_code_files.append(tmp_code_file_name) tool, valid, message = self.load_tool_from_config( repository_id, tmp_tool_config) for tmp_code_file in tmp_code_files: try: os.unlink(tmp_code_file) except Exception: pass try: os.unlink(tmp_tool_config) except Exception: pass return tool, valid, message