def main(gtk_context): FWV0 = Path.cwd() log.info("Running gear in %s", FWV0) gtk_context.log_config() # Errors and warnings will be always logged when they are detected. # Keep a list of errors and warning to print all in one place at end of log # Any errors will prevent the command from running and will cause exit(1) errors = [] warnings = [] output_dir = gtk_context.output_dir log.info("output_dir is %s", output_dir) work_dir = gtk_context.work_dir log.info("work_dir is %s", work_dir) gear_name = gtk_context.manifest["name"] # run-time configuration options from the gear's context.json config = gtk_context.config dry_run = config.get("gear-dry-run") # Given the destination container, figure out if running at the project, # subject, or session level. destination_id = gtk_context.destination["id"] hierarchy = get_analysis_run_level_and_hierarchy(gtk_context.client, destination_id) # This is the label of the project, subject or session and is used # as part of the name of the output files. run_label = make_file_name_safe(hierarchy["run_label"]) # Output will be put into a directory named as the destination id. # This allows the raw output to be deleted so that a zipped archive # can be returned. output_analysis_id_dir = output_dir / destination_id log.info("Creating output directory %s", output_analysis_id_dir) if Path(output_analysis_id_dir).exists(): log.info( "Not actually creating output directory %s because it exists. This must be a test", output_analysis_id_dir, ) else: Path(output_analysis_id_dir).mkdir() environ = get_and_log_environment() # set # threads and max memory to use config["n_cpus"], config["omp-nthreads"] = set_n_cpus( config.get("n_cpus"), config.get("omp-nthreads")) config["mem"] = set_mem_mb(config.get("mem_mb")) environ["OMP_NUM_THREADS"] = str(config["omp-nthreads"]) # All writeable directories need to be set up in the current working directory orig_subject_dir = Path(environ["SUBJECTS_DIR"]) subjects_dir = FWV0 / "freesurfer/subjects" environ["SUBJECTS_DIR"] = str(subjects_dir) if not subjects_dir.exists(): # needs to be created unless testing subjects_dir.mkdir(parents=True) (subjects_dir / "fsaverage").symlink_to(orig_subject_dir / "fsaverage") (subjects_dir / "fsaverage5").symlink_to(orig_subject_dir / "fsaverage5") (subjects_dir / "fsaverage6").symlink_to(orig_subject_dir / "fsaverage6") bids_filter_file_path = gtk_context.get_input_path("bids-filter-file") if bids_filter_file_path: paths = list(Path("input/bids-filter-file").glob("*")) log.info("Using provided PyBIDS filter file %s", str(paths[0])) config["bids-filter-file"] = str(paths[0]) previous_work_zip_file_path = gtk_context.get_input_path("work-dir") if previous_work_zip_file_path: paths = list(Path("input/work-dir").glob("*")) log.info("Using provided fMRIPrep intermediate work file %s", str(paths[0])) unzip_dir = FWV0 / "unzip-work-dir" unzip_dir.mkdir(parents=True) unzip_archive(paths[0], unzip_dir) for a_dir in unzip_dir.glob("*/*"): if ( a_dir.name == "bids" ): # skip previous bids directory so current bids data will be used log.info("Found %s, but ignoring it to use current bids data", a_dir.name) else: log.info("Found %s", a_dir.name) a_dir.rename(FWV0 / "work" / a_dir.name) hash_file = list( Path("work/fmriprep_wf/").glob("fsdir_run_*/_0x*.json"))[0] if hash_file.exists(): with open(hash_file) as json_file: data = json.load(json_file) old_tmp_path = data[0][1] old_tmp_name = old_tmp_path.split("/")[2] log.info("Found old tmp name: %s", old_tmp_name) cur_tmp_name = str(FWV0).split("/")[2] # rename the directory to the old name Path("/tmp/" + cur_tmp_name).replace( Path("/tmp/" + old_tmp_name)) # create a symbolic link using the new name to the old name just in case Path("/tmp/" + cur_tmp_name).symlink_to( Path("/tmp/" + old_tmp_name), target_is_directory=True) # update all variables to have the old directory name in them FWV0 = Path("/tmp/" + old_tmp_name + "/flywheel/v0") output_dir = str(output_dir).replace(cur_tmp_name, old_tmp_name) output_analysis_id_dir = Path( str(output_analysis_id_dir).replace( cur_tmp_name, old_tmp_name)) log.info("new output directory is: %s", output_dir) work_dir = Path( str(work_dir).replace(cur_tmp_name, old_tmp_name)) log.info("new work directory is: %s", work_dir) subjects_dir = Path( str(subjects_dir).replace(cur_tmp_name, old_tmp_name)) config["fs-subjects-dir"] = subjects_dir log.info("new FreeSurfer subjects directory is: %s", subjects_dir) # for old work to be recognized, switch to running from the old path os.chdir(FWV0) log.info("cd %s", FWV0) else: log.info("Could not find hash file") config["work-dir"] = str(FWV0 / "work") subject_zip_file_path = gtk_context.get_input_path("fs-subjects-dir") if subject_zip_file_path: paths = list(Path("input/fs-subjects-dir").glob("*")) log.info("Using provided Freesurfer subject file %s", str(paths[0])) unzip_dir = FWV0 / "unzip-fs-subjects-dir" unzip_dir.mkdir(parents=True) unzip_archive(paths[0], unzip_dir) for a_subject in unzip_dir.glob("*/*"): if (subjects_dir / a_subject.name).exists(): log.info("Found %s but using existing", a_subject.name) else: log.info("Found %s", a_subject.name) a_subject.rename(subjects_dir / a_subject.name) config["fs-subjects-dir"] = subjects_dir previous_results_zip_file_path = gtk_context.get_input_path( "previous-results") if previous_results_zip_file_path: paths = list(Path("input/previous-results").glob("*")) log.info("Using provided fMRIPrep previous results file %s", str(paths[0])) unzip_dir = FWV0 / "unzip-previous-results" unzip_dir.mkdir(parents=True) unzip_archive(paths[0], unzip_dir) for a_dir in unzip_dir.glob("*/*"): log.info("Found %s", a_dir.name) a_dir.rename(output_analysis_id_dir / a_dir.name) environ["FS_LICENSE"] = str(FWV0 / "freesurfer/license.txt") license_list = list(Path("input/freesurfer_license").glob("*")) if len(license_list) > 0: fs_license_path = license_list[0] else: fs_license_path = "" install_freesurfer_license( str(fs_license_path), config.get("gear-FREESURFER_LICENSE"), gtk_context.client, destination_id, FREESURFER_LICENSE, ) # TemplateFlow seems to be baked in to the container since 2021-10-07 16:25:12 so this is not needed # templateflow_dir = FWV0 / "templateflow" # templateflow_dir.mkdir() # environ["SINGULARITYENV_TEMPLATEFLOW_HOME"] = str(templateflow_dir) # environ["TEMPLATEFLOW_HOME"] = str(templateflow_dir) command = generate_command(config, work_dir, output_analysis_id_dir, errors, warnings) # Download BIDS Formatted data if len(errors) == 0: # Create HTML file that shows BIDS "Tree" like output tree = True tree_title = f"{gear_name} BIDS Tree" error_code = download_bids_for_runlevel( gtk_context, hierarchy, tree=tree, tree_title=tree_title, src_data=DOWNLOAD_SOURCE, folders=DOWNLOAD_MODALITIES, dry_run=dry_run, do_validate_bids=config.get("gear-run-bids-validation"), ) if error_code > 0 and not config.get("gear-ignore-bids-errors"): errors.append(f"BIDS Error(s) detected. Did not run {CONTAINER}") else: log.info("Did not download BIDS because of previous errors") print(errors) # Don't run if there were errors or if this is a dry run return_code = 0 try: if len(errors) > 0: return_code = 1 log.info("Command was NOT run because of previous errors.") elif dry_run: e = "gear-dry-run is set: Command was NOT run." log.warning(e) warnings.append(e) pretend_it_ran(destination_id) else: if config["gear-log-level"] != "INFO": # show what's in the current working directory just before running os.system("tree -al .") if "gear-timeout" in config: command = [f"timeout {config['gear-timeout']}"] + command # This is what it is all about exec_command( command, environ=environ, dry_run=dry_run, shell=True, cont_output=True, ) except RuntimeError as exc: return_code = 1 errors.append(exc) log.critical(exc) log.exception("Unable to execute command.") finally: # Save time, etc. resources used in metadata on analysis if Path("time_output.txt").exists(): # some tests won't have this file metadata = { "analysis": { "info": { "resources used": {}, }, }, } with open("time_output.txt") as file: for line in file: if ":" in line: if ( "Elapsed" in line ): # special case "Elapsed (wall clock) time (h:mm:ss or m:ss): 0:08.11" sline = re.split(r"\):", line) sline[0] += ")" else: sline = line.split(":") key = sline[0].strip() val = sline[1].strip(' "\n') metadata["analysis"]["info"]["resources used"][ key] = val with open(f"{output_dir}/.metadata.json", "w") as fff: json.dump(metadata, fff) log.info(f"Wrote {output_dir}/.metadata.json") # Cleanup, move all results to the output directory # Remove all fsaverage* directories if not config.get("gear-keep-fsaverage"): path = output_analysis_id_dir / "freesurfer" fsavg_dirs = path.glob("fsaverage*") for fsavg in fsavg_dirs: log.info("deleting %s", str(fsavg)) shutil.rmtree(fsavg) else: log.info("Keeping fsaverage directories") # zip entire output/<analysis_id> folder into # <gear_name>_<project|subject|session label>_<analysis.id>.zip zip_file_name = gear_name + f"_{run_label}_{destination_id}.zip" zip_output( str(output_dir), destination_id, zip_file_name, dry_run=False, exclude_files=None, ) # Make archives for result *.html files for easy display on platform zip_htmls(output_dir, destination_id, output_analysis_id_dir / BIDS_APP) # possibly save ALL intermediate output if config.get("gear-save-intermediate-output"): zip_all_intermediate_output(destination_id, gear_name, output_dir, work_dir, run_label) # possibly save intermediate files and folders zip_intermediate_selected( config.get("gear-intermediate-files"), config.get("gear-intermediate-folders"), destination_id, gear_name, output_dir, work_dir, run_label, ) # clean up: remove output that was zipped if Path(output_analysis_id_dir).exists(): if not config.get("gear-keep-output"): log.debug('removing output directory "%s"', str(output_analysis_id_dir)) shutil.rmtree(output_analysis_id_dir) else: log.info('NOT removing output directory "%s"', str(output_analysis_id_dir)) else: log.info("Output directory does not exist so it cannot be removed") # Report errors and warnings at the end of the log so they can be easily seen. if len(warnings) > 0: msg = "Previous warnings:\n" for warn in warnings: msg += " Warning: " + str(warn) + "\n" log.info(msg) if len(errors) > 0: msg = "Previous errors:\n" for err in errors: if str(type(err)).split("'")[1] == "str": # show string msg += " Error msg: " + str(err) + "\n" else: # show type (of error) and error message err_type = str(type(err)).split("'")[1] msg += f" {err_type}: {str(err)}\n" log.info(msg) return_code = 1 log.info("%s Gear is done. Returning %s", CONTAINER, return_code) return return_code
def main(gtk_context): gtk_context.log_config() # Errors and warnings will be always logged when they are detected. # Keep a list of errors and warning to print all in one place at end of log # Any errors will prevent the BIDS App from running and will cause exit(1) errors = [] warnings = [] output_dir = gtk_context.output_dir work_dir = gtk_context.work_dir gear_name = gtk_context.manifest["name"] # run-time configuration options from the gear's context.json config = gtk_context.config dry_run = config.get("gear-dry-run") # Given the destination container, figure out if running at the project, # subject, or session level. destination_id = gtk_context.destination["id"] hierarchy = get_analysis_run_level_and_hierarchy(gtk_context.client, destination_id) # This is the label of the project, subject or session and is used # as part of the name of the output files. run_label = make_file_name_safe(hierarchy["run_label"]) # Output will be put into a directory named as the destination id. # This allows the raw output to be deleted so that a zipped archive # can be returned. output_analysis_id_dir = output_dir / destination_id # editme: optional features -- set # threads and max memory to use config["n_cpus"] = set_n_cpus(config.get("n_cpus")) config["mem_gb"] = set_mem_gb(config.get("mem_gb")) environ = get_and_log_environment() # editme: if the command needs a Freesurfer license keep this install_freesurfer_license( gtk_context.get_input_path("freesurfer_license"), config.get("gear-FREESURFER_LICENSE"), gtk_context.client, destination_id, FREESURFER_LICENSE, ) command = generate_command(config, work_dir, output_analysis_id_dir, errors, warnings) # This is used as part of the name of output files command_name = make_file_name_safe(command[0]) # Download BIDS Formatted data if len(errors) == 0: # editme: optional feature # Create HTML file that shows BIDS "Tree" like output tree = True tree_title = f"{command_name} BIDS Tree" error_code = download_bids_for_runlevel( gtk_context, hierarchy, tree=tree, tree_title=tree_title, src_data=DOWNLOAD_SOURCE, folders=DOWNLOAD_MODALITIES, dry_run=dry_run, do_validate_bids=config.get("gear-run-bids-validation"), ) if error_code > 0 and not config.get("gear-ignore-bids-errors"): errors.append(f"BIDS Error(s) detected. Did not run {CONTAINER}") else: log.info("Did not download BIDS because of previous errors") print(errors) # Don't run if there were errors or if this is a dry run return_code = 0 try: if len(errors) > 0: return_code = 1 log.info("Command was NOT run because of previous errors.") elif dry_run: e = "gear-dry-run is set: Command was NOT run." log.warning(e) warnings.append(e) pretend_it_ran(destination_id) else: # Create output directory log.info("Creating output directory %s", output_analysis_id_dir) Path(output_analysis_id_dir).mkdir() # This is what it is all about exec_command( command, environ=environ, dry_run=dry_run, shell=True, cont_output=True, ) except RuntimeError as exc: return_code = 1 errors.append(exc) log.critical(exc) log.exception("Unable to execute command.") finally: # Cleanup, move all results to the output directory # TODO use pybids (or delete from requirements.txt) # see https://github.com/bids-standard/pybids/tree/master/examples # for any necessary work on the bids files inside the gear, perhaps # to query results or count stuff to estimate how long things will take. # zip entire output/<analysis_id> folder into # <gear_name>_<project|subject|session label>_<analysis.id>.zip zip_file_name = gear_name + f"_{run_label}_{destination_id}.zip" zip_output( str(output_dir), destination_id, zip_file_name, dry_run=False, exclude_files=None, ) # editme: optional feature # zip any .html files in output/<analysis_id>/ zip_htmls(output_dir, destination_id, output_analysis_id_dir) # editme: optional feature # possibly save ALL intermediate output if config.get("gear-save-intermediate-output"): zip_all_intermediate_output(destination_id, gear_name, output_dir, work_dir, run_label) # possibly save intermediate files and folders zip_intermediate_selected( config.get("gear-intermediate-files"), config.get("gear-intermediate-folders"), destination_id, gear_name, output_dir, work_dir, run_label, ) # clean up: remove output that was zipped if Path(output_analysis_id_dir).exists(): if not config.get("gear-keep-output"): log.debug('removing output directory "%s"', str(output_analysis_id_dir)) shutil.rmtree(output_analysis_id_dir) else: log.info('NOT removing output directory "%s"', str(output_analysis_id_dir)) else: log.info("Output directory does not exist so it cannot be removed") # editme: optional feature # save .metadata file metadata = { "project": { "info": { "test": "Hello project", f"{run_label} {destination_id}": "put this here", }, "tags": [run_label, destination_id], }, "subject": { "info": { "test": "Hello subject", f"{run_label} {destination_id}": "put this here", }, "tags": [run_label, destination_id], }, "session": { "info": { "test": "Hello session", f"{run_label} {destination_id}": "put this here", }, "tags": [run_label, destination_id], }, "analysis": { "info": { "test": "Hello analysis", f"{run_label} {destination_id}": "put this here", }, "files": [{ "name": "bids_tree.html", "info": { "value1": "foo", "value2": "bar", f"{run_label} {destination_id}": "put this here", }, "tags": ["ein", "zwei"], }], "tags": [run_label, destination_id], }, } with open(f"{output_dir}/.metadata.json", "w") as fff: json.dump(metadata, fff) log.info(f"Wrote {output_dir}/.metadata.json") # Report errors and warnings at the end of the log so they can be easily seen. if len(warnings) > 0: msg = "Previous warnings:\n" for warn in warnings: msg += " Warning: " + str(warn) + "\n" log.info(msg) if len(errors) > 0: msg = "Previous errors:\n" for err in errors: if str(type(err)).split("'")[1] == "str": # show string msg += " Error msg: " + str(err) + "\n" else: # show type (of error) and error message err_type = str(type(err)).split("'")[1] msg += f" {err_type}: {str(err)}\n" log.info(msg) return_code = 1 log.info("%s Gear is done. Returning %s", CONTAINER, return_code) return return_code
def main(gtk_context): gtk_context.init_logging("debug") gtk_context.log_config() log = gtk_context.log acquisition_id = gtk_context.config_json["inputs"]["anatomical"][ "hierarchy"]["id"] file_name = gtk_context.config_json["inputs"]["anatomical"]["location"][ "name"] log.info(f"acquisition {acquisition_id} {file_name}") fw = gtk_context.client full_file = fw.get_acquisition_file_info(acquisition_id, file_name) field_strength = full_file.info.get('MagneticFieldStrength') log.info(f"field_strength = {field_strength}") # grab environment for gear (saved in Dockerfile) with open("/tmp/gear_environ.json", "r") as f: environ = json.load(f) install_freesurfer_license(gtk_context, LICENSE_FILE) subject_id = fw.get_analysis(gtk_context.destination["id"]).parents.subject subject = fw.get_subject(subject_id) subject_id = subject.label subject_dir = Path(SUBJECTS_DIR / subject_id) work_dir = gtk_context.output_dir / subject_id if not work_dir.is_symlink(): work_dir.symlink_to(subject_dir) anat_dir = Path("/flywheel/v0/input/anatomical") anatomical_list = list(anat_dir.rglob("*.nii*")) anatomical = str(anatomical_list[0]) # The main command line command to be run: command = ["recon-all", "-i", anatomical, "-subjid", subject_id] if field_strength == 3: command.append("-3T") command += [ "-all", "&&", "segmentBS.sh", subject_id, "&&", "gtmseg", "--s", subject_id, ] try: return_code = 0 exec_command( command, environ=environ, dry_run=False, # Set to True for testing shell=True, cont_output=True, ) except RuntimeError as exc: log.critical(exc) log.exception("Unable to execute command.") return_code = 1 # zip entire output/<subject_id> folder into # <gear_name>_<subject_id>_<analysis.id>.zip zip_file_name = (gtk_context.manifest["name"] + f"_{subject_id}_{gtk_context.destination['id']}.zip") if subject_dir.exists(): log.info("Saving %s in %s as output", subject_id, SUBJECTS_DIR) zip_output(str(gtk_context.output_dir), subject_id, zip_file_name) else: log.error("Could not find %s in %s", subject_id, SUBJECTS_DIR) # clean up: remove symbolic link to subject so it won't be in output if work_dir.exists(): log.debug('removing output directory "%s"', str(work_dir)) work_dir.unlink() else: log.info("Output directory does not exist so it cannot be removed") log.info("Gear is done. Returning %d", return_code) sys.exit(return_code)
def main(gtk_context): log = gtk_context.log # Keep a list of errors and warning to print all in one place at end of log # Any errors will prevent the command from running and will cause exit(1) errors = [] warnings = [] # Given the destination container, figure out if running at the project, # subject, or session level. hierarchy = get_run_level_and_hierarchy( gtk_context.client, gtk_context.destination["id"] ) # This is the label of the project, subject or session and is used # as part of the name of the output files. run_label = make_file_name_safe(hierarchy["run_label"]) # Output will be put into a directory named as the destination id. # This allows the raw output to be deleted so that a zipped archive # can be returned. output_analysisid_dir = gtk_context.output_dir / gtk_context.destination["id"] # editme: optional feature # get # cpu's to set -openmp os_cpu_count = str(os.cpu_count()) log.info("os.cpu_count() = %s", os_cpu_count) n_cpus = gtk_context.config.get("n_cpus") if n_cpus: if n_cpus > os_cpu_count: log.warning("n_cpus > number available, using %d", os_cpu_count) gtk_context.config["n_cpus"] = os_cpu_count elif n_cpus == 0: log.info("n_cpus == 0, using %d (maximum available)", os_cpu_count) gtk_context.config["n_cpus"] = os_cpu_count else: # Default is to use all cpus available gtk_context.config["n_cpus"] = os_cpu_count # zoom zoom # editme: optional feature mem_gb = psutil.virtual_memory().available / (1024 ** 3) log.info("psutil.virtual_memory().available= {:4.1f} GiB".format(mem_gb)) # grab environment for gear (saved in Dockerfile) with open("/tmp/gear_environ.json", "r") as f: environ = json.load(f) # Add environment to log if debugging kv = "" for k, v in environ.items(): kv += k + "=" + v + " " log.debug("Environment: " + kv) # The main command line command to be run: # editme: Set the actual gear command: command = [BIDS_APP] # This is also used as part of the name of output files command_name = make_file_name_safe(command[0]) # editme: add positional arguments that the above command needs # 3 positional args: bids path, output dir, 'participant' # This should be done here in case there are nargs='*' arguments # These follow the BIDS Apps definition (https://github.com/BIDS-Apps) command.append(str(gtk_context.work_dir / "bids")) command.append(str(output_analysisid_dir)) command.append(ANALYSIS_LEVEL) # get config for command by skipping gear config parameters command_config = {} for key, val in gtk_context.config.items(): if key == "bids_app_args": bids_app_args = val.split(" ") for baa in bids_app_args: command.append(baa) elif not key.startswith("gear-"): command_config[key] = val # print("command_config:", json.dumps(command_config, indent=4)) # Validate the command parameter dictionary - make sure everything is # ready to run so errors will appear before launching the actual gear # code. Add descriptions of problems to errors & warnings lists. # print("gtk_context.config:", json.dumps(gtk_context.config, indent=4)) command = build_command_list(command, command_config) # print(command) # editme: fix --verbose argparse argument for ii, cmd in enumerate(command): if cmd.startswith("--verbose"): # handle a 'count' argparse argument where manifest gives # enumerated possibilities like v, vv, or vvv # e.g. replace "--verbose=vvv' with '-vvv' command[ii] = "-" + cmd.split("=")[1] log.info("command is: %s", str(command)) # editme: if the command needs a freesurfer license keep this if Path(FREESURFER_LICENSE).exists(): log.debug("%s exists.", FREESURFER_LICENSE) install_freesurfer_license(gtk_context, FREESURFER_LICENSE) if len(errors) == 0: # editme: optional feature # Create HTML file that shows BIDS "Tree" like output? tree = True tree_title = f"{command_name} BIDS Tree" error_code = download_bids_for_runlevel( gtk_context, hierarchy, tree=tree, tree_title=tree_title, src_data=DOWNLOAD_SOURCE, folders=DOWNLOAD_MODALITIES, dry_run=gtk_context.config.get("gear-dry-run"), do_validate_bids=gtk_context.config.get("gear-run-bids-validation"), ) if error_code > 0 and not gtk_context.config.get("gear-ignore-bids-errors"): errors.append(f"BIDS Error(s) detected. Did not run {CONTAINER}") # now that work/bids/ exists, copy in the ignore file bidsignore_path = gtk_context.get_input_path("bidsignore") if bidsignore_path: shutil.copy(bidsignore_path, "work/bids/.bidsignore") log.info("Installed .bidsignore in work/bids/") # see https://github.com/bids-standard/pybids/tree/master/examples # for any necessary work on the bids files inside the gear, perhaps # to query results or count stuff to estimate how long things will take. # Add that stuff to utils/bids.py # Don't run if there were errors or if this is a dry run ok_to_run = True if len(errors) > 0: ok_to_run = False returncode = 1 log.info("Command was NOT run because of previous errors.") if gtk_context.config.get("gear-dry-run"): ok_to_run = False returncode = 0 e = "gear-dry-run is set: Command was NOT run." log.warning(e) warnings.append(e) pretend_it_ran(gtk_context) try: if ok_to_run: returncode = 0 # Create output directory log.info("Creating output directory %s", output_analysisid_dir) Path(output_analysisid_dir).mkdir() # This is what it is all about exec_command(command, environ=environ) except RuntimeError as exc: returncode = 1 errors.append(exc) log.critical(exc) log.exception("Unable to execute command.") finally: # Cleanup, move all results to the output directory # TODO # see https://github.com/bids-standard/pybids/tree/master/examples # for any necessary work on the bids files inside the gear, perhaps # to query results or count stuff to estimate how long things will take. # Add that to utils/results.py # zip entire output/<analysis_id> folder into # <gear_name>_<project|subject|session label>_<analysis.id>.zip zip_file_name = ( gtk_context.manifest["name"] + f"_{run_label}_{gtk_context.destination['id']}.zip" ) zip_output( str(gtk_context.output_dir), gtk_context.destination["id"], zip_file_name, dry_run=False, exclude_files=None, ) # editme: optional feature # zip any .html files in output/<analysis_id>/ zip_htmls(gtk_context, output_analysisid_dir) # editme: optional feature # possibly save ALL intermediate output if gtk_context.config.get("gear-save-intermediate-output"): zip_all_intermediate_output(gtk_context, run_label) # possibly save intermediate files and folders zip_intermediate_selected(gtk_context, run_label) # clean up: remove output that was zipped if Path(output_analysisid_dir).exists(): if not gtk_context.config.get("gear-keep-output"): log.debug('removing output directory "%s"', str(output_analysisid_dir)) shutil.rmtree(output_analysisid_dir) else: log.info( 'NOT removing output directory "%s"', str(output_analysisid_dir) ) else: log.info("Output directory does not exist so it cannot be removed") # Report errors and warnings at the end of the log so they can be easily seen. if len(warnings) > 0: msg = "Previous warnings:\n" for err in warnings: if str(type(err)).split("'")[1] == "str": # show string msg += " Warning: " + str(err) + "\n" else: # show type (of warning) and warning message err_type = str(type(err)).split("'")[1] msg += f" {err_type}: {str(err)}\n" log.info(msg) if len(errors) > 0: msg = "Previous errors:\n" for err in errors: if str(type(err)).split("'")[1] == "str": # show string msg += " Error msg: " + str(err) + "\n" else: # show type (of error) and error message err_type = str(type(err)).split("'")[1] msg += f" {err_type}: {str(err)}\n" log.info(msg) returncode = 1 return returncode