def test_download_bids_for_runlevel_acquisition_exception_detected(tmp_path, caplog):
caplog.set_level(logging.DEBUG)
with patch(
"flywheel_gear_toolkit.GearToolkitContext.client", return_value=Acquisition(),
):
with patch(
"utils.bids.download_run_level.download_bids_dir",
side_effect=flywheel.ApiException("foo", "fum"),
):
with patch(
"utils.bids.download_run_level.validate_bids", return_value=0,
):
gtk_context = flywheel_gear_toolkit.GearToolkitContext(
input_args=["-d aex:analysis"], gear_path=tmp_path
)
err_code = download_bids_for_runlevel(
gtk_context,
HIERARCHY,
tree=False,
tree_title=None,
src_data=True,
folders=["anat", "func"],
dry_run=True,
)
assert len(caplog.records) == 6
assert "(foo) Reason: fum" in caplog.records[4].message
def test_download_bids_for_runlevel_no_destination_complains(tmp_path, caplog):
caplog.set_level(logging.DEBUG)
HIERARCHY["run_level"] = "no_destination"
with patch(
"flywheel_gear_toolkit.GearToolkitContext.client", return_value=Acquisition(),
):
with patch("utils.bids.download_run_level.download_bids_dir"):
with patch(
"utils.bids.download_run_level.validate_bids", return_value=0,
):
gtk_context = flywheel_gear_toolkit.GearToolkitContext(
input_args=[], gear_path=tmp_path
)
err_code = download_bids_for_runlevel(
gtk_context,
HIERARCHY,
tree=False,
tree_title=None,
src_data=True,
folders=[],
dry_run=True,
)
assert len(caplog.records) == 2
assert "Destination does not exist" in caplog.records[0].message
HIERARCHY["run_level"] = "acquisition" # fix what was broke
HIERARCHY["run_label"] = HIERARCHY[f"{HIERARCHY['run_level']}_label"]
def test_download_bids_for_runlevel_bad_destination_noted(tmp_path, caplog):
caplog.set_level(logging.DEBUG)
HIERARCHY["run_level"] = "subject"
HIERARCHY["run_label"] = HIERARCHY[f"{HIERARCHY['run_level']}_label"]
# create expected file
bids_path = Path(tmp_path) / "work/bids"
bids_path.mkdir(parents=True)
with open(bids_path / "dataset_description.json", "w") as jfp:
json.dump(DATASET_DESCRIPTION, jfp)
with patch(
"flywheel_gear_toolkit.GearToolkitContext.client",
return_value=Acquisition(),
):
with patch(
"flywheel_gear_toolkit.GearToolkitContext.download_project_bids",
return_value=bids_path,
):
with patch(
"utils.bids.download_run_level.validate_bids",
return_value=0,
):
gtk_context = flywheel_gear_toolkit.GearToolkitContext(
input_args=["-d aex:bad_destination"], gear_path=tmp_path)
err_code = download_bids_for_runlevel(
gtk_context,
HIERARCHY,
tree=False,
tree_title=None,
src_data=True,
folders=[" anat", "func"],
dry_run=True,
)
assert len(caplog.records) == 9
assert "is not an analysis or acquisition" in caplog.records[0].message
assert 'subject "TheSubjectCode"' in caplog.records[4].message
HIERARCHY["run_level"] = "acquisition" # fix what was broke
HIERARCHY["run_label"] = HIERARCHY[f"{HIERARCHY['run_level']}_label"]
def test_download_bids_for_runlevel_validate_exception_detected(
tmp_path, caplog):
caplog.set_level(logging.DEBUG)
HIERARCHY["run_level"] = "project"
# create expected file
bids_path = Path(tmp_path) / "work/bids"
bids_path.mkdir(parents=True)
with open(bids_path / "dataset_description.json", "w") as jfp:
json.dump(DATASET_DESCRIPTION, jfp)
with patch(
"flywheel_gear_toolkit.GearToolkitContext.client",
return_value=Acquisition(),
):
with patch(
"flywheel_gear_toolkit.GearToolkitContext.download_project_bids",
return_value=bids_path,
):
with patch(
"utils.bids.download_run_level.validate_bids",
side_effect=Exception("except", "what"),
):
gtk_context = flywheel_gear_toolkit.GearToolkitContext(
input_args=["-d aex:analysis"], gear_path=tmp_path)
err_code = download_bids_for_runlevel(
gtk_context,
HIERARCHY,
tree=False,
tree_title=None,
src_data=True,
folders=[],
dry_run=True,
)
assert len(caplog.records) == 9
assert "('except', 'what')" in caplog.records[7].message
HIERARCHY["run_level"] = "acquisition" # fix what was broke
def test_download_bids_for_runlevel_unknown_acqusition_detected(
tmp_path, caplog):
caplog.set_level(logging.DEBUG)
hierarchy = copy.deepcopy(HIERARCHY)
hierarchy["acquisition_label"] = "unknown acqusition"
# create expected file
bids_path = Path(tmp_path) / "work/bids"
bids_path.mkdir(parents=True)
with open(bids_path / "dataset_description.json", "w") as jfp:
json.dump(DATASET_DESCRIPTION, jfp)
with patch(
"flywheel_gear_toolkit.GearToolkitContext.client",
return_value=Acquisition(),
):
with patch(
"flywheel_gear_toolkit.GearToolkitContext.download_project_bids",
return_value=bids_path,
):
with patch(
"utils.bids.download_run_level.validate_bids",
return_value=0,
):
gtk_context = flywheel_gear_toolkit.GearToolkitContext(
input_args=["-d aex:analysis"], gear_path=tmp_path)
err_code = download_bids_for_runlevel(
gtk_context,
hierarchy,
tree=False,
tree_title=None,
src_data=True,
folders=["anat", "func"],
dry_run=True,
)
assert len(caplog.records) == 5
assert 'acquisition "unknown acqusition"' in caplog.records[3].message
def test_download_bids_for_runlevel_project_works(tmp_path, caplog):
TestCase.skipTest("", f"Can't get this test to work")
user_json = Path(Path.home() / ".config/flywheel/user.json")
if not user_json.exists():
TestCase.skipTest("", f"No API key available in {str(user_json)}")
if caplog:
caplog.set_level(logging.DEBUG)
# create expected path
work_path = Path(tmp_path) / "work"
work_path.mkdir()
with patch(
"utils.bids.download_run_level.validate_bids", return_value=0,
):
gtk_context = flywheel_gear_toolkit.GearToolkitContext(
input_args=["-d 5fa1c3668910f0873535f10e:analysis"], gear_path=tmp_path
)
hierarchy = get_analysis_run_level_and_hierarchy(
gtk_context.client, "5fa1c3668910f0873535f10e"
)
dest = gtk_context.client.get("5fa1c3668910f0873535f10e")
print(dest)
err_code = download_bids_for_runlevel(
gtk_context,
hierarchy,
tree=True,
tree_title=None,
src_data=False,
folders=[],
dry_run=False,
)
assert len(caplog.records) == 10
assert 'project "TheProjectLabel"' in caplog.records[3].message
assert 'Getting "tree" listing' in caplog.records[8].message
assert 0
def test_download_bids_for_runlevel_bidsexporterror_exception_detected(
tmp_path, caplog):
caplog.set_level(logging.DEBUG)
## create expected file
# bids_path = Path(tmp_path) / "work/bids"
# bids_path.mkdir(parents=True)
# with open(bids_path / "dataset_description.json", "w") as jfp:
# json.dump(DATASET_DESCRIPTION, jfp)
with patch(
"flywheel_gear_toolkit.GearToolkitContext.client",
return_value=Acquisition(),
):
with patch(
"utils.bids.download_run_level.download_bids_dir",
side_effect=BIDSExportError("crash", "boom"),
):
with patch(
"utils.bids.download_run_level.validate_bids",
return_value=0,
):
gtk_context = flywheel_gear_toolkit.GearToolkitContext(
input_args=["-d aex:analysis"], gear_path=tmp_path)
err_code = download_bids_for_runlevel(
gtk_context,
HIERARCHY,
tree=False,
tree_title=None,
src_data=True,
folders=["anat", "func"],
dry_run=True,
)
assert len(caplog.records) == 6
assert "crash" in caplog.records[4].message
def test_download_bids_for_runlevel_acquisition_works(tmp_path, caplog):
caplog.set_level(logging.DEBUG)
with patch(
"flywheel_gear_toolkit.GearToolkitContext.client",
return_value=Acquisition(),
):
with patch("utils.bids.download_run_level.download_bids_dir"):
with patch(
"utils.bids.download_run_level.validate_bids",
return_value=0,
):
gtk_context = flywheel_gear_toolkit.GearToolkitContext(
input_args=[], gear_path=tmp_path)
# create expected file
bids_path = Path(gtk_context.work_dir) / "bids"
bids_path.mkdir()
with open(bids_path / "dataset_description.json", "w") as jfp:
json.dump(DATASET_DESCRIPTION, jfp)
err_code = download_bids_for_runlevel(
gtk_context,
HIERARCHY,
tree=False,
tree_title=None,
src_data=False,
folders=[],
dry_run=False,
)
assert len(caplog.records) == 9
assert "Downloading BIDS data was successful" in caplog.records[8].message
def main(gtk_context):
FWV0 = Path.cwd()
log.info("Running gear in %s", FWV0)
gtk_context.log_config()
# Errors and warnings will be always logged when they are detected.
# Keep a list of errors and warning to print all in one place at end of log
# Any errors will prevent the command from running and will cause exit(1)
errors = []
warnings = []
output_dir = gtk_context.output_dir
log.info("output_dir is %s", output_dir)
work_dir = gtk_context.work_dir
log.info("work_dir is %s", work_dir)
gear_name = gtk_context.manifest["name"]
# run-time configuration options from the gear's context.json
config = gtk_context.config
dry_run = config.get("gear-dry-run")
# Given the destination container, figure out if running at the project,
# subject, or session level.
destination_id = gtk_context.destination["id"]
hierarchy = get_analysis_run_level_and_hierarchy(gtk_context.client,
destination_id)
# This is the label of the project, subject or session and is used
# as part of the name of the output files.
run_label = make_file_name_safe(hierarchy["run_label"])
# Output will be put into a directory named as the destination id.
# This allows the raw output to be deleted so that a zipped archive
# can be returned.
output_analysis_id_dir = output_dir / destination_id
log.info("Creating output directory %s", output_analysis_id_dir)
if Path(output_analysis_id_dir).exists():
log.info(
"Not actually creating output directory %s because it exists. This must be a test",
output_analysis_id_dir,
)
else:
Path(output_analysis_id_dir).mkdir()
environ = get_and_log_environment()
# set # threads and max memory to use
config["n_cpus"], config["omp-nthreads"] = set_n_cpus(
config.get("n_cpus"), config.get("omp-nthreads"))
config["mem"] = set_mem_mb(config.get("mem_mb"))
environ["OMP_NUM_THREADS"] = str(config["omp-nthreads"])
# All writeable directories need to be set up in the current working directory
orig_subject_dir = Path(environ["SUBJECTS_DIR"])
subjects_dir = FWV0 / "freesurfer/subjects"
environ["SUBJECTS_DIR"] = str(subjects_dir)
if not subjects_dir.exists(): # needs to be created unless testing
subjects_dir.mkdir(parents=True)
(subjects_dir / "fsaverage").symlink_to(orig_subject_dir / "fsaverage")
(subjects_dir / "fsaverage5").symlink_to(orig_subject_dir /
"fsaverage5")
(subjects_dir / "fsaverage6").symlink_to(orig_subject_dir /
"fsaverage6")
bids_filter_file_path = gtk_context.get_input_path("bids-filter-file")
if bids_filter_file_path:
paths = list(Path("input/bids-filter-file").glob("*"))
log.info("Using provided PyBIDS filter file %s", str(paths[0]))
config["bids-filter-file"] = str(paths[0])
previous_work_zip_file_path = gtk_context.get_input_path("work-dir")
if previous_work_zip_file_path:
paths = list(Path("input/work-dir").glob("*"))
log.info("Using provided fMRIPrep intermediate work file %s",
str(paths[0]))
unzip_dir = FWV0 / "unzip-work-dir"
unzip_dir.mkdir(parents=True)
unzip_archive(paths[0], unzip_dir)
for a_dir in unzip_dir.glob("*/*"):
if (
a_dir.name == "bids"
): # skip previous bids directory so current bids data will be used
log.info("Found %s, but ignoring it to use current bids data",
a_dir.name)
else:
log.info("Found %s", a_dir.name)
a_dir.rename(FWV0 / "work" / a_dir.name)
hash_file = list(
Path("work/fmriprep_wf/").glob("fsdir_run_*/_0x*.json"))[0]
if hash_file.exists():
with open(hash_file) as json_file:
data = json.load(json_file)
old_tmp_path = data[0][1]
old_tmp_name = old_tmp_path.split("/")[2]
log.info("Found old tmp name: %s", old_tmp_name)
cur_tmp_name = str(FWV0).split("/")[2]
# rename the directory to the old name
Path("/tmp/" + cur_tmp_name).replace(
Path("/tmp/" + old_tmp_name))
# create a symbolic link using the new name to the old name just in case
Path("/tmp/" + cur_tmp_name).symlink_to(
Path("/tmp/" + old_tmp_name), target_is_directory=True)
# update all variables to have the old directory name in them
FWV0 = Path("/tmp/" + old_tmp_name + "/flywheel/v0")
output_dir = str(output_dir).replace(cur_tmp_name,
old_tmp_name)
output_analysis_id_dir = Path(
str(output_analysis_id_dir).replace(
cur_tmp_name, old_tmp_name))
log.info("new output directory is: %s", output_dir)
work_dir = Path(
str(work_dir).replace(cur_tmp_name, old_tmp_name))
log.info("new work directory is: %s", work_dir)
subjects_dir = Path(
str(subjects_dir).replace(cur_tmp_name, old_tmp_name))
config["fs-subjects-dir"] = subjects_dir
log.info("new FreeSurfer subjects directory is: %s",
subjects_dir)
# for old work to be recognized, switch to running from the old path
os.chdir(FWV0)
log.info("cd %s", FWV0)
else:
log.info("Could not find hash file")
config["work-dir"] = str(FWV0 / "work")
subject_zip_file_path = gtk_context.get_input_path("fs-subjects-dir")
if subject_zip_file_path:
paths = list(Path("input/fs-subjects-dir").glob("*"))
log.info("Using provided Freesurfer subject file %s", str(paths[0]))
unzip_dir = FWV0 / "unzip-fs-subjects-dir"
unzip_dir.mkdir(parents=True)
unzip_archive(paths[0], unzip_dir)
for a_subject in unzip_dir.glob("*/*"):
if (subjects_dir / a_subject.name).exists():
log.info("Found %s but using existing", a_subject.name)
else:
log.info("Found %s", a_subject.name)
a_subject.rename(subjects_dir / a_subject.name)
config["fs-subjects-dir"] = subjects_dir
previous_results_zip_file_path = gtk_context.get_input_path(
"previous-results")
if previous_results_zip_file_path:
paths = list(Path("input/previous-results").glob("*"))
log.info("Using provided fMRIPrep previous results file %s",
str(paths[0]))
unzip_dir = FWV0 / "unzip-previous-results"
unzip_dir.mkdir(parents=True)
unzip_archive(paths[0], unzip_dir)
for a_dir in unzip_dir.glob("*/*"):
log.info("Found %s", a_dir.name)
a_dir.rename(output_analysis_id_dir / a_dir.name)
environ["FS_LICENSE"] = str(FWV0 / "freesurfer/license.txt")
license_list = list(Path("input/freesurfer_license").glob("*"))
if len(license_list) > 0:
fs_license_path = license_list[0]
else:
fs_license_path = ""
install_freesurfer_license(
str(fs_license_path),
config.get("gear-FREESURFER_LICENSE"),
gtk_context.client,
destination_id,
FREESURFER_LICENSE,
)
# TemplateFlow seems to be baked in to the container since 2021-10-07 16:25:12 so this is not needed
# templateflow_dir = FWV0 / "templateflow"
# templateflow_dir.mkdir()
# environ["SINGULARITYENV_TEMPLATEFLOW_HOME"] = str(templateflow_dir)
# environ["TEMPLATEFLOW_HOME"] = str(templateflow_dir)
command = generate_command(config, work_dir, output_analysis_id_dir,
errors, warnings)
# Download BIDS Formatted data
if len(errors) == 0:
# Create HTML file that shows BIDS "Tree" like output
tree = True
tree_title = f"{gear_name} BIDS Tree"
error_code = download_bids_for_runlevel(
gtk_context,
hierarchy,
tree=tree,
tree_title=tree_title,
src_data=DOWNLOAD_SOURCE,
folders=DOWNLOAD_MODALITIES,
dry_run=dry_run,
do_validate_bids=config.get("gear-run-bids-validation"),
)
if error_code > 0 and not config.get("gear-ignore-bids-errors"):
errors.append(f"BIDS Error(s) detected. Did not run {CONTAINER}")
else:
log.info("Did not download BIDS because of previous errors")
print(errors)
# Don't run if there were errors or if this is a dry run
return_code = 0
try:
if len(errors) > 0:
return_code = 1
log.info("Command was NOT run because of previous errors.")
elif dry_run:
e = "gear-dry-run is set: Command was NOT run."
log.warning(e)
warnings.append(e)
pretend_it_ran(destination_id)
else:
if config["gear-log-level"] != "INFO":
# show what's in the current working directory just before running
os.system("tree -al .")
if "gear-timeout" in config:
command = [f"timeout {config['gear-timeout']}"] + command
# This is what it is all about
exec_command(
command,
environ=environ,
dry_run=dry_run,
shell=True,
cont_output=True,
)
except RuntimeError as exc:
return_code = 1
errors.append(exc)
log.critical(exc)
log.exception("Unable to execute command.")
finally:
# Save time, etc. resources used in metadata on analysis
if Path("time_output.txt").exists(): # some tests won't have this file
metadata = {
"analysis": {
"info": {
"resources used": {},
},
},
}
with open("time_output.txt") as file:
for line in file:
if ":" in line:
if (
"Elapsed" in line
): # special case "Elapsed (wall clock) time (h:mm:ss or m:ss): 0:08.11"
sline = re.split(r"\):", line)
sline[0] += ")"
else:
sline = line.split(":")
key = sline[0].strip()
val = sline[1].strip(' "\n')
metadata["analysis"]["info"]["resources used"][
key] = val
with open(f"{output_dir}/.metadata.json", "w") as fff:
json.dump(metadata, fff)
log.info(f"Wrote {output_dir}/.metadata.json")
# Cleanup, move all results to the output directory
# Remove all fsaverage* directories
if not config.get("gear-keep-fsaverage"):
path = output_analysis_id_dir / "freesurfer"
fsavg_dirs = path.glob("fsaverage*")
for fsavg in fsavg_dirs:
log.info("deleting %s", str(fsavg))
shutil.rmtree(fsavg)
else:
log.info("Keeping fsaverage directories")
# zip entire output/<analysis_id> folder into
# <gear_name>_<project|subject|session label>_<analysis.id>.zip
zip_file_name = gear_name + f"_{run_label}_{destination_id}.zip"
zip_output(
str(output_dir),
destination_id,
zip_file_name,
dry_run=False,
exclude_files=None,
)
# Make archives for result *.html files for easy display on platform
zip_htmls(output_dir, destination_id,
output_analysis_id_dir / BIDS_APP)
# possibly save ALL intermediate output
if config.get("gear-save-intermediate-output"):
zip_all_intermediate_output(destination_id, gear_name, output_dir,
work_dir, run_label)
# possibly save intermediate files and folders
zip_intermediate_selected(
config.get("gear-intermediate-files"),
config.get("gear-intermediate-folders"),
destination_id,
gear_name,
output_dir,
work_dir,
run_label,
)
# clean up: remove output that was zipped
if Path(output_analysis_id_dir).exists():
if not config.get("gear-keep-output"):
log.debug('removing output directory "%s"',
str(output_analysis_id_dir))
shutil.rmtree(output_analysis_id_dir)
else:
log.info('NOT removing output directory "%s"',
str(output_analysis_id_dir))
else:
log.info("Output directory does not exist so it cannot be removed")
# Report errors and warnings at the end of the log so they can be easily seen.
if len(warnings) > 0:
msg = "Previous warnings:\n"
for warn in warnings:
msg += " Warning: " + str(warn) + "\n"
log.info(msg)
if len(errors) > 0:
msg = "Previous errors:\n"
for err in errors:
if str(type(err)).split("'")[1] == "str":
# show string
msg += " Error msg: " + str(err) + "\n"
else: # show type (of error) and error message
err_type = str(type(err)).split("'")[1]
msg += f" {err_type}: {str(err)}\n"
log.info(msg)
return_code = 1
log.info("%s Gear is done. Returning %s", CONTAINER, return_code)
return return_code
def main(gtk_context):
gtk_context.log_config()
# Errors and warnings will be always logged when they are detected.
# Keep a list of errors and warning to print all in one place at end of log
# Any errors will prevent the BIDS App from running and will cause exit(1)
errors = []
warnings = []
output_dir = gtk_context.output_dir
work_dir = gtk_context.work_dir
gear_name = gtk_context.manifest["name"]
# run-time configuration options from the gear's context.json
config = gtk_context.config
dry_run = config.get("gear-dry-run")
# Given the destination container, figure out if running at the project,
# subject, or session level.
destination_id = gtk_context.destination["id"]
hierarchy = get_analysis_run_level_and_hierarchy(gtk_context.client,
destination_id)
# This is the label of the project, subject or session and is used
# as part of the name of the output files.
run_label = make_file_name_safe(hierarchy["run_label"])
# Output will be put into a directory named as the destination id.
# This allows the raw output to be deleted so that a zipped archive
# can be returned.
output_analysis_id_dir = output_dir / destination_id
# editme: optional features -- set # threads and max memory to use
config["n_cpus"] = set_n_cpus(config.get("n_cpus"))
config["mem_gb"] = set_mem_gb(config.get("mem_gb"))
environ = get_and_log_environment()
# editme: if the command needs a Freesurfer license keep this
install_freesurfer_license(
gtk_context.get_input_path("freesurfer_license"),
config.get("gear-FREESURFER_LICENSE"),
gtk_context.client,
destination_id,
FREESURFER_LICENSE,
)
command = generate_command(config, work_dir, output_analysis_id_dir,
errors, warnings)
# This is used as part of the name of output files
command_name = make_file_name_safe(command[0])
# Download BIDS Formatted data
if len(errors) == 0:
# editme: optional feature
# Create HTML file that shows BIDS "Tree" like output
tree = True
tree_title = f"{command_name} BIDS Tree"
error_code = download_bids_for_runlevel(
gtk_context,
hierarchy,
tree=tree,
tree_title=tree_title,
src_data=DOWNLOAD_SOURCE,
folders=DOWNLOAD_MODALITIES,
dry_run=dry_run,
do_validate_bids=config.get("gear-run-bids-validation"),
)
if error_code > 0 and not config.get("gear-ignore-bids-errors"):
errors.append(f"BIDS Error(s) detected. Did not run {CONTAINER}")
else:
log.info("Did not download BIDS because of previous errors")
print(errors)
# Don't run if there were errors or if this is a dry run
return_code = 0
try:
if len(errors) > 0:
return_code = 1
log.info("Command was NOT run because of previous errors.")
elif dry_run:
e = "gear-dry-run is set: Command was NOT run."
log.warning(e)
warnings.append(e)
pretend_it_ran(destination_id)
else:
# Create output directory
log.info("Creating output directory %s", output_analysis_id_dir)
Path(output_analysis_id_dir).mkdir()
# This is what it is all about
exec_command(
command,
environ=environ,
dry_run=dry_run,
shell=True,
cont_output=True,
)
except RuntimeError as exc:
return_code = 1
errors.append(exc)
log.critical(exc)
log.exception("Unable to execute command.")
finally:
# Cleanup, move all results to the output directory
# TODO use pybids (or delete from requirements.txt)
# see https://github.com/bids-standard/pybids/tree/master/examples
# for any necessary work on the bids files inside the gear, perhaps
# to query results or count stuff to estimate how long things will take.
# zip entire output/<analysis_id> folder into
# <gear_name>_<project|subject|session label>_<analysis.id>.zip
zip_file_name = gear_name + f"_{run_label}_{destination_id}.zip"
zip_output(
str(output_dir),
destination_id,
zip_file_name,
dry_run=False,
exclude_files=None,
)
# editme: optional feature
# zip any .html files in output/<analysis_id>/
zip_htmls(output_dir, destination_id, output_analysis_id_dir)
# editme: optional feature
# possibly save ALL intermediate output
if config.get("gear-save-intermediate-output"):
zip_all_intermediate_output(destination_id, gear_name, output_dir,
work_dir, run_label)
# possibly save intermediate files and folders
zip_intermediate_selected(
config.get("gear-intermediate-files"),
config.get("gear-intermediate-folders"),
destination_id,
gear_name,
output_dir,
work_dir,
run_label,
)
# clean up: remove output that was zipped
if Path(output_analysis_id_dir).exists():
if not config.get("gear-keep-output"):
log.debug('removing output directory "%s"',
str(output_analysis_id_dir))
shutil.rmtree(output_analysis_id_dir)
else:
log.info('NOT removing output directory "%s"',
str(output_analysis_id_dir))
else:
log.info("Output directory does not exist so it cannot be removed")
# editme: optional feature
# save .metadata file
metadata = {
"project": {
"info": {
"test": "Hello project",
f"{run_label} {destination_id}": "put this here",
},
"tags": [run_label, destination_id],
},
"subject": {
"info": {
"test": "Hello subject",
f"{run_label} {destination_id}": "put this here",
},
"tags": [run_label, destination_id],
},
"session": {
"info": {
"test": "Hello session",
f"{run_label} {destination_id}": "put this here",
},
"tags": [run_label, destination_id],
},
"analysis": {
"info": {
"test": "Hello analysis",
f"{run_label} {destination_id}": "put this here",
},
"files": [{
"name": "bids_tree.html",
"info": {
"value1": "foo",
"value2": "bar",
f"{run_label} {destination_id}": "put this here",
},
"tags": ["ein", "zwei"],
}],
"tags": [run_label, destination_id],
},
}
with open(f"{output_dir}/.metadata.json", "w") as fff:
json.dump(metadata, fff)
log.info(f"Wrote {output_dir}/.metadata.json")
# Report errors and warnings at the end of the log so they can be easily seen.
if len(warnings) > 0:
msg = "Previous warnings:\n"
for warn in warnings:
msg += " Warning: " + str(warn) + "\n"
log.info(msg)
if len(errors) > 0:
msg = "Previous errors:\n"
for err in errors:
if str(type(err)).split("'")[1] == "str":
# show string
msg += " Error msg: " + str(err) + "\n"
else: # show type (of error) and error message
err_type = str(type(err)).split("'")[1]
msg += f" {err_type}: {str(err)}\n"
log.info(msg)
return_code = 1
log.info("%s Gear is done. Returning %s", CONTAINER, return_code)
return return_code
def main(gtk_context):
log = gtk_context.log
# Keep a list of errors and warning to print all in one place at end of log
# Any errors will prevent the command from running and will cause exit(1)
errors = []
warnings = []
# Given the destination container, figure out if running at the project,
# subject, or session level.
hierarchy = get_run_level_and_hierarchy(
gtk_context.client, gtk_context.destination["id"]
)
# This is the label of the project, subject or session and is used
# as part of the name of the output files.
run_label = make_file_name_safe(hierarchy["run_label"])
# Output will be put into a directory named as the destination id.
# This allows the raw output to be deleted so that a zipped archive
# can be returned.
output_analysisid_dir = gtk_context.output_dir / gtk_context.destination["id"]
# editme: optional feature
# get # cpu's to set -openmp
os_cpu_count = str(os.cpu_count())
log.info("os.cpu_count() = %s", os_cpu_count)
n_cpus = gtk_context.config.get("n_cpus")
if n_cpus:
if n_cpus > os_cpu_count:
log.warning("n_cpus > number available, using %d", os_cpu_count)
gtk_context.config["n_cpus"] = os_cpu_count
elif n_cpus == 0:
log.info("n_cpus == 0, using %d (maximum available)", os_cpu_count)
gtk_context.config["n_cpus"] = os_cpu_count
else: # Default is to use all cpus available
gtk_context.config["n_cpus"] = os_cpu_count # zoom zoom
# editme: optional feature
mem_gb = psutil.virtual_memory().available / (1024 ** 3)
log.info("psutil.virtual_memory().available= {:4.1f} GiB".format(mem_gb))
# grab environment for gear (saved in Dockerfile)
with open("/tmp/gear_environ.json", "r") as f:
environ = json.load(f)
# Add environment to log if debugging
kv = ""
for k, v in environ.items():
kv += k + "=" + v + " "
log.debug("Environment: " + kv)
# The main command line command to be run:
# editme: Set the actual gear command:
command = [BIDS_APP]
# This is also used as part of the name of output files
command_name = make_file_name_safe(command[0])
# editme: add positional arguments that the above command needs
# 3 positional args: bids path, output dir, 'participant'
# This should be done here in case there are nargs='*' arguments
# These follow the BIDS Apps definition (https://github.com/BIDS-Apps)
command.append(str(gtk_context.work_dir / "bids"))
command.append(str(output_analysisid_dir))
command.append(ANALYSIS_LEVEL)
# get config for command by skipping gear config parameters
command_config = {}
for key, val in gtk_context.config.items():
if key == "bids_app_args":
bids_app_args = val.split(" ")
for baa in bids_app_args:
command.append(baa)
elif not key.startswith("gear-"):
command_config[key] = val
# print("command_config:", json.dumps(command_config, indent=4))
# Validate the command parameter dictionary - make sure everything is
# ready to run so errors will appear before launching the actual gear
# code. Add descriptions of problems to errors & warnings lists.
# print("gtk_context.config:", json.dumps(gtk_context.config, indent=4))
command = build_command_list(command, command_config)
# print(command)
# editme: fix --verbose argparse argument
for ii, cmd in enumerate(command):
if cmd.startswith("--verbose"):
# handle a 'count' argparse argument where manifest gives
# enumerated possibilities like v, vv, or vvv
# e.g. replace "--verbose=vvv' with '-vvv'
command[ii] = "-" + cmd.split("=")[1]
log.info("command is: %s", str(command))
# editme: if the command needs a freesurfer license keep this
if Path(FREESURFER_LICENSE).exists():
log.debug("%s exists.", FREESURFER_LICENSE)
install_freesurfer_license(gtk_context, FREESURFER_LICENSE)
if len(errors) == 0:
# editme: optional feature
# Create HTML file that shows BIDS "Tree" like output?
tree = True
tree_title = f"{command_name} BIDS Tree"
error_code = download_bids_for_runlevel(
gtk_context,
hierarchy,
tree=tree,
tree_title=tree_title,
src_data=DOWNLOAD_SOURCE,
folders=DOWNLOAD_MODALITIES,
dry_run=gtk_context.config.get("gear-dry-run"),
do_validate_bids=gtk_context.config.get("gear-run-bids-validation"),
)
if error_code > 0 and not gtk_context.config.get("gear-ignore-bids-errors"):
errors.append(f"BIDS Error(s) detected. Did not run {CONTAINER}")
# now that work/bids/ exists, copy in the ignore file
bidsignore_path = gtk_context.get_input_path("bidsignore")
if bidsignore_path:
shutil.copy(bidsignore_path, "work/bids/.bidsignore")
log.info("Installed .bidsignore in work/bids/")
# see https://github.com/bids-standard/pybids/tree/master/examples
# for any necessary work on the bids files inside the gear, perhaps
# to query results or count stuff to estimate how long things will take.
# Add that stuff to utils/bids.py
# Don't run if there were errors or if this is a dry run
ok_to_run = True
if len(errors) > 0:
ok_to_run = False
returncode = 1
log.info("Command was NOT run because of previous errors.")
if gtk_context.config.get("gear-dry-run"):
ok_to_run = False
returncode = 0
e = "gear-dry-run is set: Command was NOT run."
log.warning(e)
warnings.append(e)
pretend_it_ran(gtk_context)
try:
if ok_to_run:
returncode = 0
# Create output directory
log.info("Creating output directory %s", output_analysisid_dir)
Path(output_analysisid_dir).mkdir()
# This is what it is all about
exec_command(command, environ=environ)
except RuntimeError as exc:
returncode = 1
errors.append(exc)
log.critical(exc)
log.exception("Unable to execute command.")
finally:
# Cleanup, move all results to the output directory
# TODO
# see https://github.com/bids-standard/pybids/tree/master/examples
# for any necessary work on the bids files inside the gear, perhaps
# to query results or count stuff to estimate how long things will take.
# Add that to utils/results.py
# zip entire output/<analysis_id> folder into
# <gear_name>_<project|subject|session label>_<analysis.id>.zip
zip_file_name = (
gtk_context.manifest["name"]
+ f"_{run_label}_{gtk_context.destination['id']}.zip"
)
zip_output(
str(gtk_context.output_dir),
gtk_context.destination["id"],
zip_file_name,
dry_run=False,
exclude_files=None,
)
# editme: optional feature
# zip any .html files in output/<analysis_id>/
zip_htmls(gtk_context, output_analysisid_dir)
# editme: optional feature
# possibly save ALL intermediate output
if gtk_context.config.get("gear-save-intermediate-output"):
zip_all_intermediate_output(gtk_context, run_label)
# possibly save intermediate files and folders
zip_intermediate_selected(gtk_context, run_label)
# clean up: remove output that was zipped
if Path(output_analysisid_dir).exists():
if not gtk_context.config.get("gear-keep-output"):
log.debug('removing output directory "%s"', str(output_analysisid_dir))
shutil.rmtree(output_analysisid_dir)
else:
log.info(
'NOT removing output directory "%s"', str(output_analysisid_dir)
)
else:
log.info("Output directory does not exist so it cannot be removed")
# Report errors and warnings at the end of the log so they can be easily seen.
if len(warnings) > 0:
msg = "Previous warnings:\n"
for err in warnings:
if str(type(err)).split("'")[1] == "str":
# show string
msg += " Warning: " + str(err) + "\n"
else: # show type (of warning) and warning message
err_type = str(type(err)).split("'")[1]
msg += f" {err_type}: {str(err)}\n"
log.info(msg)
if len(errors) > 0:
msg = "Previous errors:\n"
for err in errors:
if str(type(err)).split("'")[1] == "str":
# show string
msg += " Error msg: " + str(err) + "\n"
else: # show type (of error) and error message
err_type = str(type(err)).split("'")[1]
msg += f" {err_type}: {str(err)}\n"
log.info(msg)
returncode = 1
return returncode
