def test_run_level_no_parent_says_so(caplog):
"""Running on an acquisition as a destination e.g. (on ss.ce.flywheel.io):
run_level = no_parent, job.id = 5c92a228c488760025dc699f
classifier acquisition dicom-mr-classifier 0.8.2
{'id': '*****@*****.**', 'type': 'user'}
job.destination = {'id': '5c929faeb33891002dc06903', 'type': 'acquisition'}
parent None
parents {'acquisition': None,
'analysis': None,
'group': 'scien',
'project': '5dc48947bd690a002bdaa0c9',
'session': '5dc48debbd690a0029da9fd7',
'subject': '5dc48debbd690a0022da9fbe'}
"""
caplog.set_level(logging.DEBUG)
fw = FW("acquisition")
fw.destination.parent = None
hierarchy = get_run_level_and_hierarchy(fw, "01234")
print(f"run_level = {hierarchy['run_level']}")
print(f"hierarchy = {json.dumps(hierarchy, indent=4)}")
print(caplog.records)
assert hierarchy["run_level"] == "no_parent"
def test_run_level_unknown_project_says_so(caplog):
"""a destination that has no project id in parents probably never happens"""
caplog.set_level(logging.DEBUG)
fw = FW("acquisition")
fw.destination.parents["project"] = None
hierarchy = get_run_level_and_hierarchy(fw, "01234")
assert "unknown project" in caplog.records[2].message
def test_run_level_no_project_says_so(caplog):
"""A destination that has no parent probably never happens"""
caplog.set_level(logging.DEBUG)
fw = FW("acquisition")
fw.destination.parent = None
hierarchy = get_run_level_and_hierarchy(fw, "01234")
assert hierarchy["run_level"] == "no_parent"
def main(gtk_context):
log = gtk_context.log
hierarchy = get_run_level_and_hierarchy(gtk_context.client,
gtk_context.destination["id"])
if hierarchy['group'] and hierarchy['project_label']:
group = hierarchy['group']
project_label = hierarchy['project_label']
else:
log.exception('Unable to determine run level and hierarchy, exiting')
sys.exit(1)
#group = 'scien'
#project_label = 'Nate-BIDS-pre-curate'
config = parse_config(gtk_context)
inputs = validate_inputs(gtk_context)
project = gtk_context.client.lookup(f'{group}/{project_label}')
log.info(f'Found project {group}/{project_label}')
if inputs:
# Make the id column the index for the dataframe
acq_df = pd.read_csv(inputs[0], dtype=str)
ses_df = pd.read_csv(inputs[1], dtype=str)
sub_df = pd.read_csv(inputs[2], dtype=str)
bids_pre_curate.read_from_csv(acq_df, sub_df, ses_df, project,
config['dry_run'])
else:
fw = gtk_context.client
acqs = [
acq.to_dict() for acq in fw.get_project_acquisitions(project.id)
]
sess = [ses.to_dict() for ses in fw.get_project_sessions(project.id)]
subs = [sub.to_dict() for sub in fw.get_project_subjects(project.id)]
file_names = bids_pre_curate.build_csv(acqs,
subs,
sess,
project.label,
suggest=config['suggest'],
allows=config['allows'])
if not os.path.exists(gtk_context.output_dir):
try:
os.mkdir(gtk_context.output_dir)
except FileNotFoundError as e:
gtk_context.log.error()
sys.exit(1)
# Move tmp files to output
for filename in file_names:
output = os.path.join(gtk_context.output_dir,
os.path.basename(filename))
shutil.move(filename, output)
def test_run_level_exception_handled(caplog):
caplog.set_level(logging.DEBUG)
with patch(
"flywheel.Client.get",
MagicMock(side_effect=flywheel.ApiException("foo", "fum")),
):
fw = flywheel.Client
hierarchy = get_run_level_and_hierarchy(fw, "01234")
print(caplog.records)
assert "Unable to get level and hierarchy" in caplog.records[0].message
def test_run_level_acquisition_works(caplog):
"""Running at acquisition level means everything is defined."""
caplog.set_level(logging.DEBUG)
fw = FW("acquisition")
hierarchy = get_run_level_and_hierarchy(fw, "01234")
print(caplog.records)
assert hierarchy["run_level"] == "acquisition"
assert len(caplog.records) == 6
assert "run_level = acquisition" in caplog.records[0].message
assert "monkeyshine" in caplog.records[1].message
assert "TheProjectLabel" in caplog.records[2].message
assert "subject_label = TheSubjectCode" in caplog.records[3].message
assert "session_label = TheSessionLabel" in caplog.records[4].message
assert "acquisition_label = TheAcquisitionLabel" in caplog.records[
5].message
def test_run_level_session_works(caplog):
"""Running at session level means acquisition is undefined."""
caplog.set_level(logging.DEBUG)
fw = FW("session")
fw.destination.parents["acquisition"] = None
hierarchy = get_run_level_and_hierarchy(fw, "01234")
print(caplog.records)
assert hierarchy["run_level"] == "session"
assert len(caplog.records) == 6
assert "run_level = session" in caplog.records[0].message
assert "monkeyshine" in caplog.records[1].message
assert "TheProjectLabel" in caplog.records[2].message
assert "subject_label = TheSubjectCode" in caplog.records[3].message
assert "session_label = TheSessionLabel" in caplog.records[4].message
assert "acquisition_label = unknown acquisition" in caplog.records[
5].message
def test_run_level_project_works(caplog):
"""Running at project level means subject, session, and acquisition are undefined."""
caplog.set_level(logging.DEBUG)
fw = FW("project")
fw.destination.parents["subject"] = None
fw.destination.parents["session"] = None
fw.destination.parents["acquisition"] = None
hierarchy = get_run_level_and_hierarchy(fw, "01234")
print(caplog.records)
assert hierarchy["run_level"] == "project"
assert hierarchy["group"] == "monkeyshine"
assert len(caplog.records) == 6
assert "run_level = project" in caplog.records[0].message
assert "monkeyshine" in caplog.records[1].message
assert "TheProjectLabel" in caplog.records[2].message
assert "subject_label = unknown subject" in caplog.records[3].message
assert "session_label = unknown session" in caplog.records[4].message
assert "acquisition_label = unknown acquisition" in caplog.records[
5].message
def main(gtk_context):
log = gtk_context.log
# Keep a list of errors and warning to print all in one place at end of log
# Any errors will prevent the command from running and will cause exit(1)
errors = []
warnings = []
# Given the destination container, figure out if running at the project,
# subject, or session level.
hierarchy = get_run_level_and_hierarchy(
gtk_context.client, gtk_context.destination["id"]
)
# This is the label of the project, subject or session and is used
# as part of the name of the output files.
run_label = make_file_name_safe(hierarchy["run_label"])
# Output will be put into a directory named as the destination id.
# This allows the raw output to be deleted so that a zipped archive
# can be returned.
output_analysisid_dir = gtk_context.output_dir / gtk_context.destination["id"]
# editme: optional feature
# get # cpu's to set -openmp
os_cpu_count = str(os.cpu_count())
log.info("os.cpu_count() = %s", os_cpu_count)
n_cpus = gtk_context.config.get("n_cpus")
if n_cpus:
if n_cpus > os_cpu_count:
log.warning("n_cpus > number available, using %d", os_cpu_count)
gtk_context.config["n_cpus"] = os_cpu_count
elif n_cpus == 0:
log.info("n_cpus == 0, using %d (maximum available)", os_cpu_count)
gtk_context.config["n_cpus"] = os_cpu_count
else: # Default is to use all cpus available
gtk_context.config["n_cpus"] = os_cpu_count # zoom zoom
# editme: optional feature
mem_gb = psutil.virtual_memory().available / (1024 ** 3)
log.info("psutil.virtual_memory().available= {:4.1f} GiB".format(mem_gb))
# grab environment for gear (saved in Dockerfile)
with open("/tmp/gear_environ.json", "r") as f:
environ = json.load(f)
# Add environment to log if debugging
kv = ""
for k, v in environ.items():
kv += k + "=" + v + " "
log.debug("Environment: " + kv)
# The main command line command to be run:
# editme: Set the actual gear command:
command = [BIDS_APP]
# This is also used as part of the name of output files
command_name = make_file_name_safe(command[0])
# editme: add positional arguments that the above command needs
# 3 positional args: bids path, output dir, 'participant'
# This should be done here in case there are nargs='*' arguments
# These follow the BIDS Apps definition (https://github.com/BIDS-Apps)
command.append(str(gtk_context.work_dir / "bids"))
command.append(str(output_analysisid_dir))
command.append(ANALYSIS_LEVEL)
# get config for command by skipping gear config parameters
command_config = {}
for key, val in gtk_context.config.items():
if key == "bids_app_args":
bids_app_args = val.split(" ")
for baa in bids_app_args:
command.append(baa)
elif not key.startswith("gear-"):
command_config[key] = val
# print("command_config:", json.dumps(command_config, indent=4))
# Validate the command parameter dictionary - make sure everything is
# ready to run so errors will appear before launching the actual gear
# code. Add descriptions of problems to errors & warnings lists.
# print("gtk_context.config:", json.dumps(gtk_context.config, indent=4))
command = build_command_list(command, command_config)
# print(command)
# editme: fix --verbose argparse argument
for ii, cmd in enumerate(command):
if cmd.startswith("--verbose"):
# handle a 'count' argparse argument where manifest gives
# enumerated possibilities like v, vv, or vvv
# e.g. replace "--verbose=vvv' with '-vvv'
command[ii] = "-" + cmd.split("=")[1]
log.info("command is: %s", str(command))
# editme: if the command needs a freesurfer license keep this
if Path(FREESURFER_LICENSE).exists():
log.debug("%s exists.", FREESURFER_LICENSE)
install_freesurfer_license(gtk_context, FREESURFER_LICENSE)
if len(errors) == 0:
# editme: optional feature
# Create HTML file that shows BIDS "Tree" like output?
tree = True
tree_title = f"{command_name} BIDS Tree"
error_code = download_bids_for_runlevel(
gtk_context,
hierarchy,
tree=tree,
tree_title=tree_title,
src_data=DOWNLOAD_SOURCE,
folders=DOWNLOAD_MODALITIES,
dry_run=gtk_context.config.get("gear-dry-run"),
do_validate_bids=gtk_context.config.get("gear-run-bids-validation"),
)
if error_code > 0 and not gtk_context.config.get("gear-ignore-bids-errors"):
errors.append(f"BIDS Error(s) detected. Did not run {CONTAINER}")
# now that work/bids/ exists, copy in the ignore file
bidsignore_path = gtk_context.get_input_path("bidsignore")
if bidsignore_path:
shutil.copy(bidsignore_path, "work/bids/.bidsignore")
log.info("Installed .bidsignore in work/bids/")
# see https://github.com/bids-standard/pybids/tree/master/examples
# for any necessary work on the bids files inside the gear, perhaps
# to query results or count stuff to estimate how long things will take.
# Add that stuff to utils/bids.py
# Don't run if there were errors or if this is a dry run
ok_to_run = True
if len(errors) > 0:
ok_to_run = False
returncode = 1
log.info("Command was NOT run because of previous errors.")
if gtk_context.config.get("gear-dry-run"):
ok_to_run = False
returncode = 0
e = "gear-dry-run is set: Command was NOT run."
log.warning(e)
warnings.append(e)
pretend_it_ran(gtk_context)
try:
if ok_to_run:
returncode = 0
# Create output directory
log.info("Creating output directory %s", output_analysisid_dir)
Path(output_analysisid_dir).mkdir()
# This is what it is all about
exec_command(command, environ=environ)
except RuntimeError as exc:
returncode = 1
errors.append(exc)
log.critical(exc)
log.exception("Unable to execute command.")
finally:
# Cleanup, move all results to the output directory
# TODO
# see https://github.com/bids-standard/pybids/tree/master/examples
# for any necessary work on the bids files inside the gear, perhaps
# to query results or count stuff to estimate how long things will take.
# Add that to utils/results.py
# zip entire output/<analysis_id> folder into
# <gear_name>_<project|subject|session label>_<analysis.id>.zip
zip_file_name = (
gtk_context.manifest["name"]
+ f"_{run_label}_{gtk_context.destination['id']}.zip"
)
zip_output(
str(gtk_context.output_dir),
gtk_context.destination["id"],
zip_file_name,
dry_run=False,
exclude_files=None,
)
# editme: optional feature
# zip any .html files in output/<analysis_id>/
zip_htmls(gtk_context, output_analysisid_dir)
# editme: optional feature
# possibly save ALL intermediate output
if gtk_context.config.get("gear-save-intermediate-output"):
zip_all_intermediate_output(gtk_context, run_label)
# possibly save intermediate files and folders
zip_intermediate_selected(gtk_context, run_label)
# clean up: remove output that was zipped
if Path(output_analysisid_dir).exists():
if not gtk_context.config.get("gear-keep-output"):
log.debug('removing output directory "%s"', str(output_analysisid_dir))
shutil.rmtree(output_analysisid_dir)
else:
log.info(
'NOT removing output directory "%s"', str(output_analysisid_dir)
)
else:
log.info("Output directory does not exist so it cannot be removed")
# Report errors and warnings at the end of the log so they can be easily seen.
if len(warnings) > 0:
msg = "Previous warnings:\n"
for err in warnings:
if str(type(err)).split("'")[1] == "str":
# show string
msg += " Warning: " + str(err) + "\n"
else: # show type (of warning) and warning message
err_type = str(type(err)).split("'")[1]
msg += f" {err_type}: {str(err)}\n"
log.info(msg)
if len(errors) > 0:
msg = "Previous errors:\n"
for err in errors:
if str(type(err)).split("'")[1] == "str":
# show string
msg += " Error msg: " + str(err) + "\n"
else: # show type (of error) and error message
err_type = str(type(err)).split("'")[1]
msg += f" {err_type}: {str(err)}\n"
log.info(msg)
returncode = 1
return returncode
