def test_load_heuristic():
by_name = load_heuristic('reproin')
from_file = load_heuristic(op.join(HEURISTICS_PATH, 'reproin.py'))
assert by_name
assert by_name.filename == from_file.filename
with pytest.raises(ImportError):
load_heuristic('unknownsomething')
with pytest.raises(ImportError):
load_heuristic(op.join(HEURISTICS_PATH, 'unknownsomething.py'))
def convert_to_bids(client,
project_label,
heuristic_path,
subject_labels=None,
session_labels=None,
dry_run=False):
"""Converts a project to bids by reading the file entries from flywheel
and using the heuristics to write back to the BIDS namespace of the flywheel
containers
Args:
client (Client): The flywheel sdk client
project_label (str): The label of the project
heuristic_path (str): The path to the heuristic file or the name of a
known heuristic
subject_code (str): The subject code
session_label (str): The session label
dry_run (bool): Print the changes, don't apply them on flywheel
"""
# Make sure we can find the heuristic
logger.info("Loading heuristic file...")
try:
if os.path.isfile(heuristic_path):
heuristic = utils.load_heuristic(heuristic_path)
elif "github" in heuristic_path and validators.url(heuristic_path):
# read from github
try:
response = requests.get(heuristic_path)
if response.ok:
name = 'heuristic'
spec = importlib.util.spec_from_loader(name, loader=None)
heuristic = importlib.util.module_from_spec(spec)
code = response.text
exec(code, heuristic.__dict__)
else:
logger.error(
"Couldn't find a valid URL for this heuristic at:\n\n"
+ heuristic_path + "\n")
raise ModuleNotFoundError
except:
logger.error("Trouble retrieving the URL!")
raise ModuleNotFoundError(
"Is this a valid URL to a heuristic on Github? Please check spelling!"
)
else:
heuristic = importlib.import_module(
'fw_heudiconv.example_heuristics.{}'.format(heuristic_path))
except ModuleNotFoundError as e:
logger.error("Couldn't load the specified heuristic file!")
logger.error(e)
sys.exit(1)
logger.info("Heuristic loaded successfully!")
if dry_run:
logger.setLevel(logging.DEBUG)
logger.info("Querying Flywheel server...")
project_obj = client.projects.find_first(
'label="{}"'.format(project_label))
assert project_obj, "Project not found! Maybe check spelling...?"
logger.debug('Found project: %s (%s)', project_obj['label'],
project_obj.id)
project_obj = confirm_bids_namespace(project_obj, dry_run)
sessions = client.get_project_sessions(project_obj.id)
# filters
if subject_labels:
sessions = [
s for s in sessions if s.subject['label'] in subject_labels
]
if session_labels:
sessions = [s for s in sessions if s.label in session_labels]
assert sessions, "No sessions found!"
logger.debug(
'Found sessions:\n\t%s',
"\n\t".join(['%s (%s)' % (ses['label'], ses.id) for ses in sessions]))
# try subject/session label functions
if hasattr(heuristic, "ReplaceSubject"):
subject_rename = heuristic.ReplaceSubject
else:
subject_rename = None
if hasattr(heuristic, "ReplaceSession"):
session_rename = heuristic.ReplaceSession
else:
session_rename = None
# try attachments
if hasattr(heuristic, "AttachToProject"):
logger.info("Processing project attachments based on heuristic file")
attachments = heuristic.AttachToProject()
if not isinstance(attachments, list):
attachments = [attachments]
for at in attachments:
upload_attachment(client,
project_obj,
level='project',
attachment_dict=at,
subject_rename=subject_rename,
session_rename=session_rename,
folders=['anat', 'dwi', 'func', 'fmap', 'perf'],
dry_run=dry_run)
'''if hasattr(heuristic, "AttachToSubject"):
logger.info("Processing subject attachments based on heuristic file")
attachments = heuristic.AttachToSubject()
if not isinstance(attachments, list):
attachments = [attachments]
for at in attachments:
logger.debug(
"\tFilename: {}\n\tData: {}\n\tMIMEType: {}".format(
at['name'], at['data'], at['type']
)
)
verify_name, verify_data, verify_type = verify_attachment(at['name'], at['data'], at['type'])
if not all([verify_name, verify_data, verify_type]):
logger.warning("Attachments may not be valid for upload!")
logger.debug(
"\tFilename valid: {}\n\tData valid: {}\n\tMIMEType valid: {}".format(
verify_name, verify_data, verify_type
)
)
if not dry_run:
subjects = [x.subject for x in sessions]
file_spec = flywheel.FileSpec(at['name'], at['data'], at['type'])
[sub.upload_file(file_spec) for sub in subjects]'''
num_sessions = len(sessions)
for sesnum, session in enumerate(sessions):
# Find SeqInfos to apply the heuristic to
logger.info("Applying heuristic to %s (%d/%d)...", session.label,
sesnum + 1, num_sessions)
seq_infos = get_seq_info(client, project_label, [session])
logger.debug(
"Found SeqInfos:\n%s",
"\n\t".join([pretty_string_seqinfo(seq) for seq in seq_infos]))
# apply heuristic to seqinfos
to_rename = heuristic.infotodict(seq_infos)
if not to_rename:
logger.debug("No changes to apply!")
continue
# try intendedfors
intention_map = defaultdict(list)
if hasattr(heuristic, "IntendedFor"):
logger.info(
"Processing IntendedFor fields based on heuristic file")
intention_map.update(heuristic.IntendedFor)
logger.debug(
"Intention map: %s",
pprint.pformat([(k[0], v)
for k, v in dict(intention_map).items()]))
# try metadataextras
metadata_extras = defaultdict(list)
if hasattr(heuristic, "MetadataExtras"):
logger.info("Processing Medatata fields based on heuristic file")
metadata_extras.update(heuristic.MetadataExtras)
logger.debug("Metadata extras: %s", metadata_extras)
# try subject/session label functions
if hasattr(heuristic, "ReplaceSubject"):
subject_rename = heuristic.ReplaceSubject
else:
subject_rename = None
if hasattr(heuristic, "ReplaceSession"):
session_rename = heuristic.ReplaceSession
else:
session_rename = None
# try attachments
if hasattr(heuristic, "AttachToSession"):
logger.info(
"Processing session attachments based on heuristic file")
attachments = heuristic.AttachToSession()
if not isinstance(attachments, list):
attachments = [attachments]
for at in attachments:
upload_attachment(
client,
session,
level='session',
attachment_dict=at,
subject_rename=subject_rename,
session_rename=session_rename,
folders=['anat', 'dwi', 'func', 'fmap', 'perf'],
dry_run=dry_run)
# final prep
if not dry_run:
logger.info("Applying changes to files...")
for key, val in to_rename.items():
# assert val is list
if not isinstance(val, set):
val = set(val)
for seqitem, value in enumerate(val):
apply_heuristic(client, key, value, dry_run,
intention_map[key], metadata_extras[key],
subject_rename, session_rename, seqitem + 1)
confirm_intentions(client, session, dry_run)
print("\n")
def test_populate_intended_for(tmpdir, monkeypatch, capfd, subjects, sesID,
expected_session_folder, heuristic):
"""
Test convert
For now, I'm just going to test that the call to populate_intended_for is
done with the correct argument.
More tests can be added here.
"""
def mock_populate_intended_for(session,
matching_parameters='Shims',
criterion='Closest'):
"""
Pretend we run populate_intended_for, but just print out the arguments.
"""
print('session: {}'.format(session))
print('matching_parameters: {}'.format(matching_parameters))
print('criterion: {}'.format(criterion))
return
# mock the "populate_intended_for":
monkeypatch.setattr(heudiconv.convert, "populate_intended_for",
mock_populate_intended_for)
outdir = op.join(str(tmpdir), 'foo')
outfolder = op.join(outdir, 'sub-{sID}',
'ses-{ses}') if sesID else op.join(
outdir, 'sub-{sID}')
sub_ses = 'sub-{sID}' + ('_ses-{ses}' if sesID else '')
# items are a list of tuples, with each tuple having three elements:
# prefix, outtypes, item_dicoms
items = [(op.join(outfolder, 'anat',
sub_ses + '_T1w').format(sID=s, ses=sesID), ('', ), [])
for s in subjects]
heuristic = load_heuristic(heuristic) if heuristic else None
heudiconv.convert.convert(items,
converter='',
scaninfo_suffix='.json',
custom_callable=None,
populate_intended_for_opts=getattr(
heuristic, 'POPULATE_INTENDED_FOR_OPTS',
None),
with_prov=None,
bids_options=[],
outdir=outdir,
min_meta=True,
overwrite=False)
output = capfd.readouterr()
# if the heuristic module has a 'POPULATE_INTENDED_FOR_OPTS' field, we expect
# to get the output of the mock_populate_intended_for, otherwise, no output:
pif_cfg = getattr(heuristic, 'POPULATE_INTENDED_FOR_OPTS', None)
if pif_cfg:
assert all([
"\n".join([
"session: " + outfolder.format(sID=s, ses=sesID),
# "ImagingVolume" is defined in heuristic file; "Shims" is the default
f"matching_parameters: {pif_cfg['matching_parameters']}",
f"criterion: {pif_cfg['criterion']}"
]) in output.out for s in subjects
])
else:
# If there was no heuristic, make sure populate_intended_for was not called
assert not output.out
def convert_to_bids(client,
project_label,
heuristic_path,
subject_labels=None,
session_labels=None,
dry_run=False):
"""Converts a project to bids by reading the file entries from flywheel
and using the heuristics to write back to the BIDS namespace of the flywheel
containers
Args:
client (Client): The flywheel sdk client
project_label (str): The label of the project
heuristic_path (str): The path to the heuristic file or the name of a
known heuristic
subject_code (str): The subject code
session_label (str): The session label
dry_run (bool): Print the changes, don't apply them on flywheel
"""
if dry_run:
logger.setLevel(logging.DEBUG)
logger.info("Querying Flywheel server...")
project_obj = client.projects.find_first(
'label="{}"'.format(project_label))
assert project_obj, "Project not found! Maybe check spelling...?"
logger.debug('Found project: %s (%s)', project_obj['label'],
project_obj.id)
project_obj = confirm_bids_namespace(project_obj, dry_run)
sessions = client.get_project_sessions(project_obj.id)
# filters
if subject_labels:
sessions = [
s for s in sessions if s.subject['label'] in subject_labels
]
if session_labels:
sessions = [s for s in sessions if s.label in session_labels]
assert sessions, "No sessions found!"
logger.debug(
'Found sessions:\n\t%s',
"\n\t".join(['%s (%s)' % (ses['label'], ses.id) for ses in sessions]))
# Find SeqInfos to apply the heuristic to
seq_infos = get_seq_info(client, project_label, sessions)
logger.debug(
"Found SeqInfos:\n%s",
"\n\t".join([pretty_string_seqinfo(seq) for seq in seq_infos]))
logger.info("Loading heuristic file...")
try:
if os.path.isfile(heuristic_path):
heuristic = utils.load_heuristic(heuristic_path)
else:
heuristic = importlib.import_module(
'heudiconv.heuristics.{}'.format(heuristic_path))
except ModuleNotFoundError as e:
logger.error("Couldn't load the specified heuristic file!")
logger.error(e)
sys.exit(1)
logger.info("Applying heuristic to query results...")
to_rename = heuristic.infotodict(seq_infos)
if not to_rename:
logger.debug("No changes to apply!")
sys.exit(1)
intention_map = defaultdict(list)
if hasattr(heuristic, "IntendedFor"):
logger.info("Processing IntendedFor fields based on heuristic file")
intention_map.update(heuristic.IntendedFor)
logger.debug("Intention map: %s", intention_map)
metadata_extras = defaultdict(list)
if hasattr(heuristic, "MetadataExtras"):
logger.info("Processing Medatata fields based on heuristic file")
metadata_extras.update(heuristic.MetadataExtras)
logger.debug("Metadata extras: %s", metadata_extras)
if not dry_run:
logger.info("Applying changes to files...")
if hasattr(heuristic, "ReplaceSubject"):
subject_rename = heuristic.ReplaceSubject
else:
subject_rename = None
if hasattr(heuristic, "ReplaceSession"):
session_rename = heuristic.ReplaceSession
else:
session_rename = None
for key, val in to_rename.items():
# assert val is list
if not isinstance(val, set):
val = set(val)
for seqitem, value in enumerate(val):
apply_heuristic(client, key, value, dry_run, intention_map[key],
metadata_extras[key], subject_rename,
session_rename, seqitem + 1)
if not dry_run:
for ses in sessions:
confirm_intentions(client, ses)
