def test_conversion(tmpdir, subject, heuristic, anon_cmd):
tmpdir.chdir()
try:
datadir = fetch_data(
tmpdir.strpath,
"dbic/QA", # path from datalad database root
getpath=op.join('sourcedata', subject))
except IncompleteResultsError as exc:
pytest.skip("Failed to fetch test data: %s" % str(exc))
outdir = tmpdir.mkdir('out').strpath
args = gen_heudiconv_args(
datadir,
outdir,
subject,
heuristic,
anon_cmd,
template=op.join('sourcedata/{subject}/*/*/*.tgz'))
runner(args) # run conversion
# verify functionals were converted
assert glob('{}/{}/func/*'.format(outdir, subject)) == \
glob('{}/{}/func/*'.format(datadir, subject))
# compare some json metadata
json_ = '{}/task-rest_acq-24mm64sl1000tr32te600dyn_bold.json'.format
orig, conv = (load_json(json_(datadir)), load_json(json_(outdir)))
keys = ['EchoTime', 'MagneticFieldStrength', 'Manufacturer', 'SliceTiming']
for key in keys:
assert orig[key] == conv[key]
def json_load_patched(fp):
calls[0] += 1
if calls[0] == 1:
# just reuse bad file
load_json(str(invalid_json_file))
elif calls[0] == 2:
raise FileNotFoundError()
else:
return json_load(fp)
def test_populate_bids_templates(tmpdir):
populate_bids_templates(str(tmpdir),
defaults={'Acknowledgements': 'something'})
for f in "README", "dataset_description.json", "CHANGES":
# Just test that we have created them and they all have stuff TODO
assert "TODO" in tmpdir.join(f).read()
description_file = tmpdir.join('dataset_description.json')
assert "something" in description_file.read()
# it should also be available as a command
os.unlink(str(description_file))
# it must fail if no heuristic was provided
with pytest.raises(ValueError) as cme:
runner(['--command', 'populate-templates', '--files', str(tmpdir)])
assert str(cme.value).startswith("Specify heuristic using -f. Known are:")
assert "convertall," in str(cme.value)
assert not description_file.exists()
runner([
'--command', 'populate-templates', '-f', 'convertall', '--files',
str(tmpdir)
])
assert "something" not in description_file.read()
assert "TODO" in description_file.read()
assert load_json(tmpdir / "scans.json") == SCANS_FILE_FIELDS
def test_multiecho(tmpdir, subject='MEEPI', heuristic='bids_ME.py'):
tmpdir.chdir()
try:
datadir = fetch_data(tmpdir.strpath, "dicoms/velasco/MEEPI")
except IncompleteResultsError as exc:
pytest.skip("Failed to fetch test data: %s" % str(exc))
outdir = tmpdir.mkdir('out').strpath
args = gen_heudiconv_args(datadir, outdir, subject, heuristic)
runner(args) # run conversion
# check if we have echo functionals
echoes = glob(op.join('out', 'sub-' + subject, 'func', '*echo*nii.gz'))
assert len(echoes) == 3
# check EchoTime of each functional
# ET1 < ET2 < ET3
prev_echo = 0
for echo in sorted(echoes):
_json = echo.replace('.nii.gz', '.json')
assert _json
echotime = load_json(_json).get('EchoTime', None)
assert echotime > prev_echo
prev_echo = echotime
events = glob(op.join('out', 'sub-' + subject, 'func', '*events.tsv'))
for event in events:
assert 'echo-' not in event
def test_populate_intended_for(tmpdir, folder, expected_prefix, simulation_function):
"""
Test populate_intended_for.
Parameters:
----------
tmpdir
folder : str or os.path
path to BIDS study to be simulated, relative to tmpdir
expected_prefix : str
expected start of the "IntendedFor" elements
simulation_function : function
function to create the directory tree and expected results
"""
session_folder = op.join(str(tmpdir), folder)
session_struct, expected_result, _, _ = simulation_function(session_folder)
populate_intended_for(session_folder, matching_parameters='Shims', criterion='First')
# Now, loop through the jsons in the fmap folder and make sure it matches
# the expected result:
fmap_folder = op.join(session_folder, 'fmap')
for j in session_struct['fmap'].keys():
if j.endswith('.json'):
assert j in expected_result.keys()
data = load_json(op.join(fmap_folder, j))
if expected_result[j]:
assert data['IntendedFor'] == expected_result[j]
# Also, make sure the run with random shims is not here:
# (It is assured by the assert above, but let's make it
# explicit)
run_prefix = j.split('_acq')[0]
assert '{p}_acq-unmatched_bold.nii.gz'.format(p=run_prefix) not in data['IntendedFor']
else:
assert 'IntendedFor' not in data.keys()
def test_conversion(tmpdir, subject, heuristic, anon_cmd):
tmpdir.chdir()
try:
datadir = fetch_data(tmpdir.strpath,
"dbic/QA", # path from datalad database root
getpath=op.join('sourcedata', f'sub-{subject}'))
except IncompleteResultsError as exc:
pytest.skip("Failed to fetch test data: %s" % str(exc))
outdir = tmpdir.mkdir('out').strpath
args = gen_heudiconv_args(
datadir, outdir, subject, heuristic, anon_cmd,
template='sourcedata/sub-{subject}/*/*/*.tgz'
)
runner(args) # run conversion
# Get the possibly anonymized subject id and verify that it was
# anonymized or not:
subject_maybe_anon = glob(f'{outdir}/sub-*')
assert len(subject_maybe_anon) == 1 # just one should be there
subject_maybe_anon = op.basename(subject_maybe_anon[0])[4:]
if anon_cmd:
assert subject_maybe_anon != subject
else:
assert subject_maybe_anon == subject
# verify functionals were converted
outfiles = sorted([f[len(outdir):] for f in glob(f'{outdir}/sub-{subject_maybe_anon}/func/*')])
assert outfiles
datafiles = sorted([f[len(datadir):] for f in glob(f'{datadir}/sub-{subject}/ses-*/func/*')])
# original data has ses- but because we are converting only func, and not
# providing any session, we will not "match". Let's strip away the session
datafiles = [re.sub(r'[/\\_]ses-[^/\\_]*', '', f) for f in datafiles]
if not anon_cmd:
assert outfiles == datafiles
else:
assert outfiles != datafiles # sid was anonymized
assert len(outfiles) == len(datafiles) # but we have the same number of files
# compare some json metadata
json_ = '{}/task-rest_acq-24mm64sl1000tr32te600dyn_bold.json'.format
orig, conv = (load_json(json_(datadir)),
load_json(json_(outdir)))
keys = ['EchoTime', 'MagneticFieldStrength', 'Manufacturer', 'SliceTiming']
for key in keys:
assert orig[key] == conv[key]
def test_load_json(tmpdir, caplog):
# test invalid json
ifname = 'invalid.json'
invalid_json_file = str(tmpdir / ifname)
create_tree(str(tmpdir), {ifname: u"I'm Jason Bourne"})
with pytest.raises(JSONDecodeError):
load_json(str(invalid_json_file))
assert ifname in caplog.text
# test valid json
vcontent = {"secret": "spy"}
vfname = "valid.json"
valid_json_file = str(tmpdir / vfname)
save_json(valid_json_file, vcontent)
assert load_json(valid_json_file) == vcontent
def test_load_json(tmpdir, caplog):
# test invalid json
ifname = 'invalid.json'
invalid_json_file = str(tmpdir / ifname)
create_tree(str(tmpdir), {ifname: u"I'm Jason Bourne"})
with pytest.raises(JSONDecodeError):
load_json(str(invalid_json_file))
# and even if we ask to retry a few times -- should be the same
with pytest.raises(JSONDecodeError):
load_json(str(invalid_json_file), retry=3)
with pytest.raises(FileNotFoundError):
load_json("absent123not.there", retry=3)
assert ifname in caplog.text
# test valid json
vcontent = {"secret": "spy"}
vfname = "valid.json"
valid_json_file = str(tmpdir / vfname)
save_json(valid_json_file, vcontent)
assert load_json(valid_json_file) == vcontent
calls = [0]
json_load = json.load
def json_load_patched(fp):
calls[0] += 1
if calls[0] == 1:
# just reuse bad file
load_json(str(invalid_json_file))
elif calls[0] == 2:
raise FileNotFoundError()
else:
return json_load(fp)
with mock.patch.object(json, 'load', json_load_patched):
assert load_json(valid_json_file, retry=3) == vcontent
def custom_callable(*args):
"""
Called at the end of `heudiconv.convert.convert()` to perform clean-up
Checks to see if multiple "clean" output files were generated by
``heudiconv``. If so, assumes that this was because they had different echo
times and tries to rename them and embed metadata from the relevant dicom
files. This only needs to be done because the PPMI dicoms are a hot mess
(cf. all the lists above with different series descriptions).
"""
import glob
import re
import pydicom as dcm
import nibabel as nib
import numpy as np
from heudiconv.cli.run import get_parser
from heudiconv.dicoms import embed_metadata_from_dicoms
from heudiconv.utils import (load_json, TempDirs, treat_infofile,
set_readonly)
# unpack inputs and get command line arguments (again)
# there's gotta be a better way to do this, but c'est la vie
prefix, outtypes, item_dicoms = args[:3]
outtype = outtypes[0]
opts = get_parser().parse_args()
# if you don't want BIDS format then you're going to have to rename outputs
# on your own!
if not opts.bids:
return
# do a crappy job of checking if multiple output files were generated
# if we're only seeing one file, we're good to go
# otherwise, we need to do some fun re-naming...
res_files = glob.glob(prefix + '[1-9].' + outtype)
if len(res_files) < 2:
return
# there are few a sequences with some weird stuff that causes >2
# files to be generated, some of which are two-dimensional (one slice)
# we don't want that because that's nonsense, so let's design a check
# for 2D files and just remove them
for fname in res_files:
if len([f for f in nib.load(fname).shape if f > 1]) < 3:
os.remove(fname)
os.remove(fname.replace(outtype, 'json'))
res_files = [fname for fname in res_files if os.path.exists(fname)]
bids_pairs = [(f, f.replace(outtype, 'json')) for f in res_files]
# if there's only one file remaining don't add a needless 'echo' key
# just rename the file and be done with it
if len(bids_pairs) == 1:
safe_movefile(bids_pairs[0][0], prefix + '.' + outtype)
safe_movefile(bids_pairs[0][1], prefix + scaninfo_suffix)
return
# usually, at least two remaining files will exist
# the main reason this happens with PPMI data is dual-echo sequences
# look in the json files for EchoTime and generate a key based on that
echonums = [load_json(json).get('EchoTime') for (_, json) in bids_pairs]
if all([f is None for f in echonums]):
return
echonums = np.argsort(echonums) + 1
for echo, (nifti, json) in zip(echonums, bids_pairs):
# create new prefix with echo specifier
# this isn't *technically* BIDS compliant, yet, but we're making due...
split = re.search(r'run-(\d+)_', prefix).end()
new_prefix = (prefix[:split] + 'echo-%d_' % echo + prefix[split:])
outname, scaninfo = (new_prefix + '.' + outtype,
new_prefix + scaninfo_suffix)
# safely move files to new name
safe_movefile(nifti, outname, overwrite=False)
safe_movefile(json, scaninfo, overwrite=False)
# embed metadata from relevant dicoms (i.e., with same echo number)
dicoms = [
f for f in item_dicoms if isclose(
float(dcm.read_file(f, force=True).EchoTime) / 1000,
load_json(scaninfo).get('EchoTime'))
]
prov_file = prefix + '_prov.ttl' if opts.with_prov else None
embed_metadata_from_dicoms(opts.bids, dicoms, outname,
new_prefix + '.json', prov_file, scaninfo,
TempDirs(), opts.with_prov, opts.minmeta)
# perform the bits of heudiconv.convert.convert that were never called
if scaninfo and os.path.exists(scaninfo):
lgr.info("Post-treating %s file", scaninfo)
treat_infofile(scaninfo)
if outname and os.path.exists(outname):
set_readonly(outname)