def test_conversion(tmpdir, subject, heuristic, anon_cmd): tmpdir.chdir() try: datadir = fetch_data( tmpdir.strpath, "dbic/QA", # path from datalad database root getpath=op.join('sourcedata', subject)) except IncompleteResultsError as exc: pytest.skip("Failed to fetch test data: %s" % str(exc)) outdir = tmpdir.mkdir('out').strpath args = gen_heudiconv_args( datadir, outdir, subject, heuristic, anon_cmd, template=op.join('sourcedata/{subject}/*/*/*.tgz')) runner(args) # run conversion # verify functionals were converted assert glob('{}/{}/func/*'.format(outdir, subject)) == \ glob('{}/{}/func/*'.format(datadir, subject)) # compare some json metadata json_ = '{}/task-rest_acq-24mm64sl1000tr32te600dyn_bold.json'.format orig, conv = (load_json(json_(datadir)), load_json(json_(outdir))) keys = ['EchoTime', 'MagneticFieldStrength', 'Manufacturer', 'SliceTiming'] for key in keys: assert orig[key] == conv[key]
def json_load_patched(fp): calls[0] += 1 if calls[0] == 1: # just reuse bad file load_json(str(invalid_json_file)) elif calls[0] == 2: raise FileNotFoundError() else: return json_load(fp)
def test_populate_bids_templates(tmpdir): populate_bids_templates(str(tmpdir), defaults={'Acknowledgements': 'something'}) for f in "README", "dataset_description.json", "CHANGES": # Just test that we have created them and they all have stuff TODO assert "TODO" in tmpdir.join(f).read() description_file = tmpdir.join('dataset_description.json') assert "something" in description_file.read() # it should also be available as a command os.unlink(str(description_file)) # it must fail if no heuristic was provided with pytest.raises(ValueError) as cme: runner(['--command', 'populate-templates', '--files', str(tmpdir)]) assert str(cme.value).startswith("Specify heuristic using -f. Known are:") assert "convertall," in str(cme.value) assert not description_file.exists() runner([ '--command', 'populate-templates', '-f', 'convertall', '--files', str(tmpdir) ]) assert "something" not in description_file.read() assert "TODO" in description_file.read() assert load_json(tmpdir / "scans.json") == SCANS_FILE_FIELDS
def test_multiecho(tmpdir, subject='MEEPI', heuristic='bids_ME.py'): tmpdir.chdir() try: datadir = fetch_data(tmpdir.strpath, "dicoms/velasco/MEEPI") except IncompleteResultsError as exc: pytest.skip("Failed to fetch test data: %s" % str(exc)) outdir = tmpdir.mkdir('out').strpath args = gen_heudiconv_args(datadir, outdir, subject, heuristic) runner(args) # run conversion # check if we have echo functionals echoes = glob(op.join('out', 'sub-' + subject, 'func', '*echo*nii.gz')) assert len(echoes) == 3 # check EchoTime of each functional # ET1 < ET2 < ET3 prev_echo = 0 for echo in sorted(echoes): _json = echo.replace('.nii.gz', '.json') assert _json echotime = load_json(_json).get('EchoTime', None) assert echotime > prev_echo prev_echo = echotime events = glob(op.join('out', 'sub-' + subject, 'func', '*events.tsv')) for event in events: assert 'echo-' not in event
def test_populate_intended_for(tmpdir, folder, expected_prefix, simulation_function): """ Test populate_intended_for. Parameters: ---------- tmpdir folder : str or os.path path to BIDS study to be simulated, relative to tmpdir expected_prefix : str expected start of the "IntendedFor" elements simulation_function : function function to create the directory tree and expected results """ session_folder = op.join(str(tmpdir), folder) session_struct, expected_result, _, _ = simulation_function(session_folder) populate_intended_for(session_folder, matching_parameters='Shims', criterion='First') # Now, loop through the jsons in the fmap folder and make sure it matches # the expected result: fmap_folder = op.join(session_folder, 'fmap') for j in session_struct['fmap'].keys(): if j.endswith('.json'): assert j in expected_result.keys() data = load_json(op.join(fmap_folder, j)) if expected_result[j]: assert data['IntendedFor'] == expected_result[j] # Also, make sure the run with random shims is not here: # (It is assured by the assert above, but let's make it # explicit) run_prefix = j.split('_acq')[0] assert '{p}_acq-unmatched_bold.nii.gz'.format(p=run_prefix) not in data['IntendedFor'] else: assert 'IntendedFor' not in data.keys()
def test_conversion(tmpdir, subject, heuristic, anon_cmd): tmpdir.chdir() try: datadir = fetch_data(tmpdir.strpath, "dbic/QA", # path from datalad database root getpath=op.join('sourcedata', f'sub-{subject}')) except IncompleteResultsError as exc: pytest.skip("Failed to fetch test data: %s" % str(exc)) outdir = tmpdir.mkdir('out').strpath args = gen_heudiconv_args( datadir, outdir, subject, heuristic, anon_cmd, template='sourcedata/sub-{subject}/*/*/*.tgz' ) runner(args) # run conversion # Get the possibly anonymized subject id and verify that it was # anonymized or not: subject_maybe_anon = glob(f'{outdir}/sub-*') assert len(subject_maybe_anon) == 1 # just one should be there subject_maybe_anon = op.basename(subject_maybe_anon[0])[4:] if anon_cmd: assert subject_maybe_anon != subject else: assert subject_maybe_anon == subject # verify functionals were converted outfiles = sorted([f[len(outdir):] for f in glob(f'{outdir}/sub-{subject_maybe_anon}/func/*')]) assert outfiles datafiles = sorted([f[len(datadir):] for f in glob(f'{datadir}/sub-{subject}/ses-*/func/*')]) # original data has ses- but because we are converting only func, and not # providing any session, we will not "match". Let's strip away the session datafiles = [re.sub(r'[/\\_]ses-[^/\\_]*', '', f) for f in datafiles] if not anon_cmd: assert outfiles == datafiles else: assert outfiles != datafiles # sid was anonymized assert len(outfiles) == len(datafiles) # but we have the same number of files # compare some json metadata json_ = '{}/task-rest_acq-24mm64sl1000tr32te600dyn_bold.json'.format orig, conv = (load_json(json_(datadir)), load_json(json_(outdir))) keys = ['EchoTime', 'MagneticFieldStrength', 'Manufacturer', 'SliceTiming'] for key in keys: assert orig[key] == conv[key]
def test_load_json(tmpdir, caplog): # test invalid json ifname = 'invalid.json' invalid_json_file = str(tmpdir / ifname) create_tree(str(tmpdir), {ifname: u"I'm Jason Bourne"}) with pytest.raises(JSONDecodeError): load_json(str(invalid_json_file)) assert ifname in caplog.text # test valid json vcontent = {"secret": "spy"} vfname = "valid.json" valid_json_file = str(tmpdir / vfname) save_json(valid_json_file, vcontent) assert load_json(valid_json_file) == vcontent
def test_load_json(tmpdir, caplog): # test invalid json ifname = 'invalid.json' invalid_json_file = str(tmpdir / ifname) create_tree(str(tmpdir), {ifname: u"I'm Jason Bourne"}) with pytest.raises(JSONDecodeError): load_json(str(invalid_json_file)) # and even if we ask to retry a few times -- should be the same with pytest.raises(JSONDecodeError): load_json(str(invalid_json_file), retry=3) with pytest.raises(FileNotFoundError): load_json("absent123not.there", retry=3) assert ifname in caplog.text # test valid json vcontent = {"secret": "spy"} vfname = "valid.json" valid_json_file = str(tmpdir / vfname) save_json(valid_json_file, vcontent) assert load_json(valid_json_file) == vcontent calls = [0] json_load = json.load def json_load_patched(fp): calls[0] += 1 if calls[0] == 1: # just reuse bad file load_json(str(invalid_json_file)) elif calls[0] == 2: raise FileNotFoundError() else: return json_load(fp) with mock.patch.object(json, 'load', json_load_patched): assert load_json(valid_json_file, retry=3) == vcontent
def custom_callable(*args): """ Called at the end of `heudiconv.convert.convert()` to perform clean-up Checks to see if multiple "clean" output files were generated by ``heudiconv``. If so, assumes that this was because they had different echo times and tries to rename them and embed metadata from the relevant dicom files. This only needs to be done because the PPMI dicoms are a hot mess (cf. all the lists above with different series descriptions). """ import glob import re import pydicom as dcm import nibabel as nib import numpy as np from heudiconv.cli.run import get_parser from heudiconv.dicoms import embed_metadata_from_dicoms from heudiconv.utils import (load_json, TempDirs, treat_infofile, set_readonly) # unpack inputs and get command line arguments (again) # there's gotta be a better way to do this, but c'est la vie prefix, outtypes, item_dicoms = args[:3] outtype = outtypes[0] opts = get_parser().parse_args() # if you don't want BIDS format then you're going to have to rename outputs # on your own! if not opts.bids: return # do a crappy job of checking if multiple output files were generated # if we're only seeing one file, we're good to go # otherwise, we need to do some fun re-naming... res_files = glob.glob(prefix + '[1-9].' + outtype) if len(res_files) < 2: return # there are few a sequences with some weird stuff that causes >2 # files to be generated, some of which are two-dimensional (one slice) # we don't want that because that's nonsense, so let's design a check # for 2D files and just remove them for fname in res_files: if len([f for f in nib.load(fname).shape if f > 1]) < 3: os.remove(fname) os.remove(fname.replace(outtype, 'json')) res_files = [fname for fname in res_files if os.path.exists(fname)] bids_pairs = [(f, f.replace(outtype, 'json')) for f in res_files] # if there's only one file remaining don't add a needless 'echo' key # just rename the file and be done with it if len(bids_pairs) == 1: safe_movefile(bids_pairs[0][0], prefix + '.' + outtype) safe_movefile(bids_pairs[0][1], prefix + scaninfo_suffix) return # usually, at least two remaining files will exist # the main reason this happens with PPMI data is dual-echo sequences # look in the json files for EchoTime and generate a key based on that echonums = [load_json(json).get('EchoTime') for (_, json) in bids_pairs] if all([f is None for f in echonums]): return echonums = np.argsort(echonums) + 1 for echo, (nifti, json) in zip(echonums, bids_pairs): # create new prefix with echo specifier # this isn't *technically* BIDS compliant, yet, but we're making due... split = re.search(r'run-(\d+)_', prefix).end() new_prefix = (prefix[:split] + 'echo-%d_' % echo + prefix[split:]) outname, scaninfo = (new_prefix + '.' + outtype, new_prefix + scaninfo_suffix) # safely move files to new name safe_movefile(nifti, outname, overwrite=False) safe_movefile(json, scaninfo, overwrite=False) # embed metadata from relevant dicoms (i.e., with same echo number) dicoms = [ f for f in item_dicoms if isclose( float(dcm.read_file(f, force=True).EchoTime) / 1000, load_json(scaninfo).get('EchoTime')) ] prov_file = prefix + '_prov.ttl' if opts.with_prov else None embed_metadata_from_dicoms(opts.bids, dicoms, outname, new_prefix + '.json', prov_file, scaninfo, TempDirs(), opts.with_prov, opts.minmeta) # perform the bits of heudiconv.convert.convert that were never called if scaninfo and os.path.exists(scaninfo): lgr.info("Post-treating %s file", scaninfo) treat_infofile(scaninfo) if outname and os.path.exists(outname): set_readonly(outname)