def test_init_with_config_options(self):
root = join(DIRNAME, 'data')
dir1 = join(root, 'valuable_stamps')
dir2 = join(root, 'ordinary_stamps')
config1 = join(DIRNAME, 'specs', 'stamps.json')
config2 = join(dir1, 'USA', 'dir_config.json')
# # Fails because Domain usa_stamps is included twice
# with pytest.raises(ValueError) as e:
# layout = Layout([(root, [config1, config2])], exclude=['7t_trt'],
# config_filename='dir_config.json')
# print(dir(e))
# assert e.value.message.startswith('Config with name')
# Test with two configs
layout = Layout([(root, [config1, config2])], exclude=['7t_trt'])
files = [f.filename for f in layout.files.values()]
assert 'name=Inverted_Jenny#value=75000#country=USA.txt' in files
assert 'name=5c_Francis_E_Willard#value=1dollar.txt' in files
assert 'name=1_Lotus#value=1#country=Canada.txt' in files
# Test with two configs and on-the-fly directory remapping
layout = Layout([dir1, ([dir1, dir2], config1)], exclude=['USA/'])
files = [f.filename for f in layout.files.values()]
assert 'name=Inverted_Jenny#value=75000#country=USA.txt' in files
assert 'name=5c_Francis_E_Willard#value=1dollar.txt' not in files
assert 'name=1_Lotus#value=1#country=Canada.txt' in files
def test_entity_mapper(self):
class EntityMapper(object):
def hash_file(self, file):
return hash(file.path)
class MappingLayout(Layout):
def hash_file(self, file):
return str(hash(file.path)) + '.hsh'
root = join(DIRNAME, 'data', '7t_trt')
config = join(DIRNAME, 'specs', 'test_with_mapper.json')
# Test with external mapper
em = EntityMapper()
layout = Layout([(root, config)], regex_search=True, entity_mapper=em)
f = list(layout.files.values())[20]
assert hash(f.path) == f.entities['hash']
# Test with mapper set to self
layout = MappingLayout([(root, config)],
regex_search=True,
entity_mapper='self')
f = list(layout.files.values())[10]
assert str(hash(f.path)) + '.hsh' == f.entities['hash']
# Should fail if we use a spec with entities that have mappers but
# don't specify an entity-mapping object
with pytest.raises(ValueError):
layout = Layout([(root, config)], regex_search=True)
def test_init_with_include_arg(self, bids_layout):
root = join(DIRNAME, 'data', '7t_trt')
config = join(DIRNAME, 'specs', 'test.json')
layout = Layout([(root, config)], regex_search=True, include='sub-\d*')
target = join(root, "dataset_description.json")
assert target in bids_layout.files
assert target not in layout.files
assert join(root, "sub-01", "sub-01_sessions.tsv") in layout.files
with pytest.raises(ValueError):
layout = Layout([(root, config)], include='sub-\d*', exclude="meh")
def test_write_contents_to_file_defaults(self, layout):
contents = 'test'
data_dir = join(dirname(__file__), 'data', '7t_trt')
config = join(dirname(__file__), 'specs', 'test.json')
layout = Layout(data_dir, [
config, {
'name':
"test_writable",
'default_path_patterns': [
'sub-{subject}/ses-{session}/{subject}'
'{session}{run}{type}{task}{acquisition}'
'{bval}'
]
}
])
entities = {
'subject': 'Bob',
'session': '01',
'run': '1',
'type': 'test',
'task': 'test',
'acquisition': 'test',
'bval': 0
}
layout.write_contents_to_file(entities, contents=contents, index=True)
target = join(data_dir, 'sub-Bob/ses-01/Bob011testtesttest0')
assert exists(target)
with open(target) as f:
written = f.read()
assert written == contents
assert target in layout.files
shutil.rmtree(join(data_dir, 'sub-Bob'))
def test_excludes(self, tmpdir):
root = tmpdir.mkdir("ohmyderivatives").mkdir("ds")
config = join(DIRNAME, 'specs', 'test.json')
layout = Layout(str(root), config, regex_search=True)
assert layout._check_inclusions(str(root.join("ohmyimportantfile")))
assert not layout._check_inclusions(
str(root.join("badbadderivatives")))
def layout():
root = os.path.join(os.path.dirname(__file__), 'data', '7t_trt')
# note about test.json:
# in this test.json 'subject' regex was left to contain possible leading 0
# the other fields (run, session) has leading 0 stripped
config = os.path.join(os.path.dirname(__file__), 'specs', 'test.json')
return Layout(root, config, regex_search=True)
def test_dynamic_getters(self, data_dir, config):
if ('hdfs' in data_dir or 'hdfs' in config):
pytest.importorskip('hdfs')
layout = Layout([(data_dir, config)], dynamic_getters=True)
assert hasattr(layout, 'get_subjects')
assert '01' in getattr(layout, 'get_subjects')()
def test_init_with_exclude_arg(self, bids_layout):
root = join(DIRNAME, 'data', '7t_trt')
config = join(DIRNAME, 'specs', 'test.json')
layout = Layout([(root, config)], regex_search=True, exclude='sub-\d*')
target = join(root, "dataset_description.json")
assert target in bids_layout.files
assert target in layout.files
sub_file = join(root, "sub-01", "sub-01_sessions.tsv")
assert sub_file in bids_layout.files
assert sub_file not in layout.files
def test_absolute_paths(self, layout):
result = layout.get(subject=1, run=1, session=1)
assert result # that we got some entries
assert all([os.path.isabs(f.filename) for f in result])
root = os.path.join(os.path.dirname(__file__), 'data', '7t_trt')
root = os.path.relpath(root)
config = os.path.join(os.path.dirname(__file__), 'specs', 'test.json')
layout = Layout(root, config, absolute_paths=False)
result = layout.get(subject=1, run=1, session=1)
assert result
assert not any([os.path.isabs(f.filename) for f in result])
layout = Layout(root, config, absolute_paths=True)
result = layout.get(subject=1, run=1, session=1)
assert result
assert all([os.path.isabs(f.filename) for f in result])
def pickling_layout(self, bids_root, name, outdir, subset=None):
""" Load the requested BIDS layout and save it as a pickle.
Parameters
----------
bids_root: str
path to the BIDS folder.
name: str
the name of subfolder to be parsed (the layout name).
outdir: str
the folder where the pickle will be generated.
subset: list of str, default None
a selector to focus only on specific folders.
Returns
-------
outfile: str
the generated layout representation location.
"""
self._check_layout(name)
self._check_conf(name)
layout_root = os.path.join(bids_root, name)
if not os.path.isdir(layout_root):
raise ValueError(
"'{0}' is not a valid directory.".format(layout_root))
if subset is None:
layout = Layout([(layout_root, self.conf[name])])
else:
layout = Layout([(os.path.join(layout_root,
dirname), self.conf[name])
for dirname in subset])
self.layouts[name] = layout
now = datetime.datetime.now()
timestamp = "{0}-{1}-{2}".format(now.year, now.month, now.day)
outfile = os.path.join(
outdir, "{0}_{1}_{2}.pkl".format(self.project, name, timestamp))
with open(outfile, "wb") as open_file:
pickle.dump(layout, open_file, -1)
return outfile
def test_absolute_paths(self, bids_layout):
result = bids_layout.get(subject=1, run=1, session=1)
assert result # that we got some entries
assert all([os.path.isabs(f.filename) for f in result])
if not hasattr(bids_layout, '_hdfs_client'):
root = join(DIRNAME, 'data', '7t_trt')
root = os.path.relpath(root)
config = join(DIRNAME, 'specs', 'test.json')
layout = Layout(root, config, absolute_paths=False)
result = layout.get(subject=1, run=1, session=1)
assert result
assert not any([os.path.isabs(f.filename) for f in result])
layout = Layout(root, config, absolute_paths=True)
result = layout.get(subject=1, run=1, session=1)
assert result
assert all([os.path.isabs(f.filename) for f in result])
# Should always be absolute paths on HDFS
else:
root = psp.join('hdfs://localhost:9000{0}'.format(
layout._hdfs_client.root), 'data', '7t_trt')
config = psp.join('hdfs://localhost:9000{0}'.format(
layout._hdfs_client.root), 'specs', 'test.json')
layout = Layout(root, config, absolute_paths=False)
result = layout.get(subject=1, run=1, session=1)
assert result
assert all([os.path.isabs(f.filename) for f in result])
layout = Layout(root, config, absolute_paths=True)
result = layout.get(subject=1, run=1, session=1)
assert result
assert all([os.path.isabs(f.filename) for f in result])
def bids_layout(request):
if request.param == 'local':
root = os.path.join(DIRNAME, 'data', '7t_trt')
# note about test.json:
# in this test.json 'subject' regex was left to contain possible
# leading 0; the other fields (run, session) has leading 0 stripped
config = os.path.join(DIRNAME, 'specs', 'test.json')
return Layout(root, config, regex_search=True)
else:
hdfs = pytest.importorskip("hdfs")
from grabbit.extensions import HDFSLayout
client = hdfs.Config().get_client()
root = psp.join('hdfs://localhost:9000{0}'.format(client.root), 'data', '7t_trt')
config = psp.join('hdfs://localhost:9000{0}'.format(client.root), 'specs', 'test.json')
return HDFSLayout(root, config, regex_search=True)
def test_dynamic_getters(self):
data_dir = os.path.join(os.path.dirname(__file__), 'data', '7t_trt')
config = os.path.join(os.path.dirname(__file__), 'specs', 'test.json')
layout = Layout(data_dir, config, dynamic_getters=True)
assert hasattr(layout, 'get_subjects')
assert '01' in getattr(layout, 'get_subjects')()
def stamp_layout():
root = os.path.join(DIRNAME, 'data', 'valuable_stamps')
config = os.path.join(DIRNAME, 'specs', 'stamps.json')
return Layout(root, config)
def layout():
data_dir = join(dirname(__file__), 'data', '7t_trt')
config = join(dirname(__file__), 'specs', 'test.json')
layout = Layout(data_dir, config=config)
return layout
def stamp_layout():
root = join(DIRNAME, 'data', 'valuable_stamps')
config = join(DIRNAME, 'specs', 'stamps.json')
return Layout([(root, config)], config_filename='dir_config.json')
def layout():
data_dir = join(dirname(__file__), 'data', '7t_trt')
config = join(dirname(__file__), 'specs', 'test.json')
layout = Layout(data_dir, config, absolute_paths=False)
return layout
def layout_include(request):
root = join(DIRNAME, 'data', '7t_trt')
config = join(DIRNAME, 'specs', 'test_include.json')
return Layout([(root, config)], regex_search=True)
def collect_bids_data(dataset, subject, spec=None, session=None, run=None):
subject = str(subject)
if not subject.startswith('sub-'):
subject = 'sub-{}'.format(subject)
if spec is None:
raise RuntimeError('A spec file should be specified')
layout = Layout(dataset, config=spec)
if session:
session_list = [session]
else:
session_list = layout.unique('session')
if session_list == []:
session_list = [None]
if run:
run_list = [run]
else:
run_list = layout.unique('run')
if run_list == []:
run_list = [None]
queries = {
'fmap': {
'modality': 'fmap',
'ext': 'nii'
},
'epi': {
'modality': 'func',
'type': 'bold',
'ext': 'nii'
},
'sbref': {
'modality': 'func',
'type': 'sbref',
'ext': 'nii'
},
't1w': {
'type': 'T1w',
'ext': 'nii'
}
}
# Add a subject key pair to each query we make so that we only deal with
# files related to this workflows specific subject. Could be made opt...
for key in queries.keys():
queries[key]['subject'] = subject
imaging_data = copy.deepcopy(INPUTS_SPEC)
fieldmap_files = [x.filename for x in layout.get(**queries['fmap'])]
imaging_data['fmap'] = fieldmap_files
t1_files = [x.filename for x in layout.get(**queries['t1w'])]
imaging_data['t1w'] = t1_files
sbref_files = [x.filename for x in layout.get(**queries['sbref'])]
imaging_data['sbref'] = sbref_files
loop_on = ['session', 'run', 'acquisition', 'task']
get_kwargs = {}
for key in loop_on:
unique_list = layout.unique(key)
if unique_list:
get_kwargs[key] = unique_list
query_kwargs = []
for key in get_kwargs:
query_kwargs.append([(key, x) for x in get_kwargs[key]])
query_kwargs = itertools.product(*query_kwargs)
for elem in query_kwargs:
epi_files = [
x.filename
for x in layout.get(**dict(dict(elem), **queries['epi']))
]
if epi_files:
imaging_data['func'].append(epi_files)
return imaging_data