def test_has_files(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
assert not tmp_crumb.has_files()
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert os.path.exists(tmp_crumb.split()[0])
assert not tmp_crumb.has_files()
pa = Path(str(paths[0]))
pa.rmdir()
pa.touch()
assert pa.exists()
assert pa.is_file()
assert tmp_crumb.has_files()
def test_regex_ignorecase(tmp_crumb):
assert not os.path.exists(tmp_crumb._path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['SUBJ_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re') # re.match
assert len(crumb['subject_id']) == 0
assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
assert crumb._ignore == crumb.replace(subject_id='haensel')._ignore
assert not crumb.unfold()
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re.ignorecase') # re.match
assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
assert crumb._ignore == crumb.replace(subject_id='haensel')._ignore
ucrumb = crumb.unfold()[0]
assert crumb._re_method == ucrumb._re_method
assert crumb._ignore == ucrumb._ignore
re_subj_ids = crumb['subject_id']
assert re_subj_ids == ['SUBJ_{:03}'.format(i) for i in range(20, 30)]
def test_regex_replace(tmp_crumb):
assert not os.path.exists(tmp_crumb._path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'), regex='fnmatch') # fnmatch
assert tmp_crumb.ls('subject_id:subj_02*', make_crumbs=False) == crumb.ls('subject_id', make_crumbs=False)
anat_crumb = crumb.replace(modality='anat')
assert anat_crumb.exists()
fn_subj_ids = {cr['subject_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}
assert fn_subj_ids == set(['subj_{:03}'.format(i) for i in range(20, 30)])
sessions = {cr['session_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}
assert sessions == set(values_dict['session_id'])
def test_mktree_tuples(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
values_map = list(ParameterGrid(values_dict))
values_tups = [tuple(d.items()) for d in values_map]
nupaths = mktree(tmp_crumb, values_tups)
for path in nupaths:
for k in values_dict:
assert k in path
assert all([os.path.exists(npath.split()[0]) for npath in nupaths])
assert all([npath.exists() for npath in nupaths])
ls_session_ids = tmp_crumb.ls('session_id',
fullpath=False,
make_crumbs=False,
check_exists=False)
assert set(ls_session_ids) == set(values_dict['session_id'])
ls_subj_ids = tmp_crumb.ls('subject_id',
fullpath=False,
make_crumbs=False,
check_exists=False)
assert set(ls_subj_ids) == set(values_dict['subject_id'])
def test_setitem(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
assert not tmp_crumb.exists()
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
cr = list(tmp_crumb.ls())[0]
assert not list(cr.open_args())
assert cr['image'] == [values_dict['image'][0]]
cr['image'] = 'mprage2.nii'
assert cr['image'] == ['mprage2.nii']
assert 'mprage2.nii' in cr.path
assert 'mprage2.nii' in cr.ls()[0].path
cr.clear('image')
assert 'image' in list(cr.open_args())
assert 'mprage2.nii' not in cr.path
assert 'mprage2.nii' not in cr.ls()[0].path
def test_mktree_dicts(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_map = {
'session_id': ['session_{}'.format(i) for i in range(2)],
'subject_id': ['subj_{}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
nupaths = mktree(tmp_crumb, list(ParameterGrid(values_map)))
assert all([os.path.exists(npath.split()[0]) for npath in nupaths])
assert all([npath.exists() for npath in nupaths])
values_map['grimm'] = ['Jacob', 'Wilhelm']
pytest.raises(ValueError, mktree, tmp_crumb,
list(ParameterGrid(values_map)))
def test_parameter_grid():
# Taken from sklearn and converted to pytest
# Test basic properties of ParameterGrid.
def assert_grid_iter_equals_getitem(grid):
assert list(grid) == [grid[i] for i in range(len(grid))]
params1 = {"foo": [1, 2, 3]}
grid1 = ParameterGrid(params1)
assert isinstance(grid1, Iterable)
assert isinstance(grid1, Sized)
assert len(grid1) == 3
assert_grid_iter_equals_getitem(grid1)
params2 = {
"foo": [4, 2],
"bar": ["ham", "spam", "eggs"]
}
grid2 = ParameterGrid(params2)
assert len(grid2) == 6
# loop to assert we can iterate over the grid multiple times
for i in range(2):
# tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2)
points = set(tuple(chain(*(sorted(p.items())))) for p in grid2)
assert points == set(
("bar", x, "foo", y) for x, y in product(params2["bar"], params2["foo"])
)
assert_grid_iter_equals_getitem(grid2)
# Special case: empty grid (useful to get default estimator settings)
empty = ParameterGrid({})
assert len(empty) == 1
assert list(empty) == [{}]
assert_grid_iter_equals_getitem(empty)
pytest.raises(IndexError, lambda: empty[1])
has_empty = ParameterGrid([{'C': [1, 10]}, {}, {'C': [.5]}])
assert len(has_empty) == 4
assert list(has_empty) == [{'C': 1}, {'C': 10}, {}, {'C': .5}]
assert_grid_iter_equals_getitem(has_empty)
def test_exists2(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert not tmp_crumb.exists()
def test_mktree_raises(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_' + str(i) for i in range(2)],
'subject_id': ['subj_' + str(i) for i in range(3)],
'modality': ['anat', 'rest', 'pet'],
'image': ['mprage.nii', 'rest.nii', 'pet.nii'],
}
del values_dict['session_id']
del values_dict['modality']
pytest.raises(KeyError, mktree, tmp_crumb,
list(ParameterGrid(values_dict)))
def test_exists(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
assert not tmp_crumb.exists()
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert tmp_crumb.exists()
assert not _split_exists('/_/asdfasdfasdf?/{hansel}')
def readme_proj4_data_crumb_args():
crumb_path = "{base_dir}/proj4/{subject_id}/{session_id}/{image_type}/{image}"
values_dict = {
'session_id': ['session_1'],
'subject_id': ['00400{:02}'.format(i + 1) for i in range(9)],
'image_type': ['anat_1'],
'image': ['mprage.nii.gz'],
}
subject1 = dict(image='anatomical.nii.gz',
image_type='anat_1',
session_id='session_1',
subject_id='00400000')
values_map = [subject1]
values_map.extend(list(ParameterGrid(values_dict)))
return crumb_path, values_map
def test_regex(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re') # re.match
re_subj_ids = crumb['subject_id']
assert re_subj_ids == ['subj_{:03}'.format(i) for i in range(20, 30)]
assert crumb.ls('subject_id:^subj_02.*$') == crumb.ls('subject_id')
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'), regex='fnmatch') # fnmatch
fn_subj_ids = crumb['subject_id']
assert fn_subj_ids == re_subj_ids
cr_bkp = crumb.copy()
assert crumb.ls('subject_id:subj_02*') == crumb.ls('subject_id')
assert crumb['subject_id'][0] == crumb.get_first('subject_id')
assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']
assert not crumb.ls('subject_id:subj_03*') == crumb.ls('subject_id')
assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']
pytest.raises(ValueError,
Crumb,
tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
regex='hansel')
crumb2 = Crumb.copy(crumb)
assert crumb2._re_method == crumb._re_method
assert crumb2._re_args == crumb._re_args
assert crumb2.patterns == crumb.patterns
assert len(crumb2.patterns) == 1
assert 'subject_id' in crumb2.patterns.keys()
def make_tree_from_crumb(base_path, crumb_path,
crumb_args: [Dict, CrumbArgsSequences]):
crumb = Crumb(crumb_path)
crumb2 = crumb.replace(base_dir=base_path)
assert not os.path.exists(crumb2._path)
assert not crumb2.has_files()
if isinstance(crumb_args, dict):
values_map = list(ParameterGrid(crumb_args))
elif isinstance(crumb_args, list):
values_map = crumb_args
else:
raise TypeError(
'Expected `crumb_args` to be dict or list, got {}.'.format(
type(crumb_args)))
mktree(crumb2, values_map)
assert os.path.exists(crumb2.split()[0])
assert not crumb2.has_files()
return crumb2
def test_set_patterns(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
# a crumb without the pattern, the pattern is added later
crumb2 = Crumb(tmp_crumb.path, regex='fnmatch')
crumb3 = crumb2.copy()
crumb3.set_patterns()
assert crumb2 == crumb3
pytest.raises(KeyError, crumb2.set_patterns, somekey='somevalue')
crumb3.set_pattern('subject_id', 'subj_02*')
crumb2.set_patterns(subject_id='subj_02*')
assert crumb2['subject_id'] == crumb3['subject_id']
def test_regex_replace2(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
# a crumb with the pattern
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
regex='fnmatch') # fnmatch
# a crumb without the pattern, the pattern is added later
crumb2 = Crumb(tmp_crumb.path, regex='fnmatch')
crumb2.set_pattern('subject_id', 'subj_02*')
assert crumb['subject_id'] == crumb2['subject_id']
crumb2.clear_pattern('subject_id')
assert tmp_crumb['subject_id'] == crumb2['subject_id']
def test_ls_with_check(tmp_crumb):
assert not os.path.exists(tmp_crumb._path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
sbj_crumb = tmp_crumb.replace(subject_id='subj_000')
assert sbj_crumb.ls('subject_id', make_crumbs=False, fullpath=False) == ['subj_000']
assert os.path.exists(tmp_crumb.split()[0])
assert all([os.path.exists(p.path) for p in paths])
assert all([p.exists() for p in paths])
images = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
modalities = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
assert all([img.exists() for img in images])
assert all([mod.exists() for mod in modalities])
Path(str(images[0])).rmdir()
images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
assert images != images2
assert len(images) == len(images2) + 1
assert not all([img.exists() for img in images])
assert all([img.exists() for img in images2])
Path(str(images[1])).rmdir()
Path(str(images[2])).rmdir()
images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
assert tmp_crumb.ls('image') == tmp_crumb.ls()
assert tmp_crumb.ls() == tmp_crumb.unfold()
assert not all([img.exists() for img in images])
assert all([img.exists() for img in images2])
modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
str_modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=False, check_exists=True)
assert images != images2
assert len(images) == len(images2) + 3
assert modalities != modalities2
assert not all([mod.exists() for mod in modalities])
assert all([mod.exists() for mod in modalities2])
assert all([isinstance(smod, str) for smod in str_modalities2])
assert all([isinstance(mod, Crumb) for mod in modalities2])
assert all([mod.path == smod for mod, smod in zip(sorted(modalities2), sorted(str_modalities2))])
shutil.rmtree(modalities2[0].split()[0], ignore_errors=True)
modalities3 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
assert modalities2 != modalities3
assert not all([mod.exists() for mod in modalities2])
assert all([mod.exists() for mod in modalities3])
assert tmp_crumb.unfold() == tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
pytest.raises(IOError, modalities2[0].__getitem__, 'image')
img_crumb = tmp_crumb.replace(image='mprage1.nii')
assert 'image' in img_crumb._argval
assert img_crumb['image'] == ['mprage1.nii']
assert img_crumb['image'][0] == img_crumb.get_first('image')
img_crumb['modality'] = 'anat'
assert 'modality' in img_crumb._argval
assert img_crumb['modality'] == ['anat']
assert img_crumb.has_set('modality')
assert img_crumb['session_id'].count('session_01') == img_crumb['session_id'].count('session_00')
img_crumb['session_id'] = 'session_00'
assert 'session_id' in img_crumb._argval
assert img_crumb['session_id'] == ['session_00']
assert 'subj_000' not in img_crumb['subject_id']
unfolded_crumbs = tmp_crumb.unfold()
assert list(unfolded_crumbs[0].open_args()) == []
assert unfolded_crumbs[0].unfold() == [unfolded_crumbs[0]]
pytest.raises(AttributeError, unfolded_crumbs[0]._check_open_args, ['subject_id'])