def test_setitem(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
assert not tmp_crumb.exists()
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
cr = list(tmp_crumb.ls())[0]
assert not list(cr.open_args())
assert cr['image'] == [values_dict['image'][0]]
cr['image'] = 'mprage2.nii'
assert cr['image'] == ['mprage2.nii']
assert 'mprage2.nii' in cr.path
assert 'mprage2.nii' in cr.ls()[0].path
cr.clear('image')
assert 'image' in list(cr.open_args())
assert 'mprage2.nii' not in cr.path
assert 'mprage2.nii' not in cr.ls()[0].path
def test_regex_ignorecase(tmp_crumb):
assert not os.path.exists(tmp_crumb._path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['SUBJ_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re') # re.match
assert len(crumb['subject_id']) == 0
assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
assert crumb._ignore == crumb.replace(subject_id='haensel')._ignore
assert not crumb.unfold()
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re.ignorecase') # re.match
assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
assert crumb._ignore == crumb.replace(subject_id='haensel')._ignore
ucrumb = crumb.unfold()[0]
assert crumb._re_method == ucrumb._re_method
assert crumb._ignore == ucrumb._ignore
re_subj_ids = crumb['subject_id']
assert re_subj_ids == ['SUBJ_{:03}'.format(i) for i in range(20, 30)]
def test_regex_replace(tmp_crumb):
assert not os.path.exists(tmp_crumb._path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'), regex='fnmatch') # fnmatch
assert tmp_crumb.ls('subject_id:subj_02*', make_crumbs=False) == crumb.ls('subject_id', make_crumbs=False)
anat_crumb = crumb.replace(modality='anat')
assert anat_crumb.exists()
fn_subj_ids = {cr['subject_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}
assert fn_subj_ids == set(['subj_{:03}'.format(i) for i in range(20, 30)])
sessions = {cr['session_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}
assert sessions == set(values_dict['session_id'])
def test_exists2(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert not tmp_crumb.exists()
def tmp_tree(request):
crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{modality}/{image}")
base_dir = tempfile.mkdtemp(prefix="crumbtest_")
crumb2 = crumb.replace(base_dir=base_dir)
def fin():
print("teardown tmp_crumb")
request.addfinalizer(fin)
assert not op.exists(crumb2._path)
assert not crumb2.has_files()
values_dict = {
"session_id": ["session_{:02}".format(i) for i in range(2)],
"subject_id": ["subj_{:03}".format(i) for i in range(3)],
"modality": ["anat"],
"image": ["mprage1.nii", "mprage2.nii", "mprage3.nii"],
}
paths = mktree(crumb2, list(ParameterGrid(values_dict)))
assert op.exists(crumb2.split()[0])
assert not crumb2.has_files()
return crumb2, values_dict # provide the fixture value
def test_setitem(tmp_crumb):
assert not op.exists(tmp_crumb.split()[0])
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
assert not tmp_crumb.exists()
_ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
cr = list(tmp_crumb.ls())[0]
assert not list(cr.open_args())
assert cr['image'] == [values_dict['image'][0]]
cr['image'] = 'mprage2.nii'
assert cr['image'] == ['mprage2.nii']
assert 'mprage2.nii' in cr.path
assert 'mprage2.nii' in cr.ls()[0].path
cr.clear('image')
assert 'image' in list(cr.open_args())
assert not 'mprage2.nii' in cr.path
assert not 'mprage2.nii' in cr.ls()[0].path
def test_mktree_tuples(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
values_map = list(ParameterGrid(values_dict))
values_tups = [tuple(d.items()) for d in values_map]
nupaths = mktree(tmp_crumb, values_tups)
for path in nupaths:
for k in values_dict:
assert k in path
assert all([os.path.exists(npath.split()[0]) for npath in nupaths])
assert all([npath.exists() for npath in nupaths])
ls_session_ids = tmp_crumb.ls('session_id',
fullpath=False,
make_crumbs=False,
check_exists=False)
assert set(ls_session_ids) == set(values_dict['session_id'])
ls_subj_ids = tmp_crumb.ls('subject_id',
fullpath=False,
make_crumbs=False,
check_exists=False)
assert set(ls_subj_ids) == set(values_dict['subject_id'])
def test_has_files(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
assert not tmp_crumb.has_files()
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert os.path.exists(tmp_crumb.split()[0])
assert not tmp_crumb.has_files()
pa = Path(str(paths[0]))
pa.rmdir()
pa.touch()
assert pa.exists()
assert pa.is_file()
assert tmp_crumb.has_files()
def test_group_pattern():
crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{image}")
base_dir1 = tempfile.mkdtemp(prefix="crumbtest1_")
tmp_crumb1 = crumb.replace(base_dir=base_dir1)
assert not op.exists(tmp_crumb1._path)
assert not tmp_crumb1.has_files()
values_dict1 = {
"session_id": ["session_{:02}".format(i) for i in range(2)],
"subject_id": ["subj_{:03}".format(i) for i in range(3)],
"image": ["mprage.nii", "pet.nii", "rest.nii", "remaining"],
}
_ = mktree(tmp_crumb1, list(ParameterGrid(values_dict1)))
patterns = {"anat": "mprage*", "pet": "pet*", "rest": "rest*"}
matches = groupby_pattern(tmp_crumb1, "image", patterns)
assert patterns.keys() == matches.keys()
for name, paths in matches.items():
assert len(paths) == 6
for p in paths:
assert isinstance(p, Crumb)
assert patterns[name] in p.patterns.values()
def test_regex_replace(tmp_crumb):
assert not op.exists(tmp_crumb._path)
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
_ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
regex='fnmatch') # fnmatch
assert tmp_crumb.ls('subject_id:subj_02*', make_crumbs=False) == \
crumb.ls('subject_id', make_crumbs=False)
anat_crumb = crumb.replace(modality='anat')
assert anat_crumb.exists()
fn_subj_ids = {cr['subject_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}
assert fn_subj_ids == set(['subj_{:03}'.format(i) for i in range(20, 30)])
sessions = {cr['session_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}
assert sessions == set(values_dict['session_id'])
def test_regex_ignorecase(tmp_crumb):
assert not op.exists(tmp_crumb._path)
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)],
'subject_id': ['SUBJ_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
_ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'),
regex='re') # re.match
assert len(crumb['subject_id']) == 0
assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
assert crumb._ignore == crumb.replace(subject_id='haensel')._ignore
assert not crumb.unfold()
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'),
regex='re.ignorecase') # re.match
assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
assert crumb._ignore == crumb.replace(subject_id='haensel')._ignore
ucrumb = crumb.unfold()[0]
assert crumb._re_method == ucrumb._re_method
assert crumb._ignore == ucrumb._ignore
re_subj_ids = crumb['subject_id']
assert re_subj_ids == ['SUBJ_{:03}'.format(i) for i in range(20, 30)]
def test_mktree_tuples(tmp_crumb):
assert not op.exists(tmp_crumb.split()[0])
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
values_map = list(ParameterGrid(values_dict))
values_tups = [tuple(d.items()) for d in values_map]
nupaths = mktree(tmp_crumb, values_tups)
for path in nupaths:
for k in values_dict:
assert k in path
assert all([op.exists(npath.split()[0]) for npath in nupaths])
assert all([npath.exists() for npath in nupaths])
ls_session_ids = tmp_crumb.ls('session_id', fullpath=False, make_crumbs=False, check_exists=False)
assert set(ls_session_ids) == set(values_dict['session_id'])
ls_subj_ids = tmp_crumb.ls('subject_id', fullpath=False, make_crumbs=False, check_exists=False)
assert set(ls_subj_ids) == set(values_dict['subject_id'])
def test_mktree1(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
nupaths = mktree(tmp_crumb, None)
assert all([os.path.exists(npath) for npath in nupaths])
pytest.raises(TypeError, mktree, tmp_crumb, 'hansel')
def test_exists(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
assert not tmp_crumb.exists()
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert tmp_crumb.exists()
assert not _split_exists('/_/asdfasdfasdf?/{hansel}')
def test_mktree1(tmp_crumb):
assert not op.exists(tmp_crumb.split()[0])
nupaths = mktree(tmp_crumb, None)
assert all([op.exists(npath) for npath in nupaths])
pytest.raises(TypeError, mktree, tmp_crumb, 'hansel')
def test_regex(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re') # re.match
re_subj_ids = crumb['subject_id']
assert re_subj_ids == ['subj_{:03}'.format(i) for i in range(20, 30)]
assert crumb.ls('subject_id:^subj_02.*$') == crumb.ls('subject_id')
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'), regex='fnmatch') # fnmatch
fn_subj_ids = crumb['subject_id']
assert fn_subj_ids == re_subj_ids
cr_bkp = crumb.copy()
assert crumb.ls('subject_id:subj_02*') == crumb.ls('subject_id')
assert crumb['subject_id'][0] == crumb.get_first('subject_id')
assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']
assert not crumb.ls('subject_id:subj_03*') == crumb.ls('subject_id')
assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']
pytest.raises(ValueError,
Crumb,
tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
regex='hansel')
crumb2 = Crumb.copy(crumb)
assert crumb2._re_method == crumb._re_method
assert crumb2._re_args == crumb._re_args
assert crumb2.patterns == crumb.patterns
assert len(crumb2.patterns) == 1
assert 'subject_id' in crumb2.patterns.keys()
def test_exists2(tmp_crumb):
assert not op.exists(tmp_crumb.split()[0])
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
}
_ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert not tmp_crumb.exists()
def make_tree_from_crumb(base_path, crumb_path,
crumb_args: [Dict, CrumbArgsSequences]):
crumb = Crumb(crumb_path)
crumb2 = crumb.replace(base_dir=base_path)
assert not os.path.exists(crumb2._path)
assert not crumb2.has_files()
if isinstance(crumb_args, dict):
values_map = list(ParameterGrid(crumb_args))
elif isinstance(crumb_args, list):
values_map = crumb_args
else:
raise TypeError(
'Expected `crumb_args` to be dict or list, got {}.'.format(
type(crumb_args)))
mktree(crumb2, values_map)
assert os.path.exists(crumb2.split()[0])
assert not crumb2.has_files()
return crumb2
def test_set_patterns(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
# a crumb without the pattern, the pattern is added later
crumb2 = Crumb(tmp_crumb.path, regex='fnmatch')
crumb3 = crumb2.copy()
crumb3.set_patterns()
assert crumb2 == crumb3
pytest.raises(KeyError, crumb2.set_patterns, somekey='somevalue')
crumb3.set_pattern('subject_id', 'subj_02*')
crumb2.set_patterns(subject_id='subj_02*')
assert crumb2['subject_id'] == crumb3['subject_id']
def test_regex_replace2(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
mktree(tmp_crumb, list(ParameterGrid(values_dict)))
# a crumb with the pattern
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
regex='fnmatch') # fnmatch
# a crumb without the pattern, the pattern is added later
crumb2 = Crumb(tmp_crumb.path, regex='fnmatch')
crumb2.set_pattern('subject_id', 'subj_02*')
assert crumb['subject_id'] == crumb2['subject_id']
crumb2.clear_pattern('subject_id')
assert tmp_crumb['subject_id'] == crumb2['subject_id']
def test_exists(tmp_crumb):
assert not op.exists(tmp_crumb.split()[0])
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
assert not tmp_crumb.exists()
_ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert tmp_crumb.exists()
assert not _split_exists('/_/asdfasdfasdf?/{hansel}')
def test_mktree_dicts(tmp_crumb):
assert not op.exists(tmp_crumb.split()[0])
values_map = {'session_id': ['session_{}'.format(i) for i in range(2)],
'subject_id': ['subj_{}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
nupaths = mktree(tmp_crumb, list(ParameterGrid(values_map)))
assert all([op.exists(npath.split()[0]) for npath in nupaths])
assert all([npath.exists() for npath in nupaths])
values_map['grimm'] = ['Jacob', 'Wilhelm']
pytest.raises(ValueError, mktree, tmp_crumb, list(ParameterGrid(values_map)))
def test_regex(tmp_crumb):
assert not op.exists(tmp_crumb.path)
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
_ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'),
regex='re') # re.match
re_subj_ids = crumb['subject_id']
assert re_subj_ids == ['subj_{:03}'.format(i) for i in range(20, 30)]
assert crumb.ls('subject_id:^subj_02.*$') == crumb.ls('subject_id')
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
regex='fnmatch') # fnmatch
fn_subj_ids = crumb['subject_id']
assert fn_subj_ids == re_subj_ids
cr_bkp = crumb.copy()
assert crumb.ls('subject_id:subj_02*') == crumb.ls('subject_id')
assert crumb['subject_id'][0] == crumb.get_first('subject_id')
assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']
assert not crumb.ls('subject_id:subj_03*') == crumb.ls('subject_id')
assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']
pytest.raises(ValueError,
Crumb,
tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
regex='hansel')
crumb2 = Crumb.copy(crumb)
assert crumb2._re_method == crumb._re_method
assert crumb2._re_args == crumb._re_args
assert crumb2.patterns == crumb.patterns
assert len(crumb2.patterns) == 1
assert 'subject_id' in crumb2.patterns.keys()
def test_mktree_dicts(tmp_crumb):
assert not os.path.exists(tmp_crumb.split()[0])
values_map = {
'session_id': ['session_{}'.format(i) for i in range(2)],
'subject_id': ['subj_{}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
nupaths = mktree(tmp_crumb, list(ParameterGrid(values_map)))
assert all([os.path.exists(npath.split()[0]) for npath in nupaths])
assert all([npath.exists() for npath in nupaths])
values_map['grimm'] = ['Jacob', 'Wilhelm']
pytest.raises(ValueError, mktree, tmp_crumb,
list(ParameterGrid(values_map)))
def _test_crumb_copy(make_links=False):
crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{image}")
base_dir1 = tempfile.mkdtemp(prefix="crumb_copy_test1_")
tmp_crumb1 = crumb.replace(base_dir=base_dir1)
assert not op.exists(tmp_crumb1._path)
assert not tmp_crumb1.has_files()
values_dict1 = {
"session_id": ["session_{:02}".format(i) for i in range(2)],
"subject_id": ["subj_{:03}".format(i) for i in range(3)],
"image": ["mprage.nii", "pet.nii", "rest.nii", "remaining"],
}
_ = mktree(tmp_crumb1, list(ParameterGrid(values_dict1)))
base_dir2 = tempfile.mkdtemp(prefix="crumb_copy_test2_")
tmp_crumb2 = crumb.replace(base_dir=base_dir2)
if make_links:
copy_func = crumb_link
else:
copy_func = crumb_copy
# make first copy
copy_func(tmp_crumb1, tmp_crumb2)
assert all([cr.exists() for cr in tmp_crumb2.ls()])
# copy again without exist_ok
pytest.raises(FileExistsError, copy_func, tmp_crumb1, tmp_crumb2)
assert all([cr.exists() for cr in tmp_crumb2.ls()])
# copy again with exist_ok
copy_func(tmp_crumb1, tmp_crumb2, exist_ok=True)
assert all([cr.exists() for cr in tmp_crumb2.ls()])
if make_links:
assert all([op.islink(cr.path) for cr in tmp_crumb2.ls()])
def test_regex_replace2(tmp_crumb):
assert not op.exists(tmp_crumb.path)
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
'modality': ['anat'],
'image': ['mprage1.nii'],
}
_ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
# a crumb with the pattern
crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
regex='fnmatch') # fnmatch
# a crumb without the pattern, the pattern is added later
crumb2 = Crumb(tmp_crumb.path, regex='fnmatch')
crumb2.set_pattern('subject_id', 'subj_02*')
assert crumb['subject_id'] == crumb2['subject_id']
crumb2.clear_pattern('subject_id')
assert tmp_crumb['subject_id'] == crumb2['subject_id']
def test_has_files(tmp_crumb):
assert not op.exists(tmp_crumb.path)
assert not tmp_crumb.has_files()
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)],
'subject_id': ['subj_{:03}'.format(i) for i in range( 3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert op.exists(tmp_crumb.split()[0])
assert not tmp_crumb.has_files()
pa = Path(str(paths[0]))
pa.rmdir()
pa.touch()
assert pa.exists()
assert pa.is_file()
assert tmp_crumb.has_files()
def test_ls_with_check(tmp_crumb):
assert not os.path.exists(tmp_crumb._path)
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
sbj_crumb = tmp_crumb.replace(subject_id='subj_000')
assert sbj_crumb.ls('subject_id', make_crumbs=False, fullpath=False) == ['subj_000']
assert os.path.exists(tmp_crumb.split()[0])
assert all([os.path.exists(p.path) for p in paths])
assert all([p.exists() for p in paths])
images = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
modalities = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
assert all([img.exists() for img in images])
assert all([mod.exists() for mod in modalities])
Path(str(images[0])).rmdir()
images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
assert images != images2
assert len(images) == len(images2) + 1
assert not all([img.exists() for img in images])
assert all([img.exists() for img in images2])
Path(str(images[1])).rmdir()
Path(str(images[2])).rmdir()
images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
assert tmp_crumb.ls('image') == tmp_crumb.ls()
assert tmp_crumb.ls() == tmp_crumb.unfold()
assert not all([img.exists() for img in images])
assert all([img.exists() for img in images2])
modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
str_modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=False, check_exists=True)
assert images != images2
assert len(images) == len(images2) + 3
assert modalities != modalities2
assert not all([mod.exists() for mod in modalities])
assert all([mod.exists() for mod in modalities2])
assert all([isinstance(smod, str) for smod in str_modalities2])
assert all([isinstance(mod, Crumb) for mod in modalities2])
assert all([mod.path == smod for mod, smod in zip(sorted(modalities2), sorted(str_modalities2))])
shutil.rmtree(modalities2[0].split()[0], ignore_errors=True)
modalities3 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
assert modalities2 != modalities3
assert not all([mod.exists() for mod in modalities2])
assert all([mod.exists() for mod in modalities3])
assert tmp_crumb.unfold() == tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
pytest.raises(IOError, modalities2[0].__getitem__, 'image')
img_crumb = tmp_crumb.replace(image='mprage1.nii')
assert 'image' in img_crumb._argval
assert img_crumb['image'] == ['mprage1.nii']
assert img_crumb['image'][0] == img_crumb.get_first('image')
img_crumb['modality'] = 'anat'
assert 'modality' in img_crumb._argval
assert img_crumb['modality'] == ['anat']
assert img_crumb.has_set('modality')
assert img_crumb['session_id'].count('session_01') == img_crumb['session_id'].count('session_00')
img_crumb['session_id'] = 'session_00'
assert 'session_id' in img_crumb._argval
assert img_crumb['session_id'] == ['session_00']
assert 'subj_000' not in img_crumb['subject_id']
unfolded_crumbs = tmp_crumb.unfold()
assert list(unfolded_crumbs[0].open_args()) == []
assert unfolded_crumbs[0].unfold() == [unfolded_crumbs[0]]
pytest.raises(AttributeError, unfolded_crumbs[0]._check_open_args, ['subject_id'])
def test_intersection():
crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{modality}/{image}")
base_dir1 = tempfile.mkdtemp(prefix="crumbtest1_")
tmp_crumb1 = crumb.replace(base_dir=base_dir1)
base_dir2 = tempfile.mkdtemp(prefix="crumbtest2_")
tmp_crumb2 = crumb.replace(base_dir=base_dir2)
assert not op.exists(tmp_crumb1._path)
assert not op.exists(tmp_crumb2._path)
assert not tmp_crumb1.has_files()
assert not tmp_crumb2.has_files()
values_dict1 = {
"session_id": ["session_{:02}".format(i) for i in range(2)],
"subject_id": ["subj_{:03}".format(i) for i in range(3)],
"modality": ["anat"],
"image": ["mprage1.nii", "mprage2.nii", "mprage3.nii"],
}
values_dict2 = {
"session_id": ["session_{:02}".format(i) for i in range(20)],
"subject_id": ["subj_{:03}".format(i) for i in range(30)],
"modality": ["anat"],
"image": ["mprage1.nii", "mprage2.nii", "mprage3.nii"],
}
_ = mktree(tmp_crumb1, list(ParameterGrid(values_dict1)))
_ = mktree(tmp_crumb2, list(ParameterGrid(values_dict2)))
assert op.exists(tmp_crumb1.split()[0])
assert op.exists(tmp_crumb2.split()[0])
assert intersection(tmp_crumb1, tmp_crumb2, on=["subject_id"]) == [
(("subject_id", val),) for val in tmp_crumb1["subject_id"]
]
assert intersection(tmp_crumb1, tmp_crumb2, on=["subject_id", "modality"]) == [
(("subject_id", "subj_000"), ("modality", "anat")),
(("subject_id", "subj_001"), ("modality", "anat")),
(("subject_id", "subj_002"), ("modality", "anat")),
]
han_crumb = tmp_crumb2.replace(subject_id="hansel")
assert intersection(tmp_crumb1, han_crumb, on=["subject_id"]) == []
s0_crumb = tmp_crumb2.replace(subject_id="subj_000")
assert intersection(tmp_crumb1, s0_crumb, on=["subject_id"]) == [(("subject_id", "subj_000"),)]
assert intersection(tmp_crumb1, s0_crumb, on=["subject_id", "modality"]) == [
(("subject_id", "subj_000"), ("modality", "anat"))
]
assert intersection(tmp_crumb1, s0_crumb, on=["subject_id", "image"]) == [
(("subject_id", "subj_000"), ("image", "mprage1.nii")),
(("subject_id", "subj_000"), ("image", "mprage2.nii")),
(("subject_id", "subj_000"), ("image", "mprage3.nii")),
]
# test raises
pytest.raises(KeyError, intersection, tmp_crumb1, tmp_crumb2, on=["hansel"])
pytest.raises(KeyError, intersection, tmp_crumb1, tmp_crumb2, on=["subject_id", "modality", "hansel"])
pytest.raises(KeyError, intersection, tmp_crumb1, Crumb(op.expanduser("~/{files}")))
pytest.raises(KeyError, intersection, tmp_crumb1, Crumb(op.expanduser("~/{files}")), on=["files"])
def test_ls_with_check(tmp_crumb):
assert not op.exists(tmp_crumb._path)
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
sbj_crumb = tmp_crumb.replace(subject_id='subj_000')
assert sbj_crumb.ls('subject_id', make_crumbs=False, fullpath=False) == ['subj_000']
assert op.exists(tmp_crumb.split()[0])
assert all([op.exists(p.path) for p in paths])
assert all([p.exists() for p in paths])
images = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
modalities = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
assert all([img.exists() for img in images])
assert all([mod.exists() for mod in modalities])
Path(str(images[0])).rmdir()
images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
assert images != images2
assert len(images) == len(images2) + 1
assert not all([img.exists() for img in images])
assert all([img.exists() for img in images2])
Path(str(images[1])).rmdir()
Path(str(images[2])).rmdir()
images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
assert tmp_crumb.ls('image') == tmp_crumb.ls()
assert tmp_crumb.ls() == tmp_crumb.unfold()
assert not all([img.exists() for img in images])
assert all([img.exists() for img in images2])
modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
str_modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=False, check_exists=True)
assert images != images2
assert len(images) == len(images2) + 3
assert modalities != modalities2
assert not all([mod.exists() for mod in modalities])
assert all([mod.exists() for mod in modalities2])
assert all([isinstance(smod, str) for smod in str_modalities2])
assert all([isinstance(mod, Crumb) for mod in modalities2])
assert all([mod.path == smod for mod, smod in zip(sorted(modalities2), sorted(str_modalities2))])
shutil.rmtree(modalities2[0].split()[0], ignore_errors=True)
modalities3 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)
assert modalities2 != modalities3
assert not all([mod.exists() for mod in modalities2])
assert all([mod.exists() for mod in modalities3])
assert tmp_crumb.unfold() == tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)
pytest.raises(IOError, modalities2[0].__getitem__, 'image')
img_crumb = tmp_crumb.replace(image='mprage1.nii')
assert 'image' in img_crumb._argval
assert img_crumb['image'] == ['mprage1.nii']
assert img_crumb['image'][0] == img_crumb.get_first('image')
img_crumb['modality'] = 'anat'
assert 'modality' in img_crumb._argval
assert img_crumb['modality'] == ['anat']
assert img_crumb.has_set('modality')
assert img_crumb['session_id'].count('session_01') == img_crumb['session_id'].count('session_00')
img_crumb['session_id'] = 'session_00'
assert 'session_id' in img_crumb._argval
assert img_crumb['session_id'] == ['session_00']
assert 'subj_000' not in img_crumb['subject_id']
unfolded_crumbs = tmp_crumb.unfold()
assert list(unfolded_crumbs[0].open_args()) == []
assert unfolded_crumbs[0].unfold() == [unfolded_crumbs[0]]
pytest.raises(AttributeError, unfolded_crumbs[0]._check_open_args, ['subject_id'])
