Exemple #1
0
def test_setitem(tmp_crumb):
    assert not os.path.exists(tmp_crumb.split()[0])

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
        'modality': ['anat'],
        'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
    }

    assert not tmp_crumb.exists()

    mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    cr = list(tmp_crumb.ls())[0]

    assert not list(cr.open_args())

    assert cr['image'] == [values_dict['image'][0]]

    cr['image'] = 'mprage2.nii'

    assert cr['image'] == ['mprage2.nii']

    assert 'mprage2.nii' in cr.path
    assert 'mprage2.nii' in cr.ls()[0].path

    cr.clear('image')
    assert 'image' in list(cr.open_args())
    assert 'mprage2.nii' not in cr.path
    assert 'mprage2.nii' not in cr.ls()[0].path
Exemple #2
0
def test_regex_ignorecase(tmp_crumb):
    assert not os.path.exists(tmp_crumb._path)

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['SUBJ_{:03}'.format(i) for i in range(100)],
        'modality': ['anat'],
        'image': ['mprage1.nii'],
    }

    mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re')  # re.match

    assert len(crumb['subject_id']) == 0
    assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
    assert crumb._ignore == crumb.replace(subject_id='haensel')._ignore

    assert not crumb.unfold()

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re.ignorecase')  # re.match
    assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
    assert crumb._ignore == crumb.replace(subject_id='haensel')._ignore

    ucrumb = crumb.unfold()[0]
    assert crumb._re_method == ucrumb._re_method
    assert crumb._ignore == ucrumb._ignore

    re_subj_ids = crumb['subject_id']

    assert re_subj_ids == ['SUBJ_{:03}'.format(i) for i in range(20, 30)]
Exemple #3
0
def test_regex_replace(tmp_crumb):
    assert not os.path.exists(tmp_crumb._path)

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
        'modality': ['anat'],
        'image': ['mprage1.nii'],
    }

    mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'), regex='fnmatch')  # fnmatch

    assert tmp_crumb.ls('subject_id:subj_02*', make_crumbs=False) == crumb.ls('subject_id', make_crumbs=False)

    anat_crumb = crumb.replace(modality='anat')
    assert anat_crumb.exists()

    fn_subj_ids = {cr['subject_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}

    assert fn_subj_ids == set(['subj_{:03}'.format(i) for i in range(20, 30)])

    sessions = {cr['session_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}
    assert sessions == set(values_dict['session_id'])
Exemple #4
0
def test_exists2(tmp_crumb):
    assert not os.path.exists(tmp_crumb.split()[0])

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
    }

    mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    assert not tmp_crumb.exists()
Exemple #5
0
def tmp_tree(request):

    crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{modality}/{image}")
    base_dir = tempfile.mkdtemp(prefix="crumbtest_")
    crumb2 = crumb.replace(base_dir=base_dir)

    def fin():
        print("teardown tmp_crumb")

    request.addfinalizer(fin)

    assert not op.exists(crumb2._path)

    assert not crumb2.has_files()

    values_dict = {
        "session_id": ["session_{:02}".format(i) for i in range(2)],
        "subject_id": ["subj_{:03}".format(i) for i in range(3)],
        "modality": ["anat"],
        "image": ["mprage1.nii", "mprage2.nii", "mprage3.nii"],
    }

    paths = mktree(crumb2, list(ParameterGrid(values_dict)))

    assert op.exists(crumb2.split()[0])

    assert not crumb2.has_files()

    return crumb2, values_dict  # provide the fixture value
Exemple #6
0
def test_setitem(tmp_crumb):
    assert not op.exists(tmp_crumb.split()[0])

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range(3)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
                  }

    assert not tmp_crumb.exists()

    _ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    cr = list(tmp_crumb.ls())[0]

    assert not list(cr.open_args())

    assert cr['image'] == [values_dict['image'][0]]

    cr['image'] = 'mprage2.nii'

    assert cr['image'] == ['mprage2.nii']

    assert 'mprage2.nii' in cr.path
    assert 'mprage2.nii' in cr.ls()[0].path

    cr.clear('image')
    assert 'image' in list(cr.open_args())
    assert not 'mprage2.nii' in cr.path
    assert not 'mprage2.nii' in cr.ls()[0].path
Exemple #7
0
def test_mktree_tuples(tmp_crumb):
    assert not os.path.exists(tmp_crumb.split()[0])

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
        'modality': ['anat'],
        'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
    }

    values_map = list(ParameterGrid(values_dict))
    values_tups = [tuple(d.items()) for d in values_map]

    nupaths = mktree(tmp_crumb, values_tups)

    for path in nupaths:
        for k in values_dict:
            assert k in path

    assert all([os.path.exists(npath.split()[0]) for npath in nupaths])
    assert all([npath.exists() for npath in nupaths])

    ls_session_ids = tmp_crumb.ls('session_id',
                                  fullpath=False,
                                  make_crumbs=False,
                                  check_exists=False)
    assert set(ls_session_ids) == set(values_dict['session_id'])

    ls_subj_ids = tmp_crumb.ls('subject_id',
                               fullpath=False,
                               make_crumbs=False,
                               check_exists=False)
    assert set(ls_subj_ids) == set(values_dict['subject_id'])
Exemple #8
0
def test_has_files(tmp_crumb):
    assert not os.path.exists(tmp_crumb.path)

    assert not tmp_crumb.has_files()

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
        'modality': ['anat'],
        'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
    }

    paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    assert os.path.exists(tmp_crumb.split()[0])

    assert not tmp_crumb.has_files()

    pa = Path(str(paths[0]))
    pa.rmdir()
    pa.touch()

    assert pa.exists()
    assert pa.is_file()
    assert tmp_crumb.has_files()
Exemple #9
0
def test_group_pattern():
    crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{image}")
    base_dir1 = tempfile.mkdtemp(prefix="crumbtest1_")
    tmp_crumb1 = crumb.replace(base_dir=base_dir1)

    assert not op.exists(tmp_crumb1._path)
    assert not tmp_crumb1.has_files()

    values_dict1 = {
        "session_id": ["session_{:02}".format(i) for i in range(2)],
        "subject_id": ["subj_{:03}".format(i) for i in range(3)],
        "image": ["mprage.nii", "pet.nii", "rest.nii", "remaining"],
    }

    _ = mktree(tmp_crumb1, list(ParameterGrid(values_dict1)))

    patterns = {"anat": "mprage*", "pet": "pet*", "rest": "rest*"}

    matches = groupby_pattern(tmp_crumb1, "image", patterns)

    assert patterns.keys() == matches.keys()

    for name, paths in matches.items():
        assert len(paths) == 6
        for p in paths:
            assert isinstance(p, Crumb)
            assert patterns[name] in p.patterns.values()
Exemple #10
0
def test_regex_replace(tmp_crumb):
    assert not op.exists(tmp_crumb._path)

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(  2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range(100)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii'],
                   }

    _ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
                  regex='fnmatch')  # fnmatch

    assert tmp_crumb.ls('subject_id:subj_02*', make_crumbs=False) == \
           crumb.ls('subject_id', make_crumbs=False)

    anat_crumb = crumb.replace(modality='anat')
    assert anat_crumb.exists()

    fn_subj_ids = {cr['subject_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}

    assert fn_subj_ids == set(['subj_{:03}'.format(i) for i in range(20, 30)])

    sessions = {cr['session_id'][0] for cr in anat_crumb.ls('session_id', check_exists=True)}
    assert sessions == set(values_dict['session_id'])
Exemple #11
0
def test_regex_ignorecase(tmp_crumb):
    assert not op.exists(tmp_crumb._path)

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(  2)],
                   'subject_id': ['SUBJ_{:03}'.format(i)    for i in range(100)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii'],
                   }

    _ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'),
                  regex='re')  # re.match

    assert len(crumb['subject_id']) == 0
    assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
    assert crumb._ignore    == crumb.replace(subject_id='haensel')._ignore

    assert not crumb.unfold()

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'),
                  regex='re.ignorecase')  # re.match
    assert crumb._re_method == crumb.replace(subject_id='haensel')._re_method
    assert crumb._ignore    == crumb.replace(subject_id='haensel')._ignore

    ucrumb = crumb.unfold()[0]
    assert crumb._re_method == ucrumb._re_method
    assert crumb._ignore    == ucrumb._ignore

    re_subj_ids = crumb['subject_id']

    assert re_subj_ids == ['SUBJ_{:03}'.format(i) for i in range(20, 30)]
Exemple #12
0
def test_mktree_tuples(tmp_crumb):

    assert not op.exists(tmp_crumb.split()[0])

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range(3)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
                   }

    values_map = list(ParameterGrid(values_dict))
    values_tups = [tuple(d.items()) for d in values_map]

    nupaths = mktree(tmp_crumb, values_tups)

    for path in nupaths:
        for k in values_dict:
            assert k in path

    assert all([op.exists(npath.split()[0]) for npath in nupaths])
    assert all([npath.exists()              for npath in nupaths])

    ls_session_ids = tmp_crumb.ls('session_id', fullpath=False, make_crumbs=False, check_exists=False)
    assert set(ls_session_ids) == set(values_dict['session_id'])

    ls_subj_ids = tmp_crumb.ls('subject_id', fullpath=False, make_crumbs=False, check_exists=False)
    assert set(ls_subj_ids) == set(values_dict['subject_id'])
Exemple #13
0
def test_mktree1(tmp_crumb):
    assert not os.path.exists(tmp_crumb.split()[0])

    nupaths = mktree(tmp_crumb, None)

    assert all([os.path.exists(npath) for npath in nupaths])

    pytest.raises(TypeError, mktree, tmp_crumb, 'hansel')
Exemple #14
0
def test_exists(tmp_crumb):
    assert not os.path.exists(tmp_crumb.split()[0])

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
        'modality': ['anat'],
        'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
    }

    assert not tmp_crumb.exists()

    mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    assert tmp_crumb.exists()

    assert not _split_exists('/_/asdfasdfasdf?/{hansel}')
Exemple #15
0
def test_mktree1(tmp_crumb):

    assert not op.exists(tmp_crumb.split()[0])

    nupaths = mktree(tmp_crumb, None)

    assert all([op.exists(npath) for npath in nupaths])

    pytest.raises(TypeError, mktree, tmp_crumb, 'hansel')
Exemple #16
0
def test_regex(tmp_crumb):
    assert not os.path.exists(tmp_crumb.path)

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
        'modality': ['anat'],
        'image': ['mprage1.nii'],
    }

    mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'), regex='re')  # re.match

    re_subj_ids = crumb['subject_id']

    assert re_subj_ids == ['subj_{:03}'.format(i) for i in range(20, 30)]
    assert crumb.ls('subject_id:^subj_02.*$') == crumb.ls('subject_id')

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'), regex='fnmatch')  # fnmatch

    fn_subj_ids = crumb['subject_id']

    assert fn_subj_ids == re_subj_ids
    cr_bkp = crumb.copy()
    assert crumb.ls('subject_id:subj_02*') == crumb.ls('subject_id')
    assert crumb['subject_id'][0] == crumb.get_first('subject_id')
    assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']

    assert not crumb.ls('subject_id:subj_03*') == crumb.ls('subject_id')
    assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']

    pytest.raises(ValueError,
                  Crumb,
                  tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
                  regex='hansel')

    crumb2 = Crumb.copy(crumb)
    assert crumb2._re_method == crumb._re_method
    assert crumb2._re_args == crumb._re_args
    assert crumb2.patterns == crumb.patterns

    assert len(crumb2.patterns) == 1
    assert 'subject_id' in crumb2.patterns.keys()
Exemple #17
0
def test_exists2(tmp_crumb):
    assert not op.exists(tmp_crumb.split()[0])

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range(3)],
                  }

    _ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    assert not tmp_crumb.exists()
Exemple #18
0
def make_tree_from_crumb(base_path, crumb_path,
                         crumb_args: [Dict, CrumbArgsSequences]):
    crumb = Crumb(crumb_path)
    crumb2 = crumb.replace(base_dir=base_path)

    assert not os.path.exists(crumb2._path)
    assert not crumb2.has_files()

    if isinstance(crumb_args, dict):
        values_map = list(ParameterGrid(crumb_args))
    elif isinstance(crumb_args, list):
        values_map = crumb_args
    else:
        raise TypeError(
            'Expected `crumb_args` to be dict or list, got {}.'.format(
                type(crumb_args)))

    mktree(crumb2, values_map)
    assert os.path.exists(crumb2.split()[0])
    assert not crumb2.has_files()
    return crumb2
Exemple #19
0
def test_set_patterns(tmp_crumb):
    assert not os.path.exists(tmp_crumb.path)

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
                   'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
                   'modality': ['anat'],
                   'image': ['mprage1.nii'],
                   }

    mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    # a crumb without the pattern, the pattern is added later
    crumb2 = Crumb(tmp_crumb.path, regex='fnmatch')

    crumb3 = crumb2.copy()
    crumb3.set_patterns()
    assert crumb2 == crumb3

    pytest.raises(KeyError, crumb2.set_patterns, somekey='somevalue')

    crumb3.set_pattern('subject_id', 'subj_02*')
    crumb2.set_patterns(subject_id='subj_02*')
    assert crumb2['subject_id'] == crumb3['subject_id']
Exemple #20
0
def test_regex_replace2(tmp_crumb):
    assert not os.path.exists(tmp_crumb.path)

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(100)],
        'modality': ['anat'],
        'image': ['mprage1.nii'],
    }

    mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    # a crumb with the pattern
    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
                  regex='fnmatch')  # fnmatch

    # a crumb without the pattern, the pattern is added later
    crumb2 = Crumb(tmp_crumb.path, regex='fnmatch')

    crumb2.set_pattern('subject_id', 'subj_02*')
    assert crumb['subject_id'] == crumb2['subject_id']

    crumb2.clear_pattern('subject_id')
    assert tmp_crumb['subject_id'] == crumb2['subject_id']
Exemple #21
0
def test_exists(tmp_crumb):
    assert not op.exists(tmp_crumb.split()[0])

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range(3)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
                  }

    assert not tmp_crumb.exists()

    _ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    assert tmp_crumb.exists()

    assert not _split_exists('/_/asdfasdfasdf?/{hansel}')
Exemple #22
0
def test_mktree_dicts(tmp_crumb):

    assert not op.exists(tmp_crumb.split()[0])

    values_map = {'session_id': ['session_{}'.format(i) for i in range(2)],
                  'subject_id': ['subj_{}'.format(i)    for i in range(3)],
                  'modality':   ['anat'],
                  'image':      ['mprage1.nii'],
                  }

    nupaths = mktree(tmp_crumb, list(ParameterGrid(values_map)))

    assert all([op.exists(npath.split()[0]) for npath in nupaths])
    assert all([npath.exists()              for npath in nupaths])

    values_map['grimm'] = ['Jacob', 'Wilhelm']
    pytest.raises(ValueError, mktree, tmp_crumb, list(ParameterGrid(values_map)))
Exemple #23
0
def test_regex(tmp_crumb):
    assert not op.exists(tmp_crumb.path)

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(  2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range(100)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii'],
                   }

    _ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:^subj_02.*$}'),
                  regex='re')  # re.match

    re_subj_ids = crumb['subject_id']

    assert re_subj_ids == ['subj_{:03}'.format(i) for i in range(20, 30)]
    assert crumb.ls('subject_id:^subj_02.*$') == crumb.ls('subject_id')

    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
                  regex='fnmatch')  # fnmatch

    fn_subj_ids = crumb['subject_id']

    assert fn_subj_ids == re_subj_ids
    cr_bkp = crumb.copy()
    assert crumb.ls('subject_id:subj_02*') == crumb.ls('subject_id')
    assert crumb['subject_id'][0] == crumb.get_first('subject_id')
    assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']

    assert not crumb.ls('subject_id:subj_03*') == crumb.ls('subject_id')
    assert crumb.patterns['subject_id'] == cr_bkp.patterns['subject_id']

    pytest.raises(ValueError,
                  Crumb,
                  tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
                  regex='hansel')

    crumb2 = Crumb.copy(crumb)
    assert crumb2._re_method == crumb._re_method
    assert crumb2._re_args == crumb._re_args
    assert crumb2.patterns == crumb.patterns

    assert len(crumb2.patterns) == 1
    assert 'subject_id' in crumb2.patterns.keys()
Exemple #24
0
def test_mktree_dicts(tmp_crumb):
    assert not os.path.exists(tmp_crumb.split()[0])

    values_map = {
        'session_id': ['session_{}'.format(i) for i in range(2)],
        'subject_id': ['subj_{}'.format(i) for i in range(3)],
        'modality': ['anat'],
        'image': ['mprage1.nii'],
    }

    nupaths = mktree(tmp_crumb, list(ParameterGrid(values_map)))

    assert all([os.path.exists(npath.split()[0]) for npath in nupaths])
    assert all([npath.exists() for npath in nupaths])

    values_map['grimm'] = ['Jacob', 'Wilhelm']
    pytest.raises(ValueError, mktree, tmp_crumb,
                  list(ParameterGrid(values_map)))
Exemple #25
0
def _test_crumb_copy(make_links=False):
    crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{image}")
    base_dir1 = tempfile.mkdtemp(prefix="crumb_copy_test1_")
    tmp_crumb1 = crumb.replace(base_dir=base_dir1)

    assert not op.exists(tmp_crumb1._path)
    assert not tmp_crumb1.has_files()

    values_dict1 = {
        "session_id": ["session_{:02}".format(i) for i in range(2)],
        "subject_id": ["subj_{:03}".format(i) for i in range(3)],
        "image": ["mprage.nii", "pet.nii", "rest.nii", "remaining"],
    }

    _ = mktree(tmp_crumb1, list(ParameterGrid(values_dict1)))

    base_dir2 = tempfile.mkdtemp(prefix="crumb_copy_test2_")
    tmp_crumb2 = crumb.replace(base_dir=base_dir2)

    if make_links:
        copy_func = crumb_link
    else:
        copy_func = crumb_copy

    # make first copy
    copy_func(tmp_crumb1, tmp_crumb2)
    assert all([cr.exists() for cr in tmp_crumb2.ls()])

    # copy again without exist_ok
    pytest.raises(FileExistsError, copy_func, tmp_crumb1, tmp_crumb2)
    assert all([cr.exists() for cr in tmp_crumb2.ls()])

    # copy again with exist_ok
    copy_func(tmp_crumb1, tmp_crumb2, exist_ok=True)
    assert all([cr.exists() for cr in tmp_crumb2.ls()])

    if make_links:
        assert all([op.islink(cr.path) for cr in tmp_crumb2.ls()])
Exemple #26
0
def test_regex_replace2(tmp_crumb):
    assert not op.exists(tmp_crumb.path)

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(  2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range(100)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii'],
                   }

    _ = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    # a crumb with the pattern
    crumb = Crumb(tmp_crumb.path.replace('{subject_id}', '{subject_id:subj_02*}'),
                  regex='fnmatch')  # fnmatch

    # a crumb without the pattern, the pattern is added later
    crumb2 = Crumb(tmp_crumb.path, regex='fnmatch')

    crumb2.set_pattern('subject_id', 'subj_02*')
    assert crumb['subject_id'] == crumb2['subject_id']

    crumb2.clear_pattern('subject_id')
    assert tmp_crumb['subject_id'] == crumb2['subject_id']
Exemple #27
0
def test_has_files(tmp_crumb):
    assert not op.exists(tmp_crumb.path)

    assert not tmp_crumb.has_files()

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range( 3)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
                   }

    paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    assert op.exists(tmp_crumb.split()[0])

    assert not tmp_crumb.has_files()

    pa = Path(str(paths[0]))
    pa.rmdir()
    pa.touch()

    assert pa.exists()
    assert pa.is_file()
    assert tmp_crumb.has_files()
Exemple #28
0
def test_ls_with_check(tmp_crumb):
    assert not os.path.exists(tmp_crumb._path)

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
        'modality': ['anat'],
        'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
    }

    paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    sbj_crumb = tmp_crumb.replace(subject_id='subj_000')
    assert sbj_crumb.ls('subject_id', make_crumbs=False, fullpath=False) == ['subj_000']

    assert os.path.exists(tmp_crumb.split()[0])

    assert all([os.path.exists(p.path) for p in paths])
    assert all([p.exists() for p in paths])

    images = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)

    modalities = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)

    assert all([img.exists() for img in images])
    assert all([mod.exists() for mod in modalities])

    Path(str(images[0])).rmdir()

    images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)

    assert images != images2
    assert len(images) == len(images2) + 1
    assert not all([img.exists() for img in images])
    assert all([img.exists() for img in images2])

    Path(str(images[1])).rmdir()
    Path(str(images[2])).rmdir()

    images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)

    assert tmp_crumb.ls('image') == tmp_crumb.ls()
    assert tmp_crumb.ls() == tmp_crumb.unfold()

    assert not all([img.exists() for img in images])
    assert all([img.exists() for img in images2])

    modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)

    str_modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=False, check_exists=True)

    assert images != images2
    assert len(images) == len(images2) + 3

    assert modalities != modalities2
    assert not all([mod.exists() for mod in modalities])
    assert all([mod.exists() for mod in modalities2])

    assert all([isinstance(smod, str) for smod in str_modalities2])
    assert all([isinstance(mod, Crumb) for mod in modalities2])
    assert all([mod.path == smod for mod, smod in zip(sorted(modalities2), sorted(str_modalities2))])

    shutil.rmtree(modalities2[0].split()[0], ignore_errors=True)

    modalities3 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)

    assert modalities2 != modalities3
    assert not all([mod.exists() for mod in modalities2])
    assert all([mod.exists() for mod in modalities3])

    assert tmp_crumb.unfold() == tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)

    pytest.raises(IOError, modalities2[0].__getitem__, 'image')

    img_crumb = tmp_crumb.replace(image='mprage1.nii')
    assert 'image' in img_crumb._argval
    assert img_crumb['image'] == ['mprage1.nii']

    assert img_crumb['image'][0] == img_crumb.get_first('image')

    img_crumb['modality'] = 'anat'
    assert 'modality' in img_crumb._argval
    assert img_crumb['modality'] == ['anat']
    assert img_crumb.has_set('modality')

    assert img_crumb['session_id'].count('session_01') == img_crumb['session_id'].count('session_00')

    img_crumb['session_id'] = 'session_00'
    assert 'session_id' in img_crumb._argval
    assert img_crumb['session_id'] == ['session_00']

    assert 'subj_000' not in img_crumb['subject_id']

    unfolded_crumbs = tmp_crumb.unfold()
    assert list(unfolded_crumbs[0].open_args()) == []
    assert unfolded_crumbs[0].unfold() == [unfolded_crumbs[0]]

    pytest.raises(AttributeError, unfolded_crumbs[0]._check_open_args, ['subject_id'])
Exemple #29
0
def test_intersection():

    crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{modality}/{image}")
    base_dir1 = tempfile.mkdtemp(prefix="crumbtest1_")
    tmp_crumb1 = crumb.replace(base_dir=base_dir1)

    base_dir2 = tempfile.mkdtemp(prefix="crumbtest2_")
    tmp_crumb2 = crumb.replace(base_dir=base_dir2)

    assert not op.exists(tmp_crumb1._path)
    assert not op.exists(tmp_crumb2._path)

    assert not tmp_crumb1.has_files()
    assert not tmp_crumb2.has_files()

    values_dict1 = {
        "session_id": ["session_{:02}".format(i) for i in range(2)],
        "subject_id": ["subj_{:03}".format(i) for i in range(3)],
        "modality": ["anat"],
        "image": ["mprage1.nii", "mprage2.nii", "mprage3.nii"],
    }

    values_dict2 = {
        "session_id": ["session_{:02}".format(i) for i in range(20)],
        "subject_id": ["subj_{:03}".format(i) for i in range(30)],
        "modality": ["anat"],
        "image": ["mprage1.nii", "mprage2.nii", "mprage3.nii"],
    }

    _ = mktree(tmp_crumb1, list(ParameterGrid(values_dict1)))
    _ = mktree(tmp_crumb2, list(ParameterGrid(values_dict2)))

    assert op.exists(tmp_crumb1.split()[0])
    assert op.exists(tmp_crumb2.split()[0])

    assert intersection(tmp_crumb1, tmp_crumb2, on=["subject_id"]) == [
        (("subject_id", val),) for val in tmp_crumb1["subject_id"]
    ]

    assert intersection(tmp_crumb1, tmp_crumb2, on=["subject_id", "modality"]) == [
        (("subject_id", "subj_000"), ("modality", "anat")),
        (("subject_id", "subj_001"), ("modality", "anat")),
        (("subject_id", "subj_002"), ("modality", "anat")),
    ]

    han_crumb = tmp_crumb2.replace(subject_id="hansel")
    assert intersection(tmp_crumb1, han_crumb, on=["subject_id"]) == []

    s0_crumb = tmp_crumb2.replace(subject_id="subj_000")
    assert intersection(tmp_crumb1, s0_crumb, on=["subject_id"]) == [(("subject_id", "subj_000"),)]

    assert intersection(tmp_crumb1, s0_crumb, on=["subject_id", "modality"]) == [
        (("subject_id", "subj_000"), ("modality", "anat"))
    ]

    assert intersection(tmp_crumb1, s0_crumb, on=["subject_id", "image"]) == [
        (("subject_id", "subj_000"), ("image", "mprage1.nii")),
        (("subject_id", "subj_000"), ("image", "mprage2.nii")),
        (("subject_id", "subj_000"), ("image", "mprage3.nii")),
    ]

    # test raises
    pytest.raises(KeyError, intersection, tmp_crumb1, tmp_crumb2, on=["hansel"])

    pytest.raises(KeyError, intersection, tmp_crumb1, tmp_crumb2, on=["subject_id", "modality", "hansel"])

    pytest.raises(KeyError, intersection, tmp_crumb1, Crumb(op.expanduser("~/{files}")))

    pytest.raises(KeyError, intersection, tmp_crumb1, Crumb(op.expanduser("~/{files}")), on=["files"])
Exemple #30
0
def test_ls_with_check(tmp_crumb):
    assert not op.exists(tmp_crumb._path)

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range(2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range(3)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
                   }

    paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    sbj_crumb = tmp_crumb.replace(subject_id='subj_000')
    assert sbj_crumb.ls('subject_id', make_crumbs=False, fullpath=False) == ['subj_000']

    assert op.exists(tmp_crumb.split()[0])

    assert all([op.exists(p.path) for p in paths])
    assert all([p.exists() for p in paths])

    images = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)

    modalities = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)

    assert all([img.exists() for img in images])
    assert all([mod.exists() for mod in modalities])

    Path(str(images[0])).rmdir()

    images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)

    assert images != images2
    assert len(images) == len(images2) + 1
    assert not all([img.exists() for img in images])
    assert     all([img.exists() for img in images2])

    Path(str(images[1])).rmdir()
    Path(str(images[2])).rmdir()

    images2 = tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)

    assert tmp_crumb.ls('image') == tmp_crumb.ls()
    assert tmp_crumb.ls() == tmp_crumb.unfold()

    assert not all([img.exists() for img in images])
    assert     all([img.exists() for img in images2])

    modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)

    str_modalities2 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=False, check_exists=True)

    assert images != images2
    assert len(images) == len(images2) + 3

    assert modalities != modalities2
    assert not all([mod.exists() for mod in modalities])
    assert     all([mod.exists() for mod in modalities2])

    assert all([isinstance(smod, str)   for smod in str_modalities2])
    assert all([isinstance(mod,  Crumb) for  mod in modalities2])
    assert all([mod.path == smod for mod, smod in zip(sorted(modalities2), sorted(str_modalities2))])

    shutil.rmtree(modalities2[0].split()[0], ignore_errors=True)

    modalities3 = tmp_crumb.ls('modality', fullpath=True, make_crumbs=True, check_exists=True)

    assert modalities2 != modalities3
    assert not all([mod.exists() for mod in modalities2])
    assert     all([mod.exists() for mod in modalities3])

    assert tmp_crumb.unfold() == tmp_crumb.ls('image', fullpath=True, make_crumbs=True, check_exists=True)

    pytest.raises(IOError, modalities2[0].__getitem__, 'image')

    img_crumb = tmp_crumb.replace(image='mprage1.nii')
    assert 'image' in img_crumb._argval
    assert img_crumb['image'] == ['mprage1.nii']

    assert img_crumb['image'][0] == img_crumb.get_first('image')

    img_crumb['modality'] = 'anat'
    assert 'modality' in img_crumb._argval
    assert img_crumb['modality'] == ['anat']
    assert img_crumb.has_set('modality')

    assert img_crumb['session_id'].count('session_01') == img_crumb['session_id'].count('session_00')

    img_crumb['session_id'] = 'session_00'
    assert 'session_id' in img_crumb._argval
    assert img_crumb['session_id'] == ['session_00']

    assert 'subj_000' not in img_crumb['subject_id']

    unfolded_crumbs = tmp_crumb.unfold()
    assert list(unfolded_crumbs[0].open_args()) == []
    assert unfolded_crumbs[0].unfold() == [unfolded_crumbs[0]]

    pytest.raises(AttributeError, unfolded_crumbs[0]._check_open_args, ['subject_id'])