def test_add_subdataset(path, other):
subds = create(opj(path, 'dir'), force=True)
ds = create(path, force=True)
ok_(subds.repo.dirty)
ok_(ds.repo.dirty)
assert_not_in('dir', ds.subdatasets(result_xfm='relpaths'))
# without a base dataset the next is interpreted as "add everything
# in subds to subds"
add(subds.path)
ok_clean_git(subds.path)
assert_not_in('dir', ds.subdatasets(result_xfm='relpaths'))
# but with a base directory we add the dataset subds as a subdataset
# to ds
ds.add(subds.path)
assert_in('dir', ds.subdatasets(result_xfm='relpaths'))
# create another one
other = create(other)
# install into superdataset, but don't add
other_clone = install(source=other.path, path=opj(ds.path, 'other'))
ok_(other_clone.is_installed)
assert_not_in('other', ds.subdatasets(result_xfm='relpaths'))
# now add, it should pick up the source URL
ds.add('other')
# and that is why, we can reobtain it from origin
ds.uninstall('other')
ok_(other_clone.is_installed)
ds.get('other')
ok_(other_clone.is_installed)
def test_add_subdataset(path, other):
subds = create(opj(path, 'dir'), force=True)
ds = create(path, force=True)
ok_(subds.repo.dirty)
ok_(ds.repo.dirty)
assert_not_in('dir', ds.subdatasets(result_xfm='relpaths'))
# without a base dataset the next is interpreted as "add everything
# in subds to subds"
add(subds.path)
ok_clean_git(subds.path)
assert_not_in('dir', ds.subdatasets(result_xfm='relpaths'))
# but with a base directory we add the dataset subds as a subdataset
# to ds
ds.add(subds.path)
assert_in('dir', ds.subdatasets(result_xfm='relpaths'))
# create another one
other = create(other)
# install into superdataset, but don't add
other_clone = install(source=other.path, path=opj(ds.path, 'other'))
ok_(other_clone.is_installed)
assert_not_in('other', ds.subdatasets(result_xfm='relpaths'))
# now add, it should pick up the source URL
ds.add('other')
# and that is why, we can reobtain it from origin
ds.uninstall('other')
ok_(other_clone.is_installed)
ds.get('other')
ok_(other_clone.is_installed)
def test_relpath_add(path):
ds = Dataset(path).create(force=True)
with chpwd(opj(path, 'dir')):
eq_(add('testindir')[0]['file'], opj(ds.path, 'dir', 'testindir'))
# and now add all
add('..')
# auto-save enabled
assert_false(ds.repo.dirty)
def test_relpath_add(path):
ds = Dataset(path).create(force=True)
with chpwd(opj(path, 'dir')):
eq_(add('testindir')[0]['path'],
opj(ds.path, 'dir', 'testindir'))
# and now add all
add('..')
# auto-save enabled
assert_false(ds.repo.dirty)
def test_add_subdataset(path):
subds = create(opj(path, 'dir'), force=True)
ds = create(path, force=True)
ok_(subds.repo.dirty)
ok_(ds.repo.dirty)
assert_not_in('dir', ds.get_subdatasets())
# without a base dataset the next is interpreted as "add everything
# in subds to subds"
add(subds.path)
ok_clean_git(subds.path)
assert_not_in('dir', ds.get_subdatasets())
# but with a base directory we add the dataset subds as a subdataset
# to ds
ds.add(subds.path)
assert_in('dir', ds.get_subdatasets())
def test_add_subdataset(path):
subds = create(opj(path, 'dir'), force=True)
ds = create(path, force=True)
ok_(subds.repo.dirty)
ok_(ds.repo.dirty)
assert_not_in('dir', ds.get_subdatasets())
# without a base dataset the next is interpreted as "add everything
# in subds to subds"
add(subds.path)
ok_clean_git(subds.path)
assert_not_in('dir', ds.get_subdatasets())
# but with a base directory we add the dataset subds as a subdataset
# to ds
ds.add(subds.path)
assert_in('dir', ds.get_subdatasets())
def test_update_known_submodule(path):
def get_baseline(p):
ds = Dataset(p).create()
sub = ds.create('sub', save=False)
# subdataset saw another commit after becoming a submodule
ok_clean_git(ds.path, index_modified=['sub'])
return ds
# attempt one
ds = get_baseline(opj(path, 'wo_ref'))
with chpwd(ds.path):
add('.', recursive=True)
ok_clean_git(ds.path)
# attempt two, same as above but call add via reference dataset
ds = get_baseline(opj(path, 'w_ref'))
ds.add('.', recursive=True)
ok_clean_git(ds.path)
def test_bf2541(path):
ds = create(path)
subds = ds.create('sub')
ok_clean_git(ds.path)
os.symlink('sub', op.join(ds.path, 'symlink'))
with chpwd(ds.path):
res = add('.', recursive=True)
ok_clean_git(ds.path)
def test_bf2541(path):
ds = create(path)
subds = ds.create('sub')
ok_clean_git(ds.path)
os.symlink('sub', op.join(ds.path, 'symlink'))
with chpwd(ds.path):
res = add('.', recursive=True)
ok_clean_git(ds.path)
def test_update_known_submodule(path):
def get_baseline(p):
ds = Dataset(p).create()
sub = ds.create('sub', save=False)
# subdataset saw another commit after becoming a submodule
ok_clean_git(ds.path, index_modified=['sub'])
return ds
# attempt one
ds = get_baseline(opj(path, 'wo_ref'))
with chpwd(ds.path):
add('.', recursive=True)
ok_clean_git(ds.path)
# attempt two, same as above but call add via reference dataset
ds = get_baseline(opj(path, 'w_ref'))
ds.add('.', recursive=True)
ok_clean_git(ds.path)
def test_bf1886(path):
parent = Dataset(path).create()
sub = parent.create('sub')
ok_clean_git(parent.path)
# create a symlink pointing down to the subdataset, and add it
os.symlink('sub', opj(parent.path, 'down'))
parent.add('down')
ok_clean_git(parent.path)
# now symlink pointing up
os.makedirs(opj(parent.path, 'subdir', 'subsubdir'))
os.symlink(opj(pardir, 'sub'), opj(parent.path, 'subdir', 'up'))
parent.add(opj('subdir', 'up'))
ok_clean_git(parent.path)
# now symlink pointing 2xup, as in #1886
os.symlink(opj(pardir, pardir, 'sub'),
opj(parent.path, 'subdir', 'subsubdir', 'upup'))
parent.add(opj('subdir', 'subsubdir', 'upup'))
ok_clean_git(parent.path)
# simulatenously add a subds and a symlink pointing to it
# create subds, but don't register it
sub2 = create(opj(parent.path, 'sub2'))
os.symlink(opj(pardir, pardir, 'sub2'),
opj(parent.path, 'subdir', 'subsubdir', 'upup2'))
parent.add(['sub2', opj('subdir', 'subsubdir', 'upup2')])
ok_clean_git(parent.path)
# full replication of #1886: the above but be in subdir of symlink
# with no reference dataset
sub3 = create(opj(parent.path, 'sub3'))
os.symlink(opj(pardir, pardir, 'sub3'),
opj(parent.path, 'subdir', 'subsubdir', 'upup3'))
# need to use absolute paths
with chpwd(opj(parent.path, 'subdir', 'subsubdir')):
add([
opj(parent.path, 'sub3'),
opj(parent.path, 'subdir', 'subsubdir', 'upup3')
])
# here is where we need to disagree with the repo in #1886
# we would not expect that `add` registers sub3 as a subdataset
# of parent, because no reference dataset was given and the
# command cannot decide (with the current semantics) whether
# it should "add anything in sub3 to sub3" or "add sub3 to whatever
# sub3 is in"
ok_clean_git(parent.path, untracked=['sub3/'])
def test_bf1886(path):
parent = Dataset(path).create()
sub = parent.create('sub')
ok_clean_git(parent.path)
# create a symlink pointing down to the subdataset, and add it
os.symlink('sub', opj(parent.path, 'down'))
parent.add('down')
ok_clean_git(parent.path)
# now symlink pointing up
os.makedirs(opj(parent.path, 'subdir', 'subsubdir'))
os.symlink(opj(pardir, 'sub'), opj(parent.path, 'subdir', 'up'))
parent.add(opj('subdir', 'up'))
ok_clean_git(parent.path)
# now symlink pointing 2xup, as in #1886
os.symlink(opj(pardir, pardir, 'sub'), opj(parent.path, 'subdir', 'subsubdir', 'upup'))
parent.add(opj('subdir', 'subsubdir', 'upup'))
ok_clean_git(parent.path)
# simulatenously add a subds and a symlink pointing to it
# create subds, but don't register it
sub2 = create(opj(parent.path, 'sub2'))
os.symlink(
opj(pardir, pardir, 'sub2'),
opj(parent.path, 'subdir', 'subsubdir', 'upup2'))
parent.add(['sub2', opj('subdir', 'subsubdir', 'upup2')])
ok_clean_git(parent.path)
# full replication of #1886: the above but be in subdir of symlink
# with no reference dataset
sub3 = create(opj(parent.path, 'sub3'))
os.symlink(
opj(pardir, pardir, 'sub3'),
opj(parent.path, 'subdir', 'subsubdir', 'upup3'))
# need to use absolute paths
with chpwd(opj(parent.path, 'subdir', 'subsubdir')):
add([opj(parent.path, 'sub3'),
opj(parent.path, 'subdir', 'subsubdir', 'upup3')])
# here is where we need to disagree with the repo in #1886
# we would not expect that `add` registers sub3 as a subdataset
# of parent, because no reference dataset was given and the
# command cannot decide (with the current semantics) whether
# it should "add anything in sub3 to sub3" or "add sub3 to whatever
# sub3 is in"
ok_clean_git(parent.path, untracked=['sub3/'])
def test_update_known_submodule(path):
def get_baseline(p):
ds = Dataset(p).create()
with chpwd(ds.path):
subds = create('sub')
ds.add('sub', save=False)
create_tree(subds.path, {"staged": ""})
subds.add("staged", save=False)
# subdataset has staged changes.
ok_clean_git(ds.path, index_modified=['sub'])
return ds
# attempt one
ds = get_baseline(opj(path, 'wo_ref'))
with chpwd(ds.path):
add('.', recursive=True)
ok_clean_git(ds.path)
# attempt two, same as above but call add via reference dataset
ds = get_baseline(opj(path, 'w_ref'))
ds.add('.', recursive=True)
ok_clean_git(ds.path)
def test_update_known_submodule(path):
def get_baseline(p):
ds = Dataset(p).create()
with chpwd(ds.path):
subds = create('sub')
ds.add('sub', save=False)
create_tree(subds.path, {"staged": ""})
subds.add("staged", save=False)
# subdataset has staged changes.
ok_clean_git(ds.path, index_modified=['sub'])
return ds
# attempt one
ds = get_baseline(opj(path, 'wo_ref'))
with chpwd(ds.path):
add('.', recursive=True)
ok_clean_git(ds.path)
# attempt two, same as above but call add via reference dataset
ds = get_baseline(opj(path, 'w_ref'))
ds.add('.', recursive=True)
ok_clean_git(ds.path)
def test_add_insufficient_args(path):
# no argument:
assert_raises(InsufficientArgumentsError, add)
# no `path`, no `source`:
assert_raises(InsufficientArgumentsError, add, dataset=path)
with chpwd(path):
res = add(path="some", on_failure='ignore')
assert_status('impossible', res)
ds = Dataset(opj(path, 'ds'))
ds.create()
# non-existing path outside
assert_status('impossible', ds.add(opj(path, 'outside'), on_failure='ignore'))
# existing path outside
with open(opj(path, 'outside'), 'w') as f:
f.write('doesnt matter')
assert_status('impossible', ds.add(opj(path, 'outside'), on_failure='ignore'))
def test_add_insufficient_args(path):
# no argument:
assert_raises(InsufficientArgumentsError, add)
# no `path`, no `source`:
assert_raises(InsufficientArgumentsError, add, dataset=path)
with chpwd(path):
res = add(path="some", on_failure='ignore')
assert_status('impossible', res)
ds = Dataset(opj(path, 'ds'))
ds.create()
# non-existing path outside
assert_status('impossible', ds.add(opj(path, 'outside'), on_failure='ignore'))
# existing path outside
with open(opj(path, 'outside'), 'w') as f:
f.write('doesnt matter')
assert_status('impossible', ds.add(opj(path, 'outside'), on_failure='ignore'))
def _makeds(path, levels, ds=None, max_leading_dirs=2):
"""Create a hierarchy of datasets
Used recursively, with current invocation generating datasets for the
first level, and delegating sub-levels to recursive invocation
Parameters
----------
path : str
Path to the top directory under which dataset will be created.
If relative -- relative to current directory
levels : list of list
List of specifications for :func:`random.randint` call per each level.
ds : Dataset, optional
Super-dataset which would contain a new dataset (thus its path whould be
a parent of path. Note that ds needs to be installed.
max_leading_dirs : int, optional
Up to how many leading directories withing a dataset could lead to a
sub-dataset
Yields
------
str
Path to the generated dataset(s)
"""
# we apparently can't import api functionality within api
from datalad.api import add
# To simplify managing all the file paths etc
if not isabs(path):
path = abspath(path)
# make it a git (or annex??) repository... ok - let's do randomly one or another ;)
RepoClass = GitRepo if random.randint(0, 1) else AnnexRepo
lgr.info("Generating repo of class %s under %s", RepoClass, path)
repo = RepoClass(path, create=True)
# let's create some dummy file and add it to the beast
fn = opj(path, "file%d.dat" % random.randint(1, 1000))
with open(fn, 'w') as f:
f.write(fn)
repo.add(fn, git=True, commit=True, msg="Added %s" % fn, _datalad_msg=True)
yield path
if levels:
# make a dataset for that one since we want to add sub datasets
ds_ = Dataset(path)
# Process the levels
level, levels_ = levels[0], levels[1:]
nrepos = random.randint(*level) # how many subds to generate
for irepo in range(nrepos):
# we would like to have up to 2 leading dirs
subds_path = opj(*(['d%i' % i
for i in range(random.randint(0, max_leading_dirs+1))]
+ ['r%i' % irepo]))
subds_fpath = opj(path, subds_path)
# yield all under
for d in _makeds(subds_fpath, levels_, ds=ds_):
yield d
if ds:
assert ds.is_installed()
out = add(
path,
dataset=ds,
)
def _makeds(path, levels, ds=None, max_leading_dirs=2):
"""Create a hierarchy of datasets
Used recursively, with current invocation generating datasets for the
first level, and delegating sub-levels to recursive invocation
Parameters
----------
path : str
Path to the top directory under which dataset will be created.
If relative -- relative to current directory
levels : list of list
List of specifications for :func:`random.randint` call per each level.
ds : Dataset, optional
Super-dataset which would contain a new dataset (thus its path whould be
a parent of path. Note that ds needs to be installed.
max_leading_dirs : int, optional
Up to how many leading directories withing a dataset could lead to a
sub-dataset
Yields
------
str
Path to the generated dataset(s)
"""
# we apparently can't import api functionality within api
from datalad.api import add
# To simplify managing all the file paths etc
if not isabs(path):
path = abspath(path)
# make it a git (or annex??) repository... ok - let's do randomly one or another ;)
RepoClass = GitRepo if random.randint(0, 1) else AnnexRepo
lgr.info("Generating repo of class %s under %s", RepoClass, path)
repo = RepoClass(path, create=True)
# let's create some dummy file and add it to the beast
fn = opj(path, "file%d.dat" % random.randint(1, 1000))
with open(fn, 'w') as f:
f.write(fn)
repo.add(fn, git=True)
repo.commit(msg="Added %s" % fn)
yield path
if levels:
# make a dataset for that one since we want to add sub datasets
ds_ = Dataset(path)
# Process the levels
level, levels_ = levels[0], levels[1:]
nrepos = random.randint(*level) # how many subds to generate
for irepo in range(nrepos):
# we would like to have up to 2 leading dirs
subds_path = opj(*(['d%i' % i
for i in range(random.randint(0, max_leading_dirs+1))]
+ ['r%i' % irepo]))
subds_fpath = opj(path, subds_path)
# yield all under
for d in _makeds(subds_fpath, levels_, ds=ds_):
yield d
if ds:
assert ds.is_installed()
out = add(
path,
dataset=ds,
)
