def test_unlock_raises(path, path2, path3):
# make sure, we are not within a dataset:
_cwd = getpwd()
chpwd(path)
# no dataset and no path:
assert_raises(InsufficientArgumentsError,
unlock, dataset=None, path=None)
# no dataset and path not within a dataset:
res = unlock(dataset=None, path=path2, result_xfm=None,
on_failure='ignore', return_type='item-or-list')
eq_(res['message'], "path not associated with any dataset")
eq_(res['path'], path2)
create(path=path, no_annex=True)
ds = Dataset(path)
# no complaints
ds.unlock()
# make it annex, but call unlock with invalid path:
AnnexRepo(path, create=True)
res = ds.unlock(path="notexistent.txt", result_xfm=None,
on_failure='ignore', return_type='item-or-list')
eq_(res['message'], "path does not exist")
chpwd(_cwd)
def test_unlock_raises(path, path2, path3):
# make sure, we are not within a dataset:
_cwd = getpwd()
chpwd(path)
# no dataset and no path:
assert_raises(InsufficientArgumentsError, unlock, dataset=None, path=None)
# no dataset and path not within a dataset:
assert_raises(NoDatasetArgumentFound, unlock, dataset=None, path=path2)
create(path=path, no_annex=True)
ds = Dataset(path)
# no complaints
ds.unlock()
# make it annex, but call unlock with invalid path:
(ds.pathobj / ".noannex").unlink()
AnnexRepo(path, create=True)
# One that doesn't exist.
res = ds.unlock(path="notexistent.txt",
result_xfm=None,
on_failure='ignore',
return_type='item-or-list')
eq_(res['message'], "path does not exist")
# And one that isn't associated with a dataset.
assert_in_results(ds.unlock(path=path2, on_failure="ignore"),
status="error",
message="path not underneath this dataset")
chpwd(_cwd)
def test_subdataset_save(path):
parent = Dataset(path).create()
sub = parent.create('sub')
assert_repo_status(parent.path)
create_tree(parent.path, {
"untracked": 'ignore',
'sub': {
"new": "wanted"}})
sub.save('new')
# defined state: one untracked, modified (but clean in itself) subdataset
assert_repo_status(sub.path)
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
# `save sub` does not save the parent!!
with chpwd(parent.path):
assert_status('notneeded', save(dataset=sub.path))
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
# `save -u .` saves the state change in the subdataset,
# but leaves any untracked content alone
with chpwd(parent.path):
assert_status('ok', parent.save(updated=True))
assert_repo_status(parent.path, untracked=['untracked'])
# get back to the original modified state and check that -S behaves in
# exactly the same way
create_tree(parent.path, {
'sub': {
"new2": "wanted2"}})
sub.save('new2')
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
def test_symlinked_relpath(path):
# initially ran into on OSX https://github.com/datalad/datalad/issues/2406
os.makedirs(op.join(path, "origin"))
dspath = op.join(path, "linked")
os.symlink('origin', dspath)
ds = Dataset(dspath).create()
create_tree(
dspath,
{
"mike1": 'mike1', # will be added from topdir
"later": "later", # later from within subdir
"d": {
"mike2": 'mike2', # to be added within subdir
}
})
# in the root of ds
with chpwd(dspath):
ds.repo.add("mike1", git=True)
ds.save(message="committing", path="./mike1")
# Let's also do in subdirectory as CWD, check that relative path
# given to a plain command (not dataset method) are treated as
# relative to CWD
with chpwd(op.join(dspath, 'd')):
save(dataset=ds.path, message="committing", path="mike2")
later = op.join(op.pardir, "later")
ds.repo.add(later, git=True)
save(dataset=ds.path, message="committing", path=later)
assert_repo_status(dspath)
def test_subdataset_save(path):
parent = Dataset(path).create()
sub = parent.create('sub')
assert_repo_status(parent.path)
create_tree(parent.path, {"untracked": 'ignore', 'sub': {"new": "wanted"}})
sub.save('new')
# defined state: one untracked, modified (but clean in itself) subdataset
assert_repo_status(sub.path)
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
# `save sub` does not save the parent!!
with chpwd(parent.path):
assert_status('notneeded', save(dataset=sub.path))
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
# `save -u .` saves the state change in the subdataset,
# but leaves any untracked content alone
with chpwd(parent.path):
assert_status('ok', parent.save(updated=True))
assert_repo_status(parent.path, untracked=['untracked'])
# get back to the original modified state and check that -S behaves in
# exactly the same way
create_tree(parent.path, {'sub': {"new2": "wanted2"}})
sub.save('new2')
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
def test_gh2043p1(path):
# this tests documents the interim agreement on what should happen
# in the case documented in gh-2043
ds = Dataset(path).create(force=True)
ds.save('1')
assert_repo_status(ds.path, untracked=['2', '3'])
ds.unlock('1')
assert_repo_status(
ds.path,
# on windows we are in an unlocked branch by default, hence
# we would see no change
modified=[] if on_windows else ['1'],
untracked=['2', '3'])
# save(.) should recommit unlocked file, and not touch anything else
# this tests the second issue in #2043
with chpwd(path):
# only save modified bits
save(path='.', updated=True)
# state of the file (unlocked/locked) is committed as well, and the
# test doesn't lock the file again
assert_repo_status(ds.path, untracked=['2', '3'])
with chpwd(path):
# but when a path is given, anything that matches this path
# untracked or not is added/saved
save(path='.')
# state of the file (unlocked/locked) is committed as well, and the
# test doesn't lock the file again
assert_repo_status(ds.path)
def test_unlock_raises(path, path2, path3):
# make sure, we are not within a dataset:
_cwd = getpwd()
chpwd(path)
# no dataset and no path:
assert_raises(InsufficientArgumentsError, unlock, dataset=None, path=None)
# no dataset and path not within a dataset:
res = unlock(dataset=None,
path=path2,
result_xfm=None,
on_failure='ignore',
return_type='item-or-list')
eq_(res['message'], "path not associated with any dataset")
eq_(res['path'], path2)
create(path=path, no_annex=True)
ds = Dataset(path)
# no complaints
ds.unlock()
# make it annex, but call unlock with invalid path:
AnnexRepo(path, create=True)
res = ds.unlock(path="notexistent.txt",
result_xfm=None,
on_failure='ignore',
return_type='item-or-list')
eq_(res['message'], "path does not exist")
chpwd(_cwd)
def test_unlock_raises(path, path2, path3):
# make sure, we are not within a dataset:
_cwd = getpwd()
chpwd(path)
# no dataset and no path:
assert_raises(InsufficientArgumentsError,
unlock, dataset=None, path=None)
# no dataset and path not within a dataset:
with swallow_logs(new_level=logging.WARNING) as cml:
unlock(dataset=None, path=path2)
assert_in("ignored paths that do not belong to any dataset: ['{0}'".format(path2),
cml.out)
create(path=path, no_annex=True)
ds = Dataset(path)
# no complaints
ds.unlock()
# make it annex, but call unlock with invalid path:
AnnexRepo(path, create=True)
with swallow_logs(new_level=logging.WARNING) as cml:
ds.unlock(path="notexistent.txt")
assert_in("ignored non-existing paths", cml.out)
chpwd(_cwd)
def test_subsuperdataset_save(path):
# Verify that when invoked without recursion save does not
# cause querying of subdatasets of the subdataset
# see https://github.com/datalad/datalad/issues/4523
parent = Dataset(path).create()
# Create 3 levels of subdatasets so later to check operation
# with or without --dataset being specified
sub1 = parent.create('sub1')
sub2 = parent.create(sub1.pathobj / 'sub2')
sub3 = parent.create(sub2.pathobj / 'sub3')
assert_repo_status(path)
# now we will lobotomize that sub3 so git would fail if any query is performed.
(sub3.pathobj / '.git' / 'config').chmod(0o000)
try:
sub3.repo.call_git(['ls-files'], read_only=True)
raise SkipTest
except CommandError:
# desired outcome
pass
# the call should proceed fine since neither should care about sub3
# default is no recursion
parent.save('sub1')
sub1.save('sub2')
assert_raises(CommandError, parent.save, 'sub1', recursive=True)
# and should not fail in the top level superdataset
with chpwd(parent.path):
save('sub1')
# or in a subdataset above the problematic one
with chpwd(sub1.path):
save('sub2')
def test_symlinked_relpath(path):
# initially ran into on OSX https://github.com/datalad/datalad/issues/2406
os.makedirs(op.join(path, "origin"))
dspath = op.join(path, "linked")
os.symlink('origin', dspath)
ds = Dataset(dspath).create()
create_tree(dspath, {
"mike1": 'mike1', # will be added from topdir
"later": "later", # later from within subdir
"d": {
"mike2": 'mike2', # to be added within subdir
}
})
# in the root of ds
with chpwd(dspath):
ds.repo.add("mike1", git=True)
ds.save(message="committing", path="./mike1")
# Let's also do in subdirectory as CWD, check that relative path
# given to a plain command (not dataset method) are treated as
# relative to CWD
with chpwd(op.join(dspath, 'd')):
save(dataset=ds.path,
message="committing",
path="mike2")
later = op.join(op.pardir, "later")
ds.repo.add(later, git=True)
save(dataset=ds.path, message="committing", path=later)
assert_repo_status(dspath)
def test_unlock_raises(path, path2, path3):
# make sure, we are not within a dataset:
_cwd = getpwd()
chpwd(path)
# no dataset and no path:
assert_raises(InsufficientArgumentsError, unlock, dataset=None, path=None)
# no dataset and path not within a dataset:
with swallow_logs(new_level=logging.WARNING) as cml:
unlock(dataset=None, path=path2)
assert_in(
"ignored paths that do not belong to any dataset: ['{0}'".format(
path2), cml.out)
create(path=path, no_annex=True)
ds = Dataset(path)
# no complaints
ds.unlock()
# make it annex, but call unlock with invalid path:
AnnexRepo(path, create=True)
with swallow_logs(new_level=logging.WARNING) as cml:
ds.unlock(path="notexistent.txt")
assert_in("ignored non-existing paths", cml.out)
chpwd(_cwd)
def test_gh2043p1(path):
# this tests documents the interim agreement on what should happen
# in the case documented in gh-2043
ds = Dataset(path).create(force=True)
ds.save('1')
assert_repo_status(ds.path, untracked=['2', '3'])
ds.unlock('1')
assert_repo_status(
ds.path,
# on windows we are in an unlocked branch by default, hence
# we would see no change
modified=[] if on_windows else ['1'],
untracked=['2', '3'])
# save(.) should recommit unlocked file, and not touch anything else
# this tests the second issue in #2043
with chpwd(path):
# only save modified bits
save(path='.', updated=True)
# state of the file (unlocked/locked) is committed as well, and the
# test doesn't lock the file again
assert_repo_status(ds.path, untracked=['2', '3'])
with chpwd(path):
# but when a path is given, anything that matches this path
# untracked or not is added/saved
save(path='.')
# state of the file (unlocked/locked) is committed as well, and the
# test doesn't lock the file again
assert_repo_status(ds.path)
def test_symlinked_relpath(path):
# initially ran into on OSX https://github.com/datalad/datalad/issues/2406
os.makedirs(op.join(path, "origin"))
dspath = op.join(path, "linked")
os.symlink('origin', dspath)
ds = Dataset(dspath).rev_create()
create_tree(
dspath,
{
"mike1": 'mike1', # will be added from topdir
"later": "later", # later from within subdir
"d": {
"mike2": 'mike2', # to be added within subdir
}
})
# in the root of ds
with chpwd(dspath):
ds.repo.add("mike1", git=True)
ds.rev_save(message="committing", path="./mike1")
# Let's also do in subdirectory
with chpwd(op.join(dspath, 'd')):
ds.rev_save(message="committing", path=op.join(op.curdir, "mike2"))
later = op.join(op.pardir, "later")
ds.repo.add(later, git=True)
ds.rev_save(message="committing", path=later)
assert_repo_status(dspath)
def test_subsuperdataset_save(path):
# Verify that when invoked without recursion save does not
# cause querying of subdatasets of the subdataset
# see https://github.com/datalad/datalad/issues/4523
parent = Dataset(path).create()
# Create 3 levels of subdatasets so later to check operation
# with or without --dataset being specified
sub1 = parent.create('sub1')
sub2 = parent.create(sub1.pathobj / 'sub2')
sub3 = parent.create(sub2.pathobj / 'sub3')
assert_repo_status(path)
# now we will lobotomize that sub2 so git would fail if any query is performed.
rmtree(str(sub3.pathobj / '.git' / 'objects'))
# the call should proceed fine since neither should care about sub3
# default is no recursion
parent.save('sub1')
sub1.save('sub2')
assert_raises(CommandError, parent.save, 'sub1', recursive=True)
# and should fail if we request saving while in the parent directory
# but while not providing a dataset, since operation would run within
# pointed subdataset
with chpwd(sub1.path):
assert_raises(CommandError, save, 'sub2')
# but should not fail in the top level superdataset
with chpwd(parent.path):
save('sub1')
def test_addurls_dry_run(path):
ds = Dataset(path).rev_create(force=True)
with chpwd(path):
json_file = "links.json"
with open(json_file, "w") as jfh:
json.dump([{"url": "URL/a.dat", "name": "a", "subdir": "foo"},
{"url": "URL/b.dat", "name": "b", "subdir": "bar"},
{"url": "URL/c.dat", "name": "c", "subdir": "foo"}],
jfh)
ds.rev_save(message="setup")
with swallow_logs(new_level=logging.INFO) as cml:
ds.addurls(json_file,
"{url}",
"{subdir}//{_url_filename_root}",
dry_run=True)
for dir_ in ["foo", "bar"]:
assert_in("Would create a subdataset at {}".format(dir_),
cml.out)
assert_in(
"Would download URL/a.dat to {}".format(
os.path.join(path, "foo", "BASE")),
cml.out)
assert_in("Metadata: {}".format([u"name=a", u"subdir=foo"]),
cml.out)
def test_addurls_url_parts(self, path):
ds = Dataset(path).rev_create(force=True)
with chpwd(path):
ds.addurls(self.json_file, "{url}", "{_url0}/{_url_basename}")
for fname in ["a.dat", "b.dat", "c.dat"]:
ok_exists(op.join("udir", fname))
def test_addurls_subdataset(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
for save in True, False:
label = "save" if save else "nosave"
hexsha_before = ds.repo.get_hexsha()
ds.addurls(self.json_file,
"{url}",
"{subdir}-" + label + "//{name}",
save=save)
hexsha_after = ds.repo.get_hexsha()
for fname in ["foo-{}/a", "bar-{}/b", "foo-{}/c"]:
ok_exists(fname.format(label))
assert_true(save ^ (hexsha_before == hexsha_after))
assert_true(save ^ ds.repo.dirty)
# Now save the "--nosave" changes and check that we have
# all the subdatasets.
ds.add(".")
eq_(set(subdatasets(ds, recursive=True, result_xfm="relpaths")),
{"foo-save", "bar-save", "foo-nosave", "bar-nosave"})
# We don't try to recreate existing subdatasets.
with swallow_logs(new_level=logging.DEBUG) as cml:
ds.addurls(self.json_file, "{url}", "{subdir}-nosave//{name}")
assert_in("Not creating subdataset at existing path", cml.out)
def test_addurls_dry_run(path):
ds = Dataset(path).create(force=True)
with chpwd(path):
json_file = "links.json"
with open(json_file, "w") as jfh:
json.dump([{"url": "URL/a.dat", "name": "a", "subdir": "foo"},
{"url": "URL/b.dat", "name": "b", "subdir": "bar"},
{"url": "URL/c.dat", "name": "c", "subdir": "foo"}],
jfh)
ds.save(message="setup")
with swallow_logs(new_level=logging.INFO) as cml:
ds.addurls(json_file,
"{url}",
"{subdir}//{_url_filename_root}",
dry_run=True)
for dir_ in ["foo", "bar"]:
assert_in("Would create a subdataset at {}".format(dir_),
cml.out)
assert_in(
"Would download URL/a.dat to {}".format(
os.path.join(path, "foo", "BASE")),
cml.out)
assert_in("Metadata: {}".format([u"name=a", u"subdir=foo"]),
cml.out)
def test_addurls_subdataset(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
for save in True, False:
label = "save" if save else "nosave"
hexsha_before = ds.repo.get_hexsha()
ds.addurls(self.json_file, "{url}",
"{subdir}-" + label + "//{name}",
save=save)
hexsha_after = ds.repo.get_hexsha()
for fname in ["foo-{}/a", "bar-{}/b", "foo-{}/c"]:
ok_exists(fname.format(label))
assert_true(save ^ (hexsha_before == hexsha_after))
assert_true(save ^ ds.repo.dirty)
# Now save the "--nosave" changes and check that we have
# all the subdatasets.
ds.add(".")
eq_(set(subdatasets(ds, recursive=True,
result_xfm="relpaths")),
{"foo-save", "bar-save", "foo-nosave", "bar-nosave"})
# We don't try to recreate existing subdatasets.
with swallow_logs(new_level=logging.DEBUG) as cml:
ds.addurls(self.json_file, "{url}", "{subdir}-nosave//{name}")
assert_in("Not creating subdataset at existing path", cml.out)
def test_addurls_url_parts(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
ds.addurls(self.json_file, "{url}", "{_url0}/{_url_basename}")
for fname in ["udir/a.dat", "udir/b.dat", "udir/c.dat"]:
ok_exists(fname)
def test_no_leaks(path1, path2):
ds1 = Dataset(path1).create()
ds1.config.set('i.was.here', 'today', where='local')
assert_in('i.was.here', ds1.config.keys())
ds1.config.reload()
assert_in('i.was.here', ds1.config.keys())
# now we move into this one repo, and create another
# make sure that no config from ds1 leaks into ds2
with chpwd(path1):
ds2 = Dataset(path2)
assert_not_in('i.was.here', ds2.config.keys())
ds2.config.reload()
assert_not_in('i.was.here', ds2.config.keys())
ds2.create()
assert_not_in('i.was.here', ds2.config.keys())
# and that we do not track the wrong files
assert_not_in(ds1.pathobj / '.git' / 'config',
ds2.config._stores['git']['files'])
assert_not_in(ds1.pathobj / '.datalad' / 'config',
ds2.config._stores['dataset']['files'])
# these are the right ones
assert_in(ds2.pathobj / '.git' / 'config',
ds2.config._stores['git']['files'])
assert_in(ds2.pathobj / '.datalad' / 'config',
ds2.config._stores['dataset']['files'])
def test_addurls_url_parts(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
ds.addurls(self.json_file, "{url}", "{_url0}/{_url_basename}")
for fname in ["a.dat", "b.dat", "c.dat"]:
ok_exists(op.join("udir", fname))
def test_bf2541(path):
ds = create(path)
subds = ds.create('sub')
assert_repo_status(ds.path)
os.symlink('sub', op.join(ds.path, 'symlink'))
with chpwd(ds.path):
res = save(recursive=True)
assert_repo_status(ds.path)
def test_addurls_url_filename_fail(self, path):
ds = Dataset(path).rev_create(force=True)
with chpwd(path):
assert_raises(IncompleteResultsError,
ds.addurls,
self.json_file,
"{url}/nofilename/",
"{_url0}/{_url_filename}")
def test_relpath_add(path):
ds = Dataset(path).create(force=True)
with chpwd(op.join(path, 'dir')):
eq_(save('testindir')[0]['path'], op.join(ds.path, 'dir', 'testindir'))
# and now add all
save('..')
# auto-save enabled
assert_repo_status(ds.path)
def test_addurls_url_filename_fail(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
assert_raises(IncompleteResultsError,
ds.addurls,
self.json_file,
"{url}/nofilename/",
"{_url0}/{_url_filename}")
def test_bf2541(path):
ds = create(path)
subds = ds.create('sub')
assert_repo_status(ds.path)
os.symlink('sub', op.join(ds.path, 'symlink'))
with chpwd(ds.path):
res = save(recursive=True)
assert_repo_status(ds.path)
def test_bf2043p2(path):
ds = Dataset(path).create(force=True)
ds.repo.add('staged')
assert_repo_status(ds.path, added=['staged'], untracked=['untracked'])
# save -u does not commit untracked content
# this tests the second issue in #2043
with chpwd(path):
save(updated=True)
assert_repo_status(ds.path, untracked=['untracked'])
def test_bf2043p2(path):
ds = Dataset(path).create(force=True)
ds.repo.add('staged')
assert_repo_status(ds.path, added=['staged'], untracked=['untracked'])
# save -u does not commit untracked content
# this tests the second issue in #2043
with chpwd(path):
save(updated=True)
assert_repo_status(ds.path, untracked=['untracked'])
def test_proxying_open_testrepobased(repo):
TEST_CONTENT = "content to be annex-addurl'd"
fname = 'test-annex.dat'
fpath = opj(repo, fname)
assert_raises(IOError, open, fpath)
aio = AutomagicIO(activate=True)
try:
with swallow_outputs():
# now we should be able just to request to open this file
with open(fpath) as f:
content = f.read()
eq_(content, TEST_CONTENT)
finally:
aio.deactivate()
# and now that we have fetched it, nothing should forbid us to open it again
with open(fpath) as f:
eq_(f.read(), TEST_CONTENT)
annex = AnnexRepo(repo, create=False)
# Let's create another file deeper under the directory with the same content
# so it would point to the same key, which we would drop and repeat the drill
fpath2 = opj(repo, 'd1', 'd2', 'test2.dat')
os.makedirs(dirname(fpath2))
with open(fpath2, 'w') as f:
f.write(content)
annex.add(fpath2)
annex.drop(fpath2)
annex.commit("added and dropped")
assert_raises(IOError, open, fpath2)
# Let's use context manager form
with AutomagicIO() as aio:
ok_(isinstance(aio, AutomagicIO))
ok_(aio.active)
# swallowing output would cause trouble while testing with
# DATALAD_ASSERT_NO_OPEN_FILES mode on. Reason is not 100% clear
# on why underlying git-annex process would be dumping to stdout or err
#with swallow_outputs():
# now we should be able just to request to open this file
with open(fpath2) as f:
content = f.read()
eq_(content, TEST_CONTENT)
annex.drop(fpath2)
assert_raises(IOError, open, fpath2)
# Let's use relative path
with chpwd(opj(repo, 'd1')):
# Let's use context manager form
with AutomagicIO() as aio, \
swallow_outputs(), \
open(opj('d2', 'test2.dat')) as f:
content = f.read()
eq_(content, TEST_CONTENT)
def _test_crawl_init_error(args, template, template_func, target_value,
tmpdir):
ar = AnnexRepo(tmpdir)
with chpwd(tmpdir):
assert_raises(target_value,
crawl_init,
args=args,
template=template,
template_func=template_func)
def test_relpath_add(path):
ds = Dataset(path).create(force=True)
with chpwd(op.join(path, 'dir')):
eq_(save('testindir')[0]['path'],
op.join(ds.path, 'dir', 'testindir'))
# and now add all
save('..')
# auto-save enabled
assert_repo_status(ds.path)
def test_copy_file_specs_from(srcdir, destdir):
srcdir = Path(srcdir)
destdir = Path(destdir)
files = [p for p in srcdir.glob('**/*') if not p.is_dir()]
# plain list of absolute path objects
r_srcabs, res = _check_copy_file_specs_from(srcdir, destdir / 'srcabs',
files)
# same, but with relative paths
with chpwd(srcdir):
r_srcrel, res = _check_copy_file_specs_from(
srcdir, destdir / 'srcrel', [p.relative_to(srcdir) for p in files])
# same, but as strings
r_srcabs_str, res = _check_copy_file_specs_from(srcdir,
destdir / 'srcabs_str',
[str(p) for p in files])
with chpwd(srcdir):
r_srcrel_str, res = _check_copy_file_specs_from(
srcdir, destdir / 'srcrel_str',
[str(p.relative_to(srcdir)) for p in files])
# same, but with src/dest pairs
r_srcdestabs_str, res = _check_copy_file_specs_from(
srcdir, destdir / 'srcdestabs_str', [
'{}\0{}'.format(str(p), str(destdir / 'srcdestabs_str' / p.name))
for p in files
])
# all methods lead to the same dataset structure
for a, b in ((r_srcabs, r_srcrel), (r_srcabs, r_srcabs_str),
(r_srcabs, r_srcrel_str), (r_srcabs, r_srcdestabs_str)):
eq_(*[
sorted(
r
for r in d.status(result_xfm='relpaths', result_renderer=None))
for d in (a, b)
])
# fail on destination outside of the dest repo
res = copy_file(specs_from=[
'{}\0{}'.format(str(p),
str(destdir / 'srcdest_wrong' / p.relative_to(srcdir)))
for p in files
],
on_failure='ignore')
assert_status('error', res)
def test_addurls(self, path):
ds = Dataset(path).create(force=True)
def get_annex_commit_counts():
return int(
ds.repo.repo.git.rev_list("--count", "git-annex").strip())
n_annex_commits = get_annex_commit_counts()
with chpwd(path):
ds.addurls(self.json_file, "{url}", "{name}")
filenames = ["a", "b", "c"]
for fname in filenames:
ok_exists(fname)
for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),
["foo", "bar", "foo"]):
assert_dict_equal(meta, {"subdir": [subdir], "name": [fname]})
# Ignore this check if we're faking dates because that disables
# batch mode.
if not os.environ.get('DATALAD_FAKE__DATES'):
# We should have two new commits on the git-annex: one for the
# added urls and one for the added metadata.
eq_(n_annex_commits + 2, get_annex_commit_counts())
# Add to already existing links, overwriting.
with swallow_logs(new_level=logging.DEBUG) as cml:
ds.addurls(self.json_file,
"{url}",
"{name}",
ifexists="overwrite")
for fname in filenames:
assert_in("Removing {}".format(os.path.join(path, fname)),
cml.out)
# Add to already existing links, skipping.
assert_in_results(ds.addurls(self.json_file,
"{url}",
"{name}",
ifexists="skip"),
action="addurls",
status="notneeded")
# Add to already existing links works, as long content is the same.
ds.addurls(self.json_file, "{url}", "{name}")
# But it fails if something has changed.
ds.unlock("a")
with open("a", "w") as ofh:
ofh.write("changed")
ds.save("a")
assert_raises(IncompleteResultsError, ds.addurls, self.json_file,
"{url}", "{name}")
def check(ds, dataset_arg, url_file, fname_format):
subdir = op.join(ds.path, "subdir")
os.mkdir(subdir)
with chpwd(subdir):
shutil.copy(self.json_file, "in.json")
addurls(dataset_arg, url_file, "{url}", fname_format)
# Files specified in the CSV file are always relative to the
# dataset.
for fname in ["a", "b", "c"]:
ok_exists(op.join(ds.path, fname))
def test_wtf(path):
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path)
assert_not_in('Dataset information', cmo.out)
assert_in('Configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('Dataset information', cmo.out)
assert_in('Configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('Configuration', cmo.out)
assert_in('Dataset information', cmo.out)
assert_in('path: {}'.format(ds.path), cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(exc_str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex=", pyperclip.paste()) # but the content is there
def test_wtf(path):
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path)
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('## configuration', cmo.out)
assert_in('## dataset', cmo.out)
assert_in('path: {}'.format(ds.path), cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(exc_str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def test_addurls(self, path):
ds = Dataset(path).create(force=True)
def get_annex_commit_counts():
return int(
ds.repo.repo.git.rev_list("--count", "git-annex").strip())
n_annex_commits = get_annex_commit_counts()
with chpwd(path):
ds.addurls(self.json_file, "{url}", "{name}")
filenames = ["a", "b", "c"]
for fname in filenames:
ok_exists(fname)
for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),
["foo", "bar", "foo"]):
assert_dict_equal(meta,
{"subdir": [subdir], "name": [fname]})
# Ignore this check if we're faking dates because that disables
# batch mode.
if not os.environ.get('DATALAD_FAKE__DATES'):
# We should have two new commits on the git-annex: one for the
# added urls and one for the added metadata.
eq_(n_annex_commits + 2, get_annex_commit_counts())
# Add to already existing links, overwriting.
with swallow_logs(new_level=logging.DEBUG) as cml:
ds.addurls(self.json_file, "{url}", "{name}",
ifexists="overwrite")
for fname in filenames:
assert_in("Removing {}".format(os.path.join(path, fname)),
cml.out)
# Add to already existing links, skipping.
assert_in_results(
ds.addurls(self.json_file, "{url}", "{name}", ifexists="skip"),
action="addurls",
status="notneeded")
# Add to already existing links works, as long content is the same.
ds.addurls(self.json_file, "{url}", "{name}")
# But it fails if something has changed.
ds.unlock("a")
with open("a", "w") as ofh:
ofh.write("changed")
ds.save("a")
assert_raises(IncompleteResultsError,
ds.addurls,
self.json_file, "{url}", "{name}")
def test_addurls_repindex(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
with assert_raises(IncompleteResultsError) as raised:
ds.addurls(self.json_file, "{url}", "{subdir}")
assert_in("There are file name collisions", str(raised.exception))
ds.addurls(self.json_file, "{url}", "{subdir}-{_repindex}")
for fname in ["foo-0", "bar-0", "foo-1"]:
ok_exists(fname)
def test_not_under_git(path):
from datalad.distribution.dataset import require_dataset
dsroot = get_dataset_root(path)
assert dsroot is None, "There must be no dataset above tmp %s. Got: %s" % (
path, dsroot)
with chpwd(path):
# And require_dataset must puke also
assert_raises(Exception,
require_dataset,
None,
check_installed=True,
purpose='test')
def test_addurls_repindex(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
with assert_raises(IncompleteResultsError) as raised:
ds.addurls(self.json_file, "{url}", "{subdir}")
assert_in("There are file name collisions", str(raised.exception))
ds.addurls(self.json_file, "{url}", "{subdir}-{_repindex}")
for fname in ["foo-0", "bar-0", "foo-1"]:
ok_exists(fname)
def test_addurls_metafail(self, path):
ds = Dataset(path).create(force=True)
# Force failure by passing a non-existent file name to annex.
fn = ds.repo.set_metadata_
def set_meta(_, **kwargs):
for i in fn("wreaking-havoc-and-such", **kwargs):
yield i
with chpwd(path), patch.object(ds.repo, 'set_metadata_', set_meta):
with assert_raises(IncompleteResultsError):
ds.addurls(self.json_file, "{url}", "{name}")
def test_paths_with_forward_slashes(path):
# access file with native absolute path spec
print(path)
ok_file_has_content(op.join(path, 'subdir', 'testfile'), 'testcontent')
with chpwd(path):
# native relative path spec
ok_file_has_content(op.join('subdir', 'testfile'), 'testcontent')
# posix relative path spec
ok_file_has_content('subdir/testfile', 'testcontent')
# abspath with forward slash path sep char
ok_file_has_content(
op.join(path, 'subdir', 'testfile').replace(op.sep, '/'),
'testcontent')
def test_addurls_metafail(self, path):
ds = Dataset(path).create(force=True)
# Force failure by passing a non-existent file name to annex.
fn = ds.repo.set_metadata
def set_meta(_, **kwargs):
for i in fn("wreaking-havoc-and-such", **kwargs):
yield i
with chpwd(path), patch.object(ds.repo, 'set_metadata', set_meta):
with assert_raises(IncompleteResultsError):
ds.addurls(self.json_file, "{url}", "{name}")
def test_git_config_warning(path):
if 'GIT_AUTHOR_NAME' in os.environ:
raise SkipTest("Found existing explicit identity config")
with chpwd(path), \
patch.dict('os.environ', get_home_envvars(path)), \
swallow_logs(new_level=30) as cml:
# no configs in that empty HOME
from datalad.api import Dataset
from datalad.config import ConfigManager
# reach into the class and disable the "checked" flag that
# has already been tripped before we get here
ConfigManager._checked_git_identity = False
Dataset(path).config.reload()
assert_in("configure Git before", cml.out)
def test_dataset_systemglobal_mode(path):
ds = create(path)
# any sensible (and also our CI) test environment(s) should have this
assert_in('user.name', ds.config)
# from .datalad/config
assert_in('datalad.dataset.id', ds.config)
# from .git/config
assert_in('annex.version', ds.config)
with chpwd(path):
# now check that no config from a random dataset at PWD is picked up
# if not dataset instance was provided
cfg = ConfigManager(dataset=None, source='any')
assert_in('user.name', cfg)
assert_not_in('datalad.dataset.id', cfg)
assert_not_in('annex.version', cfg)
def test_update_known_submodule(path):
def get_baseline(p):
ds = Dataset(p).create()
sub = create(text_type(ds.pathobj / 'sub'))
assert_repo_status(ds.path, untracked=['sub'])
return ds
# attempt one
ds = get_baseline(op.join(path, 'wo_ref'))
with chpwd(ds.path):
save(recursive=True)
assert_repo_status(ds.path)
# attempt two, same as above but call add via reference dataset
ds = get_baseline(op.join(path, 'w_ref'))
ds.save(recursive=True)
assert_repo_status(ds.path)
def test_bf1886(path):
parent = Dataset(path).create()
parent.create('sub')
assert_repo_status(parent.path)
# create a symlink pointing down to the subdataset, and add it
os.symlink('sub', op.join(parent.path, 'down'))
parent.save('down')
assert_repo_status(parent.path)
# now symlink pointing up
os.makedirs(op.join(parent.path, 'subdir', 'subsubdir'))
os.symlink(op.join(op.pardir, 'sub'), op.join(parent.path, 'subdir', 'up'))
parent.save(op.join('subdir', 'up'))
# 'all' to avoid the empty dir being listed
assert_repo_status(parent.path, untracked_mode='all')
# now symlink pointing 2xup, as in #1886
os.symlink(
op.join(op.pardir, op.pardir, 'sub'),
op.join(parent.path, 'subdir', 'subsubdir', 'upup'))
parent.save(op.join('subdir', 'subsubdir', 'upup'))
assert_repo_status(parent.path)
# simulatenously add a subds and a symlink pointing to it
# create subds, but don't register it
create(op.join(parent.path, 'sub2'))
os.symlink(
op.join(op.pardir, op.pardir, 'sub2'),
op.join(parent.path, 'subdir', 'subsubdir', 'upup2'))
parent.save(['sub2', op.join('subdir', 'subsubdir', 'upup2')])
assert_repo_status(parent.path)
# full replication of #1886: the above but be in subdir of symlink
# with no reference dataset
create(op.join(parent.path, 'sub3'))
os.symlink(
op.join(op.pardir, op.pardir, 'sub3'),
op.join(parent.path, 'subdir', 'subsubdir', 'upup3'))
# need to use absolute paths
with chpwd(op.join(parent.path, 'subdir', 'subsubdir')):
save([op.join(parent.path, 'sub3'),
op.join(parent.path, 'subdir', 'subsubdir', 'upup3')])
assert_repo_status(parent.path)
def test_addurls_subdataset(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
for save in True, False:
label = "save" if save else "nosave"
ds.addurls(self.json_file, "{url}",
"{subdir}-" + label + "//{name}",
save=save)
subdirs = ["{}-{}".format(d, label) for d in ["foo", "bar"]]
subdir_files = dict(zip(subdirs, [["a", "c"], ["b"]]))
for subds, fnames in subdir_files.items():
for fname in fnames:
ok_exists(op.join(subds, fname))
if save:
assert_repo_status(path)
else:
# The datasets are create and saved ...
assert_repo_status(path, modified=subdirs)
# but the downloaded files aren't.
for subds, fnames in subdir_files.items():
assert_repo_status(subds, added=fnames)
# Now save the "--nosave" changes and check that we have
# all the subdatasets.
ds.save()
eq_(set(subdatasets(dataset=ds, recursive=True,
result_xfm="relpaths")),
{"foo-save", "bar-save", "foo-nosave", "bar-nosave"})
# We don't try to recreate existing subdatasets.
with swallow_logs(new_level=logging.DEBUG) as cml:
ds.addurls(self.json_file, "{url}", "{subdir}-nosave//{name}")
assert_in("Not creating subdataset at existing path", cml.out)
def test_addurls_url_filename(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path):
ds.addurls(self.json_file, "{url}", "{_url0}/{_url_filename}")
for fname in ["udir/a.dat", "udir/b.dat", "udir/c.dat"]:
ok_exists(fname)
def test_save(path):
ds = Dataset(path)
with open(op.join(path, "new_file.tst"), "w") as f:
f.write("something")
ds.repo.add("new_file.tst", git=True)
ok_(ds.repo.dirty)
ds.save(message="add a new file")
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
with open(op.join(path, "new_file.tst"), "w") as f:
f.write("modify")
ok_(ds.repo.dirty)
ds.save(message="modified new_file.tst")
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# save works without ds and files given in the PWD
with open(op.join(path, "new_file.tst"), "w") as f:
f.write("rapunzel")
with chpwd(path):
save(message="love rapunzel")
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# and also without `-a` when things are staged
with open(op.join(path, "new_file.tst"), "w") as f:
f.write("exotic")
ds.repo.add("new_file.tst", git=True)
with chpwd(path):
save(message="love marsians")
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
files = ['one.txt', 'two.txt']
for fn in files:
with open(op.join(path, fn), "w") as f:
f.write(fn)
ds.save([op.join(path, f) for f in files])
# superfluous call to save (alll saved it already), should not fail
# but report that nothing was saved
assert_status('notneeded', ds.save(message="set of new files"))
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# create subdataset
subds = ds.create('subds')
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# modify subds
with open(op.join(subds.path, "some_file.tst"), "w") as f:
f.write("something")
subds.save()
assert_repo_status(subds.path, annex=isinstance(subds.repo, AnnexRepo))
# ensure modified subds is committed
ds.save()
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# now introduce a change downstairs
subds.create('someotherds')
assert_repo_status(subds.path, annex=isinstance(subds.repo, AnnexRepo))
ok_(ds.repo.dirty)
# and save via subdataset path
ds.save('subds', version_tag='new_sub')
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
tags = ds.repo.get_tags()
ok_(len(tags) == 1)
eq_(tags[0], dict(hexsha=ds.repo.get_hexsha(), name='new_sub'))
# fails when retagged, like git does
res = ds.save(version_tag='new_sub', on_failure='ignore')
assert_status('error', res)
assert_result_count(
res, 1,
action='save', type='dataset', path=ds.path,
message=('cannot tag this version: %s',
"fatal: tag 'new_sub' already exists"))
def test_addurls_dropped_urls(self, path):
ds = Dataset(path).create(force=True)
with chpwd(path), swallow_logs(new_level=logging.WARNING) as cml:
ds.addurls(self.json_file, "", "{subdir}//{name}")
assert_re_in(r".*Dropped [0-9]+ row\(s\) that had an empty URL",
str(cml.out))
def test_diff(path, norepo):
with chpwd(norepo):
assert_raises(NoDatasetArgumentFound, diff)
ds = Dataset(path).create()
assert_repo_status(ds.path)
# reports stupid revision input
assert_result_count(
ds.diff(fr='WTF', on_failure='ignore'),
1,
status='impossible',
message="Git reference 'WTF' invalid")
# no diff
assert_result_count(_dirty_results(ds.diff()), 0)
assert_result_count(_dirty_results(ds.diff(fr='HEAD')), 0)
# bogus path makes no difference
assert_result_count(_dirty_results(ds.diff(path='THIS', fr='HEAD')), 0)
# let's introduce a known change
create_tree(ds.path, {'new': 'empty'})
ds.save(to_git=True)
assert_repo_status(ds.path)
res = _dirty_results(ds.diff(fr='HEAD~1'))
assert_result_count(res, 1)
assert_result_count(
res, 1, action='diff', path=op.join(ds.path, 'new'), state='added')
# we can also find the diff without going through the dataset explicitly
with chpwd(ds.path):
assert_result_count(
_dirty_results(diff(fr='HEAD~1')), 1,
action='diff', path=op.join(ds.path, 'new'), state='added')
# no diff against HEAD
assert_result_count(_dirty_results(ds.diff()), 0)
# modify known file
create_tree(ds.path, {'new': 'notempty'})
res = _dirty_results(ds.diff())
assert_result_count(res, 1)
assert_result_count(
res, 1, action='diff', path=op.join(ds.path, 'new'),
state='modified')
# but if we give another path, it doesn't show up
assert_result_count(ds.diff(path='otherpath'), 0)
# giving the right path must work though
assert_result_count(
ds.diff(path='new'), 1,
action='diff', path=op.join(ds.path, 'new'), state='modified')
# stage changes
ds.repo.add('.', git=True)
# no change in diff, staged is not commited
assert_result_count(_dirty_results(ds.diff()), 1)
ds.save()
assert_repo_status(ds.path)
assert_result_count(_dirty_results(ds.diff()), 0)
# untracked stuff
create_tree(ds.path, {'deep': {'down': 'untracked', 'down2': 'tobeadded'}})
# a plain diff should report the untracked file
# but not directly, because the parent dir is already unknown
res = _dirty_results(ds.diff())
assert_result_count(res, 1)
assert_result_count(
res, 1, state='untracked', type='directory',
path=op.join(ds.path, 'deep'))
# report of individual files is also possible
assert_result_count(
ds.diff(untracked='all'), 2, state='untracked', type='file')
# an unmatching path will hide this result
assert_result_count(ds.diff(path='somewhere'), 0)
# perfect match and anything underneath will do
assert_result_count(
ds.diff(path='deep'), 1, state='untracked',
path=op.join(ds.path, 'deep'),
type='directory')
assert_result_count(
ds.diff(path='deep'), 1,
state='untracked', path=op.join(ds.path, 'deep'))
ds.repo.add(op.join('deep', 'down2'), git=True)
# now the remaining file is the only untracked one
assert_result_count(
ds.diff(), 1, state='untracked',
path=op.join(ds.path, 'deep', 'down'),
type='file')
def test_path_diff(_path, linkpath):
# do the setup on the real path, not the symlink, to have its
# bugs not affect this test of status()
ds = get_deeply_nested_structure(str(_path))
if has_symlink_capability():
# make it more complicated by default
ut.Path(linkpath).symlink_to(_path, target_is_directory=True)
path = linkpath
else:
path = _path
ds = Dataset(path)
if not on_windows:
# TODO test should also be has_symlink_capability(), but
# something in the repo base class is not behaving yet
# check the premise of this test
assert ds.pathobj != ds.repo.pathobj
plain_recursive = ds.diff(recursive=True, annex='all')
# check integrity of individual reports with a focus on how symlinks
# are reported
for res in plain_recursive:
# anything that is an "intended" symlink should be reported
# as such. In contrast, anything that is a symlink for mere
# technical reasons (annex using it for something in some mode)
# should be reported as the thing it is representing (i.e.
# a file)
if 'link2' in text_type(res['path']):
assert res['type'] == 'symlink', res
else:
assert res['type'] != 'symlink', res
# every item must report its parent dataset
assert_in('parentds', res)
# bunch of smoke tests
# query of '.' is same as no path
eq_(plain_recursive, ds.diff(path='.', recursive=True, annex='all'))
# duplicate paths do not change things
eq_(plain_recursive, ds.diff(path=['.', '.'], recursive=True, annex='all'))
# neither do nested paths
eq_(plain_recursive,
ds.diff(path=['.', 'subds_modified'], recursive=True, annex='all'))
# when invoked in a subdir of a dataset it still reports on the full thing
# just like `git status`, as long as there are no paths specified
with chpwd(op.join(path, 'directory_untracked')):
plain_recursive = diff(recursive=True, annex='all')
# should be able to take absolute paths and yield the same
# output
eq_(plain_recursive, ds.diff(path=ds.path, recursive=True, annex='all'))
# query for a deeply nested path from the top, should just work with a
# variety of approaches
rpath = op.join('subds_modified', 'subds_lvl1_modified',
u'{}_directory_untracked'.format(OBSCURE_FILENAME))
apathobj = ds.pathobj / rpath
apath = text_type(apathobj)
for p in (rpath, apath, None):
if p is None:
# change into the realpath of the dataset and
# query with an explicit path
with chpwd(ds.path):
res = ds.diff(
path=op.join('.', rpath),
recursive=True,
annex='all')
else:
res = ds.diff(
path=p,
recursive=True,
annex='all')
assert_result_count(
res,
1,
state='untracked',
type='directory',
refds=ds.path,
# path always comes out a full path inside the queried dataset
path=apath,
)
assert_result_count(
ds.diff(
recursive=True),
1,
path=apath)
# limiting recursion will exclude this particular path
assert_result_count(
ds.diff(
recursive=True,
recursion_limit=1),
0,
path=apath)
# negative limit is unlimited limit
eq_(
ds.diff(recursive=True, recursion_limit=-1),
ds.diff(recursive=True)
)
def test_siblings(origin, repo_path):
sshurl = "ssh://push-remote.example.com"
httpurl1 = "http://remote1.example.com/location"
httpurl2 = "http://remote2.example.com/location"
# insufficient arguments
# we need a dataset to work at
with chpwd(repo_path): # not yet there
assert_raises(InsufficientArgumentsError,
siblings, 'add', url=httpurl1)
# prepare src
source = install(repo_path, source=origin, recursive=True)
# pollute config
depvar = 'remote.test-remote.datalad-publish-depends'
source.config.add(depvar, 'stupid', where='local')
# cannot configure unknown remotes as dependencies
res = siblings(
'configure',
dataset=source,
name="test-remote",
url=httpurl1,
publish_depends=['r1', 'r2'],
on_failure='ignore',
result_renderer=None)
assert_status('error', res)
eq_(res[0]['message'],
('unknown sibling(s) specified as publication dependency: %s',
set(('r1', 'r2'))))
# prior config was not changed by failed call above
eq_(source.config.get(depvar, None), 'stupid')
res = siblings('configure',
dataset=source, name="test-remote",
url=httpurl1,
result_xfm='paths',
result_renderer=None)
eq_(res, [source.path])
assert_in("test-remote", source.repo.get_remotes())
eq_(httpurl1,
source.repo.get_remote_url("test-remote"))
# reconfiguring doesn't change anything
siblings('configure', dataset=source, name="test-remote",
url=httpurl1,
result_renderer=None)
assert_in("test-remote", source.repo.get_remotes())
eq_(httpurl1,
source.repo.get_remote_url("test-remote"))
# re-adding doesn't work
res = siblings('add', dataset=source, name="test-remote",
url=httpurl1, on_failure='ignore',
result_renderer=None)
assert_status('error', res)
# only after removal
res = siblings('remove', dataset=source, name="test-remote",
result_renderer=None)
assert_status('ok', res)
assert_not_in("test-remote", source.repo.get_remotes())
res = siblings('add', dataset=source, name="test-remote",
url=httpurl1, on_failure='ignore',
result_renderer=None)
assert_status('ok', res)
# add to another remote automagically taking it from the url
# and being in the dataset directory
with chpwd(source.path):
res = siblings('add', url=httpurl2,
result_renderer=None)
assert_result_count(
res, 1,
name="remote2.example.com", type='sibling')
assert_in("remote2.example.com", source.repo.get_remotes())
# don't fail with conflicting url, when using force:
res = siblings('configure',
dataset=source, name="test-remote",
url=httpurl1 + "/elsewhere",
result_renderer=None)
assert_status('ok', res)
eq_(httpurl1 + "/elsewhere",
source.repo.get_remote_url("test-remote"))
# no longer a use case, I would need additional convincing that
# this is anyhow useful other then tripple checking other peoples
# errors. for an actual check use 'query'
# maybe it could be turned into a set of warnings when `configure`
# alters an existing setting, but then why call configure, if you
# want to keep the old values
#with assert_raises(RuntimeError) as cm:
# add_sibling(dataset=source, name="test-remote",
# url=httpurl1 + "/elsewhere")
#assert_in("""'test-remote' already exists with conflicting settings""",
# str(cm.exception))
## add a push url without force fails, since in a way the fetch url is the
## configured push url, too, in that case:
#with assert_raises(RuntimeError) as cm:
# add_sibling(dataset=source, name="test-remote",
# url=httpurl1 + "/elsewhere",
# pushurl=sshurl, force=False)
#assert_in("""'test-remote' already exists with conflicting settings""",
# str(cm.exception))
# add push url (force):
res = siblings('configure',
dataset=source, name="test-remote",
url=httpurl1 + "/elsewhere",
pushurl=sshurl,
result_renderer=None)
assert_status('ok', res)
eq_(httpurl1 + "/elsewhere",
source.repo.get_remote_url("test-remote"))
eq_(sshurl,
source.repo.get_remote_url("test-remote", push=True))
# recursively:
for r in siblings(
'configure',
dataset=source, name="test-remote",
url=httpurl1 + "/%NAME",
pushurl=sshurl + "/%NAME",
recursive=True,
# we need to disable annex queries, as it will try to access
# the fake URL configured above
get_annex_info=False):
repo = GitRepo(r['path'], create=False)
assert_in("test-remote", repo.get_remotes())
url = repo.get_remote_url("test-remote")
pushurl = repo.get_remote_url("test-remote", push=True)
ok_(url.startswith(httpurl1 + '/' + basename(source.path)))
ok_(url.endswith(basename(repo.path)))
ok_(pushurl.startswith(sshurl + '/' + basename(source.path)))
ok_(pushurl.endswith(basename(repo.path)))
eq_(url, r['url'])
eq_(pushurl, r['pushurl'])
# recursively without template:
for r in siblings(
'configure',
dataset=source, name="test-remote-2",
url=httpurl1,
pushurl=sshurl,
recursive=True,
# we need to disable annex queries, as it will try to access
# the fake URL configured above
get_annex_info=False,
result_renderer=None):
repo = GitRepo(r['path'], create=False)
assert_in("test-remote-2", repo.get_remotes())
url = repo.get_remote_url("test-remote-2")
pushurl = repo.get_remote_url("test-remote-2", push=True)
ok_(url.startswith(httpurl1))
ok_(pushurl.startswith(sshurl))
# FIXME: next condition used to compare the *Repo objects instead of
# there paths. Due to missing annex-init in
# datalad/tests/utils.py:clone_url this might not be the same, since
# `source` actually is an annex, but after flavor 'clone' in
# `with_testrepos` and then `install` any trace of an annex might be
# gone in v5 (branch 'master' only), while in direct mode it still is
# considered an annex. `repo` is forced to be a `GitRepo`, so we might
# compare two objects of different classes while they actually are
# pointing to the same repository.
# See github issue #1854
if repo.path != source.repo.path:
ok_(url.endswith('/' + basename(repo.path)))
ok_(pushurl.endswith(basename(repo.path)))
eq_(url, r['url'])
eq_(pushurl, r['pushurl'])
def test_wtf(path):
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path)
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('## configuration', cmo.out)
assert_in('## dataset', cmo.out)
assert_in('path: {}'.format(ds.path), cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
# Sections selection
#
# If we ask for no sections and there is no dataset
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
assert_not_in('## dataset', cmo.out)
for s in SECTION_CALLABLES:
assert_not_in('## %s' % s.lower(), cmo.out.lower())
# ask for a selected set
secs = ['git-annex', 'configuration']
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=secs)
for s in SECTION_CALLABLES:
(assert_in if s in secs else assert_not_in)(
'## %s' % s.lower(), cmo.out.lower()
)
# order should match our desired one, not alphabetical
assert cmo.out.index('## git-annex') < cmo.out.index('## configuration')
# not achievable from cmdline is to pass an empty list of sections.
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
eq_(cmo.out.rstrip(), '# WTF')
# and we could decorate it nicely for embedding e.g. into github issues
with swallow_outputs() as cmo:
wtf(sections=['dependencies'], decor='html_details')
ok_startswith(cmo.out, '<details><summary>DataLad %s WTF' % __version__)
assert_in('## dependencies', cmo.out)
# should result only in '# WTF'
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(exc_str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex:", pyperclip.paste()) # but the content is there