def test_gh2043p1(path=None):
# this tests documents the interim agreement on what should happen
# in the case documented in gh-2043
ds = Dataset(path).create(force=True)
ds.save('1')
assert_repo_status(ds.path, untracked=['2', '3'])
ds.unlock('1')
assert_repo_status(
ds.path,
# on windows we are in an unlocked branch by default, hence
# we would see no change
modified=[] if ds.repo.is_managed_branch() else ['1'],
untracked=['2', '3'])
# save(.) should recommit unlocked file, and not touch anything else
# this tests the second issue in #2043
with chpwd(path):
# only save modified bits
save(path='.', updated=True)
# state of the file (unlocked/locked) is committed as well, and the
# test doesn't lock the file again
assert_repo_status(ds.path, untracked=['2', '3'])
with chpwd(path):
# but when a path is given, anything that matches this path
# untracked or not is added/saved
save(path='.')
# state of the file (unlocked/locked) is committed as well, and the
# test doesn't lock the file again
assert_repo_status(ds.path)
def test_subdataset_save(path=None):
parent = Dataset(path).create()
sub = parent.create('sub')
assert_repo_status(parent.path)
create_tree(parent.path, {"untracked": 'ignore', 'sub': {"new": "wanted"}})
sub.save('new')
# defined state: one untracked, modified (but clean in itself) subdataset
assert_repo_status(sub.path)
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
# `save sub` does not save the parent!!
with chpwd(parent.path):
assert_status('notneeded', save(dataset=sub.path))
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
# `save -u .` saves the state change in the subdataset,
# but leaves any untracked content alone
with chpwd(parent.path):
assert_status('ok', parent.save(updated=True))
assert_repo_status(parent.path, untracked=['untracked'])
# get back to the original modified state and check that -S behaves in
# exactly the same way
create_tree(parent.path, {'sub': {"new2": "wanted2"}})
sub.save('new2')
assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
def test_unlock_raises(path=None, path2=None, path3=None):
# make sure, we are not within a dataset:
_cwd = getpwd()
chpwd(path)
# no dataset and no path:
assert_raises(InsufficientArgumentsError,
unlock, dataset=None, path=None)
# no dataset and path not within a dataset:
assert_raises(NoDatasetFound,
unlock, dataset=None, path=path2)
create(path=path, annex=False)
ds = Dataset(path)
# no complaints
ds.unlock()
# make it annex, but call unlock with invalid path:
(ds.pathobj / ".noannex").unlink()
AnnexRepo(path, create=True)
# One that doesn't exist.
res = ds.unlock(path="notexistent.txt", result_xfm=None,
on_failure='ignore', return_type='item-or-list')
eq_(res['message'], "path does not exist")
# And one that isn't associated with a dataset.
assert_in_results(
ds.unlock(path=path2, on_failure="ignore"),
status="error",
message=("path not underneath the reference dataset %s", ds.path))
chpwd(_cwd)
def test_symlinked_relpath(path=None):
# initially ran into on OSX https://github.com/datalad/datalad/issues/2406
os.makedirs(op.join(path, "origin"))
dspath = op.join(path, "linked")
os.symlink('origin', dspath)
ds = Dataset(dspath).create()
create_tree(
dspath,
{
"mike1": 'mike1', # will be added from topdir
"later": "later", # later from within subdir
"d": {
"mike2": 'mike2', # to be added within subdir
}
})
# in the root of ds
with chpwd(dspath):
ds.repo.add("mike1", git=True)
ds.save(message="committing", path="./mike1")
# Let's also do in subdirectory as CWD, check that relative path
# given to a plain command (not dataset method) are treated as
# relative to CWD
with chpwd(op.join(dspath, 'd')):
save(dataset=ds.path, message="committing", path="mike2")
later = op.join(op.pardir, "later")
ds.repo.add(later, git=True)
save(dataset=ds.path, message="committing", path=later)
assert_repo_status(dspath)
def test_subsuperdataset_save(path=None):
# Verify that when invoked without recursion save does not
# cause querying of subdatasets of the subdataset
# see https://github.com/datalad/datalad/issues/4523
parent = Dataset(path).create()
# Create 3 levels of subdatasets so later to check operation
# with or without --dataset being specified
sub1 = parent.create('sub1')
sub2 = parent.create(sub1.pathobj / 'sub2')
sub3 = parent.create(sub2.pathobj / 'sub3')
assert_repo_status(path)
# now we will lobotomize that sub3 so git would fail if any query is performed.
(sub3.pathobj / '.git' / 'config').chmod(0o000)
try:
sub3.repo.call_git(['ls-files'], read_only=True)
raise SkipTest
except CommandError:
# desired outcome
pass
# the call should proceed fine since neither should care about sub3
# default is no recursion
parent.save('sub1')
sub1.save('sub2')
assert_raises(CommandError, parent.save, 'sub1', recursive=True)
# and should not fail in the top level superdataset
with chpwd(parent.path):
save('sub1')
# or in a subdataset above the problematic one
with chpwd(sub1.path):
save('sub2')
def test_no_leaks(path1=None, path2=None):
ds1 = Dataset(path1).create()
ds1.config.set('i.was.here', 'today', scope='local')
assert_in('i.was.here', ds1.config.keys())
ds1.config.reload()
assert_in('i.was.here', ds1.config.keys())
# now we move into this one repo, and create another
# make sure that no config from ds1 leaks into ds2
with chpwd(path1):
ds2 = Dataset(path2)
assert_not_in('i.was.here', ds2.config.keys())
ds2.config.reload()
assert_not_in('i.was.here', ds2.config.keys())
ds2.create()
assert_not_in('i.was.here', ds2.config.keys())
# and that we do not track the wrong files
assert_not_in(ds1.pathobj / '.git' / 'config',
ds2.config._stores['git']['files'])
assert_not_in(ds1.pathobj / '.datalad' / 'config',
ds2.config._stores['branch']['files'])
# these are the right ones
assert_in(ds2.pathobj / '.git' / 'config',
ds2.config._stores['git']['files'])
assert_in(ds2.pathobj / '.datalad' / 'config',
ds2.config._stores['branch']['files'])
def test_bf2541(path=None):
ds = create(path)
subds = ds.create('sub')
assert_repo_status(ds.path)
os.symlink('sub', op.join(ds.path, 'symlink'))
with chpwd(ds.path):
res = save(recursive=True)
assert_repo_status(ds.path)
def test_relpath_add(path=None):
ds = Dataset(path).create(force=True)
with chpwd(op.join(path, 'dir')):
eq_(save('testindir')[0]['path'], op.join(ds.path, 'dir', 'testindir'))
# and now add all
save('..')
# auto-save enabled
assert_repo_status(ds.path)
def test_bf2043p2(path=None):
ds = Dataset(path).create(force=True)
ds.repo.add('staged')
assert_repo_status(ds.path, added=['staged'], untracked=['untracked'])
# save -u does not commit untracked content
# this tests the second issue in #2043
with chpwd(path):
save(updated=True)
assert_repo_status(ds.path, untracked=['untracked'])
def test_copy_file_specs_from(srcdir=None, destdir=None):
srcdir = Path(srcdir)
destdir = Path(destdir)
files = [p for p in srcdir.glob('**/*') if not p.is_dir()]
# plain list of absolute path objects
r_srcabs, res = _check_copy_file_specs_from(srcdir, destdir / 'srcabs',
files)
# same, but with relative paths
with chpwd(srcdir):
r_srcrel, res = _check_copy_file_specs_from(
srcdir, destdir / 'srcrel', [p.relative_to(srcdir) for p in files])
# same, but as strings
r_srcabs_str, res = _check_copy_file_specs_from(srcdir,
destdir / 'srcabs_str',
[str(p) for p in files])
with chpwd(srcdir):
r_srcrel_str, res = _check_copy_file_specs_from(
srcdir, destdir / 'srcrel_str',
[str(p.relative_to(srcdir)) for p in files])
# same, but with src/dest pairs
r_srcdestabs_str, res = _check_copy_file_specs_from(
srcdir, destdir / 'srcdestabs_str', [
'{}\0{}'.format(str(p), str(destdir / 'srcdestabs_str' / p.name))
for p in files
])
# all methods lead to the same dataset structure
for a, b in ((r_srcabs, r_srcrel), (r_srcabs, r_srcabs_str),
(r_srcabs, r_srcrel_str), (r_srcabs, r_srcdestabs_str)):
eq_(*[
sorted(r for r in d.status(result_xfm='relpaths',
result_renderer='disabled'))
for d in (a, b)
])
# fail on destination outside of the dest repo
res = copy_file(specs_from=[
'{}\0{}'.format(str(p),
str(destdir / 'srcdest_wrong' / p.relative_to(srcdir)))
for p in files
],
on_failure='ignore')
assert_status('error', res)
def test_not_under_git(path=None):
from datalad.distribution.dataset import require_dataset
dsroot = get_dataset_root(path)
assert dsroot is None, "There must be no dataset above tmp %s. Got: %s" % (path, dsroot)
with chpwd(path):
# And require_dataset must puke also
assert_raises(
Exception,
require_dataset,
None, check_installed=True, purpose='test'
)
def test_paths_with_forward_slashes(path=None):
# access file with native absolute path spec
print(path)
ok_file_has_content(op.join(path, 'subdir', 'testfile'), 'testcontent')
with chpwd(path):
# native relative path spec
ok_file_has_content(op.join('subdir', 'testfile'), 'testcontent')
# posix relative path spec
ok_file_has_content('subdir/testfile', 'testcontent')
# abspath with forward slash path sep char
ok_file_has_content(
op.join(path, 'subdir', 'testfile').replace(op.sep, '/'),
'testcontent')
def check(ds, dataset_arg, url_file, fname_format):
subdir = op.join(ds.path, "subdir")
os.mkdir(subdir)
with chpwd(subdir):
shutil.copy(self.json_file, "in.json")
addurls(url_file,
"{url}",
fname_format,
dataset=dataset_arg,
result_renderer='disabled')
# Files specified in the CSV file are always relative to the
# dataset.
for fname in ["a", "b", "c"]:
ok_exists(op.join(ds.path, fname))
def test_remove_subdataset_nomethod(path=None):
ds = Dataset(path).create()
ds.create('subds')
with chpwd(path):
# fails due to unique state
res = remove('subds', on_failure='ignore')
assert_in_results(res,
action='uninstall',
status='error',
type='dataset')
res = remove('subds', reckless='availability', on_failure='ignore')
assert_in_results(res, action='uninstall', status='ok', type='dataset')
assert_in_results(res, action='remove', status='ok')
assert_in_results(res, action='save', status='ok')
def test_dataset_systemglobal_mode(path=None):
ds = create(path)
# any sensible (and also our CI) test environment(s) should have this
assert_in('user.name', ds.config)
# from .datalad/config
assert_in('datalad.dataset.id', ds.config)
# from .git/config
assert_in('annex.version', ds.config)
with chpwd(path):
# now check that no config from a random dataset at PWD is picked up
# if not dataset instance was provided
cfg = ConfigManager(dataset=None, source='any')
assert_in('user.name', cfg)
assert_not_in('datalad.dataset.id', cfg)
assert_not_in('annex.version', cfg)
def test_update_known_submodule(path=None):
def get_baseline(p):
ds = Dataset(p).create()
sub = create(str(ds.pathobj / 'sub'))
assert_repo_status(ds.path, untracked=['sub'])
return ds
# attempt one
ds = get_baseline(op.join(path, 'wo_ref'))
with chpwd(ds.path):
save(recursive=True)
assert_repo_status(ds.path)
# attempt two, same as above but call add via reference dataset
ds = get_baseline(op.join(path, 'w_ref'))
ds.save(recursive=True)
assert_repo_status(ds.path)
def test_bf1886(path=None):
parent = Dataset(path).create()
parent.create('sub')
assert_repo_status(parent.path)
# create a symlink pointing down to the subdataset, and add it
os.symlink('sub', op.join(parent.path, 'down'))
parent.save('down')
assert_repo_status(parent.path)
# now symlink pointing up
os.makedirs(op.join(parent.path, 'subdir', 'subsubdir'))
os.symlink(op.join(op.pardir, 'sub'), op.join(parent.path, 'subdir', 'up'))
parent.save(op.join('subdir', 'up'))
# 'all' to avoid the empty dir being listed
assert_repo_status(parent.path, untracked_mode='all')
# now symlink pointing 2xup, as in #1886
os.symlink(op.join(op.pardir, op.pardir, 'sub'),
op.join(parent.path, 'subdir', 'subsubdir', 'upup'))
parent.save(op.join('subdir', 'subsubdir', 'upup'))
assert_repo_status(parent.path)
# simulatenously add a subds and a symlink pointing to it
# create subds, but don't register it
create(op.join(parent.path, 'sub2'))
os.symlink(op.join(op.pardir, op.pardir, 'sub2'),
op.join(parent.path, 'subdir', 'subsubdir', 'upup2'))
parent.save(['sub2', op.join('subdir', 'subsubdir', 'upup2')])
assert_repo_status(parent.path)
# full replication of #1886: the above but be in subdir of symlink
# with no reference dataset
create(op.join(parent.path, 'sub3'))
os.symlink(op.join(op.pardir, op.pardir, 'sub3'),
op.join(parent.path, 'subdir', 'subsubdir', 'upup3'))
# need to use absolute paths
with chpwd(op.join(parent.path, 'subdir', 'subsubdir')):
save([
op.join(parent.path, 'sub3'),
op.join(parent.path, 'subdir', 'subsubdir', 'upup3')
])
assert_repo_status(parent.path)
def test_git_config_warning(path=None):
if 'GIT_AUTHOR_NAME' in os.environ:
raise SkipTest("Found existing explicit identity config")
# Note: An easier way to test this, would be to just set GIT_CONFIG_GLOBAL
# to point somewhere else. However, this is not supported by git before
# 2.32. Hence, stick with changed HOME in this test, but be sure to unset a
# possible GIT_CONFIG_GLOBAL in addition.
patched_env = os.environ.copy()
patched_env.pop('GIT_CONFIG_GLOBAL', None)
patched_env.update(get_home_envvars(path))
with chpwd(path), \
patch.dict('os.environ', patched_env, clear=True), \
swallow_logs(new_level=30) as cml:
# no configs in that empty HOME
from datalad.api import Dataset
from datalad.config import ConfigManager
# reach into the class and disable the "checked" flag that
# has already been tripped before we get here
ConfigManager._checked_git_identity = False
Dataset(path).config.reload()
assert_in("configure Git before", cml.out)
def _test_create_store(host, base_path=None, ds_path=None, clone_path=None):
ds = Dataset(ds_path).create(force=True)
subds = ds.create('sub', force=True)
subds2 = ds.create('sub2', force=True, annex=False)
ds.save(recursive=True)
assert_repo_status(ds.path)
# don't specify special remote. By default should be git-remote + "-storage"
res = ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
post_update_hook=True,
new_store_ok=True)
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
# remotes exist, but only in super
siblings = ds.siblings(result_renderer='disabled')
eq_({'datastore', 'datastore-storage', 'here'},
{s['name']
for s in siblings})
sub_siblings = subds.siblings(result_renderer='disabled')
eq_({'here'}, {s['name'] for s in sub_siblings})
sub2_siblings = subds2.siblings(result_renderer='disabled')
eq_({'here'}, {s['name'] for s in sub2_siblings})
# check bare repo:
git_dir = Path(base_path) / ds.id[:3] / ds.id[3:]
# The post-update hook was enabled.
ok_exists(git_dir / "hooks" / "post-update")
# And create_sibling_ria took care of an initial call to
# git-update-server-info.
ok_exists(git_dir / "info" / "refs")
git_config = git_dir / 'config'
ok_exists(git_config)
content = git_config.read_text()
assert_in("[datalad \"ora-remote\"]", content)
super_uuid = ds.config.get(
"remote.{}.annex-uuid".format('datastore-storage'))
assert_in("uuid = {}".format(super_uuid), content)
# implicit test of success by ria-installing from store:
ds.push(to="datastore")
with chpwd(clone_path):
if host:
# note, we are not using the "test-store"-label here
clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id),
path='test_install')
else:
# TODO: Whenever ria+file supports special remote config (label),
# change here:
clone('ria+file://{}#{}'.format(base_path, ds.id),
path='test_install')
installed_ds = Dataset(op.join(clone_path, 'test_install'))
assert installed_ds.is_installed()
assert_repo_status(installed_ds.repo)
eq_(installed_ds.id, ds.id)
# Note: get_annexed_files() always reports POSIX paths.
assert_in('ds/file1.txt', installed_ds.repo.get_annexed_files())
assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')),
1,
status='ok',
action='get',
path=op.join(installed_ds.path, 'ds', 'file1.txt'))
# now, again but recursive.
res = ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
recursive=True,
existing='reconfigure',
new_store_ok=True)
assert_result_count(res,
1,
path=str(ds.pathobj),
status='ok',
action="create-sibling-ria")
assert_result_count(res,
1,
path=str(subds.pathobj),
status='ok',
action="create-sibling-ria")
assert_result_count(res,
1,
path=str(subds2.pathobj),
status='ok',
action="create-sibling-ria")
# remotes now exist in super and sub
siblings = ds.siblings(result_renderer='disabled')
eq_({'datastore', 'datastore-storage', 'here'},
{s['name']
for s in siblings})
sub_siblings = subds.siblings(result_renderer='disabled')
eq_({'datastore', 'datastore-storage', 'here'},
{s['name']
for s in sub_siblings})
# but no special remote in plain git subdataset:
sub2_siblings = subds2.siblings(result_renderer='disabled')
eq_({'datastore', 'here'}, {s['name'] for s in sub2_siblings})
# for testing trust_level parameter, redo for each label:
for trust in ['trust', 'semitrust', 'untrust']:
ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
existing='reconfigure',
trust_level=trust,
new_store_ok=True)
res = ds.repo.repo_info()
assert_in(
'[datastore-storage]',
[r['description'] for r in res['{}ed repositories'.format(trust)]])
def test_save(path=None):
ds = Dataset(path).create(annex=False)
with open(op.join(path, "new_file.tst"), "w") as f:
f.write("something")
ds.repo.add("new_file.tst", git=True)
ok_(ds.repo.dirty)
ds.save(message="add a new file")
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
with open(op.join(path, "new_file.tst"), "w") as f:
f.write("modify")
ok_(ds.repo.dirty)
ds.save(message="modified new_file.tst")
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# save works without ds and files given in the PWD
with open(op.join(path, "new_file.tst"), "w") as f:
f.write("rapunzel")
with chpwd(path):
save(message="love rapunzel")
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# and also without `-a` when things are staged
with open(op.join(path, "new_file.tst"), "w") as f:
f.write("exotic")
ds.repo.add("new_file.tst", git=True)
with chpwd(path):
save(message="love marsians")
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
files = ['one.txt', 'two.txt']
for fn in files:
with open(op.join(path, fn), "w") as f:
f.write(fn)
ds.save([op.join(path, f) for f in files])
# superfluous call to save (alll saved it already), should not fail
# but report that nothing was saved
assert_status('notneeded', ds.save(message="set of new files"))
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# create subdataset
subds = ds.create('subds')
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# modify subds
with open(op.join(subds.path, "some_file.tst"), "w") as f:
f.write("something")
subds.save()
assert_repo_status(subds.path, annex=isinstance(subds.repo, AnnexRepo))
# ensure modified subds is committed
ds.save()
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
# now introduce a change downstairs
subds.create('someotherds')
assert_repo_status(subds.path, annex=isinstance(subds.repo, AnnexRepo))
ok_(ds.repo.dirty)
# and save via subdataset path
ds.save('subds', version_tag='new_sub')
assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo))
tags = ds.repo.get_tags()
ok_(len(tags) == 1)
eq_(tags[0], dict(hexsha=ds.repo.get_hexsha(), name='new_sub'))
# fails when retagged, like git does
res = ds.save(version_tag='new_sub', on_failure='ignore')
assert_status('error', res)
assert_result_count(res,
1,
action='save',
type='dataset',
path=ds.path,
message=('cannot tag this version: %s',
"fatal: tag 'new_sub' already exists"))
def test_no_local_write_if_no_dataset(path=None):
Dataset(path).create()
with chpwd(path):
cfg = ConfigManager()
with assert_raises(CommandError):
cfg.set('a.b.c', 'd', scope='local')
def test_path_diff(_path=None, linkpath=None):
# do the setup on the real path, not the symlink, to have its
# bugs not affect this test of status()
ds = get_deeply_nested_structure(str(_path))
if has_symlink_capability():
# make it more complicated by default
ut.Path(linkpath).symlink_to(_path, target_is_directory=True)
path = linkpath
else:
path = _path
ds = Dataset(path)
if has_symlink_capability():
assert ds.pathobj != ds.repo.pathobj
plain_recursive = ds.diff(recursive=True,
annex='all',
result_renderer='disabled')
# check integrity of individual reports with a focus on how symlinks
# are reported
for res in plain_recursive:
# anything that is an "intended" symlink should be reported
# as such. In contrast, anything that is a symlink for mere
# technical reasons (annex using it for something in some mode)
# should be reported as the thing it is representing (i.e.
# a file)
if 'link2' in str(res['path']):
assert res['type'] == 'symlink', res
else:
assert res['type'] != 'symlink', res
# every item must report its parent dataset
assert_in('parentds', res)
# bunch of smoke tests
# query of '.' is same as no path
eq_(
plain_recursive,
ds.diff(path='.',
recursive=True,
annex='all',
result_renderer='disabled'))
# duplicate paths do not change things
eq_(
plain_recursive,
ds.diff(path=['.', '.'],
recursive=True,
annex='all',
result_renderer='disabled'))
# neither do nested paths
if not "2.24.0" <= ds.repo.git_version < "2.25.0":
# Release 2.24.0 contained a regression that was fixed with 072a231016
# (2019-12-10).
eq_(
plain_recursive,
ds.diff(path=['.', 'subds_modified'],
recursive=True,
annex='all',
result_renderer='disabled'))
# when invoked in a subdir of a dataset it still reports on the full thing
# just like `git status`, as long as there are no paths specified
with chpwd(op.join(path, 'directory_untracked')):
plain_recursive = diff(recursive=True,
annex='all',
result_renderer='disabled')
# should be able to take absolute paths and yield the same
# output
eq_(
plain_recursive,
ds.diff(path=ds.path,
recursive=True,
annex='all',
result_renderer='disabled'))
# query for a deeply nested path from the top, should just work with a
# variety of approaches
rpath = op.join('subds_modified', 'subds_lvl1_modified',
u'{}_directory_untracked'.format(OBSCURE_FILENAME))
apathobj = ds.pathobj / rpath
apath = str(apathobj)
for p in (rpath, apath, None):
if p is None:
# change into the realpath of the dataset and
# query with an explicit path
with chpwd(ds.path):
res = ds.diff(path=op.join('.', rpath),
recursive=True,
annex='all',
result_renderer='disabled')
else:
res = ds.diff(path=p,
recursive=True,
annex='all',
result_renderer='disabled')
assert_result_count(
res,
1,
state='untracked',
type='directory',
refds=ds.path,
# path always comes out a full path inside the queried dataset
path=apath,
)
assert_result_count(ds.diff(recursive=True, result_renderer='disabled'),
1,
path=apath)
# limiting recursion will exclude this particular path
assert_result_count(ds.diff(recursive=True,
recursion_limit=1,
result_renderer='disabled'),
0,
path=apath)
# negative limit is unlimited limit
eq_(
ds.diff(recursive=True, recursion_limit=-1,
result_renderer='disabled'),
ds.diff(recursive=True, result_renderer='disabled'))
def test_wtf(topdir=None):
path = opj(topdir, OBSCURE_FILENAME)
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path, on_failure="ignore")
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('## configuration', cmo.out)
assert_in('## dataset', cmo.out)
assert_in(u'path: {}'.format(ds.path), ensure_unicode(cmo.out))
assert_in('branches', cmo.out)
assert_in(DEFAULT_BRANCH + '@', cmo.out)
assert_in('git-annex@', cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
# Sections selection
#
# If we ask for no sections and there is no dataset
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
assert_not_in('## dataset', cmo.out)
for s in SECTION_CALLABLES:
assert_not_in('## %s' % s.lower(), cmo.out.lower())
# ask for a selected set
secs = ['git-annex', 'configuration']
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=secs)
for s in SECTION_CALLABLES:
(assert_in if s in secs else assert_not_in)('## %s' %
s.lower(),
cmo.out.lower())
# order should match our desired one, not alphabetical
# but because of https://github.com/datalad/datalad/issues/3915
# alphanum is now desired
assert cmo.out.index('## git-annex') > cmo.out.index(
'## configuration')
# not achievable from cmdline is to pass an empty list of sections.
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
eq_(cmo.out.rstrip(), '# WTF')
# and we could decorate it nicely for embedding e.g. into github issues
with swallow_outputs() as cmo:
wtf(sections=['dependencies'], decor='html_details')
ok_startswith(cmo.out,
'<details><summary>DataLad %s WTF' % __version__)
assert_in('## dependencies', cmo.out)
# short flavor
with swallow_outputs() as cmo:
wtf(flavor='short')
assert_in("- datalad: version=%s" % __version__, cmo.out)
assert_in("- dependencies: ", cmo.out)
eq_(len(cmo.out.splitlines()),
4) # #WTF, datalad, dependencies, trailing new line
with swallow_outputs() as cmo:
wtf(flavor='short', sections='*')
assert_greater(len(cmo.out.splitlines()), 10) # many more
# check that wtf of an unavailable section yields impossible result (#6712)
res = wtf(sections=['murkie'], on_failure='ignore')
eq_(res[0]["status"], "impossible")
# should result only in '# WTF'
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def test_diff(path=None, norepo=None):
with chpwd(norepo):
assert_raises(NoDatasetFound, diff)
ds = Dataset(path).create()
assert_repo_status(ds.path)
# reports stupid revision input
assert_result_count(ds.diff(fr='WTF',
on_failure='ignore',
result_renderer='disabled'),
1,
status='impossible',
message="Git reference 'WTF' invalid")
# no diff
assert_result_count(_dirty_results(ds.diff(result_renderer='disabled')), 0)
assert_result_count(
_dirty_results(ds.diff(fr='HEAD', result_renderer='disabled')), 0)
# bogus path makes no difference
assert_result_count(
_dirty_results(
ds.diff(path='THIS', fr='HEAD', result_renderer='disabled')), 0)
# let's introduce a known change
create_tree(ds.path, {'new': 'empty'})
ds.save(to_git=True)
assert_repo_status(ds.path)
if ds.repo.is_managed_branch():
fr_base = DEFAULT_BRANCH
to = DEFAULT_BRANCH
else:
fr_base = "HEAD"
to = None
res = _dirty_results(
ds.diff(fr=fr_base + '~1', to=to, result_renderer='disabled'))
assert_result_count(res, 1)
assert_result_count(res,
1,
action='diff',
path=op.join(ds.path, 'new'),
state='added')
# we can also find the diff without going through the dataset explicitly
with chpwd(ds.path):
assert_result_count(_dirty_results(
diff(fr=fr_base + '~1', to=to, result_renderer='disabled')),
1,
action='diff',
path=op.join(ds.path, 'new'),
state='added')
# no diff against HEAD
assert_result_count(_dirty_results(ds.diff(result_renderer='disabled')), 0)
# modify known file
create_tree(ds.path, {'new': 'notempty'})
res = _dirty_results(ds.diff(result_renderer='disabled'))
assert_result_count(res, 1)
assert_result_count(res,
1,
action='diff',
path=op.join(ds.path, 'new'),
state='modified')
# but if we give another path, it doesn't show up
assert_result_count(ds.diff(path='otherpath', result_renderer='disabled'),
0)
# giving the right path must work though
assert_result_count(ds.diff(path='new', result_renderer='disabled'),
1,
action='diff',
path=op.join(ds.path, 'new'),
state='modified')
# stage changes
ds.repo.add('.', git=True)
# no change in diff, staged is not committed
assert_result_count(_dirty_results(ds.diff(result_renderer='disabled')), 1)
ds.save()
assert_repo_status(ds.path)
assert_result_count(_dirty_results(ds.diff(result_renderer='disabled')), 0)
# untracked stuff
create_tree(ds.path, {'deep': {'down': 'untracked', 'down2': 'tobeadded'}})
# a plain diff should report the untracked file
# but not directly, because the parent dir is already unknown
res = _dirty_results(ds.diff(result_renderer='disabled'))
assert_result_count(res, 1)
assert_result_count(res,
1,
state='untracked',
type='directory',
path=op.join(ds.path, 'deep'))
# report of individual files is also possible
assert_result_count(ds.diff(untracked='all', result_renderer='disabled'),
2,
state='untracked',
type='file')
# an unmatching path will hide this result
assert_result_count(ds.diff(path='somewhere', result_renderer='disabled'),
0)
# perfect match and anything underneath will do
assert_result_count(ds.diff(path='deep', result_renderer='disabled'),
1,
state='untracked',
path=op.join(ds.path, 'deep'),
type='directory')
assert_result_count(ds.diff(path='deep', result_renderer='disabled'),
1,
state='untracked',
path=op.join(ds.path, 'deep'))
ds.repo.add(op.join('deep', 'down2'), git=True)
# now the remaining file is the only untracked one
assert_result_count(ds.diff(result_renderer='disabled'),
1,
state='untracked',
path=op.join(ds.path, 'deep', 'down'),
type='file')
def test_siblings(origin=None, repo_path=None, local_clone_path=None):
ca = dict(result_renderer='disabled')
# a remote dataset with a subdataset underneath
origds = Dataset(origin).create(**ca)
_ = origds.create('subm 1', **ca)
sshurl = "ssh://push-remote.example.com"
httpurl1 = "http://remote1.example.com/location"
httpurl2 = "http://remote2.example.com/location"
# insufficient arguments
# we need a dataset to work at
with chpwd(repo_path): # not yet there
assert_raises(InsufficientArgumentsError,
siblings,
'add',
url=httpurl1,
**ca)
# prepare src
source = install(repo_path, source=origin, recursive=True, **ca)
# pollute config
depvar = 'remote.test-remote.datalad-publish-depends'
source.config.add(depvar, 'stupid', scope='local')
# cannot configure unknown remotes as dependencies
res = siblings('configure',
dataset=source,
name="test-remote",
url=httpurl1,
publish_depends=['r1', 'r2'],
on_failure='ignore',
**ca)
assert_status('error', res)
eq_(res[0]['message'],
('unknown sibling(s) specified as publication dependency: %s',
set(('r1', 'r2'))))
# prior config was not changed by failed call above
eq_(source.config.get(depvar, None), 'stupid')
res = siblings('configure',
dataset=source,
name="test-remote",
url=httpurl1,
result_xfm='paths',
**ca)
eq_(res, [source.path])
assert_in("test-remote", source.repo.get_remotes())
eq_(httpurl1, source.repo.get_remote_url("test-remote"))
# reconfiguring doesn't change anything
siblings('configure',
dataset=source,
name="test-remote",
url=httpurl1,
**ca)
assert_in("test-remote", source.repo.get_remotes())
eq_(httpurl1, source.repo.get_remote_url("test-remote"))
# re-adding doesn't work
res = siblings('add',
dataset=source,
name="test-remote",
url=httpurl1,
on_failure='ignore',
**ca)
assert_status('error', res)
# only after removal
res = siblings('remove', dataset=source, name="test-remote", **ca)
assert_status('ok', res)
assert_not_in("test-remote", source.repo.get_remotes())
# remove again (with result renderer to smoke-test a renderer
# special case for this too)
res = siblings('remove', dataset=source, name="test-remote", **ca)
assert_status('notneeded', res)
res = siblings('add',
dataset=source,
name="test-remote",
url=httpurl1,
on_failure='ignore',
**ca)
assert_status('ok', res)
# add another remove with a publication dependency
# again pre-pollute config
depvar = 'remote.test-remote2.datalad-publish-depends'
pushvar = 'remote.test-remote2.push'
source.config.add(depvar, 'stupid', scope='local')
source.config.add(pushvar, 'senseless', scope='local')
res = siblings(
'configure',
dataset=source,
name="test-remote2",
url=httpurl2,
on_failure='ignore',
publish_depends='test-remote',
# just for smoke testing
publish_by_default=DEFAULT_BRANCH,
**ca)
assert_status('ok', res)
# config replaced with new setup
#source.config.reload(force=True)
eq_(source.config.get(depvar, None), 'test-remote')
eq_(source.config.get(pushvar, None), DEFAULT_BRANCH)
# add to another remote automagically taking it from the url
# and being in the dataset directory
with chpwd(source.path):
res = siblings('add', url=httpurl2, **ca)
assert_result_count(res, 1, name="remote2.example.com", type='sibling')
assert_in("remote2.example.com", source.repo.get_remotes())
# don't fail with conflicting url, when using force:
res = siblings('configure',
dataset=source,
name="test-remote",
url=httpurl1 + "/elsewhere",
**ca)
assert_status('ok', res)
eq_(httpurl1 + "/elsewhere", source.repo.get_remote_url("test-remote"))
# no longer a use case, I would need additional convincing that
# this is anyhow useful other then triple checking other peoples
# errors. for an actual check use 'query'
# maybe it could be turned into a set of warnings when `configure`
# alters an existing setting, but then why call configure, if you
# want to keep the old values
#with assert_raises(RuntimeError) as cm:
# add_sibling(dataset=source, name="test-remote",
# url=httpurl1 + "/elsewhere")
#assert_in("""'test-remote' already exists with conflicting settings""",
# str(cm.value))
## add a push url without force fails, since in a way the fetch url is the
## configured push url, too, in that case:
#with assert_raises(RuntimeError) as cm:
# add_sibling(dataset=source, name="test-remote",
# url=httpurl1 + "/elsewhere",
# pushurl=sshurl, force=False)
#assert_in("""'test-remote' already exists with conflicting settings""",
# str(cm.value))
# add push url (force):
res = siblings('configure',
dataset=source,
name="test-remote",
url=httpurl1 + "/elsewhere",
pushurl=sshurl,
**ca)
assert_status('ok', res)
eq_(httpurl1 + "/elsewhere", source.repo.get_remote_url("test-remote"))
eq_(sshurl, source.repo.get_remote_url("test-remote", push=True))
# recursively:
for r in siblings(
'configure',
dataset=source,
name="test-remote",
url=httpurl1 + "/%NAME",
pushurl=sshurl + "/%NAME",
recursive=True,
# we need to disable annex queries, as it will try to access
# the fake URL configured above
get_annex_info=False,
**ca):
repo = GitRepo(r['path'], create=False)
assert_in("test-remote", repo.get_remotes())
url = repo.get_remote_url("test-remote")
pushurl = repo.get_remote_url("test-remote", push=True)
ok_(url.startswith(httpurl1 + '/' + basename(source.path)))
ok_(url.endswith(basename(repo.path)))
ok_(pushurl.startswith(sshurl + '/' + basename(source.path)))
ok_(pushurl.endswith(basename(repo.path)))
eq_(url, r['url'])
eq_(pushurl, r['pushurl'])
# recursively without template:
for r in siblings(
'configure',
dataset=source,
name="test-remote-2",
url=httpurl1,
pushurl=sshurl,
recursive=True,
# we need to disable annex queries, as it will try to access
# the fake URL configured above
get_annex_info=False,
**ca):
repo = GitRepo(r['path'], create=False)
assert_in("test-remote-2", repo.get_remotes())
url = repo.get_remote_url("test-remote-2")
pushurl = repo.get_remote_url("test-remote-2", push=True)
ok_(url.startswith(httpurl1))
ok_(pushurl.startswith(sshurl))
# FIXME: next condition used to compare the *Repo objects instead of
# there paths. Due to missing annex-init in
# datalad/tests/utils.py:clone_url this might not be the same, since
# `source` actually is an annex, but after flavor 'clone' in
# `with_testrepos` and then `install` any trace of an annex might be
# gone in v5 (branch 'master' only), while in direct mode it still is
# considered an annex. `repo` is forced to be a `GitRepo`, so we might
# compare two objects of different classes while they actually are
# pointing to the same repository.
# See github issue #1854
if repo.path != source.repo.path:
ok_(url.endswith('/' + basename(repo.path)))
ok_(pushurl.endswith(basename(repo.path)))
eq_(url, r['url'])
eq_(pushurl, r['pushurl'])
# recursively without template and pushurl but full "hierarchy"
# to a local clone
for r in siblings(
'configure',
dataset=source,
name="test-remote-3",
url=local_clone_path,
recursive=True,
# we need to disable annex queries, as it will try to access
# the fake URL configured above
get_annex_info=False,
**ca):
repo = GitRepo(r['path'], create=False)
assert_in("test-remote-3", repo.get_remotes())
url = repo.get_remote_url("test-remote-3")
pushurl = repo.get_remote_url("test-remote-3", push=True)
eq_(
normpath(url),
normpath(
opj(local_clone_path, relpath(str(r['path']), source.path))))
# https://github.com/datalad/datalad/issues/3951
ok_(not pushurl) # no pushurl should be defined
# 5621: Users shouldn't pass identical names for remote & common data source
assert_raises(ValueError,
siblings,
'add',
dataset=source,
name='howdy',
url=httpurl1,
as_common_datasrc='howdy')
