def test_interface():
di = Demo()
import argparse
parser = argparse.ArgumentParser()
di.setup_parser(parser)
with swallow_outputs() as cmo:
assert_equal(parser.print_help(), None)
assert (cmo.out)
assert_equal(cmo.err, '')
args = parser.parse_args(['42', '11', '1', '2', '--demoarg', '23'])
assert_is(args.demoarg, 23)
assert_equal(args.demoposarg, [42, 11])
assert_equal(args.demooptposarg1, 1)
assert_equal(args.demooptposarg2, 2)
# wrong type
with swallow_outputs() as cmo:
assert_raises(SystemExit, parser.parse_args, ['--demoarg', 'abc'])
# that is what we dump upon folks atm. TODO: improve reporting of illspecified options
assert_re_in(".*invalid constraint:int value:.*", cmo.err, re.DOTALL)
# missing argument to option
with swallow_outputs() as cmo:
assert_raises(SystemExit, parser.parse_args, ['--demoarg'])
assert_re_in(".*--demoarg: expected one argument", cmo.err, re.DOTALL)
# missing positional argument
with swallow_outputs() as cmo:
assert_raises(SystemExit, parser.parse_args, [''])
# PY2|PY3
assert_re_in(
".*error: (too few arguments|the following arguments are required: demoposarg)",
cmo.err, re.DOTALL)
def test_ExecutionTimeExternalsProtocol(path1, path2):
timer_protocol = ExecutionTimeExternalsProtocol()
runner = Runner(protocol=timer_protocol)
# test external command:
cmd = ['git', 'init']
os.mkdir(path1)
runner.run(cmd, cwd=path1)
assert_equal(len(timer_protocol), 1, str(runner.protocol))
assert_equal(cmd, timer_protocol[0]['command'])
ok_(timer_protocol[0]['end'] >= timer_protocol[0]['start'])
ok_(timer_protocol[0]['duration'] >= 0)
assert_is(timer_protocol[0]['exception'], None)
# now with exception, since path2 doesn't exist yet:
try:
with swallow_logs() as cml:
runner.run(cmd, cwd=path2)
except Exception as e:
catched_exception = e
finally:
assert_equal(len(timer_protocol), 2)
assert_equal(cmd, timer_protocol[1]['command'])
ok_(timer_protocol[1]['end'] >= timer_protocol[1]['start'])
ok_(timer_protocol[1]['duration'] >= 0)
assert_is(timer_protocol[1]['exception'], catched_exception)
# test callable (no entry added):
new_runner = Runner(cwd=path2, protocol=timer_protocol)
new_runner(os.mkdir, path2)
assert_true(os.path.exists(path2))
assert_equal(len(timer_protocol), 2)
def test_property_reevaluation(repo1):
from os.path import lexists
from datalad.tests.utils import ok_clean_git
ds = Dataset(repo1)
assert_is_none(ds.repo)
assert_is_not_none(ds.config)
first_config = ds.config
assert_false(ds._cfg_bound)
assert_is_none(ds.id)
ds.create()
ok_clean_git(repo1)
# after creation, we have `repo`, and `config` was reevaluated to point
# to the repo's config:
assert_is_not_none(ds.repo)
assert_is_not_none(ds.config)
second_config = ds.config
assert_true(ds._cfg_bound)
assert_is(ds.config, ds.repo.config)
assert_is_not(first_config, second_config)
assert_is_not_none(ds.id)
first_id = ds.id
ds.remove()
# repo is gone, and config is again reevaluated to only provide user/system
# level config:
assert_false(lexists(ds.path))
assert_is_none(ds.repo)
assert_is_not_none(ds.config)
third_config = ds.config
assert_false(ds._cfg_bound)
assert_is_not(second_config, third_config)
assert_is_none(ds.id)
ds.create()
ok_clean_git(repo1)
# after recreation everything is sane again:
assert_is_not_none(ds.repo)
assert_is_not_none(ds.config)
assert_is(ds.config, ds.repo.config)
forth_config = ds.config
assert_true(ds._cfg_bound)
assert_is_not(third_config, forth_config)
assert_is_not_none(ds.id)
assert_not_equal(ds.id, first_id)
def test_property_reevaluation(repo1):
from os.path import lexists
from datalad.tests.utils import ok_clean_git
ds = Dataset(repo1)
assert_is_none(ds.repo)
assert_is_not_none(ds.config)
first_config = ds.config
assert_false(ds._cfg_bound)
assert_is_none(ds.id)
ds.create()
ok_clean_git(repo1)
# after creation, we have `repo`, and `config` was reevaluated to point
# to the repo's config:
assert_is_not_none(ds.repo)
assert_is_not_none(ds.config)
second_config = ds.config
assert_true(ds._cfg_bound)
assert_is(ds.config, ds.repo.config)
assert_is_not(first_config, second_config)
assert_is_not_none(ds.id)
first_id = ds.id
ds.remove()
# repo is gone, and config is again reevaluated to only provide user/system
# level config:
assert_false(lexists(ds.path))
assert_is_none(ds.repo)
assert_is_not_none(ds.config)
third_config = ds.config
assert_false(ds._cfg_bound)
assert_is_not(second_config, third_config)
assert_is_none(ds.id)
ds.create()
ok_clean_git(repo1)
# after recreation everything is sane again:
assert_is_not_none(ds.repo)
assert_is_not_none(ds.config)
assert_is(ds.config, ds.repo.config)
forth_config = ds.config
assert_true(ds._cfg_bound)
assert_is_not(third_config, forth_config)
assert_is_not_none(ds.id)
assert_not_equal(ds.id, first_id)
def test_ExecutionTimeProtocol(path1, path2):
timer_protocol = ExecutionTimeProtocol()
runner = Runner(protocol=timer_protocol)
# test external command:
cmd = ['git', 'init']
os.mkdir(path1)
runner.run(cmd, cwd=path1)
assert_equal(len(timer_protocol), 1, str(runner.protocol))
assert_equal(cmd, timer_protocol[0]['command'])
ok_(timer_protocol[0]['end'] >= timer_protocol[0]['start'])
ok_(timer_protocol[0]['duration'] >= 0)
assert_is(timer_protocol[0]['exception'], None)
# now with exception, since path2 doesn't exist yet:
try:
with swallow_logs() as cml:
runner.run(cmd, cwd=path2)
except Exception as e:
catched_exception = e
finally:
assert_equal(len(timer_protocol), 2)
assert_equal(cmd, timer_protocol[1]['command'])
ok_(timer_protocol[1]['end'] >= timer_protocol[1]['start'])
ok_(timer_protocol[1]['duration'] >= 0)
assert_is(timer_protocol[1]['exception'], catched_exception)
# test callable:
new_runner = Runner(cwd=path2, protocol=timer_protocol)
new_runner(os.mkdir, path2)
assert_equal(len(timer_protocol), 3)
assert_in('mkdir', timer_protocol[2]['command'][0])
# extract path from args and compare
# note: simple string concatenation for comparison doesn't work
# on windows due to path conversion taking place
ok_(timer_protocol[2]['command'][1].startswith("args=('"))
extracted_path = timer_protocol[2]['command'][1].split(',')[0][7:-1]
assert_equal(normpath(extracted_path), normpath(path2))
# kwargs needs to be in protocol, but order isn't relevant:
ok_("kwargs={}" in timer_protocol[2]['command'][2])
ok_(timer_protocol[2]['end'] >= timer_protocol[2]['start'])
ok_(timer_protocol[2]['duration'] >= 0)
def test_runner_dry(tempfile):
dry = DryRunProtocol()
runner = Runner(protocol=dry)
# test dry command call
cmd = 'echo Testing äöü東 dry run > %s' % tempfile
with swallow_logs(new_level=5) as cml:
ret = runner.run(cmd)
cml.assert_logged("{DryRunProtocol} Running: %s" % cmd, regex=False)
assert_equal(("DRY", "DRY"), ret,
"Output of dry run (%s): %s" % (cmd, ret))
assert_equal(split_cmdline(cmd), dry[0]['command'])
assert_false(os.path.exists(tempfile))
# test dry python function call
output = runner.call(os.path.join, 'foo', 'bar')
assert_is(None, output, "Dry call of: os.path.join, 'foo', 'bar' "
"returned: %s" % output)
assert_in('join', dry[1]['command'][0])
assert_equal("args=('foo', 'bar')", dry[1]['command'][1])
def test_symlinked_dataset_properties(repo1, repo2, repo3, non_repo, symlink):
ds = Dataset(repo1).create()
# now, let ds be a symlink and change that symlink to point to different
# things:
ar2 = AnnexRepo(repo2)
ar3 = AnnexRepo(repo3)
assert_true(os.path.isabs(non_repo))
os.symlink(repo1, symlink)
ds_link = Dataset(symlink)
assert_is(ds_link.repo, ds.repo) # same Repo instance
assert_is_not(ds_link, ds) # but not the same Dataset instance
assert_is(ds_link.config, ds.repo.config)
assert_true(ds_link._cfg_bound)
assert_is_not_none(ds_link.id)
# same id, although different Dataset instance:
assert_equal(ds_link.id, ds.id)
os.unlink(symlink)
os.symlink(repo2, symlink)
assert_is(ds_link.repo, ar2) # same Repo instance
assert_is(ds_link.config, ar2.config)
assert_true(ds_link._cfg_bound)
# id is None again, since this repository is an annex but there was no
# Dataset.create() called yet.
assert_is_none(ds_link.id)
os.unlink(symlink)
os.symlink(repo3, symlink)
assert_is(ds_link.repo, ar3) # same Repo instance
assert_is(ds_link.config, ar3.config)
assert_true(ds_link._cfg_bound)
# id is None again, since this repository is an annex but there was no
# Dataset.create() called yet.
assert_is_none(ds_link.id)
os.unlink(symlink)
os.symlink(non_repo, symlink)
assert_is_none(ds_link.repo)
assert_is_not(ds_link.config, ar3.config)
assert_false(ds_link._cfg_bound)
assert_is_none(ds_link.id)
def test_get_cached_dataset(cache_dir):
# patch DATALAD_TESTS_CACHE to not use the actual cache with
# the test testing that very cache.
cache_dir = Path(cache_dir)
# store file-based values for testrepo-minimalds for readability:
annexed_file = opj('inannex', 'animated.gif')
annexed_file_key = "MD5E-s144625--4c458c62b7ac8ec8e19c8ff14b2e34ad.gif"
with patch(CACHE_PATCH_STR, new=cache_dir):
# tuples to test (url, version, keys, class):
test_cases = [
# a simple testrepo
("https://github.com/datalad/testrepo--minimalds",
"541cf855d13c2a338ff2803d4488daf0035e568f", None, AnnexRepo),
# Same repo, but request paths to be present. This should work
# with a subsequent call, although the first one did not already
# request any:
("https://github.com/datalad/testrepo--minimalds",
"9dd8b56cc706ab56185f2ceb75fbe9de9b606724", annexed_file_key,
AnnexRepo),
# Same repo again, but invalid version
(
"https://github.com/datalad/testrepo--minimalds",
"nonexistent",
"irrelevantkey", # invalid version; don't even try to get the key
AnnexRepo),
# same thing with different name should be treated as a new thing:
("https://github.com/datalad/testrepo--minimalds", "git-annex",
None, AnnexRepo),
# try a plain git repo to make sure we can deal with that:
# Note, that we first need a test case w/o a `key` parameter to not
# blow up the test when Clone is patched, resulting in a MagicMock
# instead of a Dataset instance within get_cached_dataset. In the
# second case it's already cached then, so the patched Clone is
# never executed.
("https://github.com/datalad/datalad.org", None, None, GitRepo),
(
"https://github.com/datalad/datalad.org",
"gh-pages",
"ignored-key", # it's a git repo; don't even try to get a key
GitRepo),
]
for url, version, keys, cls in test_cases:
target = cache_dir / url2filename(url)
# assuming it doesn't exist yet - patched cache dir!
in_cache_before = target.exists()
with patch(CLONE_PATCH_STR) as exec_clone:
try:
ds = get_cached_dataset(url, version, keys)
invalid_version = False
except AssertionError:
# should happen only if `version` wasn't found. Implies
# that the dataset exists in cache (although not returned
# due to exception)
assert_true(version)
assert_false(Dataset(target).repo.commit_exists(version))
# mark for later assertions (most of them should still hold
# true)
invalid_version = True
assert_equal(exec_clone.call_count, 0 if in_cache_before else 1)
# Patch prevents actual execution. Now do it for real. Note, that
# this might be necessary for content retrieval even if dataset was
# in cache before.
try:
ds = get_cached_dataset(url, version, keys)
except AssertionError:
# see previous call
assert_true(invalid_version)
assert_is_instance(ds, Dataset)
assert_true(ds.is_installed())
assert_equal(target, ds.pathobj)
assert_is_instance(ds.repo, cls)
if keys and not invalid_version and \
AnnexRepo.is_valid_repo(ds.path):
# Note: it's not supposed to get that content if passed
# `version` wasn't available. get_cached_dataset would then
# raise before and not download anything only to raise
# afterwards.
here = ds.config.get("annex.uuid")
where = ds.repo.whereis(ensure_list(keys), key=True)
assert_true(all(here in remotes for remotes in where))
# version check. Note, that all `get_cached_dataset` is supposed to
# do, is verifying, that specified version exists - NOT check it
# out"
if version and not invalid_version:
assert_true(ds.repo.commit_exists(version))
# re-execution
with patch(CLONE_PATCH_STR) as exec_clone:
try:
ds2 = get_cached_dataset(url, version, keys)
except AssertionError:
assert_true(invalid_version)
exec_clone.assert_not_called()
# returns the same Dataset as before:
assert_is(ds, ds2)
def test_symlinked_dataset_properties(repo1, repo2, repo3, non_repo, symlink):
ds = Dataset(repo1).create()
# now, let ds be a symlink and change that symlink to point to different
# things:
ar2 = AnnexRepo(repo2)
ar3 = AnnexRepo(repo3)
assert_true(os.path.isabs(non_repo))
os.symlink(repo1, symlink)
ds_link = Dataset(symlink)
assert_is(ds_link.repo, ds.repo) # same Repo instance
assert_is_not(ds_link, ds) # but not the same Dataset instance
assert_is(ds_link.config, ds.repo.config)
assert_true(ds_link._cfg_bound)
assert_is_not_none(ds_link.id)
# same id, although different Dataset instance:
assert_equal(ds_link.id, ds.id)
os.unlink(symlink)
os.symlink(repo2, symlink)
assert_is(ds_link.repo, ar2) # same Repo instance
assert_is(ds_link.config, ar2.config)
assert_true(ds_link._cfg_bound)
# id is None again, since this repository is an annex but there was no
# Dataset.create() called yet.
assert_is_none(ds_link.id)
os.unlink(symlink)
os.symlink(repo3, symlink)
assert_is(ds_link.repo, ar3) # same Repo instance
assert_is(ds_link.config, ar3.config)
assert_true(ds_link._cfg_bound)
# id is None again, since this repository is an annex but there was no
# Dataset.create() called yet.
assert_is_none(ds_link.id)
os.unlink(symlink)
os.symlink(non_repo, symlink)
assert_is_none(ds_link.repo)
assert_is_not(ds_link.config, ar3.config)
assert_false(ds_link._cfg_bound)
assert_is_none(ds_link.id)
def assert_repo_status(path, annex=None, untracked_mode='normal', **kwargs):
"""Compare a repo status against (optional) exceptions.
Anything file/directory that is not explicitly indicated must have
state 'clean', i.e. no modifications and recorded in Git.
This is an alternative to the traditional `ok_clean_git` helper.
Parameters
----------
path: str or Repo
in case of a str: path to the repository's base dir;
Note, that passing a Repo instance prevents detecting annex. This might
be useful in case of a non-initialized annex, a GitRepo is pointing to.
annex: bool or None
explicitly set to True or False to indicate, that an annex is (not)
expected; set to None to autodetect, whether there is an annex.
Default: None.
untracked_mode: {'no', 'normal', 'all'}
If and how untracked content is reported. The specification of untracked
files that are OK to be found must match this mode. See `Repo.status()`
**kwargs
Files/directories that are OK to not be in 'clean' state. Each argument
must be one of 'added', 'untracked', 'deleted', 'modified' and each
value must be a list of filenames (relative to the root of the
repository.
"""
r = None
if isinstance(path, AnnexRepo):
if annex is None:
annex = True
# if `annex` was set to False, but we find an annex => fail
assert_is(annex, True)
r = path
elif isinstance(path, GitRepo):
if annex is None:
annex = False
# explicitly given GitRepo instance doesn't make sense with
# 'annex' True
assert_is(annex, False)
r = path
else:
# 'path' is an actual path
try:
r = AnnexRepo(path, init=False, create=False)
if annex is None:
annex = True
# if `annex` was set to False, but we find an annex => fail
assert_is(annex, True)
except Exception:
# Instantiation failed => no annex
try:
r = GitRepo(path, init=False, create=False)
except Exception:
raise AssertionError("Couldn't find an annex or a git "
"repository at {}.".format(path))
if annex is None:
annex = False
# explicitly given GitRepo instance doesn't make sense with
# 'annex' True
assert_is(annex, False)
status = r.status(untracked=untracked_mode)
# for any file state that indicates some kind of change (all but 'clean)
for state in ('added', 'untracked', 'deleted', 'modified'):
oktobefound = sorted(kwargs.get(state, []))
state_files = sorted(k for k, v in iteritems(status)
if v.get('state', None) == state)
eq_(
state_files, oktobefound,
'unexpected content of state "%s": %r != %r' %
(state, state_files, oktobefound))
