def test_rerun_branch(path=None):
ds = Dataset(path).create()
if ds.repo.is_managed_branch():
assert_status('impossible',
ds.rerun(branch="triggers-abort", on_failure="ignore"))
raise SkipTest("Test incompatible with adjusted branch")
ds.repo.tag("prerun")
outfile = op.join(path, "run-file")
with swallow_outputs():
ds.run(f'echo x$({cat_command} run-file) > run-file')
ds.rerun()
eq_('xx\n', open(outfile).read())
with open(op.join(path, "nonrun-file"), "w") as f:
f.write("foo")
ds.save("nonrun-file")
# Rerun the commands on a new branch that starts at the parent
# commit of the first run.
with swallow_outputs():
ds.rerun(since="prerun", onto="prerun", branch="rerun")
eq_(ds.repo.get_active_branch(), "rerun")
eq_('xx\n', open(outfile).read())
# NOTE: This test depends on the non-run commit above following a run
# commit. Otherwise, all the metadata (e.g., author date) aside from the
# parent commit that is used to generate the commit ID may be set when
# running the tests, which would result in two commits rather than three.
for revrange in ["rerun.." + DEFAULT_BRANCH, DEFAULT_BRANCH + "..rerun"]:
eq_(len(ds.repo.get_revisions(revrange)), 3)
eq_(ds.repo.get_merge_base([DEFAULT_BRANCH, "rerun"]),
ds.repo.get_hexsha("prerun"))
# Start rerun branch at tip of current branch.
ds.repo.checkout(DEFAULT_BRANCH)
ds.rerun(since="prerun", branch="rerun2")
eq_(ds.repo.get_active_branch(), "rerun2")
eq_('xxxx\n', open(outfile).read())
eq_(len(ds.repo.get_revisions(DEFAULT_BRANCH + "..rerun2")), 2)
eq_(len(ds.repo.get_revisions("rerun2.." + DEFAULT_BRANCH)), 0)
# Using an existing branch name fails.
ds.repo.checkout(DEFAULT_BRANCH)
assert_raises(IncompleteResultsError,
ds.rerun, since="prerun", branch="rerun2")
def test_shell_completion_python():
# largely a smoke test for our print("hello world")
with swallow_outputs() as cmo:
res = shell_completion()
out = cmo.out.rstrip()
# we get it printed and returned for double-pleasure
eq_(out, res[0]['content'].rstrip())
def test_add_archive_use_archive_dir(repo_path=None):
ds = Dataset(repo_path).create(force=True)
with chpwd(repo_path):
# Let's add first archive to the repo with default setting
archive_path = opj('4u', '1.tar.gz')
# check it gives informative error if archive is not already added
res = add_archive_content(archive_path, on_failure='ignore')
message = \
"Can not add an untracked archive. Run 'datalad save 4u\\1.tar.gz'"\
if on_windows else \
"Can not add an untracked archive. Run 'datalad save 4u/1.tar.gz'"
assert_in_results(res,
action='add-archive-content',
message=message,
status='impossible')
with swallow_outputs():
ds.save(archive_path)
ok_archives_caches(ds.path, 0)
add_archive_content(archive_path,
strip_leading_dirs=True,
use_current_dir=True)
ok_(not exists(opj('4u', '1 f.txt')))
ok_file_under_git(ds.path, '1 f.txt', annexed=True)
ok_archives_caches(ds.path, 0)
# and now let's extract under archive dir
add_archive_content(archive_path, strip_leading_dirs=True)
ok_file_under_git(ds.path, opj('4u', '1 f.txt'), annexed=True)
ok_archives_caches(ds.path, 0)
add_archive_content(opj('4u', 'sub.tar.gz'))
ok_file_under_git(ds.path, opj('4u', 'sub', '2 f.txt'), annexed=True)
ok_archives_caches(ds.path, 0)
def check_decompress_file(leading_directories, path=None):
outdir = op.join(path, 'simple-extracted')
with swallow_outputs() as cmo:
decompress_file(op.join(path, fn_archive_obscure_ext),
outdir,
leading_directories=leading_directories)
eq_(cmo.out, "")
eq_(cmo.err, "")
path_archive_obscure = op.join(outdir, fn_archive_obscure)
if leading_directories == 'strip':
assert_false(op.exists(path_archive_obscure))
testpath = outdir
elif leading_directories is None:
assert_true(op.exists(path_archive_obscure))
testpath = path_archive_obscure
else:
raise NotImplementedError("Dunno about this strategy: %s" %
leading_directories)
assert_true(op.exists(op.join(testpath, '3.txt')))
assert_true(op.exists(op.join(testpath, fn_in_archive_obscure)))
with open(op.join(testpath, '3.txt')) as f:
eq_(f.read(), '3 load')
def test_run_cmdline_disambiguation(path=None):
Dataset(path).create()
with chpwd(path):
# Without a positional argument starting a command, any option is
# treated as an option to 'datalad run'.
with swallow_outputs() as cmo:
with patch("datalad.core.local.run._execute_command") as exec_cmd:
with assert_raises(SystemExit):
main(["datalad", "run", "--message"])
exec_cmd.assert_not_called()
assert_in("message: expected one", cmo.err)
# If we want to pass an option as the first value of a command (e.g.,
# because we are using a runscript with containers-run), we can do this
# with "--".
with patch("datalad.core.local.run._execute_command") as exec_cmd:
with assert_raises(SystemExit):
main(["datalad", "run", "--", "--message"])
exec_cmd.assert_called_once_with(
'"--message"' if on_windows else "--message", path)
# Our parser used to mishandle --version (gh-3067),
# treating 'datalad run CMD --version' as 'datalad --version'.
# but that is no longer the case and echo --version should work with or
# without explicit "--" separator
for sep in [[], ['--']]:
with patch("datalad.core.local.run._execute_command") as exec_cmd:
with assert_raises(SystemExit):
main(["datalad", "run"] + sep + ["echo", "--version"])
exec_cmd.assert_called_once_with(
'"echo" "--version"' if on_windows else "echo --version",
path)
def test_add_archive_content_zip(repo_path=None):
ds = Dataset(repo_path).create(force=True)
with chpwd(repo_path):
with swallow_outputs():
ds.save("1.zip", message="add 1.zip")
add_archive_content("1.zip")
ok_file_under_git(ds.pathobj / "1" / "foo", annexed=True)
ok_file_under_git(ds.pathobj / "1" / "dir" / "bar", annexed=True)
ok_archives_caches(ds.path, 0)
def _test_BasicAnnexTestRepo(repodir):
trepo = BasicAnnexTestRepo(repodir)
trepo.create()
assert_repo_status(trepo.path)
ok_file_under_git(trepo.path, 'test.dat')
ok_file_under_git(trepo.path, 'INFO.txt')
ok_file_under_git(trepo.path, 'test-annex.dat', annexed=True)
ok_(trepo.repo.file_has_content('test-annex.dat') is False)
with swallow_outputs():
trepo.repo.get('test-annex.dat')
ok_(trepo.repo.file_has_content('test-annex.dat'))
def test_addurls_subdataset(self=None, path=None):
ds = Dataset(path).create(force=True)
for save in True, False:
label = "save" if save else "nosave"
with swallow_outputs() as cmo:
ds.addurls(self.json_file,
"{url}",
"{subdir}-" + label + "//{name}",
save=save,
cfg_proc=["yoda"])
# The custom result renderer transforms the subdataset
# action=create results into something more informative than
# "create(ok): . (dataset)"...
assert_in("create(ok): foo-{} (dataset)".format(label),
cmo.out)
# ... and that doesn't lose the standard summary.
assert_in("create (ok: 2)", cmo.out)
subdirs = [
op.join(ds.path, "{}-{}".format(d, label))
for d in ["foo", "bar"]
]
subdir_files = dict(zip(subdirs, [["a", "c"], ["b"]]))
for subds, fnames in subdir_files.items():
for fname in fnames:
ok_exists(op.join(subds, fname))
# cfg_proc was applied generated subdatasets.
ok_exists(op.join(subds, "code"))
if save:
assert_repo_status(path)
else:
# The datasets are create but not saved (since asked not to)
assert_repo_status(path, untracked=subdirs)
# but the downloaded files aren't.
for subds, fnames in subdir_files.items():
assert_repo_status(subds, added=fnames)
# Now save the "--nosave" changes and check that we have
# all the subdatasets.
ds.save()
eq_(
set(subdatasets(dataset=ds, recursive=True,
result_xfm="relpaths")),
{"foo-save", "bar-save", "foo-nosave", "bar-nosave"})
# We don't try to recreate existing subdatasets.
with swallow_logs(new_level=logging.DEBUG) as cml:
ds.addurls(self.json_file,
"{url}",
"{subdir}-nosave//{name}",
result_renderer='disabled')
assert_in("Not creating subdataset at existing path", cml.out)
def test_exit_code():
# will relay actual exit code on CommandError
cmd = ['datalad', 'sshrun', 'datalad-test', 'exit 42']
with assert_raises(SystemExit) as cme:
# running nosetests without -s
if isinstance(sys.stdout, StringIO): # pragma: no cover
with swallow_outputs(): # need to give smth with .fileno ;)
main(cmd)
else:
# to test both scenarios
main(cmd)
assert_equal(cme.value.code, 42)
def test_add_archive_dirs(path_orig=None, url=None, repo_path=None):
# change to repo_path
with chpwd(repo_path):
# create annex repo
ds = Dataset(repo_path).create(force=True)
repo = ds.repo
# add archive to the repo so we could test
with swallow_outputs():
repo.add_url_to_file('1.tar.gz', opj(url, '1.tar.gz'))
repo.commit("added 1.tar.gz")
# test with excludes and annex options
add_archive_content(
'1.tar.gz',
existing='archive-suffix',
# Since inconsistent and seems in many cases no
# leading dirs to strip, keep them as provided
strip_leading_dirs=True,
delete=True,
leading_dirs_consider=['crcns.*', '1'],
leading_dirs_depth=2,
use_current_dir=False,
exclude='.*__MACOSX.*') # some junk penetrates
eq_(
repo.get_description(
uuid=DATALAD_SPECIAL_REMOTES_UUIDS[ARCHIVES_SPECIAL_REMOTE]),
'[%s]' % ARCHIVES_SPECIAL_REMOTE)
all_files = sorted(find_files('.'))
# posixify paths to make it work on Windows as well
all_files = [Path(file).as_posix() for file in all_files]
target_files = {
'CR24A/behaving1/1 f.txt',
'CR24C/behaving3/3 f.txt',
'CR24D/behaving2/2 f.txt',
'.datalad/config',
}
eq_(set(all_files), target_files)
# regression test: the subdir in MACOSX wasn't excluded and its name was
# getting stripped by leading_dir_len
# if stripping and exclude didn't work this fails
assert_false(exists('__MACOSX'))
# if exclude doesn't work then name of subdir gets stripped by
# leading_dir_len
assert_false(exists('c-1_data'))
# if exclude doesn't work but everything else works this fails
assert_false(exists('CR24B'))
def test_interface():
di = Demo()
import argparse
parser = argparse.ArgumentParser()
from datalad.cli.parser import setup_parser_for_interface
setup_parser_for_interface(parser, di)
with swallow_outputs() as cmo:
assert_equal(parser.print_help(), None)
assert (cmo.out)
assert_equal(cmo.err, '')
args = parser.parse_args(['42', '11', '1', '2', '--demoarg', '23'])
assert_is(args.demoarg, 23)
assert_equal(args.demoposarg, [42, 11])
assert_equal(args.demooptposarg1, 1)
assert_equal(args.demooptposarg2, 2)
# wrong type
with swallow_outputs() as cmo:
assert_raises(SystemExit, parser.parse_args, ['--demoarg', 'abc'])
# that is what we dump upon folks atm. TODO: improve reporting of illspecified options
assert_re_in(".*invalid constraint:int value:.*", cmo.err, re.DOTALL)
# missing argument to option
with swallow_outputs() as cmo:
assert_raises(SystemExit, parser.parse_args, ['--demoarg'])
assert_re_in(".*--demoarg: expected one argument", cmo.err, re.DOTALL)
# missing positional argument
with swallow_outputs() as cmo:
assert_raises(SystemExit, parser.parse_args, [''])
# PY2|PY3
assert_re_in(
".*error: (too few arguments|the following arguments are required: demoposarg)",
cmo.err, re.DOTALL)
def test_add_archive_content_strip_leading(path_orig=None,
url=None,
repo_path=None):
with chpwd(repo_path):
ds = Dataset(repo_path).create(force=True)
repo = ds.repo
# Let's add first archive to the repo so we could test
with swallow_outputs():
repo.add_url_to_file('1.tar.gz', opj(url, '1.tar.gz'))
repo.commit("added 1.tar.gz")
add_archive_content('1.tar.gz', strip_leading_dirs=True)
ok_(not exists('1'))
ok_file_under_git(ds.path, '1 f.txt', annexed=True)
ok_file_under_git('d', '1d', annexed=True)
ok_archives_caches(ds.path, 0)
def test_ssh_option():
# This test is hacky in that detecting the sent value depends on systems
# commonly configuring `AcceptEnv LC_*` in their sshd_config. If we get
# back an empty value, assume that isn't configured, and skip the test.
with patch.dict('os.environ', {"LC_DATALAD_HACK": 'hackbert'}):
with swallow_outputs() as cmo:
with assert_raises(SystemExit):
main([
"datalad", "sshrun", "-oSendEnv=LC_DATALAD_HACK",
"datalad-test", "echo $LC_DATALAD_HACK"
])
out = cmo.out.strip()
if not out:
raise SkipTest(
"SSH target probably does not accept LC_* variables. "
"Skipping")
assert_equal(out, "hackbert")
def test_rerun_chain(path=None):
ds = Dataset(path).create()
commits = []
with swallow_outputs():
ds.run(f'echo x$({cat_command} grows) > grows')
ds.repo.tag("first-run", commit=DEFAULT_BRANCH)
for _ in range(3):
commits.append(ds.repo.get_hexsha(DEFAULT_BRANCH))
ds.rerun()
_, info = get_run_info(ds, last_commit_msg(ds.repo))
eq_(info["chain"], commits)
ds.rerun(revision="first-run")
_, info = get_run_info(ds, last_commit_msg(ds.repo))
eq_(info["chain"], commits[:1])
def test_status_custom_summary_no_repeats(path=None):
from datalad.api import Dataset
from datalad.core.local.status import Status
# This regression test depends on the command having a custom summary
# renderer *and* the particular call producing summary output. status()
# having this method doesn't guarantee that it is still an appropriate
# command for this test, but it's at least a necessary condition.
ok_(hasattr(Status, "custom_result_summary_renderer"))
ds = Dataset(path).create()
out = WitlessRunner(cwd=path).run(
["datalad", "--output-format=tailored", "status"],
protocol=StdOutCapture)
out_lines = out['stdout'].splitlines()
ok_(out_lines)
eq_(len(out_lines), len(set(out_lines)))
with swallow_outputs() as cmo:
ds.status(return_type="list", result_renderer="tailored")
eq_(out_lines, cmo.out.splitlines())
def test_rerun_just_one_commit(path=None):
ds = Dataset(path).create()
if ds.repo.is_managed_branch():
assert_status('impossible',
ds.rerun(branch="triggers-abort", on_failure="ignore"))
raise SkipTest("Test incompatible with adjusted branch")
ds.repo.checkout("orph", options=["--orphan"])
ds.repo.call_git(["reset", "--hard"])
ds.repo.config.reload()
ds.run('echo static-content > static')
eq_(len(ds.repo.get_revisions("HEAD")), 1)
assert_raises(IncompleteResultsError, ds.rerun)
assert_raises(IncompleteResultsError, ds.rerun, since="", onto="")
# --script propagates the error.
with swallow_outputs():
assert_raises(IncompleteResultsError,
ds.rerun, since="", onto="", script="-")
# --dry-run propagates the error.
assert_raises(IncompleteResultsError,
ds.rerun, since="", onto="",
report=True, return_type="list")
def test_generic_result_renderer():
# a bunch of bad cases of results
testcases = [
# an empty result will surface
({}, ['<action-unspecified>(<status-unspecified>)']),
# non-standard status makes it out again
(dict(status='funky'), ['<action-unspecified>(funky)']),
# just an action result is enough to get some output
(dict(action='funky'), ['funky(<status-unspecified>)']),
# a plain path produces output, although
(dict(path='funky'),
['<action-unspecified>(<status-unspecified>): funky']),
# plain type makes it through
(dict(type='funky'),
['<action-unspecified>(<status-unspecified>): (funky)']),
# plain message makes it through
(dict(message='funky', error_message='extra-funky'),
['<action-unspecified>(<status-unspecified>): [funky] [extra-funky]'
]),
]
if on_windows:
testcases.extend([
# if relpath'ing is not possible, takes the path verbatim
(dict(path='C:\\funky', refds='D:\\medina'),
['<action-unspecified>(<status-unspecified>): C:\\funky']),
])
else:
testcases.extend([
(dict(path='/funky/cold/medina', refds='/funky'),
['<action-unspecified>(<status-unspecified>): cold/medina']),
])
for result, contenttests in testcases:
with swallow_outputs() as cmo:
generic_result_renderer(result)
for ctest in contenttests:
assert_in(ctest, cmo.out)
def check_create_obscure(create_kwargs, path):
with chpwd(path):
with swallow_outputs():
ds = create(result_renderer="default", **create_kwargs)
ok_(ds.is_installed())
def test_rerun(path=None, nodspath=None):
ds = Dataset(path).create()
sub = ds.create('sub')
probe_path = op.join(sub.path, 'sequence')
# run inside the dataset
with chpwd(path), \
swallow_outputs():
ds.run(f'echo x$({cat_command} sub/sequence) > sub/sequence')
# command ran once, all clean
assert_repo_status(ds.path)
eq_('x\n', open(probe_path).read())
# now, for a rerun we can be anywhere, PWD and all are recorded
# moreover, rerun must figure out which bits to unlock, even in
# subdatasets
with chpwd(nodspath), \
swallow_outputs():
ds.rerun()
assert_repo_status(ds.path)
# ran twice now
eq_('xx\n', open(probe_path).read())
# Rerunning from a subdataset skips the command.
_, sub_info = get_run_info(ds, last_commit_msg(sub.repo))
eq_(ds.id, sub_info["dsid"])
assert_result_count(
sub.rerun(return_type="list", on_failure="ignore"),
1, status="impossible", action="run", rerun_action="skip")
eq_('xx\n', open(probe_path).read())
# Rerun fails with a dirty repo.
dirt = op.join(path, "dirt")
with open(dirt, "w") as fh:
fh.write("")
assert_status('impossible', ds.rerun(on_failure="ignore"))
remove(dirt)
assert_repo_status(ds.path)
# Make a non-run commit.
with open(op.join(path, "nonrun-file"), "w") as f:
f.write("foo")
ds.save("nonrun-file")
# Now rerun the buried command.
ds.rerun(revision=DEFAULT_BRANCH + "~", message="rerun buried")
eq_('xxx\n', open(probe_path).read())
# Also check that the messasge override worked.
eq_(last_commit_msg(ds.repo).splitlines()[0],
"[DATALAD RUNCMD] rerun buried")
# Or a range of commits, skipping non-run commits.
ds.rerun(since=DEFAULT_BRANCH + "~3")
eq_('xxxxx\n', open(probe_path).read())
# Or --since= to run all reachable commits.
ds.rerun(since="")
eq_('xxxxxxxxxx\n', open(probe_path).read())
# We can get back a report of what would happen rather than actually
# rerunning anything.
report = ds.rerun(since="", report=True, return_type="list")
# The "diff" section of the report doesn't include the unchanged files that
# would come in "-f json diff" output.
for entry in report:
if entry["rerun_action"] == "run":
# None of the run commits touch .datalad/config or any other config
# file.
assert_false(any(r["path"].endswith("config")
for r in entry["diff"]))
# Nothing changed.
eq_('xxxxxxxxxx\n', open(probe_path).read())
assert_result_count(report, 1, rerun_action="skip-or-pick")
report[-1]["commit"] == ds.repo.get_hexsha()
# If a file is dropped, we remove it instead of unlocking it.
ds.drop(probe_path, reckless='kill')
with swallow_outputs():
ds.rerun()
eq_('x\n', open(probe_path).read())
def test_save_obscure_name(path=None):
ds = Dataset(path).create(force=True)
fname = OBSCURE_FILENAME
# Just check that we don't fail with a unicode error.
with swallow_outputs():
ds.save(path=fname, result_renderer="default")
def test_something(path=None, new_home=None):
ds = Dataset(opj(path, 'ds')).create(force=True)
ds.save()
# catches unsupported argument combinations
assert_raises(ValueError, ds.configuration, 'dump', scope='branch')
assert_raises(ValueError, ds.configuration, 'set', spec=('onlyname', ))
assert_raises(ValueError, ds.configuration, 'set', spec='nosection=value')
# we also get that from the internal helper
from datalad.local.configuration import configuration as cfghelper
assert_in_results(
cfghelper('set', 'global', [('nosection', 'value')], {}),
status='error',
)
assert_raises(ValueError, ds.configuration, 'invalid')
res = ds.configuration(result_renderer='disabled')
assert_in_results(res, name='something.user.name', value='Jane Doe')
# UTF handling
assert_in_results(res,
name=u'onemore.{}.findme'.format(complicated_str),
value='5.0')
res = ds.configuration(
'set',
spec='some.more=test',
result_renderer='disabled',
)
assert_in_results(res, name='some.more', value='test')
# Python tuple specs
# swallow outputs to be able to execise the result renderer
with swallow_outputs():
res = ds.configuration(
'set',
spec=[
('some.more.still', 'test2'),
# value is non-str -- will be converted
('lonely.val', 4)
],
)
assert_in_results(res, name='some.more.still', value='test2')
assert_in_results(res, name='lonely.val', value='4')
assert_in_results(
ds.configuration('get', spec='lonely.val'),
status='ok',
name='lonely.val',
value='4',
)
# remove something that does not exist in the specified scope
assert_in_results(ds.configuration('unset',
scope='branch',
spec='lonely.val',
result_renderer='disabled',
on_failure='ignore'),
status='error')
# remove something that does not exist in the specified scope
assert_in_results(ds.configuration('unset',
spec='lonely.val',
result_renderer='disabled'),
status='ok')
assert_not_in('lonely.val', ds.config)
# errors if done again
assert_in_results(ds.configuration('unset',
spec='lonely.val',
result_renderer='disabled',
on_failure='ignore'),
status='error')
# add a subdataset to test recursive operation
subds = ds.create('subds')
with swallow_outputs():
res = ds.configuration('set', spec='rec.test=done', recursive=True)
assert_result_count(
res,
2,
name='rec.test',
value='done',
)
# exercise the result renderer
with swallow_outputs() as cml:
ds.configuration(recursive=True)
# we get something on the subds with the desired markup
assert_in('<ds>/subds:rec.test=done', cml.out)
def test_wtf(topdir=None):
path = opj(topdir, OBSCURE_FILENAME)
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path, on_failure="ignore")
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('## configuration', cmo.out)
assert_in('## dataset', cmo.out)
assert_in(u'path: {}'.format(ds.path), ensure_unicode(cmo.out))
assert_in('branches', cmo.out)
assert_in(DEFAULT_BRANCH + '@', cmo.out)
assert_in('git-annex@', cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
# Sections selection
#
# If we ask for no sections and there is no dataset
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
assert_not_in('## dataset', cmo.out)
for s in SECTION_CALLABLES:
assert_not_in('## %s' % s.lower(), cmo.out.lower())
# ask for a selected set
secs = ['git-annex', 'configuration']
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=secs)
for s in SECTION_CALLABLES:
(assert_in if s in secs else assert_not_in)('## %s' %
s.lower(),
cmo.out.lower())
# order should match our desired one, not alphabetical
# but because of https://github.com/datalad/datalad/issues/3915
# alphanum is now desired
assert cmo.out.index('## git-annex') > cmo.out.index(
'## configuration')
# not achievable from cmdline is to pass an empty list of sections.
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
eq_(cmo.out.rstrip(), '# WTF')
# and we could decorate it nicely for embedding e.g. into github issues
with swallow_outputs() as cmo:
wtf(sections=['dependencies'], decor='html_details')
ok_startswith(cmo.out,
'<details><summary>DataLad %s WTF' % __version__)
assert_in('## dependencies', cmo.out)
# short flavor
with swallow_outputs() as cmo:
wtf(flavor='short')
assert_in("- datalad: version=%s" % __version__, cmo.out)
assert_in("- dependencies: ", cmo.out)
eq_(len(cmo.out.splitlines()),
4) # #WTF, datalad, dependencies, trailing new line
with swallow_outputs() as cmo:
wtf(flavor='short', sections='*')
assert_greater(len(cmo.out.splitlines()), 10) # many more
# check that wtf of an unavailable section yields impossible result (#6712)
res = wtf(sections=['murkie'], on_failure='ignore')
eq_(res[0]["status"], "impossible")
# should result only in '# WTF'
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def test_add_archive_content(path_orig=None, url=None, repo_path=None):
with chpwd(repo_path):
# TODO we need to be able to pass path into add_archive_content
# We could mock but I mean for the API
# no repo yet
assert_raises(NoDatasetFound, add_archive_content,
"nonexisting.tar.gz")
ds = Dataset(repo_path).create()
res = ds.add_archive_content("nonexisting.tar.gz", on_failure='ignore')
assert_in_results(res,
action='add-archive-content',
status='impossible')
repo = ds.repo
# we can't add a file from outside the repo ATM
res = ds.add_archive_content(Path(path_orig) / '1.tar.gz',
on_failure='ignore')
assert_in_results(res,
action='add-archive-content',
status='impossible',
type="dataset",
message="Can not add archive outside of the dataset")
# Let's add first archive to the repo so we could test
with swallow_outputs():
repo.add_url_to_file('1.tar.gz', opj(url, '1.tar.gz'))
for s in range(1, 5):
repo.add_url_to_file('%du/1.tar.gz' % s,
opj(url, '%du/1.tar.gz' % s))
repo.commit("added 1.tar.gz")
key_1tar = repo.get_file_annexinfo('1.tar.gz')[
'key'] # will be used in the test later
def d1_basic_checks():
ok_(exists('1'))
ok_file_under_git('1', '1 f.txt', annexed=True)
ok_file_under_git(opj('1', 'd', '1d'), annexed=True)
ok_archives_caches(repo_path, 0)
# and by default it just does it, everything goes to annex
res = add_archive_content('1.tar.gz')
assert_in_results(res, action='add-archive-content', status='ok')
d1_basic_checks()
# If ran again, should proceed just fine since the content is the same
# so no changes would be made really
res = add_archive_content('1.tar.gz')
assert_in_results(res, action='add-archive-content', status='ok')
# But that other one carries updated file, so should fail due to
# overwrite
res = add_archive_content(Path('1u') / '1.tar.gz',
use_current_dir=True,
on_failure='ignore')
assert_in_results(
res,
action='add-archive-content',
status='error',
)
assert_in('exists, but would be overwritten by new file',
res[0]['message'])
# but should do fine if overrides are allowed
add_archive_content(Path('1u') / '1.tar.gz',
existing='overwrite',
use_current_dir=True)
add_archive_content(Path('2u') / '1.tar.gz',
existing='archive-suffix',
use_current_dir=True)
add_archive_content(Path('3u') / '1.tar.gz',
existing='archive-suffix',
use_current_dir=True)
add_archive_content(Path('4u') / '1.tar.gz',
existing='archive-suffix',
use_current_dir=True)
# rudimentary test
assert_equal(sorted(map(basename, glob(opj(repo_path, '1', '1*')))),
['1 f-1.1.txt', '1 f-1.2.txt', '1 f-1.txt', '1 f.txt'])
whereis = repo.whereis(glob(opj(repo_path, '1', '1*')))
# they all must be the same
assert (all([x == whereis[0] for x in whereis[1:]]))
# and we should be able to reference it while under subdirectory
subdir = opj(repo_path, 'subdir')
with chpwd(subdir, mkdir=True):
add_archive_content(opj(pardir, '1.tar.gz'),
dataset=ds.path,
use_current_dir=True)
d1_basic_checks()
# or we could keep relative path and also demand to keep the archive prefix
# while extracting under original (annex root) dir
add_archive_content(opj(pardir, '1.tar.gz'),
dataset=ds.path,
add_archive_leading_dir=True)
with chpwd(opj(repo_path, '1')):
d1_basic_checks()
with chpwd(repo_path):
# test with excludes and renames and annex options
ds.add_archive_content(
'1.tar.gz',
exclude=['d'],
rename=['/ /_', '/^1/2'],
annex_options="-c annex.largefiles=exclude=*.txt",
delete=True)
# no conflicts since new name
ok_file_under_git('2', '1_f.txt', annexed=False)
assert_false(exists(opj('2', 'd')))
assert_false(exists('1.tar.gz')) # delete was in effect
# now test ability to extract within subdir
with chpwd(opj(repo_path, 'd1'), mkdir=True):
# Let's add first archive to the repo so we could test
# named the same way but different content
with swallow_outputs():
repo.add_url_to_file('d1/1.tar.gz', opj(url, 'd1', '1.tar.gz'))
repo.commit("added 1.tar.gz in d1")
def d2_basic_checks():
ok_(exists('1'))
ok_file_under_git('1', '2 f.txt', annexed=True)
ok_file_under_git(opj('1', 'd2', '2d'), annexed=True)
ok_archives_caches(repo.path, 0)
add_archive_content('1.tar.gz', dataset=ds.path)
d2_basic_checks()
# in manual tests ran into the situation of inability to obtain on a single run
# a file from an archive which was coming from a dropped key. I thought it was
# tested in custom remote tests, but I guess not sufficiently well enough
repo.drop(opj('1', '1 f.txt')) # should be all kosher
repo.get(opj('1', '1 f.txt'))
ok_archives_caches(repo.path, 1, persistent=True)
ok_archives_caches(repo.path, 0, persistent=False)
repo.drop(opj('1', '1 f.txt')) # should be all kosher
repo.drop(key_1tar,
key=True) # is available from the URL -- should be kosher
repo.get(opj('1', '1 f.txt')) # that what managed to not work
# TODO: check if persistent archive is there for the 1.tar.gz
# We should be able to drop everything since available online
with swallow_outputs():
clean(dataset=ds)
repo.drop(key_1tar,
key=True) # is available from the URL -- should be kosher
ds.drop(opj('1', '1 f.txt')) # should be all kosher
ds.get(opj('1', '1 f.txt')) # and should be able to get it again
# bug was that dropping didn't work since archive was dropped first
repo.call_annex(["drop", "--all"])
# verify that we can't drop a file if archive key was dropped and online archive was removed or changed size! ;)
repo.get(key_1tar, key=True)
unlink(opj(path_orig, '1.tar.gz'))
with assert_raises(CommandError) as e:
repo.drop(key_1tar, key=True)
assert_equal(e.kwargs['stdout_json'][0]['success'], False)
assert_result_values_cond(
e.kwargs['stdout_json'], 'note', lambda x:
'(Use --force to override this check, or adjust numcopies.)' in x)
assert exists(opj(repo.path, repo.get_contentlocation(key_1tar)))
def test_rerun_onto(path=None):
ds = Dataset(path).create()
if ds.repo.is_managed_branch():
assert_status('impossible',
ds.rerun(onto="triggers-abort", on_failure="ignore"))
raise SkipTest("Test incompatible with adjusted branch")
# Make sure we have more than one commit. The one commit case is checked
# elsewhere.
ds.repo.commit(msg="noop commit", options=["--allow-empty"])
grow_file = op.join(path, "grows")
# Make sure we can handle range-specifications that yield no results.
for since in ["", "HEAD"]:
assert_result_count(
ds.rerun("HEAD", onto="", since=since, on_failure="ignore"),
1, status="impossible", action="run")
ds.run('echo static-content > static')
ds.repo.tag("static")
with swallow_outputs():
ds.run(f'echo x$({cat_command} grows) > grows')
ds.rerun()
eq_('xx\n', open(grow_file).read())
# If we run the "static" change on top of itself, we end up in the
# same (but detached) place.
ds.rerun(revision="static", onto="static")
ok_(ds.repo.get_active_branch() is None)
eq_(ds.repo.get_hexsha(),
ds.repo.get_hexsha("static"))
# If we run the "static" change from the same "base", we end up
# with a new commit.
ds.repo.checkout(DEFAULT_BRANCH)
with swallow_outputs():
ds.rerun(revision="static", onto="static^")
ok_(ds.repo.get_active_branch() is None)
neq_(ds.repo.get_hexsha(),
ds.repo.get_hexsha("static"))
ok_(all(r["state"] == "clean" for r in ds.diff(fr="HEAD", to="static")))
for revrange in ["..static", "static.."]:
eq_(len(ds.repo.get_revisions(revrange)), 1)
# Unlike the static change, if we run the ever-growing change on
# top of itself, we end up with a new commit.
ds.repo.checkout(DEFAULT_BRANCH)
ds.rerun(onto="HEAD")
ok_(ds.repo.get_active_branch() is None)
neq_(ds.repo.get_hexsha(),
ds.repo.get_hexsha(DEFAULT_BRANCH))
# An empty `onto` means use the parent of the first revision.
ds.repo.checkout(DEFAULT_BRANCH)
with swallow_outputs():
ds.rerun(since="static^", onto="")
ok_(ds.repo.get_active_branch() is None)
for revrange in [".." + DEFAULT_BRANCH, DEFAULT_BRANCH + ".."]:
eq_(len(ds.repo.get_revisions(revrange)), 3)
# An empty `onto` means use the parent of the first revision that
# has a run command.
ds.repo.checkout(DEFAULT_BRANCH)
with swallow_outputs():
ds.rerun(since="", onto="", branch="from-base")
eq_(ds.repo.get_active_branch(), "from-base")
ok_(all(r["state"] == "clean"
for r in ds.diff(fr=DEFAULT_BRANCH, to="from-base")))
eq_(ds.repo.get_merge_base(["static", "from-base"]),
ds.repo.get_hexsha("static^"))
# We abort when an explicitly specified `onto` doesn't exist.
ds.repo.checkout(DEFAULT_BRANCH)
assert_result_count(
ds.rerun(since="", onto="doesnotexist", branch="from-base",
on_failure="ignore"),
1, status="error", action="run")
def test_basics(path=None, nodspath=None):
ds = Dataset(path).create()
last_state = ds.repo.get_hexsha()
# run inside the dataset
with chpwd(path), \
swallow_outputs():
# provoke command failure
res = ds.run('7i3amhmuch9invalid',
on_failure="ignore",
result_renderer=None)
assert_result_count(res, 1, action="run", status="error")
run_res = [r for r in res if r["action"] == "run"][0]
# let's not speculate that the exit code is always 127
ok_(run_res["run_info"]["exit"] > 0)
eq_(last_state, ds.repo.get_hexsha())
# now one that must work
res = ds.run('cd .> empty', message='TEST')
assert_repo_status(ds.path)
assert_result_count(res, 3)
# TODO 'state' is still untracked!!!
assert_result_count(res,
1,
action='add',
path=op.join(ds.path, 'empty'),
type='file')
assert_result_count(res, 1, action='save', path=ds.path)
commit_msg = last_commit_msg(ds.repo)
ok_(commit_msg.startswith('[DATALAD RUNCMD] TEST'))
# crude test that we have a record for the PWD
assert_in('"pwd": "."', commit_msg)
last_state = ds.repo.get_hexsha()
# now run a command that will not alter the dataset
noop_cmd = ':'
res = ds.run(noop_cmd, message='NOOP_TEST')
assert_result_count(res, 1, action='save', status='notneeded')
eq_(last_state, ds.repo.get_hexsha())
# We can also run the command via a single-item list because this is
# what the CLI interface passes in for quoted commands.
res = ds.run([noop_cmd], message='NOOP_TEST')
assert_result_count(res, 1, action='save', status='notneeded')
# run outside the dataset, should still work but with limitations
with chpwd(nodspath), \
swallow_outputs():
res = ds.run('cd . > empty2', message='TEST')
assert_result_count(res,
1,
action='add',
path=op.join(ds.path, 'empty2'),
type='file',
status='ok')
assert_result_count(res, 1, action='save', status='ok')
# running without a command is a noop
with chpwd(path):
with swallow_logs(new_level=logging.WARN) as cml:
ds.run()
assert_in("No command given", cml.out)
# running without a command is a noop
with chpwd(path):
with swallow_logs(new_level=logging.INFO) as cml:
assert_raises(
IncompleteResultsError,
ds.run,
'7i3amhmuch9invalid',
# this is on_failure=stop by default
)
# must give recovery hint in Python notation
assert_in("can save the changes with \"Dataset(", cml.out)
with chpwd(path):
# make sure that an invalid input declaration prevents command
# execution by default
assert_raises(IncompleteResultsError,
ds.run,
'cd .> dummy0',
inputs=['not-here'])
ok_(not (ds.pathobj / 'dummy0').exists())
# but the default behavior can be changed
assert_raises(IncompleteResultsError,
ds.run,
'cd .> dummy0',
inputs=['not-here'],
on_failure='continue')
# it has stilled failed, but the command got executed nevertheless
ok_((ds.pathobj / 'dummy0').exists())
def test_dry_run(path=None):
ds = Dataset(path).create(force=True)
# The dataset is reported as dirty, and the custom result render relays
# that to the default renderer.
with swallow_outputs() as cmo:
with assert_raises(IncompleteResultsError):
ds.run("blah ", dry_run="basic")
assert_in("run(impossible)", cmo.out)
assert_not_in("blah", cmo.out)
ds.save()
# unknown dry-run mode
assert_raises(ValueError, ds.run, 'blah', dry_run='absurd')
with swallow_outputs() as cmo:
ds.run("blah ", dry_run="basic")
assert_in("Dry run", cmo.out)
assert_in("location", cmo.out)
assert_in("blah", cmo.out)
assert_not_in("expanded inputs", cmo.out)
assert_not_in("expanded outputs", cmo.out)
with swallow_outputs() as cmo:
ds.run("blah {inputs} {outputs}",
dry_run="basic",
inputs=["fo*"],
outputs=["b*r"])
assert_in('blah "foo" "bar"' if on_windows else "blah foo bar",
cmo.out)
assert_in("expanded inputs", cmo.out)
assert_in("['foo']", cmo.out)
assert_in("expanded outputs", cmo.out)
assert_in("['bar']", cmo.out)
# Just the command.
with swallow_outputs() as cmo:
ds.run("blah ", dry_run="command")
assert_not_in("Dry run", cmo.out)
assert_in("blah", cmo.out)
assert_not_in("inputs", cmo.out)
# The output file wasn't unlocked.
assert_repo_status(ds.path)
# Subdaset handling
subds = ds.create("sub")
(subds.pathobj / "baz").write_text("z")
ds.save(recursive=True)
# If a subdataset is installed, it works as usual.
with swallow_outputs() as cmo:
ds.run("blah {inputs}", dry_run="basic", inputs=["sub/b*"])
assert_in('blah "sub\\baz"' if on_windows else 'blah sub/baz', cmo.out)
# However, a dry run will not do the install/reglob procedure.
ds.drop("sub", what='all', reckless='kill', recursive=True)
with swallow_outputs() as cmo:
ds.run("blah {inputs}", dry_run="basic", inputs=["sub/b*"])
assert_in("sub/b*", cmo.out)
assert_not_in("baz", cmo.out)
