def test_run_inputs_no_annex_repo(path=None):
ds = Dataset(path).create(annex=False, force=True)
ds.save()
# Running --input in a plain Git repo doesn't fail.
ds.run("cd .> dummy", inputs=["*"])
ok_exists(op.join(ds.path, "dummy"))
ds.rerun()
def test_rerun_script(path=None):
ds = Dataset(path).create()
ds.run("echo a >foo")
ds.run([touch_command + "bar"], message='BAR', sidecar=True)
# a run record sidecar file was added with the last commit
assert(any(d['path'].startswith(op.join(ds.path, '.datalad', 'runinfo'))
for d in ds.rerun(report=True, return_type='item-or-list')['diff']))
bar_hexsha = ds.repo.get_hexsha(DEFAULT_BRANCH)
script_file = op.join(path, "commands.sh")
ds.rerun(script=script_file)
ok_exists(script_file)
with open(script_file) as sf:
lines = sf.readlines()
assert_in(touch_command + "bar\n", lines)
# The commit message is there too.
assert_in("# BAR\n", lines)
assert_in("# (record: {})\n".format(bar_hexsha), lines)
assert_not_in("echo a >foo\n", lines)
ds.rerun(since="", script=script_file)
with open(script_file) as sf:
lines = sf.readlines()
assert_in(touch_command + "bar\n", lines)
# Automatic commit messages aren't included.
assert_not_in("# echo a >foo\n", lines)
assert_in("echo a >foo\n", lines)
# --script=- writes to stdout.
with patch("sys.stdout", new_callable=StringIO) as cmout:
ds.rerun(script="-")
assert_in(touch_command + "bar",
cmout.getvalue().splitlines())
def test_addurls_url_on_collision_error_if_different(self=None, path=None):
ds = Dataset(path).create(force=True)
data = [self.data[0].copy(), self.data[0].copy()]
data[0]["some_metadata"] = "1"
data[1]["some_metadata"] = "2"
with patch("sys.stdin", new=StringIO(json.dumps(data))):
assert_in_results(ds.addurls("-",
"{url}",
"{name}",
on_failure="ignore"),
action="addurls",
status="error")
with patch("sys.stdin", new=StringIO(json.dumps(data))):
assert_in_results(ds.addurls("-",
"{url}",
"{name}",
on_collision="error-if-different",
on_failure="ignore"),
action="addurls",
status="error")
with patch("sys.stdin", new=StringIO(json.dumps(data))):
ds.addurls("-",
"{url}",
"{name}",
exclude_autometa="*",
on_collision="error-if-different")
ok_exists(op.join(ds.path, "a"))
def test_addurls_version(self=None, path=None):
ds = Dataset(path).create(force=True)
def version_fn(url):
if url.endswith("b.dat"):
raise ValueError("Scheme error")
return url + ".v1"
with patch("datalad.local.addurls.get_versioned_url", version_fn):
with swallow_logs(new_level=logging.WARNING) as cml:
ds.addurls(self.json_file,
"{url}",
"{name}",
version_urls=True,
result_renderer='disabled')
assert_in("b.dat", str(cml.out))
names = ["a", "c"]
for fname in names:
ok_exists(os.path.join(path, fname))
whereis = ds.repo.whereis(names, output="full")
for fname, info in whereis.items():
eq_(info[WEB_SPECIAL_REMOTE_UUID]['urls'],
["{}udir/{}.dat.v1".format(self.url, fname)])
def test_addurls_url_filename(self=None, path=None):
ds = Dataset(path).create(force=True)
ds.addurls(self.json_file,
"{url}",
"{_url0}/{_url_filename}",
result_renderer='disabled')
for fname in ["a.dat", "b.dat", "c.dat"]:
ok_exists(op.join(ds.path, "udir", fname))
def test_addurls_create_newdataset(self=None, path=None):
dspath = os.path.join(path, "ds")
addurls(self.json_file,
"{url}",
"{name}",
dataset=dspath,
cfg_proc=["yoda"],
result_renderer='disabled')
for fname in ["a", "b", "c", "code"]:
ok_exists(os.path.join(dspath, fname))
def test_addurls_stdin_input_command_line(self=None, path=None):
# The previous test checks all the cases, but it overrides sys.stdin.
# Do a simple check that's closer to a command line call.
Dataset(path).create(force=True)
runner = WitlessRunner(cwd=path)
with open(self.json_file) as jfh:
runner.run(["datalad", "addurls", '-', '{url}', '{name}'],
stdin=jfh)
for fname in ["a", "b", "c"]:
ok_exists(op.join(path, fname))
def check_addurls_stdin_input(self, input_text, input_type, path):
ds = Dataset(path).create(force=True)
with patch("sys.stdin", new=StringIO(input_text)):
ds.addurls("-",
"{url}",
"{name}",
input_type=input_type,
result_renderer='disabled')
for fname in ["a", "b", "c"]:
ok_exists(op.join(ds.path, fname))
def test_download_url_dataset(toppath=None, topurl=None, path=None):
# Non-dataset directory.
file1_fullpath = opj(path, "file1.txt")
with chpwd(path):
download_url(topurl + "file1.txt")
ok_exists(file1_fullpath)
os.remove(file1_fullpath)
files_tosave = ['file1.txt', 'file2.txt']
urls_tosave = [topurl + f for f in files_tosave]
ds = Dataset(opj(path, "ds")).create()
# By default, files are saved when called in a dataset.
ds.download_url(urls_tosave)
for fname in files_tosave:
ok_(ds.repo.file_has_content(fname))
eq_(ds.repo.get_urls("file1.txt"), [urls_tosave[0]])
eq_(ds.repo.get_urls("file2.txt"), [urls_tosave[1]])
ds.download_url([topurl + "file3.txt"], save=False)
assert_false(ds.repo.file_has_content("file3.txt"))
# Leading paths for target are created if needed.
subdir_target = opj("l1", "l2", "f")
ds.download_url([opj(topurl, "file1.txt")], path=subdir_target)
ok_(ds.repo.file_has_content(subdir_target))
subdir_path = opj(ds.path, "subdir", "")
os.mkdir(subdir_path)
with chpwd(subdir_path):
download_url(topurl + "file4.txt")
download_url(topurl + "file5.txt", path="five.txt")
ds.download_url(topurl + "file6.txt")
download_url(topurl + "file7.txt", dataset=ds.path)
# download_url calls within a subdirectory save the file there
ok_(ds.repo.file_has_content(opj("subdir", "file4.txt")))
ok_(ds.repo.file_has_content(opj("subdir", "five.txt")))
# ... unless the dataset instance is provided
ok_(ds.repo.file_has_content("file6.txt"))
# ... but a string for the dataset (as it would be from the command line)
# still uses CWD semantics
ok_(ds.repo.file_has_content(opj("subdir", "file7.txt")))
with chpwd(path):
# We're in a non-dataset path and pass in a string as the dataset. The
# path is taken as relative to the current working directory, so we get
# an error when trying to save it.
assert_in_results(download_url(topurl + "file8.txt",
dataset=ds.path,
on_failure="ignore"),
status="error",
action="status")
assert_false((ds.pathobj / "file8.txt").exists())
def test_run_empty_repo(path=None):
ds = Dataset(path).create()
cmd = [sys.executable, "-c", "open('foo', 'w').write('')"]
# Using "*" in a completely empty repo will fail.
with assert_raises(IncompleteResultsError):
ds.run(cmd, inputs=["*"], on_failure="stop")
assert_repo_status(ds.path)
# "." will work okay, though.
assert_status("ok", ds.run(cmd, inputs=["."]))
assert_repo_status(ds.path)
ok_exists(str(ds.pathobj / "foo"))
def test_addurls_subdataset(self=None, path=None):
ds = Dataset(path).create(force=True)
for save in True, False:
label = "save" if save else "nosave"
with swallow_outputs() as cmo:
ds.addurls(self.json_file,
"{url}",
"{subdir}-" + label + "//{name}",
save=save,
cfg_proc=["yoda"])
# The custom result renderer transforms the subdataset
# action=create results into something more informative than
# "create(ok): . (dataset)"...
assert_in("create(ok): foo-{} (dataset)".format(label),
cmo.out)
# ... and that doesn't lose the standard summary.
assert_in("create (ok: 2)", cmo.out)
subdirs = [
op.join(ds.path, "{}-{}".format(d, label))
for d in ["foo", "bar"]
]
subdir_files = dict(zip(subdirs, [["a", "c"], ["b"]]))
for subds, fnames in subdir_files.items():
for fname in fnames:
ok_exists(op.join(subds, fname))
# cfg_proc was applied generated subdatasets.
ok_exists(op.join(subds, "code"))
if save:
assert_repo_status(path)
else:
# The datasets are create but not saved (since asked not to)
assert_repo_status(path, untracked=subdirs)
# but the downloaded files aren't.
for subds, fnames in subdir_files.items():
assert_repo_status(subds, added=fnames)
# Now save the "--nosave" changes and check that we have
# all the subdatasets.
ds.save()
eq_(
set(subdatasets(dataset=ds, recursive=True,
result_xfm="relpaths")),
{"foo-save", "bar-save", "foo-nosave", "bar-nosave"})
# We don't try to recreate existing subdatasets.
with swallow_logs(new_level=logging.DEBUG) as cml:
ds.addurls(self.json_file,
"{url}",
"{subdir}-nosave//{name}",
result_renderer='disabled')
assert_in("Not creating subdataset at existing path", cml.out)
def check(ds, dataset_arg, url_file, fname_format):
subdir = op.join(ds.path, "subdir")
os.mkdir(subdir)
with chpwd(subdir):
shutil.copy(self.json_file, "in.json")
addurls(url_file,
"{url}",
fname_format,
dataset=dataset_arg,
result_renderer='disabled')
# Files specified in the CSV file are always relative to the
# dataset.
for fname in ["a", "b", "c"]:
ok_exists(op.join(ds.path, fname))
def test_run_failure(path=None):
ds = Dataset(path).create()
subds = ds.create("sub")
hexsha_initial = ds.repo.get_hexsha()
if on_windows:
# this does not do exactly the same as the cmd on other systems
# but is close enough to make running the test worthwhile
cmd_failing = "echo x>{} & false".format(op.join("sub", "grows"))
else:
cmd_failing = ("echo x$(cat {0}) > {0} && false"
.format(op.join("sub", "grows")))
with assert_raises(IncompleteResultsError):
ds.run(cmd_failing, result_renderer=None)
eq_(hexsha_initial, ds.repo.get_hexsha())
ok_(ds.repo.dirty)
msgfile = op.join(path, ds.repo.get_git_dir(ds.repo), "COMMIT_EDITMSG")
ok_exists(msgfile)
ds.save(recursive=True, message_file=msgfile)
assert_repo_status(ds.path)
neq_(hexsha_initial, ds.repo.get_hexsha())
outfile = op.join(subds.path, "grows")
eq_('x \n' if on_windows else 'x\n', open(outfile).read())
if on_windows:
# FIXME: Make the remaining code compatible with Windows.
return
# There is no CommandError on rerun if the non-zero error matches the
# original code.
ds.rerun()
eq_('xx\n', open(outfile).read())
# On the other hand, we fail if we rerun a command and there is a non-zero
# error that doesn't match.
ds.run("[ ! -e bar ] && echo c >bar")
assert_repo_status(ds.path)
assert_in_results(ds.rerun(result_renderer=None, on_failure="ignore"),
action="run", status="error")
# We don't show instructions if the caller specified us not to save.
remove(msgfile)
with assert_raises(IncompleteResultsError):
ds.run("false", explicit=True, outputs=None, on_failure="stop")
assert_false(op.exists(msgfile))
def test_addurls_repindex(self=None, path=None):
ds = Dataset(path).create(force=True)
with assert_raises(IncompleteResultsError) as raised:
ds.addurls(self.json_file,
"{url}",
"{subdir}",
result_renderer='disabled')
assert_in("collided", str(raised.value))
ds.addurls(self.json_file,
"{url}",
"{subdir}-{_repindex}",
result_renderer='disabled')
for fname in ["foo-0", "bar-0", "foo-1"]:
ok_exists(op.join(ds.path, fname))
def check_create_initopts_form(form, path=None):
path = Path(path)
template_dir = path / "templates"
template_dir.mkdir()
(template_dir / "foo").write_text("")
forms = {
"list": [f"--template={template_dir}"],
"dict": {
"template": str(template_dir)
}
}
ds = Dataset(path / "ds")
ds.create(initopts=forms[form])
ok_exists(ds.repo.dot_git / "foo")
def test_lock_if_check_fails(tempfile=None):
# basic test, should never try to lock so filename is not important
with lock_if_check_fails(True, None) as (check, lock):
assert check is True
assert lock is None
assert check # still available outside
# and with a callable
with lock_if_check_fails(lambda: "valuable", None) as (check, lock):
eq_(check, "valuable")
assert lock is None
eq_(check, "valuable")
# basic test, should never try to lock so filename is not important
with lock_if_check_fails(False, tempfile) as (check, lock):
ok_(lock)
ok_exists(tempfile + '.lck')
assert not op.exists(tempfile + '.lck') # and it gets removed after
# the same with providing operation
# basic test, should never try to lock so filename is not important
with lock_if_check_fails(False, tempfile,
operation='get') as (check, lock):
ok_(lock)
ok_exists(tempfile + '.get-lck')
assert not op.exists(tempfile + '.get-lck') # and it gets removed after
from multiprocessing import (
Process,
Queue,
)
q = Queue()
p = Process(target=Subproc(tempfile), args=(q, ))
# now we need somehow to actually check the bloody lock functioning
with lock_if_check_fails((op.exists, (tempfile, )),
tempfile,
_return_acquired=True) as (check, lock, acquired):
eq_(check, False)
ok_(lock)
ok_(acquired)
# but now we will try to lock again, but we need to do it in another
# process
p.start()
assert q.get() is False
p.join()
with open(tempfile, 'w') as f:
pass
ok_exists(tempfile)
ok_exists(tempfile)
# and we redo -- it will acquire it
p = Process(target=Subproc(tempfile), args=(q, ))
p.start()
ok_(q.get())
p.join()
def test_addurls_deeper(self=None, path=None):
ds = Dataset(path).create(force=True)
ds.addurls(self.json_file,
"{url}",
"{subdir}//adir/{subdir}-again//other-ds//bdir/{name}",
jobs=3,
result_renderer='disabled')
eq_(
set(ds.subdatasets(recursive=True, result_xfm="relpaths")), {
"foo", "bar",
op.join("foo", "adir", "foo-again"),
op.join("bar", "adir", "bar-again"),
op.join("foo", "adir", "foo-again", "other-ds"),
op.join("bar", "adir", "bar-again", "other-ds")
})
ok_exists(
os.path.join(ds.path, "foo", "adir", "foo-again", "other-ds",
"bdir", "a"))
def test_run_path_semantics(path=None):
# Test that we follow path resolution from gh-3435: paths are relative to
# dataset if a dataset instance is given and relative to the current
# working directory otherwise.
ds0 = Dataset(op.join(path, "ds0")).create()
ds0_subdir = op.join(ds0.path, "s0")
os.mkdir(ds0_subdir)
# Although not useful, we can specify `dataset` as a string that lines up
# with the one from the current directory.
with chpwd(ds0_subdir):
run("cd .> one", dataset="..")
run("cd .> one", outputs=["one"], dataset=ds0.path)
ok_exists(op.join(ds0_subdir, "one"))
assert_repo_status(ds0.path)
# Specify string dataset argument, running from another dataset ...
ds1 = Dataset(op.join(path, "ds1")).create()
ds1_subdir = op.join(ds1.path, "s1")
os.mkdir(ds1_subdir)
# ... producing output file in specified dataset
with chpwd(ds1_subdir):
run("cd .> {}".format(op.join(ds0.path, "two")), dataset=ds0.path)
ok_exists(op.join(ds0.path, "two"))
assert_repo_status(ds0.path)
# ... producing output file in specified dataset and passing output file as
# relative to current directory
with chpwd(ds1_subdir):
out = op.join(ds0.path, "three")
run("cd .> {}".format(out),
dataset=ds0.path,
explicit=True,
outputs=[op.relpath(out, ds1_subdir)])
ok_exists(op.join(ds0.path, "three"))
assert_repo_status(ds0.path)
# ... producing output file outside of specified dataset, leaving it
# untracked in the other dataset
assert_repo_status(ds1.path)
with chpwd(ds1_subdir):
run("cd .> four", dataset=ds0.path)
assert_repo_status(ds1.path, untracked=[ds1_subdir])
# If we repeat above with an instance instead of the string, the directory
# for the run is the specified dataset.
with chpwd(ds1_subdir):
run("cd .> five", dataset=ds0)
ok_exists(op.join(ds0.path, "five"))
assert_repo_status(ds0.path)
def test_run_reglob_outputs(path=None):
ds = Dataset(path).create()
repo = ds.repo
(ds.pathobj / "write_text.py").write_text("""
import sys
assert len(sys.argv) == 2
name = sys.argv[1]
with open(name + ".txt", "w") as fh:
fh.write(name)
""")
ds.save(to_git=True)
cmd = [sys.executable, "write_text.py"]
ds.run(cmd + ["foo"], outputs=["*.txt"], expand="outputs")
assert_in("foo.txt", last_commit_msg(repo))
ds.run(cmd + ["bar"], outputs=["*.txt"], explicit=True)
ok_exists(str(ds.pathobj / "bar.txt"))
assert_repo_status(ds.path)
def test_py2_unicode_command(path=None):
# Avoid OBSCURE_FILENAME to avoid windows-breakage (gh-2929).
ds = Dataset(path).create()
touch_cmd = "import sys; open(sys.argv[1], 'w').write('')"
cmd_str = u"{} -c \"{}\" {}".format(sys.executable, touch_cmd, u"bβ0.dat")
ds.run(cmd_str)
assert_repo_status(ds.path)
ok_exists(op.join(path, u"bβ0.dat"))
# somewhat desperate attempt to detect our own Github CI tests on a
# crippled filesystem (VFAT) that is so crippled that it doesn't handle
# what is needed here. It just goes mad with encoded bytestrings:
# CommandError: ''python -c '"'"'import sys; open(sys.argv[1], '"'"'"'"'"'"'"'"'w'"'"'"'"'"'"'"'"').write('"'"'"'"'"'"'"'"''"'"'"'"'"'"'"'"')'"'"' '"'"' β1 '"'"''' failed with exitcode 1 under /crippledfs/
if not on_windows and os.environ.get('TMPDIR',
None) != '/crippledfs': # FIXME
ds.run([sys.executable, "-c", touch_cmd, u"bβ1.dat"])
assert_repo_status(ds.path)
ok_exists(op.join(path, u"bβ1.dat"))
# Send in a list of byte-strings to mimic a py2 command-line
# invocation.
ds.run([
s.encode("utf-8")
for s in [sys.executable, "-c", touch_cmd, u" β1 "]
])
assert_repo_status(ds.path)
ok_exists(op.join(path, u" β1 "))
assert_in_results(ds.run(u"bβ2.dat",
result_renderer=None,
on_failure="ignore"),
status="error",
action="run")
def test_run_from_subds_gh3551(path=None):
ds = Dataset(path).create(force=True)
ds.save()
ds.create("output")
with chpwd(op.join(ds.path, "sub")):
assert_in_results(run("echo",
inputs=[op.join(op.pardir, "sub", "input")],
outputs=[op.join(op.pardir, "output")],
return_type="list",
result_filter=None,
result_xfm=None),
action="get",
status="notneeded")
assert_repo_status(ds.path)
subds_path = op.join("output", "subds")
ds.create(subds_path)
with chpwd(op.join(ds.path, "sub")):
output_dir = op.join(op.pardir, "output", "subds")
# The below command is trying to be compatible. It could be made better
# (e.g., actually using the input file) by someone that knows something
# about Windows.
assert_in_results(run("cd .> {}".format(op.join(output_dir, "f")),
inputs=[op.join(op.pardir, "sub", "input")],
outputs=[output_dir],
return_type="list",
result_filter=None,
result_xfm=None),
action="save",
status="ok")
assert_repo_status(ds.path)
subds = Dataset(op.join(ds.path, subds_path))
ok_exists(op.join(subds.path, "f"))
if not ds.repo.is_managed_branch(): # FIXME
# This check fails on Windows:
# https://github.com/datalad/datalad/pull/3747/checks?check_run_id=248506560#step:8:254
ok_(subds.repo.file_has_content("f"))
def test_addurls(self=None, path=None):
ds = Dataset(path).create(force=True)
def get_annex_commit_counts():
return len(ds.repo.get_revisions("git-annex"))
n_annex_commits = get_annex_commit_counts()
# Meanwhile also test that we can specify path relative
# to the top of the dataset, as we generally treat paths in
# Python API, and it will be the one saved in commit
# message record
json_file = op.relpath(self.json_file, ds.path)
ds.addurls(json_file,
"{url}",
"{name}",
exclude_autometa="(md5sum|size)",
result_renderer='disabled')
ok_startswith(ds.repo.format_commit('%b', DEFAULT_BRANCH),
f"url_file='{json_file}'")
filenames = ["a", "b", "c"]
for fname in filenames:
ok_exists(op.join(ds.path, fname))
for (fname, meta), subdir in zip(ds.repo.get_metadata(filenames),
["foo", "bar", "foo"]):
assert_dict_equal(meta, {"subdir": [subdir], "name": [fname]})
# Ignore this check if we're faking dates because that disables
# batch mode.
# Also ignore if on Windows as it seems as if a git-annex bug
# leads to separate meta data commits:
# https://github.com/datalad/datalad/pull/5202#discussion_r535429704
if not (dl_cfg.get('datalad.fake-dates') or on_windows):
# We should have two new commits on the git-annex: one for the
# added urls and one for the added metadata.
eq_(n_annex_commits + 2, get_annex_commit_counts())
# Add to already existing links, overwriting.
with swallow_logs(new_level=logging.DEBUG) as cml:
ds.addurls(self.json_file,
"{url}",
"{name}",
ifexists="overwrite",
result_renderer='disabled')
for fname in filenames:
assert_in("Removing {}".format(os.path.join(path, fname)),
cml.out)
# Add to already existing links, skipping.
assert_in_results(ds.addurls(self.json_file,
"{url}",
"{name}",
ifexists="skip",
result_renderer='disabled'),
action="addurls",
status="notneeded")
# Add to already existing links works, as long content is the same.
ds.addurls(self.json_file,
"{url}",
"{name}",
result_renderer='disabled')
# But it fails if something has changed.
ds.unlock("a")
with open(op.join(ds.path, "a"), "w") as ofh:
ofh.write("changed")
ds.save("a")
assert_raises(IncompleteResultsError,
ds.addurls,
self.json_file,
"{url}",
"{name}",
result_renderer='disabled')
def test_placeholders(path=None):
ds = Dataset(path).create(force=True)
ds.save()
assert_repo_status(ds.path)
# ATTN windows is sensitive to spaces before redirect symbol
ds.run("echo {inputs}>{outputs}", inputs=[".", "*.in"], outputs=["c.out"])
ok_file_has_content(op.join(path, "c.out"), "a.in b.in\n")
hexsha_before = ds.repo.get_hexsha()
ds.rerun()
eq_(hexsha_before, ds.repo.get_hexsha())
# ATTN windows is sensitive to spaces before redirect symbol
ds.run("echo {inputs[0]}>getitem", inputs=["*.in"])
ok_file_has_content(op.join(path, "getitem"), "a.in\n")
ds.run("echo {pwd} >expanded-pwd")
ok_file_has_content(op.join(path, "expanded-pwd"), path,
strip=True)
ds.run("echo {dspath} >expanded-dspath")
ok_file_has_content(op.join(path, "expanded-dspath"), ds.path,
strip=True)
subdir_path = op.join(path, "subdir")
with chpwd(subdir_path):
run("echo {pwd} >expanded-pwd")
ok_file_has_content(op.join(path, "subdir", "expanded-pwd"), subdir_path,
strip=True)
eq_(get_run_info(ds, last_commit_msg(ds.repo))[1]["pwd"],
"subdir")
# Double brackets can be used to escape placeholders.
ds.run("cd .> {{inputs}}", inputs=["*.in"])
ok_exists(op.join(path, "{inputs}"))
# rerun --script expands the placeholders.
with patch("sys.stdout", new_callable=StringIO) as cmout:
ds.rerun(script="-", since="")
script_out = cmout.getvalue()
assert_in("echo a.in b.in>c.out", script_out)
assert_in("echo {} >expanded-pwd".format(subdir_path),
script_out)
assert_in("echo {} >expanded-dspath".format(ds.path),
script_out)
assert_result_count(
ds.run("{unknown_placeholder}", on_failure="ignore"),
1, status="impossible", action="run")
# Configured placeholders.
ds.config.add("datalad.run.substitutions.license", "gpl3", scope="local")
ds.run("echo {license} >configured-license")
ok_file_has_content(op.join(path, "configured-license"), "gpl3",
strip=True)
# --script handles configured placeholders.
with patch("sys.stdout", new_callable=StringIO) as cmout:
ds.rerun(script="-")
assert_in("gpl3", cmout.getvalue())
ds.run("echo {tmpdir} >tout")
ok_file_has_content(op.join(path, "tout"), ".*datalad-run.*", re_=True)
def _test_create_store(host, base_path=None, ds_path=None, clone_path=None):
ds = Dataset(ds_path).create(force=True)
subds = ds.create('sub', force=True)
subds2 = ds.create('sub2', force=True, annex=False)
ds.save(recursive=True)
assert_repo_status(ds.path)
# don't specify special remote. By default should be git-remote + "-storage"
res = ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
post_update_hook=True,
new_store_ok=True)
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
# remotes exist, but only in super
siblings = ds.siblings(result_renderer='disabled')
eq_({'datastore', 'datastore-storage', 'here'},
{s['name']
for s in siblings})
sub_siblings = subds.siblings(result_renderer='disabled')
eq_({'here'}, {s['name'] for s in sub_siblings})
sub2_siblings = subds2.siblings(result_renderer='disabled')
eq_({'here'}, {s['name'] for s in sub2_siblings})
# check bare repo:
git_dir = Path(base_path) / ds.id[:3] / ds.id[3:]
# The post-update hook was enabled.
ok_exists(git_dir / "hooks" / "post-update")
# And create_sibling_ria took care of an initial call to
# git-update-server-info.
ok_exists(git_dir / "info" / "refs")
git_config = git_dir / 'config'
ok_exists(git_config)
content = git_config.read_text()
assert_in("[datalad \"ora-remote\"]", content)
super_uuid = ds.config.get(
"remote.{}.annex-uuid".format('datastore-storage'))
assert_in("uuid = {}".format(super_uuid), content)
# implicit test of success by ria-installing from store:
ds.push(to="datastore")
with chpwd(clone_path):
if host:
# note, we are not using the "test-store"-label here
clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id),
path='test_install')
else:
# TODO: Whenever ria+file supports special remote config (label),
# change here:
clone('ria+file://{}#{}'.format(base_path, ds.id),
path='test_install')
installed_ds = Dataset(op.join(clone_path, 'test_install'))
assert installed_ds.is_installed()
assert_repo_status(installed_ds.repo)
eq_(installed_ds.id, ds.id)
# Note: get_annexed_files() always reports POSIX paths.
assert_in('ds/file1.txt', installed_ds.repo.get_annexed_files())
assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')),
1,
status='ok',
action='get',
path=op.join(installed_ds.path, 'ds', 'file1.txt'))
# now, again but recursive.
res = ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
recursive=True,
existing='reconfigure',
new_store_ok=True)
assert_result_count(res,
1,
path=str(ds.pathobj),
status='ok',
action="create-sibling-ria")
assert_result_count(res,
1,
path=str(subds.pathobj),
status='ok',
action="create-sibling-ria")
assert_result_count(res,
1,
path=str(subds2.pathobj),
status='ok',
action="create-sibling-ria")
# remotes now exist in super and sub
siblings = ds.siblings(result_renderer='disabled')
eq_({'datastore', 'datastore-storage', 'here'},
{s['name']
for s in siblings})
sub_siblings = subds.siblings(result_renderer='disabled')
eq_({'datastore', 'datastore-storage', 'here'},
{s['name']
for s in sub_siblings})
# but no special remote in plain git subdataset:
sub2_siblings = subds2.siblings(result_renderer='disabled')
eq_({'datastore', 'here'}, {s['name'] for s in sub2_siblings})
# for testing trust_level parameter, redo for each label:
for trust in ['trust', 'semitrust', 'untrust']:
ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
existing='reconfigure',
trust_level=trust,
new_store_ok=True)
res = ds.repo.repo_info()
assert_in(
'[datastore-storage]',
[r['description'] for r in res['{}ed repositories'.format(trust)]])
def test_addurls_from_list(self=None, path=None):
ds = Dataset(path).create()
ds.addurls(self.data, "{url}", "{name}", result_renderer='disabled')
for fname in ["a", "b", "c"]:
ok_exists(op.join(path, fname))