def test_rerun_subdir(path):
ds = Dataset(path).create()
subdir = opj(path, 'subdir')
mkdir(subdir)
with chpwd(subdir):
run("touch test.dat")
ok_clean_git(ds.path)
ok_file_under_git(opj(subdir, "test.dat"), annexed=True)
rec_msg, runinfo = get_run_info(ds.repo.repo.head.commit.message)
eq_(runinfo['pwd'], 'subdir')
# now, rerun within root of the dataset
with chpwd(ds.path):
ds.rerun()
ok_clean_git(ds.path)
ok_file_under_git(opj(subdir, "test.dat"), annexed=True)
# and not on top
assert_raises(AssertionError,
ok_file_under_git,
opj(ds.path, "test.dat"),
annexed=True)
# but if we run ds.run -- runs within top of the dataset
with chpwd(subdir):
ds.run("touch test2.dat")
ok_clean_git(ds.path)
ok_file_under_git(opj(ds.path, "test2.dat"), annexed=True)
rec_msg, runinfo = get_run_info(ds.repo.repo.head.commit.message)
eq_(runinfo['pwd'], '.')
# now, rerun within subdir -- smoke for now
with chpwd(subdir):
ds.rerun()
def test_placeholders(path):
ds = Dataset(path).create(force=True)
ds.save()
assert_repo_status(ds.path)
# ATTN windows is sensitive to spaces before redirect symbol
ds.run("echo {inputs}>{outputs}", inputs=[".", "*.in"], outputs=["c.out"])
ok_file_has_content(op.join(path, "c.out"), "a.in b.in\n")
hexsha_before = ds.repo.get_hexsha()
ds.rerun()
eq_(hexsha_before, ds.repo.get_hexsha())
# ATTN windows is sensitive to spaces before redirect symbol
ds.run("echo {inputs[0]}>getitem", inputs=["*.in"])
ok_file_has_content(op.join(path, "getitem"), "a.in\n")
ds.run("echo {pwd} >expanded-pwd")
ok_file_has_content(op.join(path, "expanded-pwd"), path, strip=True)
ds.run("echo {dspath} >expanded-dspath")
ok_file_has_content(op.join(path, "expanded-dspath"), ds.path, strip=True)
subdir_path = op.join(path, "subdir")
with chpwd(subdir_path):
run("echo {pwd} >expanded-pwd")
ok_file_has_content(op.join(path, "subdir", "expanded-pwd"),
subdir_path,
strip=True)
eq_(get_run_info(ds, ds.repo.format_commit("%B"))[1]["pwd"], "subdir")
# Double brackets can be used to escape placeholders.
ds.run("cd .> {{inputs}}", inputs=["*.in"])
ok_exists(op.join(path, "{inputs}"))
# rerun --script expands the placeholders.
with patch("sys.stdout", new_callable=StringIO) as cmout:
ds.rerun(script="-", since="")
script_out = cmout.getvalue()
assert_in("echo a.in b.in>c.out", script_out)
assert_in("echo {} >expanded-pwd".format(subdir_path), script_out)
assert_in("echo {} >expanded-dspath".format(ds.path), script_out)
assert_result_count(ds.run("{unknown_placeholder}", on_failure="ignore"),
1,
status="impossible",
action="run")
# Configured placeholders.
ds.config.add("datalad.run.substitutions.license", "gpl3", where="local")
ds.run("echo {license} >configured-license")
ok_file_has_content(op.join(path, "configured-license"),
"gpl3",
strip=True)
# --script handles configured placeholders.
with patch("sys.stdout", new_callable=StringIO) as cmout:
ds.rerun(script="-")
assert_in("gpl3", cmout.getvalue())
ds.run("echo {tmpdir} >tout")
ok_file_has_content(op.join(path, "tout"), ".*datalad-run.*", re_=True)
def test_rerun_chain(path):
ds = Dataset(path).create()
commits = []
grow_file = opj(path, "grows")
ds.run('echo x$(cat grows) > grows')
ds.repo.tag("first-run")
for _ in range(3):
commits.append(ds.repo.get_hexsha())
ds.rerun()
_, info = get_run_info(ds.repo.repo.head.commit.message)
assert info["chain"] == commits
ds.rerun(revision="first-run")
_, info = get_run_info(ds.repo.repo.head.commit.message)
assert info["chain"] == commits[:1]
def test_rerun_chain(path):
ds = Dataset(path).create()
commits = []
with swallow_outputs():
ds.run('echo x$(cat grows) > grows')
ds.repo.tag("first-run", commit=DEFAULT_BRANCH)
for _ in range(3):
commits.append(ds.repo.get_hexsha(DEFAULT_BRANCH))
ds.rerun()
_, info = get_run_info(ds, last_commit_msg(ds.repo))
eq_(info["chain"], commits)
ds.rerun(revision="first-run")
_, info = get_run_info(ds, last_commit_msg(ds.repo))
eq_(info["chain"], commits[:1])
def test_rerun_chain(path):
ds = Dataset(path).create()
commits = []
with swallow_outputs():
ds.run('echo x$(cat grows) > grows')
ds.repo.tag("first-run")
for _ in range(3):
commits.append(ds.repo.get_hexsha())
ds.rerun()
_, info = get_run_info(ds, ds.repo.format_commit("%B"))
assert info["chain"] == commits
ds.rerun(revision="first-run")
_, info = get_run_info(ds, ds.repo.format_commit("%B"))
assert info["chain"] == commits[:1]
def test_rerun_chain(path):
ds = Dataset(path).create()
commits = []
grow_file = opj(path, "grows")
ds.run('echo x$(cat grows) > grows')
ds.repo.tag("first-run")
for _ in range(3):
commits.append(ds.repo.get_hexsha())
ds.rerun()
_, info = get_run_info(ds, ds.repo.format_commit("%B"))
assert info["chain"] == commits
ds.rerun(revision="first-run")
_, info = get_run_info(ds, ds.repo.format_commit("%B"))
assert info["chain"] == commits[:1]
def test_placeholders(path):
ds = Dataset(path).create(force=True)
ds.add(".")
ds.run("echo {inputs} >{outputs}", inputs=[".", "*.in"], outputs=["c.out"])
ok_file_has_content(opj(path, "c.out"), "a.in b.in\n")
hexsha_before = ds.repo.get_hexsha()
ds.rerun()
eq_(hexsha_before, ds.repo.get_hexsha())
ds.run("echo {inputs[0]} >getitem", inputs=["*.in"])
ok_file_has_content(opj(path, "getitem"), "a.in\n")
ds.run("echo {pwd} >expanded-pwd")
ok_file_has_content(opj(path, "expanded-pwd"), path,
strip=True)
ds.run("echo {dspath} >expanded-dspath")
ok_file_has_content(opj(path, "expanded-dspath"), ds.path,
strip=True)
subdir_path = opj(path, "subdir")
with chpwd(subdir_path):
run("echo {pwd} >expanded-pwd")
ok_file_has_content(opj(path, "subdir", "expanded-pwd"), subdir_path,
strip=True)
eq_(get_run_info(ds, ds.repo.format_commit("%B"))[1]["pwd"],
"subdir")
# Double brackets can be used to escape placeholders.
ds.run("touch {{inputs}}", inputs=["*.in"])
ok_exists(opj(path, "{inputs}"))
# rerun --script expands the placeholders.
with patch("sys.stdout", new_callable=StringIO) as cmout:
ds.rerun(script="-", since="")
script_out = cmout.getvalue()
assert_in("echo a.in b.in >c.out", script_out)
assert_in("echo {} >expanded-pwd".format(subdir_path),
script_out)
assert_in("echo {} >expanded-dspath".format(ds.path),
script_out)
assert_result_count(
ds.run("{unknown_placeholder}", on_failure="ignore"),
1, status="impossible", action="run")
# Configured placeholders.
ds.config.add("datalad.run.substitutions.license", "gpl3", where="local")
ds.run("echo {license} >configured-license")
ok_file_has_content(opj(path, "configured-license"), "gpl3", strip=True)
# --script handles configured placeholders.
with patch("sys.stdout", new_callable=StringIO) as cmout:
ds.rerun(script="-")
assert_in("gpl3", cmout.getvalue())
def test_rerun_subdir(path):
# Note: Using with_tree rather than with_tempfile is matters. The latter
# calls realpath on the path, which masks a failure in the
# TMPDIR="/var/tmp/sym link" test case
ds = Dataset(path).create(force=True)
subdir = op.join(path, 'subdir')
with chpwd(subdir):
run("touch test.dat")
assert_repo_status(ds.path)
# FIXME: A plain ok_file_under_git call doesn't properly resolve the file
# in the TMPDIR="/var/tmp/sym link" test case. Temporarily call realpath.
def ok_file_under_git_kludge(path, basename):
ok_file_under_git(op.join(str(Path(path).resolve()), basename),
annexed=True)
ok_file_under_git_kludge(subdir, "test.dat")
rec_msg, runinfo = get_run_info(ds, ds.repo.format_commit("%B"))
eq_(runinfo['pwd'], 'subdir')
# now, rerun within root of the dataset
with chpwd(ds.path):
ds.rerun()
assert_repo_status(ds.path)
ok_file_under_git_kludge(subdir, "test.dat")
# and not on top
assert_raises(AssertionError,
ok_file_under_git,
op.join(ds.path, "test.dat"),
annexed=True)
# but if we run ds.run -- runs within top of the dataset
with chpwd(subdir):
ds.run("touch test2.dat")
assert_repo_status(ds.path)
ok_file_under_git_kludge(ds.path, "test2.dat")
rec_msg, runinfo = get_run_info(ds, ds.repo.format_commit("%B"))
eq_(runinfo['pwd'], '.')
# now, rerun within subdir -- smoke for now
with chpwd(subdir):
ds.rerun()
def test_rerun_subdir(path):
# Note: Using with_tree rather than with_tempfile is matters. The latter
# calls realpath on the path, which masks a failure in the
# TMPDIR="/var/tmp/sym link" test case
ds = Dataset(path).create(force=True)
subdir = opj(path, 'subdir')
with chpwd(subdir):
run("touch test.dat")
ok_clean_git(ds.path)
# FIXME: A plain ok_file_under_git call doesn't properly resolve the file
# in the TMPDIR="/var/tmp/sym link" test case. Temporarily call realpath.
def ok_file_under_git_kludge(path, basename):
ok_file_under_git(opj(op.realpath(path), basename), annexed=True)
ok_file_under_git_kludge(subdir, "test.dat")
rec_msg, runinfo = get_run_info(ds, ds.repo.format_commit("%B"))
eq_(runinfo['pwd'], 'subdir')
# now, rerun within root of the dataset
with chpwd(ds.path):
ds.rerun()
ok_clean_git(ds.path)
ok_file_under_git_kludge(subdir, "test.dat")
# and not on top
assert_raises(AssertionError, ok_file_under_git, opj(ds.path, "test.dat"), annexed=True)
# but if we run ds.run -- runs within top of the dataset
with chpwd(subdir):
ds.run("touch test2.dat")
ok_clean_git(ds.path)
ok_file_under_git_kludge(ds.path, "test2.dat")
rec_msg, runinfo = get_run_info(ds, ds.repo.format_commit("%B"))
eq_(runinfo['pwd'], '.')
# now, rerun within subdir -- smoke for now
with chpwd(subdir):
ds.rerun()
def test_rerun_commit_message_check():
assert_raises(ValueError,
get_run_info,
None,
"""\
[DATALAD RUNCMD] no command
=== Do not change lines below ===
{
"pwd": ".",
"exit": 0
}
^^^ Do not change lines above ^^^""")
assert_raises(ValueError,
get_run_info,
None,
"""\
[DATALAD RUNCMD] junk json
=== Do not change lines below ===
{
"pwd": ".,
"cmd": "echo ok >okfile",
"exit": 0
}
^^^ Do not change lines above ^^^""")
subject, info = get_run_info(
None,
"""\
[DATALAD RUNCMD] fine
=== Do not change lines below ===
{
"pwd": ".",
"cmd": "echo ok >okfile",
"exit": 0
}
^^^ Do not change lines above ^^^""")
eq_(subject, "fine")
assert_dict_equal(info,
{"pwd": ".", "cmd": "echo ok >okfile", "exit": 0})
def test_rerun_commit_message_check():
assert_raises(ValueError,
get_run_info,
None,
"""\
[DATALAD RUNCMD] no command
=== Do not change lines below ===
{
"pwd": ".",
"exit": 0
}
^^^ Do not change lines above ^^^""")
assert_raises(ValueError,
get_run_info,
None,
"""\
[DATALAD RUNCMD] junk json
=== Do not change lines below ===
{
"pwd": ".,
"cmd": "echo ok >okfile",
"exit": 0
}
^^^ Do not change lines above ^^^""")
subject, info = get_run_info(
None,
"""\
[DATALAD RUNCMD] fine
=== Do not change lines below ===
{
"pwd": ".",
"cmd": "echo ok >okfile",
"exit": 0
}
^^^ Do not change lines above ^^^""")
eq_(subject, "fine")
assert_dict_equal(info,
{"pwd": ".", "cmd": "echo ok >okfile", "exit": 0})
def test_placeholders(path):
ds = Dataset(path).create(force=True)
ds.add(".")
ds.run("echo {inputs} >{outputs}", inputs=[".", "*.in"], outputs=["c.out"])
ok_file_has_content(opj(path, "c.out"), "a.in b.in\n")
hexsha_before = ds.repo.get_hexsha()
ds.rerun()
eq_(hexsha_before, ds.repo.get_hexsha())
ds.run("echo {inputs[0]} >getitem", inputs=["*.in"])
ok_file_has_content(opj(path, "getitem"), "a.in\n")
ds.run("echo {pwd} >expanded-pwd")
ok_file_has_content(opj(path, "expanded-pwd"), path,
strip=True)
ds.run("echo {dspath} >expanded-dspath")
ok_file_has_content(opj(path, "expanded-dspath"), ds.path,
strip=True)
subdir_path = opj(path, "subdir")
with chpwd(subdir_path):
run("echo {pwd} >expanded-pwd")
ok_file_has_content(opj(path, "subdir", "expanded-pwd"), subdir_path,
strip=True)
eq_(get_run_info(ds, ds.repo.repo.head.commit.message)[1]["pwd"],
"subdir")
# Double brackets can be used to escape placeholders.
ds.run("touch {{inputs}}", inputs=["*.in"])
ok_exists(opj(path, "{inputs}"))
# rerun --script expands the placeholders.
with patch("sys.stdout", new_callable=StringIO) as cmout:
ds.rerun(script="-", since="")
script_out = cmout.getvalue()
assert_in("echo a.in b.in >c.out", script_out)
assert_in("echo {} >expanded-pwd".format(subdir_path),
script_out)
assert_in("echo {} >expanded-dspath".format(ds.path),
script_out)
def test_rerun(path, nodspath):
ds = Dataset(path).create()
sub = ds.create('sub')
probe_path = opj(sub.path, 'sequence')
# run inside the dataset
with chpwd(path), \
swallow_outputs():
ds.run('echo x$(cat sub/sequence) > sub/sequence')
# command ran once, all clean
ok_clean_git(ds.path)
eq_('x\n', open(probe_path).read())
# now, for a rerun we can be anywhere, PWD and all are recorded
# moreover, rerun must figure out which bits to unlock, even in
# subdatasets
with chpwd(nodspath), \
swallow_outputs():
ds.rerun()
ok_clean_git(ds.path)
# ran twice now
eq_('xx\n', open(probe_path).read())
# Rerunning from a subdataset skips the command.
_, sub_info = get_run_info(ds, sub.repo.repo.head.commit.message)
eq_(ds.id, sub_info["dsid"])
assert_result_count(sub.rerun(return_type="list", on_failure="ignore"),
1,
status="impossible",
action="run",
rerun_action="skip")
eq_('xx\n', open(probe_path).read())
# Rerun fails with a dirty repo.
dirt = opj(path, "dirt")
with open(dirt, "w") as fh:
fh.write("")
assert_status('impossible', ds.rerun(on_failure="ignore"))
remove(dirt)
ok_clean_git(ds.path)
# Make a non-run commit.
with open(opj(path, "nonrun-file"), "w") as f:
f.write("foo")
ds.add("nonrun-file")
# Now rerun the buried command.
ds.rerun(revision="HEAD~", message="rerun buried")
eq_('xxx\n', open(probe_path).read())
# Also check that the messasge override worked.
eq_(
ds.repo.format_commit("%B").splitlines()[0],
"[DATALAD RUNCMD] rerun buried")
# Or a range of commits, skipping non-run commits.
ds.rerun(since="HEAD~3")
eq_('xxxxx\n', open(probe_path).read())
# Or --since= to run all reachable commits.
ds.rerun(since="")
eq_('xxxxxxxxxx\n', open(probe_path).read())
# We can get back a report of what would happen rather than actually
# rerunning anything.
report = ds.rerun(since="", report=True, return_type="list")
# Nothing changed.
eq_('xxxxxxxxxx\n', open(probe_path).read())
assert_result_count(report, 1, rerun_action="skip")
report[-1]["commit"] == ds.repo.get_hexsha()
# If a file is dropped, we remove it instead of unlocking it.
ds.drop(probe_path, check=False)
with swallow_outputs():
ds.rerun()
eq_('x\n', open(probe_path).read())
# If the history to rerun has a merge commit, we abort.
ds.repo.checkout("HEAD~3", options=["-b", "topic"])
with open(opj(path, "topic-file"), "w") as f:
f.write("topic")
ds.add("topic-file")
ds.repo.checkout("master")
ds.repo.merge("topic")
ok_clean_git(ds.path)
assert_raises(IncompleteResultsError, ds.rerun)
def test_rerun(path, nodspath):
ds = Dataset(path).create()
sub = ds.create('sub')
probe_path = op.join(sub.path, 'sequence')
# run inside the dataset
with chpwd(path), \
swallow_outputs():
ds.run('echo x$(cat sub/sequence) > sub/sequence')
# command ran once, all clean
assert_repo_status(ds.path)
eq_('x\n', open(probe_path).read())
# now, for a rerun we can be anywhere, PWD and all are recorded
# moreover, rerun must figure out which bits to unlock, even in
# subdatasets
with chpwd(nodspath), \
swallow_outputs():
ds.rerun()
assert_repo_status(ds.path)
# ran twice now
eq_('xx\n', open(probe_path).read())
# Rerunning from a subdataset skips the command.
_, sub_info = get_run_info(ds, sub.repo.format_commit("%B"))
eq_(ds.id, sub_info["dsid"])
assert_result_count(sub.rerun(return_type="list", on_failure="ignore"),
1,
status="impossible",
action="run",
rerun_action="skip")
eq_('xx\n', open(probe_path).read())
# Rerun fails with a dirty repo.
dirt = op.join(path, "dirt")
with open(dirt, "w") as fh:
fh.write("")
assert_status('impossible', ds.rerun(on_failure="ignore"))
remove(dirt)
assert_repo_status(ds.path)
# Make a non-run commit.
with open(op.join(path, "nonrun-file"), "w") as f:
f.write("foo")
ds.save("nonrun-file")
# Now rerun the buried command.
ds.rerun(revision="HEAD~", message="rerun buried")
eq_('xxx\n', open(probe_path).read())
# Also check that the messasge override worked.
eq_(
ds.repo.format_commit("%B").splitlines()[0],
"[DATALAD RUNCMD] rerun buried")
# Or a range of commits, skipping non-run commits.
ds.rerun(since="HEAD~3")
eq_('xxxxx\n', open(probe_path).read())
# Or --since= to run all reachable commits.
ds.rerun(since="")
eq_('xxxxxxxxxx\n', open(probe_path).read())
# We can get back a report of what would happen rather than actually
# rerunning anything.
report = ds.rerun(since="", report=True, return_type="list")
# The "diff" section of the report doesn't include the unchanged files that
# would come in "-f json diff" output.
for entry in report:
if entry["rerun_action"] == "run":
# None of the run commits touch .datalad/config or any other config
# file.
assert_false(
any(r["path"].endswith("config") for r in entry["diff"]))
# Nothing changed.
eq_('xxxxxxxxxx\n', open(probe_path).read())
assert_result_count(report, 1, rerun_action="skip-or-pick")
report[-1]["commit"] == ds.repo.get_hexsha()
# If a file is dropped, we remove it instead of unlocking it.
ds.drop(probe_path, check=False)
with swallow_outputs():
ds.rerun()
eq_('x\n', open(probe_path).read())
def test_rerun(path, nodspath):
ds = Dataset(path).create()
sub = ds.create('sub')
probe_path = opj(sub.path, 'sequence')
# run inside the dataset
with chpwd(path), \
swallow_outputs():
ds.run('echo x$(cat sub/sequence) > sub/sequence')
# command ran once, all clean
ok_clean_git(ds.path)
eq_('x\n', open(probe_path).read())
# now, for a rerun we can be anywhere, PWD and all are recorded
# moreover, rerun must figure out which bits to unlock, even in
# subdatasets
with chpwd(nodspath), \
swallow_outputs():
ds.rerun()
ok_clean_git(ds.path)
# ran twice now
eq_('xx\n', open(probe_path).read())
# Rerunning from a subdataset skips the command.
_, sub_info = get_run_info(ds, sub.repo.repo.head.commit.message)
eq_(ds.id, sub_info["dsid"])
assert_result_count(
sub.rerun(return_type="list", on_failure="ignore"),
1, status="impossible", action="run", rerun_action="skip")
eq_('xx\n', open(probe_path).read())
# Rerun fails with a dirty repo.
dirt = opj(path, "dirt")
with open(dirt, "w") as fh:
fh.write("")
assert_status('impossible', ds.rerun(on_failure="ignore"))
remove(dirt)
ok_clean_git(ds.path)
# Make a non-run commit.
with open(opj(path, "nonrun-file"), "w") as f:
f.write("foo")
ds.add("nonrun-file")
# Now rerun the buried command.
ds.rerun(revision="HEAD~", message="rerun buried")
eq_('xxx\n', open(probe_path).read())
# Also check that the messasge override worked.
eq_(ds.repo.format_commit("%B").splitlines()[0],
"[DATALAD RUNCMD] rerun buried")
# Or a range of commits, skipping non-run commits.
ds.rerun(since="HEAD~3")
eq_('xxxxx\n', open(probe_path).read())
# Or --since= to run all reachable commits.
ds.rerun(since="")
eq_('xxxxxxxxxx\n', open(probe_path).read())
# We can get back a report of what would happen rather than actually
# rerunning anything.
report = ds.rerun(since="", report=True, return_type="list")
# Nothing changed.
eq_('xxxxxxxxxx\n', open(probe_path).read())
assert_result_count(report, 1, rerun_action="skip")
report[-1]["commit"] == ds.repo.get_hexsha()
# If a file is dropped, we remove it instead of unlocking it.
ds.drop(probe_path, check=False)
ds.rerun()
eq_('x\n', open(probe_path).read())
# If the history to rerun has a merge commit, we abort.
ds.repo.checkout("HEAD~3", options=["-b", "topic"])
with open(opj(path, "topic-file"), "w") as f:
f.write("topic")
ds.add("topic-file")
ds.repo.checkout("master")
ds.repo.merge("topic")
ok_clean_git(ds.path)
assert_raises(IncompleteResultsError, ds.rerun)
