def test_rerun_subdir(path): ds = Dataset(path).create() subdir = opj(path, 'subdir') mkdir(subdir) with chpwd(subdir): run("touch test.dat") ok_clean_git(ds.path) ok_file_under_git(opj(subdir, "test.dat"), annexed=True) rec_msg, runinfo = get_run_info(ds.repo.repo.head.commit.message) eq_(runinfo['pwd'], 'subdir') # now, rerun within root of the dataset with chpwd(ds.path): ds.rerun() ok_clean_git(ds.path) ok_file_under_git(opj(subdir, "test.dat"), annexed=True) # and not on top assert_raises(AssertionError, ok_file_under_git, opj(ds.path, "test.dat"), annexed=True) # but if we run ds.run -- runs within top of the dataset with chpwd(subdir): ds.run("touch test2.dat") ok_clean_git(ds.path) ok_file_under_git(opj(ds.path, "test2.dat"), annexed=True) rec_msg, runinfo = get_run_info(ds.repo.repo.head.commit.message) eq_(runinfo['pwd'], '.') # now, rerun within subdir -- smoke for now with chpwd(subdir): ds.rerun()
def test_placeholders(path): ds = Dataset(path).create(force=True) ds.save() assert_repo_status(ds.path) # ATTN windows is sensitive to spaces before redirect symbol ds.run("echo {inputs}>{outputs}", inputs=[".", "*.in"], outputs=["c.out"]) ok_file_has_content(op.join(path, "c.out"), "a.in b.in\n") hexsha_before = ds.repo.get_hexsha() ds.rerun() eq_(hexsha_before, ds.repo.get_hexsha()) # ATTN windows is sensitive to spaces before redirect symbol ds.run("echo {inputs[0]}>getitem", inputs=["*.in"]) ok_file_has_content(op.join(path, "getitem"), "a.in\n") ds.run("echo {pwd} >expanded-pwd") ok_file_has_content(op.join(path, "expanded-pwd"), path, strip=True) ds.run("echo {dspath} >expanded-dspath") ok_file_has_content(op.join(path, "expanded-dspath"), ds.path, strip=True) subdir_path = op.join(path, "subdir") with chpwd(subdir_path): run("echo {pwd} >expanded-pwd") ok_file_has_content(op.join(path, "subdir", "expanded-pwd"), subdir_path, strip=True) eq_(get_run_info(ds, ds.repo.format_commit("%B"))[1]["pwd"], "subdir") # Double brackets can be used to escape placeholders. ds.run("cd .> {{inputs}}", inputs=["*.in"]) ok_exists(op.join(path, "{inputs}")) # rerun --script expands the placeholders. with patch("sys.stdout", new_callable=StringIO) as cmout: ds.rerun(script="-", since="") script_out = cmout.getvalue() assert_in("echo a.in b.in>c.out", script_out) assert_in("echo {} >expanded-pwd".format(subdir_path), script_out) assert_in("echo {} >expanded-dspath".format(ds.path), script_out) assert_result_count(ds.run("{unknown_placeholder}", on_failure="ignore"), 1, status="impossible", action="run") # Configured placeholders. ds.config.add("datalad.run.substitutions.license", "gpl3", where="local") ds.run("echo {license} >configured-license") ok_file_has_content(op.join(path, "configured-license"), "gpl3", strip=True) # --script handles configured placeholders. with patch("sys.stdout", new_callable=StringIO) as cmout: ds.rerun(script="-") assert_in("gpl3", cmout.getvalue()) ds.run("echo {tmpdir} >tout") ok_file_has_content(op.join(path, "tout"), ".*datalad-run.*", re_=True)
def test_rerun_chain(path): ds = Dataset(path).create() commits = [] grow_file = opj(path, "grows") ds.run('echo x$(cat grows) > grows') ds.repo.tag("first-run") for _ in range(3): commits.append(ds.repo.get_hexsha()) ds.rerun() _, info = get_run_info(ds.repo.repo.head.commit.message) assert info["chain"] == commits ds.rerun(revision="first-run") _, info = get_run_info(ds.repo.repo.head.commit.message) assert info["chain"] == commits[:1]
def test_rerun_chain(path): ds = Dataset(path).create() commits = [] with swallow_outputs(): ds.run('echo x$(cat grows) > grows') ds.repo.tag("first-run", commit=DEFAULT_BRANCH) for _ in range(3): commits.append(ds.repo.get_hexsha(DEFAULT_BRANCH)) ds.rerun() _, info = get_run_info(ds, last_commit_msg(ds.repo)) eq_(info["chain"], commits) ds.rerun(revision="first-run") _, info = get_run_info(ds, last_commit_msg(ds.repo)) eq_(info["chain"], commits[:1])
def test_rerun_chain(path): ds = Dataset(path).create() commits = [] with swallow_outputs(): ds.run('echo x$(cat grows) > grows') ds.repo.tag("first-run") for _ in range(3): commits.append(ds.repo.get_hexsha()) ds.rerun() _, info = get_run_info(ds, ds.repo.format_commit("%B")) assert info["chain"] == commits ds.rerun(revision="first-run") _, info = get_run_info(ds, ds.repo.format_commit("%B")) assert info["chain"] == commits[:1]
def test_rerun_chain(path): ds = Dataset(path).create() commits = [] grow_file = opj(path, "grows") ds.run('echo x$(cat grows) > grows') ds.repo.tag("first-run") for _ in range(3): commits.append(ds.repo.get_hexsha()) ds.rerun() _, info = get_run_info(ds, ds.repo.format_commit("%B")) assert info["chain"] == commits ds.rerun(revision="first-run") _, info = get_run_info(ds, ds.repo.format_commit("%B")) assert info["chain"] == commits[:1]
def test_placeholders(path): ds = Dataset(path).create(force=True) ds.add(".") ds.run("echo {inputs} >{outputs}", inputs=[".", "*.in"], outputs=["c.out"]) ok_file_has_content(opj(path, "c.out"), "a.in b.in\n") hexsha_before = ds.repo.get_hexsha() ds.rerun() eq_(hexsha_before, ds.repo.get_hexsha()) ds.run("echo {inputs[0]} >getitem", inputs=["*.in"]) ok_file_has_content(opj(path, "getitem"), "a.in\n") ds.run("echo {pwd} >expanded-pwd") ok_file_has_content(opj(path, "expanded-pwd"), path, strip=True) ds.run("echo {dspath} >expanded-dspath") ok_file_has_content(opj(path, "expanded-dspath"), ds.path, strip=True) subdir_path = opj(path, "subdir") with chpwd(subdir_path): run("echo {pwd} >expanded-pwd") ok_file_has_content(opj(path, "subdir", "expanded-pwd"), subdir_path, strip=True) eq_(get_run_info(ds, ds.repo.format_commit("%B"))[1]["pwd"], "subdir") # Double brackets can be used to escape placeholders. ds.run("touch {{inputs}}", inputs=["*.in"]) ok_exists(opj(path, "{inputs}")) # rerun --script expands the placeholders. with patch("sys.stdout", new_callable=StringIO) as cmout: ds.rerun(script="-", since="") script_out = cmout.getvalue() assert_in("echo a.in b.in >c.out", script_out) assert_in("echo {} >expanded-pwd".format(subdir_path), script_out) assert_in("echo {} >expanded-dspath".format(ds.path), script_out) assert_result_count( ds.run("{unknown_placeholder}", on_failure="ignore"), 1, status="impossible", action="run") # Configured placeholders. ds.config.add("datalad.run.substitutions.license", "gpl3", where="local") ds.run("echo {license} >configured-license") ok_file_has_content(opj(path, "configured-license"), "gpl3", strip=True) # --script handles configured placeholders. with patch("sys.stdout", new_callable=StringIO) as cmout: ds.rerun(script="-") assert_in("gpl3", cmout.getvalue())
def test_rerun_subdir(path): # Note: Using with_tree rather than with_tempfile is matters. The latter # calls realpath on the path, which masks a failure in the # TMPDIR="/var/tmp/sym link" test case ds = Dataset(path).create(force=True) subdir = op.join(path, 'subdir') with chpwd(subdir): run("touch test.dat") assert_repo_status(ds.path) # FIXME: A plain ok_file_under_git call doesn't properly resolve the file # in the TMPDIR="/var/tmp/sym link" test case. Temporarily call realpath. def ok_file_under_git_kludge(path, basename): ok_file_under_git(op.join(str(Path(path).resolve()), basename), annexed=True) ok_file_under_git_kludge(subdir, "test.dat") rec_msg, runinfo = get_run_info(ds, ds.repo.format_commit("%B")) eq_(runinfo['pwd'], 'subdir') # now, rerun within root of the dataset with chpwd(ds.path): ds.rerun() assert_repo_status(ds.path) ok_file_under_git_kludge(subdir, "test.dat") # and not on top assert_raises(AssertionError, ok_file_under_git, op.join(ds.path, "test.dat"), annexed=True) # but if we run ds.run -- runs within top of the dataset with chpwd(subdir): ds.run("touch test2.dat") assert_repo_status(ds.path) ok_file_under_git_kludge(ds.path, "test2.dat") rec_msg, runinfo = get_run_info(ds, ds.repo.format_commit("%B")) eq_(runinfo['pwd'], '.') # now, rerun within subdir -- smoke for now with chpwd(subdir): ds.rerun()
def test_rerun_subdir(path): # Note: Using with_tree rather than with_tempfile is matters. The latter # calls realpath on the path, which masks a failure in the # TMPDIR="/var/tmp/sym link" test case ds = Dataset(path).create(force=True) subdir = opj(path, 'subdir') with chpwd(subdir): run("touch test.dat") ok_clean_git(ds.path) # FIXME: A plain ok_file_under_git call doesn't properly resolve the file # in the TMPDIR="/var/tmp/sym link" test case. Temporarily call realpath. def ok_file_under_git_kludge(path, basename): ok_file_under_git(opj(op.realpath(path), basename), annexed=True) ok_file_under_git_kludge(subdir, "test.dat") rec_msg, runinfo = get_run_info(ds, ds.repo.format_commit("%B")) eq_(runinfo['pwd'], 'subdir') # now, rerun within root of the dataset with chpwd(ds.path): ds.rerun() ok_clean_git(ds.path) ok_file_under_git_kludge(subdir, "test.dat") # and not on top assert_raises(AssertionError, ok_file_under_git, opj(ds.path, "test.dat"), annexed=True) # but if we run ds.run -- runs within top of the dataset with chpwd(subdir): ds.run("touch test2.dat") ok_clean_git(ds.path) ok_file_under_git_kludge(ds.path, "test2.dat") rec_msg, runinfo = get_run_info(ds, ds.repo.format_commit("%B")) eq_(runinfo['pwd'], '.') # now, rerun within subdir -- smoke for now with chpwd(subdir): ds.rerun()
def test_rerun_commit_message_check(): assert_raises(ValueError, get_run_info, None, """\ [DATALAD RUNCMD] no command === Do not change lines below === { "pwd": ".", "exit": 0 } ^^^ Do not change lines above ^^^""") assert_raises(ValueError, get_run_info, None, """\ [DATALAD RUNCMD] junk json === Do not change lines below === { "pwd": "., "cmd": "echo ok >okfile", "exit": 0 } ^^^ Do not change lines above ^^^""") subject, info = get_run_info( None, """\ [DATALAD RUNCMD] fine === Do not change lines below === { "pwd": ".", "cmd": "echo ok >okfile", "exit": 0 } ^^^ Do not change lines above ^^^""") eq_(subject, "fine") assert_dict_equal(info, {"pwd": ".", "cmd": "echo ok >okfile", "exit": 0})
def test_rerun_commit_message_check(): assert_raises(ValueError, get_run_info, None, """\ [DATALAD RUNCMD] no command === Do not change lines below === { "pwd": ".", "exit": 0 } ^^^ Do not change lines above ^^^""") assert_raises(ValueError, get_run_info, None, """\ [DATALAD RUNCMD] junk json === Do not change lines below === { "pwd": "., "cmd": "echo ok >okfile", "exit": 0 } ^^^ Do not change lines above ^^^""") subject, info = get_run_info( None, """\ [DATALAD RUNCMD] fine === Do not change lines below === { "pwd": ".", "cmd": "echo ok >okfile", "exit": 0 } ^^^ Do not change lines above ^^^""") eq_(subject, "fine") assert_dict_equal(info, {"pwd": ".", "cmd": "echo ok >okfile", "exit": 0})
def test_placeholders(path): ds = Dataset(path).create(force=True) ds.add(".") ds.run("echo {inputs} >{outputs}", inputs=[".", "*.in"], outputs=["c.out"]) ok_file_has_content(opj(path, "c.out"), "a.in b.in\n") hexsha_before = ds.repo.get_hexsha() ds.rerun() eq_(hexsha_before, ds.repo.get_hexsha()) ds.run("echo {inputs[0]} >getitem", inputs=["*.in"]) ok_file_has_content(opj(path, "getitem"), "a.in\n") ds.run("echo {pwd} >expanded-pwd") ok_file_has_content(opj(path, "expanded-pwd"), path, strip=True) ds.run("echo {dspath} >expanded-dspath") ok_file_has_content(opj(path, "expanded-dspath"), ds.path, strip=True) subdir_path = opj(path, "subdir") with chpwd(subdir_path): run("echo {pwd} >expanded-pwd") ok_file_has_content(opj(path, "subdir", "expanded-pwd"), subdir_path, strip=True) eq_(get_run_info(ds, ds.repo.repo.head.commit.message)[1]["pwd"], "subdir") # Double brackets can be used to escape placeholders. ds.run("touch {{inputs}}", inputs=["*.in"]) ok_exists(opj(path, "{inputs}")) # rerun --script expands the placeholders. with patch("sys.stdout", new_callable=StringIO) as cmout: ds.rerun(script="-", since="") script_out = cmout.getvalue() assert_in("echo a.in b.in >c.out", script_out) assert_in("echo {} >expanded-pwd".format(subdir_path), script_out) assert_in("echo {} >expanded-dspath".format(ds.path), script_out)
def test_rerun(path, nodspath): ds = Dataset(path).create() sub = ds.create('sub') probe_path = opj(sub.path, 'sequence') # run inside the dataset with chpwd(path), \ swallow_outputs(): ds.run('echo x$(cat sub/sequence) > sub/sequence') # command ran once, all clean ok_clean_git(ds.path) eq_('x\n', open(probe_path).read()) # now, for a rerun we can be anywhere, PWD and all are recorded # moreover, rerun must figure out which bits to unlock, even in # subdatasets with chpwd(nodspath), \ swallow_outputs(): ds.rerun() ok_clean_git(ds.path) # ran twice now eq_('xx\n', open(probe_path).read()) # Rerunning from a subdataset skips the command. _, sub_info = get_run_info(ds, sub.repo.repo.head.commit.message) eq_(ds.id, sub_info["dsid"]) assert_result_count(sub.rerun(return_type="list", on_failure="ignore"), 1, status="impossible", action="run", rerun_action="skip") eq_('xx\n', open(probe_path).read()) # Rerun fails with a dirty repo. dirt = opj(path, "dirt") with open(dirt, "w") as fh: fh.write("") assert_status('impossible', ds.rerun(on_failure="ignore")) remove(dirt) ok_clean_git(ds.path) # Make a non-run commit. with open(opj(path, "nonrun-file"), "w") as f: f.write("foo") ds.add("nonrun-file") # Now rerun the buried command. ds.rerun(revision="HEAD~", message="rerun buried") eq_('xxx\n', open(probe_path).read()) # Also check that the messasge override worked. eq_( ds.repo.format_commit("%B").splitlines()[0], "[DATALAD RUNCMD] rerun buried") # Or a range of commits, skipping non-run commits. ds.rerun(since="HEAD~3") eq_('xxxxx\n', open(probe_path).read()) # Or --since= to run all reachable commits. ds.rerun(since="") eq_('xxxxxxxxxx\n', open(probe_path).read()) # We can get back a report of what would happen rather than actually # rerunning anything. report = ds.rerun(since="", report=True, return_type="list") # Nothing changed. eq_('xxxxxxxxxx\n', open(probe_path).read()) assert_result_count(report, 1, rerun_action="skip") report[-1]["commit"] == ds.repo.get_hexsha() # If a file is dropped, we remove it instead of unlocking it. ds.drop(probe_path, check=False) with swallow_outputs(): ds.rerun() eq_('x\n', open(probe_path).read()) # If the history to rerun has a merge commit, we abort. ds.repo.checkout("HEAD~3", options=["-b", "topic"]) with open(opj(path, "topic-file"), "w") as f: f.write("topic") ds.add("topic-file") ds.repo.checkout("master") ds.repo.merge("topic") ok_clean_git(ds.path) assert_raises(IncompleteResultsError, ds.rerun)
def test_rerun(path, nodspath): ds = Dataset(path).create() sub = ds.create('sub') probe_path = op.join(sub.path, 'sequence') # run inside the dataset with chpwd(path), \ swallow_outputs(): ds.run('echo x$(cat sub/sequence) > sub/sequence') # command ran once, all clean assert_repo_status(ds.path) eq_('x\n', open(probe_path).read()) # now, for a rerun we can be anywhere, PWD and all are recorded # moreover, rerun must figure out which bits to unlock, even in # subdatasets with chpwd(nodspath), \ swallow_outputs(): ds.rerun() assert_repo_status(ds.path) # ran twice now eq_('xx\n', open(probe_path).read()) # Rerunning from a subdataset skips the command. _, sub_info = get_run_info(ds, sub.repo.format_commit("%B")) eq_(ds.id, sub_info["dsid"]) assert_result_count(sub.rerun(return_type="list", on_failure="ignore"), 1, status="impossible", action="run", rerun_action="skip") eq_('xx\n', open(probe_path).read()) # Rerun fails with a dirty repo. dirt = op.join(path, "dirt") with open(dirt, "w") as fh: fh.write("") assert_status('impossible', ds.rerun(on_failure="ignore")) remove(dirt) assert_repo_status(ds.path) # Make a non-run commit. with open(op.join(path, "nonrun-file"), "w") as f: f.write("foo") ds.save("nonrun-file") # Now rerun the buried command. ds.rerun(revision="HEAD~", message="rerun buried") eq_('xxx\n', open(probe_path).read()) # Also check that the messasge override worked. eq_( ds.repo.format_commit("%B").splitlines()[0], "[DATALAD RUNCMD] rerun buried") # Or a range of commits, skipping non-run commits. ds.rerun(since="HEAD~3") eq_('xxxxx\n', open(probe_path).read()) # Or --since= to run all reachable commits. ds.rerun(since="") eq_('xxxxxxxxxx\n', open(probe_path).read()) # We can get back a report of what would happen rather than actually # rerunning anything. report = ds.rerun(since="", report=True, return_type="list") # The "diff" section of the report doesn't include the unchanged files that # would come in "-f json diff" output. for entry in report: if entry["rerun_action"] == "run": # None of the run commits touch .datalad/config or any other config # file. assert_false( any(r["path"].endswith("config") for r in entry["diff"])) # Nothing changed. eq_('xxxxxxxxxx\n', open(probe_path).read()) assert_result_count(report, 1, rerun_action="skip-or-pick") report[-1]["commit"] == ds.repo.get_hexsha() # If a file is dropped, we remove it instead of unlocking it. ds.drop(probe_path, check=False) with swallow_outputs(): ds.rerun() eq_('x\n', open(probe_path).read())
def test_rerun(path, nodspath): ds = Dataset(path).create() sub = ds.create('sub') probe_path = opj(sub.path, 'sequence') # run inside the dataset with chpwd(path), \ swallow_outputs(): ds.run('echo x$(cat sub/sequence) > sub/sequence') # command ran once, all clean ok_clean_git(ds.path) eq_('x\n', open(probe_path).read()) # now, for a rerun we can be anywhere, PWD and all are recorded # moreover, rerun must figure out which bits to unlock, even in # subdatasets with chpwd(nodspath), \ swallow_outputs(): ds.rerun() ok_clean_git(ds.path) # ran twice now eq_('xx\n', open(probe_path).read()) # Rerunning from a subdataset skips the command. _, sub_info = get_run_info(ds, sub.repo.repo.head.commit.message) eq_(ds.id, sub_info["dsid"]) assert_result_count( sub.rerun(return_type="list", on_failure="ignore"), 1, status="impossible", action="run", rerun_action="skip") eq_('xx\n', open(probe_path).read()) # Rerun fails with a dirty repo. dirt = opj(path, "dirt") with open(dirt, "w") as fh: fh.write("") assert_status('impossible', ds.rerun(on_failure="ignore")) remove(dirt) ok_clean_git(ds.path) # Make a non-run commit. with open(opj(path, "nonrun-file"), "w") as f: f.write("foo") ds.add("nonrun-file") # Now rerun the buried command. ds.rerun(revision="HEAD~", message="rerun buried") eq_('xxx\n', open(probe_path).read()) # Also check that the messasge override worked. eq_(ds.repo.format_commit("%B").splitlines()[0], "[DATALAD RUNCMD] rerun buried") # Or a range of commits, skipping non-run commits. ds.rerun(since="HEAD~3") eq_('xxxxx\n', open(probe_path).read()) # Or --since= to run all reachable commits. ds.rerun(since="") eq_('xxxxxxxxxx\n', open(probe_path).read()) # We can get back a report of what would happen rather than actually # rerunning anything. report = ds.rerun(since="", report=True, return_type="list") # Nothing changed. eq_('xxxxxxxxxx\n', open(probe_path).read()) assert_result_count(report, 1, rerun_action="skip") report[-1]["commit"] == ds.repo.get_hexsha() # If a file is dropped, we remove it instead of unlocking it. ds.drop(probe_path, check=False) ds.rerun() eq_('x\n', open(probe_path).read()) # If the history to rerun has a merge commit, we abort. ds.repo.checkout("HEAD~3", options=["-b", "topic"]) with open(opj(path, "topic-file"), "w") as f: f.write("topic") ds.add("topic-file") ds.repo.checkout("master") ds.repo.merge("topic") ok_clean_git(ds.path) assert_raises(IncompleteResultsError, ds.rerun)