def test_globbedpaths_misses(path):
gp = GlobbedPaths(["amiss"], pwd=path)
eq_(gp.expand_strict(), [])
eq_(gp.misses, ["amiss"])
eq_(gp.expand(include_misses=True), ["amiss"])
# miss at beginning
gp = GlobbedPaths(["amiss", "*.txt", "*.dat"], pwd=path)
eq_(gp.expand_strict(), ["1.txt", "3.txt", "2.dat"])
eq_(gp.expand(include_misses=True), ["amiss", "1.txt", "3.txt", "2.dat"])
# miss in middle
gp = GlobbedPaths(["*.txt", "amiss", "*.dat"], pwd=path)
eq_(gp.expand_strict(), ["1.txt", "3.txt", "2.dat"])
eq_(gp.misses, ["amiss"])
eq_(gp.expand(include_misses=True), ["1.txt", "3.txt", "amiss", "2.dat"])
# miss at end
gp = GlobbedPaths(["*.txt", "*.dat", "amiss"], pwd=path)
eq_(gp.expand_strict(), ["1.txt", "3.txt", "2.dat"])
eq_(gp.misses, ["amiss"])
eq_(gp.expand(include_misses=True), ["1.txt", "3.txt", "2.dat", "amiss"])
# miss at beginning, middle, and end
gp = GlobbedPaths(
["amiss1", "amiss2", "*.txt", "amiss3", "*.dat", "amiss4"], pwd=path)
eq_(gp.expand_strict(), ["1.txt", "3.txt", "2.dat"])
eq_(gp.misses, ["amiss1", "amiss2", "amiss3", "amiss4"])
eq_(gp.expand(include_misses=True),
["amiss1", "amiss2", "1.txt", "3.txt", "amiss3", "2.dat", "amiss4"])
# Property expands if needed.
gp = GlobbedPaths(["amiss"], pwd=path)
eq_(gp.misses, ["amiss"])
def test_globbedpaths_cached(path):
# Smoke test to trigger cache handling.
gp = GlobbedPaths([op.join("?", ".dat"), "*.txt"], pwd=path)
for full, partial, misses in product([False, True], repeat=3):
eq_(gp.expand(full=full,
include_misses=misses,
include_partial=partial),
gp.expand(full=full,
include_misses=misses,
include_partial=partial))
def test_globbedpaths(path):
dotdir = op.curdir + op.sep
for patterns, expected in [
(["1.txt", "2.dat"], {"1.txt", "2.dat"}),
([dotdir + "1.txt", "2.dat"], {dotdir + "1.txt", "2.dat"}),
(["*.txt", "*.dat"], {"1.txt", "2.dat", u"bβ.dat", "3.txt"}),
([dotdir + "*.txt", "*.dat"],
{dotdir + "1.txt", "2.dat", u"bβ.dat", dotdir + "3.txt"}),
(["subdir/*.txt"], {"subdir/1.txt", "subdir/2.txt"}),
([dotdir + "subdir/*.txt"],
{dotdir + p for p in ["subdir/1.txt", "subdir/2.txt"]}),
(["*.txt"], {"1.txt", "3.txt"})]:
gp = GlobbedPaths(patterns, pwd=path)
eq_(set(gp.expand()), expected)
eq_(set(gp.expand(full=True)),
{op.join(path, p) for p in expected})
pardir = op.pardir + op.sep
subdir_path = op.join(path, "subdir")
for patterns, expected in [
(["*.txt"], {"1.txt", "2.txt"}),
([dotdir + "*.txt"], {dotdir + p for p in ["1.txt", "2.txt"]}),
([pardir + "*.txt"], {pardir + p for p in ["1.txt", "3.txt"]}),
([dotdir + pardir + "*.txt"],
{dotdir + pardir + p for p in ["1.txt", "3.txt"]}),
(["subdir/"], {"subdir/"})]:
gp = GlobbedPaths(patterns, pwd=subdir_path)
eq_(set(gp.expand()), expected)
eq_(set(gp.expand(full=True)),
{op.join(subdir_path, p) for p in expected})
# Full patterns still get returned as relative to pwd.
gp = GlobbedPaths([op.join(path, "*.dat")], pwd=path)
eq_(gp.expand(), ["2.dat", u"bβ.dat"])
# "." gets special treatment.
gp = GlobbedPaths([".", "*.dat"], pwd=path)
eq_(set(gp.expand()), {"2.dat", u"bβ.dat", "."})
eq_(gp.expand(dot=False), ["2.dat", u"bβ.dat"])
gp = GlobbedPaths(["."], pwd=path, expand=False)
eq_(gp.expand(), ["."])
eq_(gp.paths, ["."])
# We can the glob outputs.
glob_results = {"z": "z",
"a": ["x", "d", "b"]}
with patch('glob.glob', glob_results.get):
gp = GlobbedPaths(["z", "a"])
eq_(gp.expand(), ["z", "b", "d", "x"])
# glob expansion for paths property is determined by expand argument.
for expand, expected in [(True, ["2.dat", u"bβ.dat"]),
(False, ["*.dat"])]:
gp = GlobbedPaths(["*.dat"], pwd=path, expand=expand)
eq_(gp.paths, expected)
with swallow_logs(new_level=logging.DEBUG) as cml:
GlobbedPaths(["not here"], pwd=path).expand()
assert_in("No matching files found for 'not here'", cml.out)
def format_command(dset, command, **kwds):
"""Plug in placeholders in `command`.
Parameters
----------
dset : Dataset
command : str or list
`kwds` is passed to the `format` call. `inputs` and `outputs` are converted
to GlobbedPaths if necessary.
Returns
-------
formatted command (str)
"""
command = normalize_command(command)
sfmt = SequenceFormatter()
for k, v in dset.config.items("datalad.run.substitutions"):
sub_key = k.replace("datalad.run.substitutions.", "")
if sub_key not in kwds:
kwds[sub_key] = v
for name in ["inputs", "outputs"]:
io_val = kwds.pop(name, None)
if not isinstance(io_val, GlobbedPaths):
io_val = GlobbedPaths(io_val, pwd=kwds.get("pwd"))
kwds[name] = list(map(quote_cmdlinearg, io_val.expand(dot=False)))
return sfmt.format(command, **kwds)
def format_command(dset, command, **kwds):
"""Plug in placeholders in `command`.
Parameters
----------
dset : Dataset
command : str or list
`kwds` is passed to the `format` call. `inputs` and `outputs` are converted
to GlobbedPaths if necessary.
Returns
-------
formatted command (str)
"""
command = normalize_command(command)
sfmt = SequenceFormatter()
for k, v in dset.config.items("datalad.run.substitutions"):
sub_key = k.replace("datalad.run.substitutions.", "")
if sub_key not in kwds:
kwds[sub_key] = v
for name in ["inputs", "outputs"]:
io_val = kwds.pop(name, None)
if not isinstance(io_val, GlobbedPaths):
io_val = GlobbedPaths(io_val, pwd=kwds.get("pwd"))
kwds[name] = list(map(shlex_quote, io_val.expand(dot=False)))
return sfmt.format(command, **kwds)
def test_globbedpaths_partial_matches(path):
gp = GlobbedPaths([op.join("?dir", "*.txt"), "*.txt"], pwd=path)
eq_(gp.expand_strict(), ["1.txt", "3.txt"])
expected_partial = ["adir" + op.sep, "bdir" + op.sep]
eq_(gp.partial_hits, expected_partial)
eq_(gp.expand(include_partial=True), expected_partial + ["1.txt", "3.txt"])
# Property expands if needed.
gp = GlobbedPaths([op.join("?dir", "*.txt")], pwd=path)
eq_(gp.partial_hits, expected_partial)
def test_globbedpaths(path):
dotdir = op.curdir + op.sep
for patterns, expected in [
(["1.txt", "2.dat"], {"1.txt", "2.dat"}),
([dotdir + "1.txt", "2.dat"], {dotdir + "1.txt", "2.dat"}),
(["*.txt", "*.dat"], {"1.txt", "2.dat", u"bβ.dat", "3.txt"}),
([dotdir + "*.txt",
"*.dat"], {dotdir + "1.txt", "2.dat", u"bβ.dat", dotdir + "3.txt"}),
(["subdir/*.txt"], {"subdir/1.txt", "subdir/2.txt"}),
([dotdir + "subdir/*.txt"],
{dotdir + p
for p in ["subdir/1.txt", "subdir/2.txt"]}),
(["*.txt"], {"1.txt", "3.txt"})
]:
gp = GlobbedPaths(patterns, pwd=path)
eq_(set(gp.expand()), expected)
eq_(set(gp.expand(full=True)), {op.join(path, p) for p in expected})
pardir = op.pardir + op.sep
subdir_path = op.join(path, "subdir")
for patterns, expected in [
(["*.txt"], {"1.txt", "2.txt"}),
([dotdir + "*.txt"], {dotdir + p
for p in ["1.txt", "2.txt"]}),
([pardir + "*.txt"], {pardir + p
for p in ["1.txt", "3.txt"]}),
([dotdir + pardir + "*.txt"],
{dotdir + pardir + p
for p in ["1.txt", "3.txt"]}), (["subdir/"], {"subdir/"})
]:
gp = GlobbedPaths(patterns, pwd=subdir_path)
eq_(set(gp.expand()), expected)
eq_(set(gp.expand(full=True)),
{op.join(subdir_path, p)
for p in expected})
# Full patterns still get returned as relative to pwd.
gp = GlobbedPaths([op.join(path, "*.dat")], pwd=path)
eq_(gp.expand(), ["2.dat", u"bβ.dat"])
# "." gets special treatment.
gp = GlobbedPaths([".", "*.dat"], pwd=path)
eq_(set(gp.expand()), {"2.dat", u"bβ.dat", "."})
eq_(gp.expand(dot=False), ["2.dat", u"bβ.dat"])
gp = GlobbedPaths(["."], pwd=path, expand=False)
eq_(gp.expand(), ["."])
eq_(gp.paths, ["."])
# We can the glob outputs.
glob_results = {"z": "z", "a": ["x", "d", "b"]}
with patch('glob.glob', glob_results.get):
gp = GlobbedPaths(["z", "a"])
eq_(gp.expand(), ["z", "b", "d", "x"])
# glob expansion for paths property is determined by expand argument.
for expand, expected in [(True, ["2.dat", u"bβ.dat"]), (False, ["*.dat"])]:
gp = GlobbedPaths(["*.dat"], pwd=path, expand=expand)
eq_(gp.paths, expected)
with swallow_logs(new_level=logging.DEBUG) as cml:
GlobbedPaths(["not here"], pwd=path).expand()
assert_in("No matching files found for 'not here'", cml.out)
def run_command(cmd,
dataset=None,
inputs=None,
outputs=None,
expand=None,
explicit=False,
message=None,
sidecar=None,
extra_info=None,
rerun_info=None,
extra_inputs=None,
rerun_outputs=None,
inject=False):
"""Run `cmd` in `dataset` and record the results.
`Run.__call__` is a simple wrapper over this function. Aside from backward
compatibility kludges, the only difference is that `Run.__call__` doesn't
expose all the parameters of this function. The unexposed parameters are
listed below.
Parameters
----------
extra_info : dict, optional
Additional information to dump with the json run record. Any value
given here will take precedence over the standard run key. Warning: To
avoid collisions with future keys added by `run`, callers should try to
use fairly specific key names and are encouraged to nest fields under a
top-level "namespace" key (e.g., the project or extension name).
rerun_info : dict, optional
Record from a previous run. This is used internally by `rerun`.
extra_inputs : list, optional
Inputs to use in addition to those specified by `inputs`. Unlike
`inputs`, these will not be injected into the {inputs} format field.
rerun_outputs : list, optional
Outputs, in addition to those in `outputs`, determined automatically
from a previous run. This is used internally by `rerun`.
inject : bool, optional
Record results as if a command was run, skipping input and output
preparation and command execution. In this mode, the caller is
responsible for ensuring that the state of the working tree is
appropriate for recording the command's results.
Yields
------
Result records for the run.
"""
if not cmd:
lgr.warning("No command given")
return
rel_pwd = rerun_info.get('pwd') if rerun_info else None
if rel_pwd and dataset:
# recording is relative to the dataset
pwd = normpath(opj(dataset.path, rel_pwd))
rel_pwd = relpath(pwd, dataset.path)
else:
pwd, rel_pwd = get_command_pwds(dataset)
ds = require_dataset(dataset,
check_installed=True,
purpose='tracking outcomes of a command')
ds_path = ds.path
lgr.debug('tracking command output underneath %s', ds)
if not (rerun_info or inject): # Rerun already takes care of this.
# For explicit=True, we probably want to check whether any inputs have
# modifications. However, we can't just do is_dirty(..., path=inputs)
# because we need to consider subdatasets and untracked files.
# MIH: is_dirty() is gone, but status() can do all of the above!
if not explicit and ds.repo.dirty:
yield get_status_dict(
'run',
ds=ds,
status='impossible',
message=(
'clean dataset required to detect changes from command; '
'use `datalad status` to inspect unsaved changes'))
return
cmd = normalize_command(cmd)
inputs = GlobbedPaths(inputs, pwd=pwd, expand=expand in ["inputs", "both"])
extra_inputs = GlobbedPaths(
extra_inputs,
pwd=pwd,
# Follow same expansion rules as `inputs`.
expand=expand in ["inputs", "both"])
outputs = GlobbedPaths(outputs,
pwd=pwd,
expand=expand in ["outputs", "both"])
# ATTN: For correct path handling, all dataset commands call should be
# unbound. They should (1) receive a string dataset argument, (2) receive
# relative paths, and (3) happen within a chpwd(pwd) context.
if not inject:
with chpwd(pwd):
for res in prepare_inputs(ds_path, inputs, extra_inputs):
yield res
if outputs:
for res in _install_and_reglob(ds_path, outputs):
yield res
for res in _unlock_or_remove(ds_path, outputs.expand()):
yield res
if rerun_outputs is not None:
for res in _unlock_or_remove(ds_path, rerun_outputs):
yield res
else:
# If an inject=True caller wants to override the exit code, they can do
# so in extra_info.
cmd_exitcode = 0
exc = None
try:
cmd_expanded = format_command(
ds,
cmd,
pwd=pwd,
dspath=ds_path,
# Check if the command contains "{tmpdir}" to avoid creating an
# unnecessary temporary directory in most but not all cases.
tmpdir=mkdtemp(prefix="datalad-run-") if "{tmpdir}" in cmd else "",
inputs=inputs,
outputs=outputs)
except KeyError as exc:
yield get_status_dict(
'run',
ds=ds,
status='impossible',
message=('command has an unrecognized placeholder: %s', exc))
return
if not inject:
cmd_exitcode, exc = _execute_command(
cmd_expanded,
pwd,
expected_exit=rerun_info.get("exit", 0) if rerun_info else None)
# amend commit message with `run` info:
# - pwd if inside the dataset
# - the command itself
# - exit code of the command
run_info = {
'cmd': cmd,
'exit': cmd_exitcode,
'chain': rerun_info["chain"] if rerun_info else [],
'inputs': inputs.paths,
'extra_inputs': extra_inputs.paths,
'outputs': outputs.paths,
}
if rel_pwd is not None:
# only when inside the dataset to not leak information
run_info['pwd'] = rel_pwd
if ds.id:
run_info["dsid"] = ds.id
if extra_info:
run_info.update(extra_info)
record = json.dumps(run_info, indent=1, sort_keys=True, ensure_ascii=False)
if sidecar is None:
use_sidecar = ds.config.get('datalad.run.record-sidecar',
default=False)
# If ConfigManager gets the ability to say "return single value",
# update this code to use that.
if isinstance(use_sidecar, tuple):
# Use same precedence as 'git config'.
use_sidecar = use_sidecar[-1]
use_sidecar = anything2bool(use_sidecar)
else:
use_sidecar = sidecar
if use_sidecar:
# record ID is hash of record itself
from hashlib import md5
record_id = md5(record.encode('utf-8')).hexdigest()
record_dir = ds.config.get('datalad.run.record-directory',
default=op.join('.datalad', 'runinfo'))
record_path = op.join(ds_path, record_dir, record_id)
if not op.lexists(record_path):
# go for compression, even for minimal records not much difference, despite offset cost
# wrap in list -- there is just one record
dump2stream([run_info], record_path, compressed=True)
# compose commit message
msg = u"""\
[DATALAD RUNCMD] {}
=== Do not change lines below ===
{}
^^^ Do not change lines above ^^^
"""
msg = msg.format(
message if message is not None else _format_cmd_shorty(cmd_expanded),
'"{}"'.format(record_id) if use_sidecar else record)
outputs_to_save = outputs.expand() if explicit else None
do_save = outputs_to_save is None or outputs_to_save
if not rerun_info and cmd_exitcode:
if do_save:
repo = ds.repo
msg_path = relpath(opj(str(repo.dot_git), "COMMIT_EDITMSG"))
with open(msg_path, "wb") as ofh:
ofh.write(assure_bytes(msg))
lgr.info(
"The command had a non-zero exit code. "
"If this is expected, you can save the changes with "
"'datalad save -d . -r -F %s'", msg_path)
raise exc
elif do_save:
with chpwd(pwd):
for r in Save.__call__(dataset=ds_path,
path=outputs_to_save,
recursive=True,
message=msg,
return_type='generator'):
yield r
def run_command(cmd, dataset=None, inputs=None, outputs=None, expand=None,
explicit=False, message=None, sidecar=None,
extra_info=None,
rerun_info=None,
extra_inputs=None,
rerun_outputs=None,
inject=False,
saver=_save_outputs):
"""Run `cmd` in `dataset` and record the results.
`Run.__call__` is a simple wrapper over this function. Aside from backward
compatibility kludges, the only difference is that `Run.__call__` doesn't
expose all the parameters of this function. The unexposed parameters are
listed below.
Parameters
----------
extra_info : dict, optional
Additional information to dump with the json run record. Any value
given here will take precedence over the standard run key. Warning: To
avoid collisions with future keys added by `run`, callers should try to
use fairly specific key names and are encouraged to nest fields under a
top-level "namespace" key (e.g., the project or extension name).
rerun_info : dict, optional
Record from a previous run. This is used internally by `rerun`.
extra_inputs : list, optional
Inputs to use in addition to those specified by `inputs`. Unlike
`inputs`, these will not be injected into the {inputs} format field.
rerun_outputs : list, optional
Outputs, in addition to those in `outputs`, determined automatically
from a previous run. This is used internally by `rerun`.
inject : bool, optional
Record results as if a command was run, skipping input and output
preparation and command execution. In this mode, the caller is
responsible for ensuring that the state of the working tree is
appropriate for recording the command's results.
saver : callable, optional
Must take a dataset instance, a list of paths to save, and a
message string as arguments and must record any changes done
to any content matching an entry in the path list. Must yield
result dictionaries as a generator.
Yields
------
Result records for the run.
"""
if not cmd:
lgr.warning("No command given")
return
rel_pwd = rerun_info.get('pwd') if rerun_info else None
if rel_pwd and dataset:
# recording is relative to the dataset
pwd = normpath(opj(dataset.path, rel_pwd))
rel_pwd = relpath(pwd, dataset.path)
else:
pwd, rel_pwd = get_command_pwds(dataset)
ds = require_dataset(
dataset, check_installed=True,
purpose='tracking outcomes of a command')
# not needed ATM
#refds_path = ds.path
lgr.debug('tracking command output underneath %s', ds)
if not (rerun_info or inject): # Rerun already takes care of this.
# For explicit=True, we probably want to check whether any inputs have
# modifications. However, we can't just do is_dirty(..., path=inputs)
# because we need to consider subdatasets and untracked files.
if not explicit and ds.repo.dirty:
yield get_status_dict(
'run',
ds=ds,
status='impossible',
message=('unsaved modifications present, '
'cannot detect changes by command'))
return
cmd = normalize_command(cmd)
inputs = GlobbedPaths(inputs, pwd=pwd,
expand=expand in ["inputs", "both"])
extra_inputs = GlobbedPaths(extra_inputs, pwd=pwd,
# Follow same expansion rules as `inputs`.
expand=expand in ["inputs", "both"])
outputs = GlobbedPaths(outputs, pwd=pwd,
expand=expand in ["outputs", "both"])
if not inject:
for res in prepare_inputs(ds, inputs, extra_inputs):
yield res
if outputs:
for res in _install_and_reglob(ds, outputs):
yield res
for res in _unlock_or_remove(ds, outputs.expand(full=True)):
yield res
if rerun_outputs is not None:
# These are files we need to unlock/remove for a rerun that aren't
# included in the explicit outputs. Unlike inputs/outputs, these are
# full paths, so we can pass them directly to unlock.
for res in _unlock_or_remove(ds, rerun_outputs):
yield res
else:
# If an inject=True caller wants to override the exit code, they can do
# so in extra_info.
cmd_exitcode = 0
exc = None
try:
cmd_expanded = format_command(ds, cmd,
pwd=pwd,
dspath=ds.path,
inputs=inputs,
outputs=outputs)
except KeyError as exc:
yield get_status_dict(
'run',
ds=ds,
status='impossible',
message=('command has an unrecognized placeholder: %s',
exc))
return
if not inject:
cmd_exitcode, exc = _execute_command(
cmd_expanded, pwd,
expected_exit=rerun_info.get("exit", 0) if rerun_info else None)
# amend commit message with `run` info:
# - pwd if inside the dataset
# - the command itself
# - exit code of the command
run_info = {
'cmd': cmd,
'exit': cmd_exitcode,
'chain': rerun_info["chain"] if rerun_info else [],
'inputs': inputs.paths,
'extra_inputs': extra_inputs.paths,
'outputs': outputs.paths,
}
if rel_pwd is not None:
# only when inside the dataset to not leak information
run_info['pwd'] = rel_pwd
if ds.id:
run_info["dsid"] = ds.id
if extra_info:
run_info.update(extra_info)
record = json.dumps(run_info, indent=1, sort_keys=True, ensure_ascii=False)
if sidecar is None:
use_sidecar = ds.config.get('datalad.run.record-sidecar', default=False)
# If ConfigManager gets the ability to say "return single value",
# update this code to use that.
if isinstance(use_sidecar, tuple):
# Use same precedence as 'git config'.
use_sidecar = use_sidecar[-1]
use_sidecar = anything2bool(use_sidecar)
else:
use_sidecar = sidecar
if use_sidecar:
# record ID is hash of record itself
from hashlib import md5
record_id = md5(record.encode('utf-8')).hexdigest()
record_dir = ds.config.get('datalad.run.record-directory', default=op.join('.datalad', 'runinfo'))
record_path = op.join(ds.path, record_dir, record_id)
if not op.lexists(record_path):
# go for compression, even for minimal records not much difference, despite offset cost
# wrap in list -- there is just one record
dump2stream([run_info], record_path, compressed=True)
# compose commit message
msg = u"""\
[DATALAD RUNCMD] {}
=== Do not change lines below ===
{}
^^^ Do not change lines above ^^^
"""
msg = msg.format(
message if message is not None else _format_cmd_shorty(cmd_expanded),
'"{}"'.format(record_id) if use_sidecar else record)
outputs_to_save = outputs.expand(full=True) if explicit else '.'
if not rerun_info and cmd_exitcode:
if outputs_to_save:
msg_path = relpath(opj(ds.repo.path, ds.repo.get_git_dir(ds.repo),
"COMMIT_EDITMSG"))
with open(msg_path, "wb") as ofh:
ofh.write(assure_bytes(msg))
lgr.info("The command had a non-zero exit code. "
"If this is expected, you can save the changes with "
"'datalad add -d . -r -F %s .'",
msg_path)
raise exc
elif outputs_to_save:
for r in saver(ds, outputs_to_save, msg):
yield r