def normalize_command(command):
"""Convert `command` to the string representation.
"""
if isinstance(command, list):
command = list(map(assure_unicode, command))
if len(command) == 1 and command[0] != "--":
# This is either a quoted compound shell command or a simple
# one-item command. Pass it as is.
#
# FIXME: This covers the predominant command-line case, but, for
# Python API callers, it means values like ["./script with spaces"]
# requires additional string-like escaping, which is inconsistent
# with the handling of multi-item lists (and subprocess's
# handling). Once we have a way to detect "running from Python API"
# (discussed in gh-2986), update this.
command = command[0]
else:
if command and command[0] == "--":
# Strip disambiguation marker. Note: "running from Python API"
# FIXME from below applies to this too.
command = command[1:]
command = join_cmdline(command)
else:
command = assure_unicode(command)
return command
def _format_cmd_shorty(cmd):
"""Get short string representation from a cmd argument list"""
cmd_shorty = (join_cmdline(cmd) if isinstance(cmd, list) else cmd)
cmd_shorty = u'{}{}'.format(
cmd_shorty[:40],
'...' if len(cmd_shorty) > 40 else '')
return cmd_shorty
def test_splitjoin_cmdline():
# Do full round trip on a number of tricky samples
for args in (
['cmd', '-o1', 'simple'],
['c o', r'\m', ''],
['c o', ' '],
):
cmdline = join_cmdline(args)
assert isinstance(cmdline, str)
eq_(split_cmdline(cmdline), args)
# assure that there is no needless quoting
if on_windows:
# in quote_cmdlinearg we always quote on Windows
eq_(join_cmdline(['abc', 'def']), '"abc" "def"')
else:
eq_(join_cmdline(['abc', 'def']), 'abc def')
def to_str(self, include_output=True):
from datalad.utils import (
ensure_unicode,
join_cmdline,
)
to_str = "{}: ".format(self.__class__.__name__)
cmd = self.cmd
if cmd:
to_str += "'{}'".format(
# go for a compact, normal looking, properly quoted
# command rendering if the command is in list form
join_cmdline(cmd) if isinstance(cmd, list) else cmd)
if self.code:
to_str += " failed with exitcode {}".format(self.code)
if self.cwd:
# only if not under standard PWD
to_str += " under {}".format(self.cwd)
if self.msg:
# typically a command error has no specific idea
to_str += " [{}]".format(ensure_unicode(self.msg))
if not include_output:
return to_str
if self.stdout:
to_str += " [out: '{}']".format(
ensure_unicode(self.stdout).strip())
if self.stderr:
to_str += " [err: '{}']".format(
ensure_unicode(self.stderr).strip())
if self.kwargs:
to_str += " [info keys: {}]".format(', '.join(self.kwargs.keys()))
return to_str
def compress_files(files, archive, path=None, overwrite=True):
"""Compress `files` into an `archive` file
Parameters
----------
files : list of str
archive : str
path : str
Alternative directory under which compressor will be invoked, to e.g.
take into account relative paths of files and/or archive
overwrite : bool
Whether to allow overwriting the target archive file if one already exists
"""
runner = Runner(cwd=path)
apath = Path(archive)
if apath.exists():
if overwrite:
apath.unlink()
else:
raise ValueError(
'Target archive {} already exists and overwrite is forbidden'.
format(apath))
suffixes = _normalize_fname_suffixes(apath.suffixes)
if len(suffixes) > 1 and suffixes[-2] == '.tar':
cmd = '7z u .tar -so -- {} | 7z u -si -- {}'.format(
join_cmdline(files),
quote_cmdlinearg(str(apath)),
)
else:
cmd = ['7z', 'u', str(apath), '--'] + files
runner.run(cmd, protocol=KillOutput)
def to_str(self, include_output=True):
from datalad.utils import (
ensure_unicode,
join_cmdline,
)
to_str = "{}: ".format(self.__class__.__name__)
cmd = self.cmd
if cmd:
to_str += "'{}'".format(
# go for a compact, normal looking, properly quoted
# command rendering if the command is in list form
join_cmdline(cmd) if isinstance(cmd, list) else cmd)
if self.code:
to_str += " failed with exitcode {}".format(self.code)
if self.cwd:
# only if not under standard PWD
to_str += " under {}".format(self.cwd)
if self.msg:
# typically a command error has no specific idea
to_str += " [{}]".format(ensure_unicode(self.msg))
if not include_output:
return to_str
if self.stdout:
to_str += " [out: '{}']".format(
ensure_unicode(self.stdout).strip())
if self.stderr:
to_str += " [err: '{}']".format(
ensure_unicode(self.stderr).strip())
if self.kwargs:
if 'stdout_json' in self.kwargs:
src_keys = ('note', 'error-messages')
from datalad.utils import unique
json_errors = unique('; '.join(
str(m[key]) for key in src_keys
if m.get(key)) for m in self.kwargs['stdout_json'] if any(
m.get(k) for k in src_keys))
if json_errors:
to_str += " [errors from JSON records: {}]".format(
json_errors)
to_str += " [info keys: {}]".format(', '.join(self.kwargs.keys()))
return to_str
def __call__(spec=None, *, dataset=None, discover=False, help_proc=False):
if not spec and not discover:
raise InsufficientArgumentsError(
'requires at least a procedure name')
if help_proc and not spec:
raise InsufficientArgumentsError('requires a procedure name')
try:
ds = require_dataset(dataset,
check_installed=False,
purpose='run a procedure')
except NoDatasetFound:
ds = None
if discover:
# specific path of procedures that were already reported
reported = set()
# specific names of procedure for which an active one has been
# found
active = set()
for m, cmd_name, cmd_tmpl, cmd_help in \
_get_procedure_implementation('*', ds=ds):
if m in reported:
continue
ex = _guess_exec(m)
# configured template (call-format string) takes precedence:
if cmd_tmpl:
ex['template'] = cmd_tmpl
if ex['state'] is None:
# doesn't seem like a match
lgr.debug("%s does not look like a procedure, ignored.", m)
continue
state = 'overridden' if cmd_name in active else ex['state']
message = ex['type'] if ex['type'] else 'unknown type'
message += ' ({})'.format(
state) if state != 'executable' else ''
res = get_status_dict(action='discover_procedure',
path=m,
type='file',
logger=lgr,
refds=ds.path if ds else None,
status='ok',
state=state,
procedure_name=cmd_name,
procedure_type=ex['type'],
procedure_callfmt=ex['template'],
procedure_help=cmd_help,
message=message)
reported.add(m)
if state == 'executable':
active.add(cmd_name)
yield res
return
if isinstance(spec, dict):
# Skip getting procedure implementation if called with a
# dictionary (presumably coming from --discover)
procedure_file = spec['path']
cmd_name = spec['procedure_name']
cmd_tmpl = spec['procedure_callfmt']
cmd_help = spec['procedure_help']
name = cmd_name
args = []
else:
if not isinstance(spec, (tuple, list)):
# maybe coming from config
spec = split_cmdline(spec)
name = spec[0]
args = spec[1:]
try:
# get the first match an run with it
procedure_file, cmd_name, cmd_tmpl, cmd_help = \
next(_get_procedure_implementation(name, ds=ds))
except StopIteration:
raise ValueError("Cannot find procedure with name '%s'" % name)
ex = _guess_exec(procedure_file)
# configured template (call-format string) takes precedence:
if cmd_tmpl:
ex['template'] = cmd_tmpl
if help_proc:
if cmd_help:
res = get_status_dict(action='procedure_help',
path=procedure_file,
type='file',
logger=lgr,
refds=ds.path if ds else None,
status='ok',
state=ex['state'],
procedure_name=cmd_name,
procedure_type=ex['type'],
procedure_callfmt=ex['template'],
message=cmd_help)
else:
res = get_status_dict(action='procedure_help',
path=procedure_file,
type='file',
logger=lgr,
refds=ds.path if ds else None,
status='impossible',
state=ex['state'],
procedure_name=cmd_name,
procedure_type=ex['type'],
procedure_callfmt=ex['template'],
message="No help available for '%s'" %
name)
yield res
return
if not ex['template']:
raise ValueError("No idea how to execute procedure %s. "
"Missing 'execute' permissions?" % procedure_file)
cmd = ex['template'].format(
script=guard_for_format(quote_cmdlinearg(procedure_file)),
ds=guard_for_format(quote_cmdlinearg(ds.path)) if ds else '',
args=join_cmdline(args) if args else '')
lgr.info(u"Running procedure %s", name)
lgr.debug(u'Full procedure command: %r', cmd)
for r in Run.__call__(
cmd=cmd,
dataset=ds,
explicit=True,
inputs=None,
outputs=None,
# pass through here
on_failure='ignore',
return_type='generator',
result_renderer='disabled'):
yield r
if ds:
# the procedure ran and we have to anticipate that it might have
# changed the dataset config, so we need to trigger an unforced
# reload.
# we have to do this despite "being done here", because
# run_procedure() runs in the same process and reuses dataset (config
# manager) instances, and the next interaction with a dataset should
# be able to count on an up-to-date config
ds.config.reload()
def __call__(spec=None, dataset=None, discover=False, help_proc=False):
if not spec and not discover:
raise InsufficientArgumentsError(
'requires at least a procedure name')
if help_proc and not spec:
raise InsufficientArgumentsError('requires a procedure name')
try:
ds = require_dataset(dataset,
check_installed=False,
purpose='run a procedure')
except NoDatasetFound:
ds = None
if discover:
# specific path of procedures that were already reported
reported = set()
# specific names of procedure for which an active one has been
# found
active = set()
for m, cmd_name, cmd_tmpl, cmd_help in \
_get_procedure_implementation('*', ds=ds):
if m in reported:
continue
ex = _guess_exec(m)
# configured template (call-format string) takes precedence:
if cmd_tmpl:
ex['template'] = cmd_tmpl
if ex['state'] is None:
# doesn't seem like a match
lgr.debug("%s does not look like a procedure, ignored.", m)
continue
state = 'overridden' if cmd_name in active else ex['state']
message = ex['type'] if ex['type'] else 'unknown type'
message += ' ({})'.format(
state) if state != 'executable' else ''
res = get_status_dict(action='discover_procedure',
path=m,
type='file',
logger=lgr,
refds=ds.path if ds else None,
status='ok',
state=state,
procedure_name=cmd_name,
procedure_type=ex['type'],
procedure_callfmt=ex['template'],
procedure_help=cmd_help,
message=message)
reported.add(m)
if state == 'executable':
active.add(cmd_name)
yield res
return
if not isinstance(spec, (tuple, list)):
# maybe coming from config
spec = split_cmdline(spec)
name = spec[0]
args = spec[1:]
try:
# get the first match an run with it
procedure_file, cmd_name, cmd_tmpl, cmd_help = \
next(_get_procedure_implementation(name, ds=ds))
except StopIteration:
res = get_status_dict(
action='run_procedure',
# TODO: Default renderer requires a key "path" to exist.
# Doesn't make a lot of sense in this case
path=name,
logger=lgr,
refds=ds.path if ds else None,
status='impossible',
message="Cannot find procedure with name '%s'" % name)
yield res
return
ex = _guess_exec(procedure_file)
# configured template (call-format string) takes precedence:
if cmd_tmpl:
ex['template'] = cmd_tmpl
if help_proc:
if cmd_help:
res = get_status_dict(action='procedure_help',
path=procedure_file,
type='file',
logger=lgr,
refds=ds.path if ds else None,
status='ok',
state=ex['state'],
procedure_name=cmd_name,
procedure_type=ex['type'],
procedure_callfmt=ex['template'],
message=cmd_help)
else:
res = get_status_dict(action='procedure_help',
path=procedure_file,
type='file',
logger=lgr,
refds=ds.path if ds else None,
status='impossible',
state=ex['state'],
procedure_name=cmd_name,
procedure_type=ex['type'],
procedure_callfmt=ex['template'],
message="No help available for '%s'" %
name)
yield res
return
if not ex['template']:
raise ValueError("No idea how to execute procedure %s. "
"Missing 'execute' permissions?" % procedure_file)
cmd = ex['template'].format(script=quote_cmdlinearg(procedure_file),
ds=quote_cmdlinearg(ds.path) if ds else '',
args=join_cmdline(args) if args else '')
lgr.info(u"Running procedure %s", name)
lgr.debug(u'Full procedure command: %r', cmd)
for r in Run.__call__(
cmd=cmd,
dataset=ds,
explicit=True,
inputs=None,
outputs=None,
# pass through here
on_failure='ignore',
return_type='generator'):
yield r
