def normalize_command(command): """Convert `command` to the string representation. """ if isinstance(command, list): command = list(map(assure_unicode, command)) if len(command) == 1 and command[0] != "--": # This is either a quoted compound shell command or a simple # one-item command. Pass it as is. # # FIXME: This covers the predominant command-line case, but, for # Python API callers, it means values like ["./script with spaces"] # requires additional string-like escaping, which is inconsistent # with the handling of multi-item lists (and subprocess's # handling). Once we have a way to detect "running from Python API" # (discussed in gh-2986), update this. command = command[0] else: if command and command[0] == "--": # Strip disambiguation marker. Note: "running from Python API" # FIXME from below applies to this too. command = command[1:] command = join_cmdline(command) else: command = assure_unicode(command) return command
def _format_cmd_shorty(cmd): """Get short string representation from a cmd argument list""" cmd_shorty = (join_cmdline(cmd) if isinstance(cmd, list) else cmd) cmd_shorty = u'{}{}'.format( cmd_shorty[:40], '...' if len(cmd_shorty) > 40 else '') return cmd_shorty
def test_splitjoin_cmdline(): # Do full round trip on a number of tricky samples for args in ( ['cmd', '-o1', 'simple'], ['c o', r'\m', ''], ['c o', ' '], ): cmdline = join_cmdline(args) assert isinstance(cmdline, str) eq_(split_cmdline(cmdline), args) # assure that there is no needless quoting if on_windows: # in quote_cmdlinearg we always quote on Windows eq_(join_cmdline(['abc', 'def']), '"abc" "def"') else: eq_(join_cmdline(['abc', 'def']), 'abc def')
def to_str(self, include_output=True): from datalad.utils import ( ensure_unicode, join_cmdline, ) to_str = "{}: ".format(self.__class__.__name__) cmd = self.cmd if cmd: to_str += "'{}'".format( # go for a compact, normal looking, properly quoted # command rendering if the command is in list form join_cmdline(cmd) if isinstance(cmd, list) else cmd) if self.code: to_str += " failed with exitcode {}".format(self.code) if self.cwd: # only if not under standard PWD to_str += " under {}".format(self.cwd) if self.msg: # typically a command error has no specific idea to_str += " [{}]".format(ensure_unicode(self.msg)) if not include_output: return to_str if self.stdout: to_str += " [out: '{}']".format( ensure_unicode(self.stdout).strip()) if self.stderr: to_str += " [err: '{}']".format( ensure_unicode(self.stderr).strip()) if self.kwargs: to_str += " [info keys: {}]".format(', '.join(self.kwargs.keys())) return to_str
def compress_files(files, archive, path=None, overwrite=True): """Compress `files` into an `archive` file Parameters ---------- files : list of str archive : str path : str Alternative directory under which compressor will be invoked, to e.g. take into account relative paths of files and/or archive overwrite : bool Whether to allow overwriting the target archive file if one already exists """ runner = Runner(cwd=path) apath = Path(archive) if apath.exists(): if overwrite: apath.unlink() else: raise ValueError( 'Target archive {} already exists and overwrite is forbidden'. format(apath)) suffixes = _normalize_fname_suffixes(apath.suffixes) if len(suffixes) > 1 and suffixes[-2] == '.tar': cmd = '7z u .tar -so -- {} | 7z u -si -- {}'.format( join_cmdline(files), quote_cmdlinearg(str(apath)), ) else: cmd = ['7z', 'u', str(apath), '--'] + files runner.run(cmd, protocol=KillOutput)
def to_str(self, include_output=True): from datalad.utils import ( ensure_unicode, join_cmdline, ) to_str = "{}: ".format(self.__class__.__name__) cmd = self.cmd if cmd: to_str += "'{}'".format( # go for a compact, normal looking, properly quoted # command rendering if the command is in list form join_cmdline(cmd) if isinstance(cmd, list) else cmd) if self.code: to_str += " failed with exitcode {}".format(self.code) if self.cwd: # only if not under standard PWD to_str += " under {}".format(self.cwd) if self.msg: # typically a command error has no specific idea to_str += " [{}]".format(ensure_unicode(self.msg)) if not include_output: return to_str if self.stdout: to_str += " [out: '{}']".format( ensure_unicode(self.stdout).strip()) if self.stderr: to_str += " [err: '{}']".format( ensure_unicode(self.stderr).strip()) if self.kwargs: if 'stdout_json' in self.kwargs: src_keys = ('note', 'error-messages') from datalad.utils import unique json_errors = unique('; '.join( str(m[key]) for key in src_keys if m.get(key)) for m in self.kwargs['stdout_json'] if any( m.get(k) for k in src_keys)) if json_errors: to_str += " [errors from JSON records: {}]".format( json_errors) to_str += " [info keys: {}]".format(', '.join(self.kwargs.keys())) return to_str
def __call__(spec=None, *, dataset=None, discover=False, help_proc=False): if not spec and not discover: raise InsufficientArgumentsError( 'requires at least a procedure name') if help_proc and not spec: raise InsufficientArgumentsError('requires a procedure name') try: ds = require_dataset(dataset, check_installed=False, purpose='run a procedure') except NoDatasetFound: ds = None if discover: # specific path of procedures that were already reported reported = set() # specific names of procedure for which an active one has been # found active = set() for m, cmd_name, cmd_tmpl, cmd_help in \ _get_procedure_implementation('*', ds=ds): if m in reported: continue ex = _guess_exec(m) # configured template (call-format string) takes precedence: if cmd_tmpl: ex['template'] = cmd_tmpl if ex['state'] is None: # doesn't seem like a match lgr.debug("%s does not look like a procedure, ignored.", m) continue state = 'overridden' if cmd_name in active else ex['state'] message = ex['type'] if ex['type'] else 'unknown type' message += ' ({})'.format( state) if state != 'executable' else '' res = get_status_dict(action='discover_procedure', path=m, type='file', logger=lgr, refds=ds.path if ds else None, status='ok', state=state, procedure_name=cmd_name, procedure_type=ex['type'], procedure_callfmt=ex['template'], procedure_help=cmd_help, message=message) reported.add(m) if state == 'executable': active.add(cmd_name) yield res return if isinstance(spec, dict): # Skip getting procedure implementation if called with a # dictionary (presumably coming from --discover) procedure_file = spec['path'] cmd_name = spec['procedure_name'] cmd_tmpl = spec['procedure_callfmt'] cmd_help = spec['procedure_help'] name = cmd_name args = [] else: if not isinstance(spec, (tuple, list)): # maybe coming from config spec = split_cmdline(spec) name = spec[0] args = spec[1:] try: # get the first match an run with it procedure_file, cmd_name, cmd_tmpl, cmd_help = \ next(_get_procedure_implementation(name, ds=ds)) except StopIteration: raise ValueError("Cannot find procedure with name '%s'" % name) ex = _guess_exec(procedure_file) # configured template (call-format string) takes precedence: if cmd_tmpl: ex['template'] = cmd_tmpl if help_proc: if cmd_help: res = get_status_dict(action='procedure_help', path=procedure_file, type='file', logger=lgr, refds=ds.path if ds else None, status='ok', state=ex['state'], procedure_name=cmd_name, procedure_type=ex['type'], procedure_callfmt=ex['template'], message=cmd_help) else: res = get_status_dict(action='procedure_help', path=procedure_file, type='file', logger=lgr, refds=ds.path if ds else None, status='impossible', state=ex['state'], procedure_name=cmd_name, procedure_type=ex['type'], procedure_callfmt=ex['template'], message="No help available for '%s'" % name) yield res return if not ex['template']: raise ValueError("No idea how to execute procedure %s. " "Missing 'execute' permissions?" % procedure_file) cmd = ex['template'].format( script=guard_for_format(quote_cmdlinearg(procedure_file)), ds=guard_for_format(quote_cmdlinearg(ds.path)) if ds else '', args=join_cmdline(args) if args else '') lgr.info(u"Running procedure %s", name) lgr.debug(u'Full procedure command: %r', cmd) for r in Run.__call__( cmd=cmd, dataset=ds, explicit=True, inputs=None, outputs=None, # pass through here on_failure='ignore', return_type='generator', result_renderer='disabled'): yield r if ds: # the procedure ran and we have to anticipate that it might have # changed the dataset config, so we need to trigger an unforced # reload. # we have to do this despite "being done here", because # run_procedure() runs in the same process and reuses dataset (config # manager) instances, and the next interaction with a dataset should # be able to count on an up-to-date config ds.config.reload()
def __call__(spec=None, dataset=None, discover=False, help_proc=False): if not spec and not discover: raise InsufficientArgumentsError( 'requires at least a procedure name') if help_proc and not spec: raise InsufficientArgumentsError('requires a procedure name') try: ds = require_dataset(dataset, check_installed=False, purpose='run a procedure') except NoDatasetFound: ds = None if discover: # specific path of procedures that were already reported reported = set() # specific names of procedure for which an active one has been # found active = set() for m, cmd_name, cmd_tmpl, cmd_help in \ _get_procedure_implementation('*', ds=ds): if m in reported: continue ex = _guess_exec(m) # configured template (call-format string) takes precedence: if cmd_tmpl: ex['template'] = cmd_tmpl if ex['state'] is None: # doesn't seem like a match lgr.debug("%s does not look like a procedure, ignored.", m) continue state = 'overridden' if cmd_name in active else ex['state'] message = ex['type'] if ex['type'] else 'unknown type' message += ' ({})'.format( state) if state != 'executable' else '' res = get_status_dict(action='discover_procedure', path=m, type='file', logger=lgr, refds=ds.path if ds else None, status='ok', state=state, procedure_name=cmd_name, procedure_type=ex['type'], procedure_callfmt=ex['template'], procedure_help=cmd_help, message=message) reported.add(m) if state == 'executable': active.add(cmd_name) yield res return if not isinstance(spec, (tuple, list)): # maybe coming from config spec = split_cmdline(spec) name = spec[0] args = spec[1:] try: # get the first match an run with it procedure_file, cmd_name, cmd_tmpl, cmd_help = \ next(_get_procedure_implementation(name, ds=ds)) except StopIteration: res = get_status_dict( action='run_procedure', # TODO: Default renderer requires a key "path" to exist. # Doesn't make a lot of sense in this case path=name, logger=lgr, refds=ds.path if ds else None, status='impossible', message="Cannot find procedure with name '%s'" % name) yield res return ex = _guess_exec(procedure_file) # configured template (call-format string) takes precedence: if cmd_tmpl: ex['template'] = cmd_tmpl if help_proc: if cmd_help: res = get_status_dict(action='procedure_help', path=procedure_file, type='file', logger=lgr, refds=ds.path if ds else None, status='ok', state=ex['state'], procedure_name=cmd_name, procedure_type=ex['type'], procedure_callfmt=ex['template'], message=cmd_help) else: res = get_status_dict(action='procedure_help', path=procedure_file, type='file', logger=lgr, refds=ds.path if ds else None, status='impossible', state=ex['state'], procedure_name=cmd_name, procedure_type=ex['type'], procedure_callfmt=ex['template'], message="No help available for '%s'" % name) yield res return if not ex['template']: raise ValueError("No idea how to execute procedure %s. " "Missing 'execute' permissions?" % procedure_file) cmd = ex['template'].format(script=quote_cmdlinearg(procedure_file), ds=quote_cmdlinearg(ds.path) if ds else '', args=join_cmdline(args) if args else '') lgr.info(u"Running procedure %s", name) lgr.debug(u'Full procedure command: %r', cmd) for r in Run.__call__( cmd=cmd, dataset=ds, explicit=True, inputs=None, outputs=None, # pass through here on_failure='ignore', return_type='generator'): yield r