def custom_result_renderer(res, **kwargs): # pragma: no cover
if not (res['status'] == 'ok' \
and res['action'] in ('status', 'diff') \
and res.get('state', None) != 'clean'):
# logging reported already
return
from datalad.ui import ui
# when to render relative paths:
# 1) if a dataset arg was given
# 2) if CWD is the refds
refds = res.get('refds', None)
refds = refds if kwargs.get('dataset', None) is not None \
or refds == os.getcwd() else None
# Note: We have to force unicode for res['path'] because
# interface.utils encodes it on py2 before passing it to
# custom_result_renderer().
path = assure_unicode(res['path']) if refds is None \
else text_type(ut.Path(res['path']).relative_to(refds))
type_ = res.get('type', res.get('type_src', ''))
max_len = len('untracked')
state = res.get('state', 'unknown')
ui.message(u'{fill}{state}: {path}{type_}'.format(
fill=' ' * max(0, max_len - len(state)),
state=ac.color_word(
state,
STATE_COLOR_MAP.get(res.get('state', 'unknown'))),
path=path,
type_=' ({})'.format(
ac.color_word(type_, ac.MAGENTA) if type_ else '')))
def test_color_word():
s = 'word'
green_s = '\033[1;32mword\033[0m'
for enabled in (True, False):
with patch('datalad.support.ansi_colors.color_enabled', lambda: enabled):
assert_equal(colors.color_word(s, colors.GREEN, force=True), green_s)
with patch('datalad.support.ansi_colors.color_enabled', lambda: True):
assert_equal(colors.color_word(s, colors.GREEN), green_s)
assert_equal(colors.color_word(s, colors.GREEN, force=False), green_s)
with patch('datalad.support.ansi_colors.color_enabled', lambda: False):
assert_equal(colors.color_word(s, colors.GREEN), s)
assert_equal(colors.color_word(s, colors.GREEN, force=False), s)
def _display_basic(res):
ui.message(ac.color_word("Dry run information", ac.MAGENTA))
def fmt_line(key, value, multiline=False):
return (" {key}:{sep}{value}".format(key=ac.color_word(key, ac.BOLD),
sep=os.linesep +
" " if multiline else " ",
value=value))
dry_run_info = res["dry_run_info"]
lines = [fmt_line("location", dry_run_info["pwd_full"])]
# TODO: Inputs and outputs could be pretty long. These may be worth
# truncating.
inputs = dry_run_info["inputs"]
if inputs:
lines.append(fmt_line("expanded inputs", inputs, multiline=True))
outputs = dry_run_info["outputs"]
if outputs:
lines.append(fmt_line("expanded outputs", outputs, multiline=True))
cmd = res["run_info"]["cmd"]
cmd_expanded = dry_run_info["cmd_expanded"]
lines.append(fmt_line("command", cmd, multiline=True))
if cmd != cmd_expanded:
lines.append(fmt_line("expanded command", cmd_expanded,
multiline=True))
ui.message(os.linesep.join(lines))
def custom_result_renderer(res, **kwargs):
if res["action"] != "containers":
default_result_renderer(res)
else:
ui.message("{name} -> {path}".format(
name=ac.color_word(res["name"], ac.MAGENTA),
path=op.relpath(res["path"], res["refds"])))
def default_result_renderer(res):
if res.get('status', None) != 'notneeded':
path = res.get('path', None)
path = ' {}'.format(path) if path else ''
ui.message('{action}({status}):{path}{type}{msg}'.format(
action=ac.color_word(res['action'], ac.BOLD),
status=ac.color_status(res['status']),
path=relpath(path, res['refds']) if path and res.get('refds') else path,
type=' ({})'.format(
ac.color_word(res['type'], ac.MAGENTA)
) if 'type' in res else '',
msg=' [{}]'.format(
res['message'][0] % res['message'][1:]
if isinstance(res['message'], tuple) else res[
'message'])
if res.get('message', None) else ''))
def custom_result_renderer(res, **kwargs):
from datalad.ui import ui
# should we attempt to remove an unknown sibling, complain like Git does
if res['status'] == 'notneeded' and res['action'] == 'remove-sibling':
ui.message('{warn}: No sibling "{name}" in dataset {path}'.format(
warn=ac.color_word('Warning', ac.LOG_LEVEL_COLORS['WARNING']),
**res))
return
if res['status'] != 'ok' or not res.get('action',
'').endswith('-sibling'):
generic_result_renderer(res)
return
path = op.relpath(res['path'], res['refds']) if res.get(
'refds', None) else res['path']
got_url = 'url' in res
spec = '{}{}{}{}'.format(res.get('url', ''), ' (' if got_url else '',
res.get('annex-externaltype', 'git'),
')' if got_url else '')
ui.message('{path}: {name}({with_annex}) [{spec}]'.format(
**dict(
res,
path=path,
# TODO report '+' for special remotes
with_annex='+' if 'annex-uuid' in res \
else ('-' if res.get('annex-ignore', None) else '?'),
spec=spec)))
def default_result_renderer(res):
if res.get('status', None) != 'notneeded':
ui.message('{action}({status}): {path}{type}{msg}'.format(
action=ac.color_word(res['action'], ac.BOLD),
status=ac.color_status(res['status']),
path=relpath(res['path'],
res['refds']) if res.get('refds', None) else res[
'path'],
type=' ({})'.format(
ac.color_word(res['type'], ac.MAGENTA)
) if 'type' in res else '',
msg=' [{}]'.format(
res['message'][0] % res['message'][1:]
if isinstance(res['message'], tuple) else res[
'message'])
if 'message' in res else ''))
def custom_result_summary_renderer(results,
action_summary): # pragma: more cover
render_action_summary(action_summary)
# report on any hints at the end
# get all unique hints
hints = set([r.get('hints', None) for r in results])
hints = [hint for hint in hints if hint is not None]
if hints:
from datalad.ui import ui
from datalad.support import ansi_colors
intro = ansi_colors.color_word("Hints: ", ansi_colors.YELLOW)
ui.message(intro)
[
ui.message("{}: {}".format(
ansi_colors.color_word(id + 1, ansi_colors.YELLOW), hint))
for id, hint in enumerate(hints)
]
def result_renderer_cmdline(res, cmdlineargs):
from datalad.ui import ui
if res is None:
res = []
format = cmdlineargs.format or 'custom'
if format == 'custom':
if cmdlineargs.report in ('*', ['*']) \
or cmdlineargs.report_matched \
or (cmdlineargs.report is not None
and len(cmdlineargs.report) > 1):
# multiline if multiple were requested and we need to disambiguate
ichr = jchr = '\n'
fmt = ' {k}: {v}'
else:
jchr = ', '
ichr = ' '
fmt = '{v}'
anything = False
for location, r in res:
# XXX Yarik thinks that Match should be replaced with actual path to the dataset
ui.message('{}{}{}{}'.format(
ansi_colors.color_word(location, ansi_colors.DATASET),
':' if r else '',
ichr,
jchr.join(
[
fmt.format(
k=ansi_colors.color_word(k, ansi_colors.FIELD),
v=pretty_bytes(r[k]))
for k in sorted(r)
])))
anything = True
if not anything:
ui.message("Nothing to report")
elif format == 'json':
import json
ui.message(json.dumps(list(map(itemgetter(1), res)), indent=2))
elif format == 'yaml':
import yaml
lgr.warning("yaml output support is not yet polished")
ui.message(yaml.safe_dump(list(map(itemgetter(1), res)),
allow_unicode=True))
def result_renderer_cmdline(res, cmdlineargs):
from datalad.ui import ui
if res is None:
res = []
format = cmdlineargs.format or 'custom'
if format == 'custom':
if cmdlineargs.report in ('*', ['*']) \
or cmdlineargs.report_matched \
or (cmdlineargs.report is not None
and len(cmdlineargs.report) > 1):
# multiline if multiple were requested and we need to disambiguate
ichr = jchr = '\n'
fmt = ' {k}: {v}'
else:
jchr = ', '
ichr = ' '
fmt = '{v}'
anything = False
for location, r in res:
# XXX Yarik thinks that Match should be replaced with actual path to the dataset
ui.message('{}{}{}{}'.format(
ansi_colors.color_word(location, ansi_colors.DATASET),
':' if r else '',
ichr,
jchr.join(
[
fmt.format(
k=ansi_colors.color_word(k, ansi_colors.FIELD),
v=pretty_bytes(r[k]))
for k in sorted(r)
])))
anything = True
if not anything:
ui.message("Nothing to report")
elif format == 'json':
import json
ui.message(json.dumps(list(map(itemgetter(1), res)), indent=2))
elif format == 'yaml':
import yaml
lgr.warning("yaml output support is not yet polished")
ui.message(yaml.safe_dump(list(map(itemgetter(1), res)),
allow_unicode=True))
def custom_result_renderer(res, **kwargs):
if res['status'] != 'ok' or not res.get('action', None) == 'metadata':
# logging complained about this already
return
# list the path, available metadata keys, and tags
path = op.relpath(res['path'],
res['refds']) if res.get('refds', None) else res['path']
meta = res.get('metadata', {})
ui.message('{path}{type}:{spacer}{meta}{tags}'.format(
path=ac.color_word(path, ac.BOLD),
type=' ({})'.format(
ac.color_word(res['type'], ac.MAGENTA)) if 'type' in res else '',
spacer=' ' if len([m for m in meta if m != 'tag']) else '',
meta=','.join(k for k in sorted(meta.keys())
if k not in ('tag', '@context', '@id'))
if meta else ' -' if 'metadata' in res else ' aggregated',
tags='' if 'tag' not in meta else ' [{}]'.format(
','.join(assure_list(meta['tag'])))))
def custom_result_renderer(res, **kwargs):
if res['status'] != 'ok' or not res.get('action', None) == 'metadata':
# logging complained about this already
return
# list the path, available metadata keys, and tags
path = relpath(res['path'], res['refds']) if res.get(
'refds', None) else res['path']
meta = res.get('metadata', {})
ui.message('{path}{type}:{spacer}{meta}{tags}'.format(
path=ac.color_word(path, ac.BOLD),
type=' ({})'.format(ac.color_word(res['type'], ac.MAGENTA))
if 'type' in res else '',
spacer=' ' if len([m for m in meta if m != 'tag']) else '',
meta=','.join(k for k in sorted(meta.keys())
if k not in ('tag', '@context', '@id'))
if meta else ' -' if 'metadata' in res else ' aggregated',
tags='' if 'tag' not in meta else ' [{}]'.format(','.join(
assure_list(meta['tag'])))))
def custom_result_renderer(res, **kwargs): # pragma: no cover
from datalad.ui import ui
if not res['status'] == 'ok' or not res['action'].startswith('htc_'):
# logging reported already
return
action = res['action'].split('_')[-1]
ui.message('{action} {sub}{job}{state}{cmd}'.format(
action=ac.color_word(action, kw_color_map.get(action, ac.WHITE))
if action != 'list' else '',
sub=res['submission'],
job=' :{}'.format(res['job']) if 'job' in res else '',
state=' [{}]'.format(
ac.color_word(res['state'],
kw_color_map.get(res['state'], ac.MAGENTA)
) if res.get('state', None) else 'unknown')
if action == 'list' else '',
cmd=': {}'.format(_format_cmd_shorty(res['cmd']))
if 'cmd' in res else '',
))
def custom_result_renderer(res, **kwargs):
from datalad.ui import ui
from datalad.interface.utils import default_result_renderer
if res['status'] != 'ok':
# logging complained about this already
return
if 'procedure' not in res.get('action', ''):
# it's not our business
default_result_renderer(res)
return
if kwargs.get('discover', None):
ui.message('{name} ({path}){msg}'.format(
name=ac.color_word(res['procedure_name'], ac.BOLD),
path=op.relpath(
res['path'],
res['refds'])
if res.get('refds', None) else res['path'],
msg=' [{}]'.format(
res['message'][0] % res['message'][1:]
if isinstance(res['message'], tuple) else res['message'])
if 'message' in res else ''
))
elif kwargs.get('help_proc', None):
ui.message('{name} ({path}){help}'.format(
name=ac.color_word(res['procedure_name'], ac.BOLD),
path=op.relpath(
res['path'],
res['refds'])
if res.get('refds', None) else res['path'],
help='{nl}{msg}'.format(
nl=os.linesep,
msg=res['message'][0] % res['message'][1:]
if isinstance(res['message'], tuple) else res['message'])
if 'message' in res else ''
))
else:
default_result_renderer(res)
def default_result_renderer(res):
if res.get('status', None) != 'notneeded':
path = res.get('path', None)
if path and res.get('refds'):
try:
path = relpath(path, res['refds'])
except ValueError:
# can happen, e.g., on windows with paths from different
# drives. just go with the original path in this case
pass
ui.message('{action}({status}):{path}{type}{msg}'.format(
action=ac.color_word(res.get('action', '<action-unspecified>'),
ac.BOLD),
status=ac.color_status(res.get('status', '<status-unspecified>')),
path=' {}'.format(path) if path else '',
type=' ({})'.format(ac.color_word(res['type'], ac.MAGENTA))
if 'type' in res else '',
msg=' [{}]'.format(res['message'][0] %
res['message'][1:] if isinstance(
res['message'], tuple) else res['message'])
if res.get('message', None) else ''))
def custom_result_renderer(res, **kwargs):
if res['status'] != 'ok' or not res.get('action', None) == 'meta_dump':
# logging complained about this already
return
if kwargs.get('reporton', None) == 'jsonld':
# special case of a JSON-LD report request
# all reports are consolidated into a single
# graph, dumps just that (no pretty printing, can
# be done outside)
ui.message(
jsondumps(
res['metadata'],
# support utf-8 output
ensure_ascii=False,
# this cannot happen, spare the checks
check_circular=False,
# this will cause the output to not necessarily be
# JSON compliant, but at least contain all info that went
# in, and be usable for javascript consumers
allow_nan=True,
))
return
# list the path, available metadata keys, and tags
path = op.relpath(res['path'], res['refds']) if res.get(
'refds', None) else res['path']
meta = res.get('metadata', {})
ui.message('{path}{type}:{spacer}{meta}{tags}'.format(
path=ac.color_word(path, ac.BOLD),
type=' ({})'.format(ac.color_word(res['type'], ac.MAGENTA))
if 'type' in res else '',
spacer=' ' if len([m for m in meta if m != 'tag']) else '',
meta=','.join(k for k in sorted(meta.keys())
if k not in ('tag', '@context', '@id'))
if meta else ' -' if 'metadata' in res else ' {}'.format(','.join(
e for e in res['extractors']
if e not in ('datalad_core', 'metalad_core',
'metalad_annex'))) if 'extractors' in res else '',
tags='' if 'tag' not in meta else ' [{}]'.format(','.join(
assure_list(meta['tag'])))))
def custom_result_renderer(res, **kwargs): # pragma: more cover
if not (res['status'] == 'ok' and res['action'] in ('status', 'diff')
and res.get('state', None) != 'clean'):
# logging reported already
return
from datalad.ui import ui
# when to render relative paths:
# 1) if a dataset arg was given
# 2) if CWD is the refds
refds = res.get('refds', None)
refds = refds if kwargs.get('dataset', None) is not None \
or refds == os.getcwd() else None
path = res['path'] if refds is None \
else str(ut.Path(res['path']).relative_to(refds))
type_ = res.get('type', res.get('type_src', ''))
max_len = len('untracked')
state = res.get('state', 'unknown')
ui.message(u'{fill}{state}: {path}{type_}'.format(
fill=' ' * max(0, max_len - len(state)),
state=ac.color_word(
state, STATE_COLOR_MAP.get(res.get('state', 'unknown'))),
path=path,
type_=' ({})'.format(
ac.color_word(type_, ac.MAGENTA) if type_ else '')))
def custom_result_renderer(res, **kwargs):
if (res['status'] != 'ok' or res['action']
not in ('get_configuration', 'dump_configuration')):
if 'message' not in res and 'name' in res:
suffix = '={}'.format(res['value']) if 'value' in res else ''
res['message'] = '{}{}'.format(res['name'], suffix)
default_result_renderer(res)
return
# TODO source
from datalad.ui import ui
name = res['name']
if res['action'] == 'dump_configuration':
for key in ('purpose', 'description'):
s = res.get(key)
if s:
ui.message('\n'.join(
wrap(
s,
initial_indent='# ',
subsequent_indent='# ',
)))
if kwargs.get('recursive', False):
have_subds = res['path'] != res['refds']
# we need to mark up from which dataset results are reported
prefix = '<ds>{}{}:'.format(
'/' if have_subds else '',
Path(res['path']).relative_to(res['refds']).as_posix()
if have_subds else '',
)
else:
prefix = ''
if kwargs.get('action', None) == 'dump':
ui.message('{}{}={}'.format(
prefix,
ac.color_word(name, ac.BOLD),
res['value'] if res['value'] is not None else '',
))
else:
ui.message('{}{}'.format(
prefix,
res['value'] if res['value'] is not None else '',
))
def custom_result_renderer(res, **kwargs): # pragma: more cover
# Don't render things like 'status' for clean-info messages -
# seems rather meaningless.
from os import getcwd
import datalad.support.ansi_colors as ac
from datalad.interface.utils import generic_result_renderer
from datalad.utils import Path
if res['action'] == 'clean':
# default renderer is just fine
return generic_result_renderer(res)
elif res['action'] != 'clean [dry-run]':
# Result didn't come from within `clean`.
# Should be handled elsewhere.
return
assert res['action'] == 'clean [dry-run]'
if res.get('status', None) == 'ok':
from datalad.ui import ui
# when to render relative paths:
# 1) if a dataset arg was given
# 2) if CWD is the refds
refds = res.get('refds', None)
refds = refds if kwargs.get('dataset', None) is not None \
or refds == getcwd() else None
path = res['path'] if refds is None \
else str(Path(res['path']).relative_to(refds))
ui.message(u"{path}: {message}".format(
path=ac.color_word(path, ac.BOLD),
message=(res['message'][0] % res['message'][1:] if isinstance(
res['message'], tuple) else res['message']) if res.get(
'message', None) else ''))
else:
# Any other status than 'ok' is reported the default way.
return generic_result_renderer(res)
def count_str(count, verb, omg=False):
if count:
msg = "{:d} {}".format(count, verb)
if omg:
msg = ansi_colors.color_word(msg, ansi_colors.RED)
return msg
def __call__(loc, recursive=False, fast=False, all_=False, long_=False,
config_file=None, list_content=False, json=None):
if json:
from datalad_deprecated.ls_webui import _ls_json
if isinstance(loc, list) and not len(loc):
# nothing given, CWD assumed -- just like regular ls
loc = '.'
kw = dict(fast=fast, recursive=recursive, all_=all_, long_=long_)
if isinstance(loc, list):
return [Ls.__call__(loc_, config_file=config_file,
list_content=list_content, json=json, **kw)
for loc_ in loc]
# TODO: do some clever handling of kwargs as to remember what were defaults
# and what any particular implementation actually needs, and then issuing
# warning if some custom value/option was specified which doesn't apply to the
# given url
# rename to not angry Python gods who took all_ good words
kw['long_'] = kw.pop('long_')
loc_type = "unknown"
if loc.startswith('s3://'):
return _ls_s3(loc, config_file=config_file, list_content=list_content,
**kw)
elif lexists(loc):
if isdir(loc):
ds = Dataset(loc)
if ds.is_installed():
return _ls_json(loc, json=json, **kw) if json else _ls_dataset(loc, **kw)
#loc_type = False
else:
loc_type = "dir" # we know that so far for sure
# it might have been an uninstalled dataset within super-dataset
superds = ds.get_superdataset()
if superds:
try:
subdatasets = Ls._cached_subdatasets[superds.path]
except KeyError:
subdatasets = Ls._cached_subdatasets[superds.path] \
= superds.subdatasets(result_xfm='relpaths')
if relpath(ds.path, superds.path) in subdatasets:
loc_type = "not installed"
else:
loc_type = "file"
# could list properties -- under annex or git, either clean/dirty
# etc
# repo = get_repo_instance(dirname(loc))
if loc_type:
#raise ValueError("ATM supporting only s3:// URLs and paths to local datasets")
# TODO: unify all_ the output here -- _ls functions should just return something
# to be displayed
ui.message(
"{} {}".format(
ansi_colors.color_word(loc, ansi_colors.DATASET),
ansi_colors.color_word(
loc_type,
ansi_colors.RED
if loc_type in {'unknown', 'not installed'}
else ansi_colors.BLUE)
)
)
def fmt_line(key, value, multiline=False):
return (" {key}:{sep}{value}"
.format(key=ac.color_word(key, ac.BOLD),
sep=os.linesep + " " if multiline else " ",
value=value))
def show_hint(msg):
from datalad.ui import ui
ui.message("{}".format(ansi_colors.color_word(msg, ansi_colors.YELLOW)))
def _process_results(
results, cmd_class,
action_summary, on_failure, incomplete_results,
result_renderer, result_xfm, result_filter, **kwargs):
# private helper pf @eval_results
# loop over results generated from some source and handle each
# of them according to the requested behavior (logging, rendering, ...)
for res in results:
if not res or 'action' not in res:
# XXX Yarik has to no clue on how to track the origin of the
# record to figure out WTF, so he just skips it
continue
if PY2:
for k, v in res.items():
if isinstance(v, unicode):
res[k] = v.encode('utf-8')
actsum = action_summary.get(res['action'], {})
if res['status']:
actsum[res['status']] = actsum.get(res['status'], 0) + 1
action_summary[res['action']] = actsum
## log message, if a logger was given
# remove logger instance from results, as it is no longer useful
# after logging was done, it isn't serializable, and generally
# pollutes the output
res_lgr = res.pop('logger', None)
if isinstance(res_lgr, logging.Logger):
# didn't get a particular log function, go with default
res_lgr = getattr(res_lgr, default_logchannels[res['status']])
if res_lgr and 'message' in res:
msg = res['message']
msgargs = None
if isinstance(msg, tuple):
msgargs = msg[1:]
msg = msg[0]
if 'path' in res:
msg = '{} [{}({})]'.format(
msg, res['action'], res['path'])
if msgargs:
# support string expansion of logging to avoid runtime cost
res_lgr(msg, *msgargs)
else:
res_lgr(msg)
## error handling
# looks for error status, and report at the end via
# an exception
if on_failure in ('continue', 'stop') \
and res['status'] in ('impossible', 'error'):
incomplete_results.append(res)
if on_failure == 'stop':
# first fail -> that's it
# raise will happen after the loop
break
if result_filter:
try:
if not result_filter(res):
raise ValueError('excluded by filter')
except ValueError as e:
lgr.debug('not reporting result (%s)', exc_str(e))
continue
## output rendering
# TODO RF this in a simple callable that gets passed into this function
if result_renderer is None or result_renderer == 'disabled':
pass
elif result_renderer == 'default':
# TODO have a helper that can expand a result message
ui.message('{action}({status}): {path}{type}{msg}'.format(
action=ac.color_word(res['action'], ac.BOLD),
status=ac.color_status(res['status']),
path=relpath(res['path'],
res['refds']) if res.get('refds', None) else res['path'],
type=' ({})'.format(
ac.color_word(res['type'], ac.MAGENTA)
) if 'type' in res else '',
msg=' [{}]'.format(
res['message'][0] % res['message'][1:]
if isinstance(res['message'], tuple) else res['message'])
if 'message' in res else ''))
elif result_renderer in ('json', 'json_pp'):
ui.message(json.dumps(
{k: v for k, v in res.items()
if k not in ('message', 'logger')},
sort_keys=True,
indent=2 if result_renderer.endswith('_pp') else None))
elif result_renderer == 'tailored':
if hasattr(cmd_class, 'custom_result_renderer'):
cmd_class.custom_result_renderer(res, **kwargs)
elif hasattr(result_renderer, '__call__'):
try:
result_renderer(res, **kwargs)
except Exception as e:
lgr.warn('Result rendering failed for: %s [%s]',
res, exc_str(e))
else:
raise ValueError('unknown result renderer "{}"'.format(result_renderer))
if PY2:
for k, v in res.items():
if isinstance(v, str):
res[k] = v.decode('utf-8')
if result_xfm:
res = result_xfm(res)
if res is None:
continue
yield res
