def custom_result_renderer(res, **kwargs): # pragma: no cover if not (res['status'] == 'ok' \ and res['action'] in ('status', 'diff') \ and res.get('state', None) != 'clean'): # logging reported already return from datalad.ui import ui # when to render relative paths: # 1) if a dataset arg was given # 2) if CWD is the refds refds = res.get('refds', None) refds = refds if kwargs.get('dataset', None) is not None \ or refds == os.getcwd() else None # Note: We have to force unicode for res['path'] because # interface.utils encodes it on py2 before passing it to # custom_result_renderer(). path = assure_unicode(res['path']) if refds is None \ else text_type(ut.Path(res['path']).relative_to(refds)) type_ = res.get('type', res.get('type_src', '')) max_len = len('untracked') state = res.get('state', 'unknown') ui.message(u'{fill}{state}: {path}{type_}'.format( fill=' ' * max(0, max_len - len(state)), state=ac.color_word( state, STATE_COLOR_MAP.get(res.get('state', 'unknown'))), path=path, type_=' ({})'.format( ac.color_word(type_, ac.MAGENTA) if type_ else '')))
def test_color_word(): s = 'word' green_s = '\033[1;32mword\033[0m' for enabled in (True, False): with patch('datalad.support.ansi_colors.color_enabled', lambda: enabled): assert_equal(colors.color_word(s, colors.GREEN, force=True), green_s) with patch('datalad.support.ansi_colors.color_enabled', lambda: True): assert_equal(colors.color_word(s, colors.GREEN), green_s) assert_equal(colors.color_word(s, colors.GREEN, force=False), green_s) with patch('datalad.support.ansi_colors.color_enabled', lambda: False): assert_equal(colors.color_word(s, colors.GREEN), s) assert_equal(colors.color_word(s, colors.GREEN, force=False), s)
def _display_basic(res): ui.message(ac.color_word("Dry run information", ac.MAGENTA)) def fmt_line(key, value, multiline=False): return (" {key}:{sep}{value}".format(key=ac.color_word(key, ac.BOLD), sep=os.linesep + " " if multiline else " ", value=value)) dry_run_info = res["dry_run_info"] lines = [fmt_line("location", dry_run_info["pwd_full"])] # TODO: Inputs and outputs could be pretty long. These may be worth # truncating. inputs = dry_run_info["inputs"] if inputs: lines.append(fmt_line("expanded inputs", inputs, multiline=True)) outputs = dry_run_info["outputs"] if outputs: lines.append(fmt_line("expanded outputs", outputs, multiline=True)) cmd = res["run_info"]["cmd"] cmd_expanded = dry_run_info["cmd_expanded"] lines.append(fmt_line("command", cmd, multiline=True)) if cmd != cmd_expanded: lines.append(fmt_line("expanded command", cmd_expanded, multiline=True)) ui.message(os.linesep.join(lines))
def custom_result_renderer(res, **kwargs): if res["action"] != "containers": default_result_renderer(res) else: ui.message("{name} -> {path}".format( name=ac.color_word(res["name"], ac.MAGENTA), path=op.relpath(res["path"], res["refds"])))
def default_result_renderer(res): if res.get('status', None) != 'notneeded': path = res.get('path', None) path = ' {}'.format(path) if path else '' ui.message('{action}({status}):{path}{type}{msg}'.format( action=ac.color_word(res['action'], ac.BOLD), status=ac.color_status(res['status']), path=relpath(path, res['refds']) if path and res.get('refds') else path, type=' ({})'.format( ac.color_word(res['type'], ac.MAGENTA) ) if 'type' in res else '', msg=' [{}]'.format( res['message'][0] % res['message'][1:] if isinstance(res['message'], tuple) else res[ 'message']) if res.get('message', None) else ''))
def custom_result_renderer(res, **kwargs): from datalad.ui import ui # should we attempt to remove an unknown sibling, complain like Git does if res['status'] == 'notneeded' and res['action'] == 'remove-sibling': ui.message('{warn}: No sibling "{name}" in dataset {path}'.format( warn=ac.color_word('Warning', ac.LOG_LEVEL_COLORS['WARNING']), **res)) return if res['status'] != 'ok' or not res.get('action', '').endswith('-sibling'): generic_result_renderer(res) return path = op.relpath(res['path'], res['refds']) if res.get( 'refds', None) else res['path'] got_url = 'url' in res spec = '{}{}{}{}'.format(res.get('url', ''), ' (' if got_url else '', res.get('annex-externaltype', 'git'), ')' if got_url else '') ui.message('{path}: {name}({with_annex}) [{spec}]'.format( **dict( res, path=path, # TODO report '+' for special remotes with_annex='+' if 'annex-uuid' in res \ else ('-' if res.get('annex-ignore', None) else '?'), spec=spec)))
def default_result_renderer(res): if res.get('status', None) != 'notneeded': ui.message('{action}({status}): {path}{type}{msg}'.format( action=ac.color_word(res['action'], ac.BOLD), status=ac.color_status(res['status']), path=relpath(res['path'], res['refds']) if res.get('refds', None) else res[ 'path'], type=' ({})'.format( ac.color_word(res['type'], ac.MAGENTA) ) if 'type' in res else '', msg=' [{}]'.format( res['message'][0] % res['message'][1:] if isinstance(res['message'], tuple) else res[ 'message']) if 'message' in res else ''))
def custom_result_summary_renderer(results, action_summary): # pragma: more cover render_action_summary(action_summary) # report on any hints at the end # get all unique hints hints = set([r.get('hints', None) for r in results]) hints = [hint for hint in hints if hint is not None] if hints: from datalad.ui import ui from datalad.support import ansi_colors intro = ansi_colors.color_word("Hints: ", ansi_colors.YELLOW) ui.message(intro) [ ui.message("{}: {}".format( ansi_colors.color_word(id + 1, ansi_colors.YELLOW), hint)) for id, hint in enumerate(hints) ]
def result_renderer_cmdline(res, cmdlineargs): from datalad.ui import ui if res is None: res = [] format = cmdlineargs.format or 'custom' if format == 'custom': if cmdlineargs.report in ('*', ['*']) \ or cmdlineargs.report_matched \ or (cmdlineargs.report is not None and len(cmdlineargs.report) > 1): # multiline if multiple were requested and we need to disambiguate ichr = jchr = '\n' fmt = ' {k}: {v}' else: jchr = ', ' ichr = ' ' fmt = '{v}' anything = False for location, r in res: # XXX Yarik thinks that Match should be replaced with actual path to the dataset ui.message('{}{}{}{}'.format( ansi_colors.color_word(location, ansi_colors.DATASET), ':' if r else '', ichr, jchr.join( [ fmt.format( k=ansi_colors.color_word(k, ansi_colors.FIELD), v=pretty_bytes(r[k])) for k in sorted(r) ]))) anything = True if not anything: ui.message("Nothing to report") elif format == 'json': import json ui.message(json.dumps(list(map(itemgetter(1), res)), indent=2)) elif format == 'yaml': import yaml lgr.warning("yaml output support is not yet polished") ui.message(yaml.safe_dump(list(map(itemgetter(1), res)), allow_unicode=True))
def custom_result_renderer(res, **kwargs): if res['status'] != 'ok' or not res.get('action', None) == 'metadata': # logging complained about this already return # list the path, available metadata keys, and tags path = op.relpath(res['path'], res['refds']) if res.get('refds', None) else res['path'] meta = res.get('metadata', {}) ui.message('{path}{type}:{spacer}{meta}{tags}'.format( path=ac.color_word(path, ac.BOLD), type=' ({})'.format( ac.color_word(res['type'], ac.MAGENTA)) if 'type' in res else '', spacer=' ' if len([m for m in meta if m != 'tag']) else '', meta=','.join(k for k in sorted(meta.keys()) if k not in ('tag', '@context', '@id')) if meta else ' -' if 'metadata' in res else ' aggregated', tags='' if 'tag' not in meta else ' [{}]'.format( ','.join(assure_list(meta['tag'])))))
def custom_result_renderer(res, **kwargs): if res['status'] != 'ok' or not res.get('action', None) == 'metadata': # logging complained about this already return # list the path, available metadata keys, and tags path = relpath(res['path'], res['refds']) if res.get( 'refds', None) else res['path'] meta = res.get('metadata', {}) ui.message('{path}{type}:{spacer}{meta}{tags}'.format( path=ac.color_word(path, ac.BOLD), type=' ({})'.format(ac.color_word(res['type'], ac.MAGENTA)) if 'type' in res else '', spacer=' ' if len([m for m in meta if m != 'tag']) else '', meta=','.join(k for k in sorted(meta.keys()) if k not in ('tag', '@context', '@id')) if meta else ' -' if 'metadata' in res else ' aggregated', tags='' if 'tag' not in meta else ' [{}]'.format(','.join( assure_list(meta['tag'])))))
def custom_result_renderer(res, **kwargs): # pragma: no cover from datalad.ui import ui if not res['status'] == 'ok' or not res['action'].startswith('htc_'): # logging reported already return action = res['action'].split('_')[-1] ui.message('{action} {sub}{job}{state}{cmd}'.format( action=ac.color_word(action, kw_color_map.get(action, ac.WHITE)) if action != 'list' else '', sub=res['submission'], job=' :{}'.format(res['job']) if 'job' in res else '', state=' [{}]'.format( ac.color_word(res['state'], kw_color_map.get(res['state'], ac.MAGENTA) ) if res.get('state', None) else 'unknown') if action == 'list' else '', cmd=': {}'.format(_format_cmd_shorty(res['cmd'])) if 'cmd' in res else '', ))
def custom_result_renderer(res, **kwargs): from datalad.ui import ui from datalad.interface.utils import default_result_renderer if res['status'] != 'ok': # logging complained about this already return if 'procedure' not in res.get('action', ''): # it's not our business default_result_renderer(res) return if kwargs.get('discover', None): ui.message('{name} ({path}){msg}'.format( name=ac.color_word(res['procedure_name'], ac.BOLD), path=op.relpath( res['path'], res['refds']) if res.get('refds', None) else res['path'], msg=' [{}]'.format( res['message'][0] % res['message'][1:] if isinstance(res['message'], tuple) else res['message']) if 'message' in res else '' )) elif kwargs.get('help_proc', None): ui.message('{name} ({path}){help}'.format( name=ac.color_word(res['procedure_name'], ac.BOLD), path=op.relpath( res['path'], res['refds']) if res.get('refds', None) else res['path'], help='{nl}{msg}'.format( nl=os.linesep, msg=res['message'][0] % res['message'][1:] if isinstance(res['message'], tuple) else res['message']) if 'message' in res else '' )) else: default_result_renderer(res)
def default_result_renderer(res): if res.get('status', None) != 'notneeded': path = res.get('path', None) if path and res.get('refds'): try: path = relpath(path, res['refds']) except ValueError: # can happen, e.g., on windows with paths from different # drives. just go with the original path in this case pass ui.message('{action}({status}):{path}{type}{msg}'.format( action=ac.color_word(res.get('action', '<action-unspecified>'), ac.BOLD), status=ac.color_status(res.get('status', '<status-unspecified>')), path=' {}'.format(path) if path else '', type=' ({})'.format(ac.color_word(res['type'], ac.MAGENTA)) if 'type' in res else '', msg=' [{}]'.format(res['message'][0] % res['message'][1:] if isinstance( res['message'], tuple) else res['message']) if res.get('message', None) else ''))
def custom_result_renderer(res, **kwargs): if res['status'] != 'ok' or not res.get('action', None) == 'meta_dump': # logging complained about this already return if kwargs.get('reporton', None) == 'jsonld': # special case of a JSON-LD report request # all reports are consolidated into a single # graph, dumps just that (no pretty printing, can # be done outside) ui.message( jsondumps( res['metadata'], # support utf-8 output ensure_ascii=False, # this cannot happen, spare the checks check_circular=False, # this will cause the output to not necessarily be # JSON compliant, but at least contain all info that went # in, and be usable for javascript consumers allow_nan=True, )) return # list the path, available metadata keys, and tags path = op.relpath(res['path'], res['refds']) if res.get( 'refds', None) else res['path'] meta = res.get('metadata', {}) ui.message('{path}{type}:{spacer}{meta}{tags}'.format( path=ac.color_word(path, ac.BOLD), type=' ({})'.format(ac.color_word(res['type'], ac.MAGENTA)) if 'type' in res else '', spacer=' ' if len([m for m in meta if m != 'tag']) else '', meta=','.join(k for k in sorted(meta.keys()) if k not in ('tag', '@context', '@id')) if meta else ' -' if 'metadata' in res else ' {}'.format(','.join( e for e in res['extractors'] if e not in ('datalad_core', 'metalad_core', 'metalad_annex'))) if 'extractors' in res else '', tags='' if 'tag' not in meta else ' [{}]'.format(','.join( assure_list(meta['tag'])))))
def custom_result_renderer(res, **kwargs): # pragma: more cover if not (res['status'] == 'ok' and res['action'] in ('status', 'diff') and res.get('state', None) != 'clean'): # logging reported already return from datalad.ui import ui # when to render relative paths: # 1) if a dataset arg was given # 2) if CWD is the refds refds = res.get('refds', None) refds = refds if kwargs.get('dataset', None) is not None \ or refds == os.getcwd() else None path = res['path'] if refds is None \ else str(ut.Path(res['path']).relative_to(refds)) type_ = res.get('type', res.get('type_src', '')) max_len = len('untracked') state = res.get('state', 'unknown') ui.message(u'{fill}{state}: {path}{type_}'.format( fill=' ' * max(0, max_len - len(state)), state=ac.color_word( state, STATE_COLOR_MAP.get(res.get('state', 'unknown'))), path=path, type_=' ({})'.format( ac.color_word(type_, ac.MAGENTA) if type_ else '')))
def custom_result_renderer(res, **kwargs): if (res['status'] != 'ok' or res['action'] not in ('get_configuration', 'dump_configuration')): if 'message' not in res and 'name' in res: suffix = '={}'.format(res['value']) if 'value' in res else '' res['message'] = '{}{}'.format(res['name'], suffix) default_result_renderer(res) return # TODO source from datalad.ui import ui name = res['name'] if res['action'] == 'dump_configuration': for key in ('purpose', 'description'): s = res.get(key) if s: ui.message('\n'.join( wrap( s, initial_indent='# ', subsequent_indent='# ', ))) if kwargs.get('recursive', False): have_subds = res['path'] != res['refds'] # we need to mark up from which dataset results are reported prefix = '<ds>{}{}:'.format( '/' if have_subds else '', Path(res['path']).relative_to(res['refds']).as_posix() if have_subds else '', ) else: prefix = '' if kwargs.get('action', None) == 'dump': ui.message('{}{}={}'.format( prefix, ac.color_word(name, ac.BOLD), res['value'] if res['value'] is not None else '', )) else: ui.message('{}{}'.format( prefix, res['value'] if res['value'] is not None else '', ))
def custom_result_renderer(res, **kwargs): # pragma: more cover # Don't render things like 'status' for clean-info messages - # seems rather meaningless. from os import getcwd import datalad.support.ansi_colors as ac from datalad.interface.utils import generic_result_renderer from datalad.utils import Path if res['action'] == 'clean': # default renderer is just fine return generic_result_renderer(res) elif res['action'] != 'clean [dry-run]': # Result didn't come from within `clean`. # Should be handled elsewhere. return assert res['action'] == 'clean [dry-run]' if res.get('status', None) == 'ok': from datalad.ui import ui # when to render relative paths: # 1) if a dataset arg was given # 2) if CWD is the refds refds = res.get('refds', None) refds = refds if kwargs.get('dataset', None) is not None \ or refds == getcwd() else None path = res['path'] if refds is None \ else str(Path(res['path']).relative_to(refds)) ui.message(u"{path}: {message}".format( path=ac.color_word(path, ac.BOLD), message=(res['message'][0] % res['message'][1:] if isinstance( res['message'], tuple) else res['message']) if res.get( 'message', None) else '')) else: # Any other status than 'ok' is reported the default way. return generic_result_renderer(res)
def count_str(count, verb, omg=False): if count: msg = "{:d} {}".format(count, verb) if omg: msg = ansi_colors.color_word(msg, ansi_colors.RED) return msg
def __call__(loc, recursive=False, fast=False, all_=False, long_=False, config_file=None, list_content=False, json=None): if json: from datalad_deprecated.ls_webui import _ls_json if isinstance(loc, list) and not len(loc): # nothing given, CWD assumed -- just like regular ls loc = '.' kw = dict(fast=fast, recursive=recursive, all_=all_, long_=long_) if isinstance(loc, list): return [Ls.__call__(loc_, config_file=config_file, list_content=list_content, json=json, **kw) for loc_ in loc] # TODO: do some clever handling of kwargs as to remember what were defaults # and what any particular implementation actually needs, and then issuing # warning if some custom value/option was specified which doesn't apply to the # given url # rename to not angry Python gods who took all_ good words kw['long_'] = kw.pop('long_') loc_type = "unknown" if loc.startswith('s3://'): return _ls_s3(loc, config_file=config_file, list_content=list_content, **kw) elif lexists(loc): if isdir(loc): ds = Dataset(loc) if ds.is_installed(): return _ls_json(loc, json=json, **kw) if json else _ls_dataset(loc, **kw) #loc_type = False else: loc_type = "dir" # we know that so far for sure # it might have been an uninstalled dataset within super-dataset superds = ds.get_superdataset() if superds: try: subdatasets = Ls._cached_subdatasets[superds.path] except KeyError: subdatasets = Ls._cached_subdatasets[superds.path] \ = superds.subdatasets(result_xfm='relpaths') if relpath(ds.path, superds.path) in subdatasets: loc_type = "not installed" else: loc_type = "file" # could list properties -- under annex or git, either clean/dirty # etc # repo = get_repo_instance(dirname(loc)) if loc_type: #raise ValueError("ATM supporting only s3:// URLs and paths to local datasets") # TODO: unify all_ the output here -- _ls functions should just return something # to be displayed ui.message( "{} {}".format( ansi_colors.color_word(loc, ansi_colors.DATASET), ansi_colors.color_word( loc_type, ansi_colors.RED if loc_type in {'unknown', 'not installed'} else ansi_colors.BLUE) ) )
def fmt_line(key, value, multiline=False): return (" {key}:{sep}{value}" .format(key=ac.color_word(key, ac.BOLD), sep=os.linesep + " " if multiline else " ", value=value))
def show_hint(msg): from datalad.ui import ui ui.message("{}".format(ansi_colors.color_word(msg, ansi_colors.YELLOW)))
def _process_results( results, cmd_class, action_summary, on_failure, incomplete_results, result_renderer, result_xfm, result_filter, **kwargs): # private helper pf @eval_results # loop over results generated from some source and handle each # of them according to the requested behavior (logging, rendering, ...) for res in results: if not res or 'action' not in res: # XXX Yarik has to no clue on how to track the origin of the # record to figure out WTF, so he just skips it continue if PY2: for k, v in res.items(): if isinstance(v, unicode): res[k] = v.encode('utf-8') actsum = action_summary.get(res['action'], {}) if res['status']: actsum[res['status']] = actsum.get(res['status'], 0) + 1 action_summary[res['action']] = actsum ## log message, if a logger was given # remove logger instance from results, as it is no longer useful # after logging was done, it isn't serializable, and generally # pollutes the output res_lgr = res.pop('logger', None) if isinstance(res_lgr, logging.Logger): # didn't get a particular log function, go with default res_lgr = getattr(res_lgr, default_logchannels[res['status']]) if res_lgr and 'message' in res: msg = res['message'] msgargs = None if isinstance(msg, tuple): msgargs = msg[1:] msg = msg[0] if 'path' in res: msg = '{} [{}({})]'.format( msg, res['action'], res['path']) if msgargs: # support string expansion of logging to avoid runtime cost res_lgr(msg, *msgargs) else: res_lgr(msg) ## error handling # looks for error status, and report at the end via # an exception if on_failure in ('continue', 'stop') \ and res['status'] in ('impossible', 'error'): incomplete_results.append(res) if on_failure == 'stop': # first fail -> that's it # raise will happen after the loop break if result_filter: try: if not result_filter(res): raise ValueError('excluded by filter') except ValueError as e: lgr.debug('not reporting result (%s)', exc_str(e)) continue ## output rendering # TODO RF this in a simple callable that gets passed into this function if result_renderer is None or result_renderer == 'disabled': pass elif result_renderer == 'default': # TODO have a helper that can expand a result message ui.message('{action}({status}): {path}{type}{msg}'.format( action=ac.color_word(res['action'], ac.BOLD), status=ac.color_status(res['status']), path=relpath(res['path'], res['refds']) if res.get('refds', None) else res['path'], type=' ({})'.format( ac.color_word(res['type'], ac.MAGENTA) ) if 'type' in res else '', msg=' [{}]'.format( res['message'][0] % res['message'][1:] if isinstance(res['message'], tuple) else res['message']) if 'message' in res else '')) elif result_renderer in ('json', 'json_pp'): ui.message(json.dumps( {k: v for k, v in res.items() if k not in ('message', 'logger')}, sort_keys=True, indent=2 if result_renderer.endswith('_pp') else None)) elif result_renderer == 'tailored': if hasattr(cmd_class, 'custom_result_renderer'): cmd_class.custom_result_renderer(res, **kwargs) elif hasattr(result_renderer, '__call__'): try: result_renderer(res, **kwargs) except Exception as e: lgr.warn('Result rendering failed for: %s [%s]', res, exc_str(e)) else: raise ValueError('unknown result renderer "{}"'.format(result_renderer)) if PY2: for k, v in res.items(): if isinstance(v, str): res[k] = v.decode('utf-8') if result_xfm: res = result_xfm(res) if res is None: continue yield res