def write(self, obj):
formatted_json = json.dumps(obj, default=serialize, sort_keys=True)
formatted_yaml = yaml.dump(json.loads(formatted_json))
colorful_yaml = highlight(formatted_yaml, lexers.YamlLexer(),
formatters.TerminalFormatter())
return colorful_yaml
def _get_lexer(codesyntax):
if codesyntax in ("cpp", "javascript"):
return lexers.JavascriptLexer()
elif codesyntax == "python":
return lexers.PythonLexer()
elif codesyntax == "json":
return lexers.JsonLexer()
elif codesyntax == "xml" or codesyntax == "html":
return lexers.HtmlLexer()
elif codesyntax == "yml" or codesyntax == "yaml":
return lexers.YamlLexer()
elif codesyntax == "css":
return lexers.CssLexer()
elif codesyntax == "sql":
return lexers.SqlLexer()
elif codesyntax == "bash" or codesyntax == "sh":
return lexers.BashLexer()
elif codesyntax == "go":
return lexers.GoLexer()
elif codesyntax == "diff":
return lexers.DiffLexer()
elif codesyntax == "emacslisp":
return lexers.EmacsLispLexer()
elif codesyntax == "lisp":
return lexers.CommonLispLexer()
elif codesyntax == "rust":
return lexers.RustLexer()
elif codesyntax == "jsx":
return BabylonLexer()
elif codesyntax:
raise NotImplementedError(codesyntax)
else:
return lexers.TextLexer()
def get_pretty_metadata(experiment_name: str, paths: BuddyPaths) -> str:
try:
with open(paths.get_metadata_file(experiment_name), "r") as f:
return highlight(
f.read().strip(),
lexers.YamlLexer(),
formatters.TerminalFormatter(),
)
except FileNotFoundError:
return termcolor.colored("No metadata", "blue")
def __yamlprint(self, data):
data = yaml.safe_dump(data, default_flow_style=False)
if self.color == 1:
l = lexers.YamlLexer()
l.add_filter(YamlFilter())
#for i in l.get_tokens(data):
# print i
from pygments import lex
#for item in lex(data, l):
# print item
print highlight(data, l, Terminal256Formatter(style=YamlStyle))
else:
print data
def pretty_print(value, raw=False, as_code=None, raw_format=None):
if isinstance(value, bytes):
if sys.stdout.isatty():
raise RuntimeError(
'can not write binary data to the text console')
else:
sys.stdout.buffer.write(value)
elif raw and raw_format in ['yml', 'yaml']:
j = yaml.dump(value, default_flow_style=False)
if sys.stdout.isatty():
from pygments import highlight, lexers, formatters
j = highlight(j, lexers.YamlLexer(),
formatters.TerminalFormatter())
print(j)
elif (isinstance(value, list) or
(isinstance(value, dict) and raw)) or (raw
and raw_format == 'json'):
import json
j = json.dumps(value, indent=4, sort_keys=True)
if sys.stdout.isatty():
from pygments import highlight, lexers, formatters
j = highlight(j, lexers.JsonLexer(),
formatters.TerminalFormatter())
print(j)
elif isinstance(value, dict):
pretty_print_table(
sorted([{
'field': k,
'type': type_name(v),
'value': yedb._format_debug_value(v),
} for k, v in value.items()],
key=lambda k: k['field']))
else:
if as_code == 'python' and sys.stdout.isatty():
from pygments import highlight, lexers, formatters
value = highlight(value, lexers.Python3Lexer(),
formatters.TerminalFormatter())
print(value)
def output(self, outdata, tablefmt):
# log.debugv(f"data passed to output():\n{pprint(outdata, indent=4)}")
if tablefmt == "json":
# from pygments import highlight, lexers, formatters
json_data = json.dumps(outdata, sort_keys=True, indent=2)
table_data = highlight(
bytes(json_data, 'UTF-8'), lexers.JsonLexer(),
formatters.Terminal256Formatter(style='solarized-dark'))
elif tablefmt == "csv":
table_data = "\n".join([
",".join([
k if outdata.index(d) == 0 else str(v)
for k, v in d.items() if k not in CUST_KEYS
]) for d in outdata
])
elif tablefmt in ["yml", "yaml"]:
table_data = highlight(
bytes(yaml.dump(
outdata,
sort_keys=True,
), 'UTF-8'), lexers.YamlLexer(),
formatters.Terminal256Formatter(style='solarized-dark'))
else:
customer_id = customer_name = ""
outdata = self.listify(outdata)
if outdata and isinstance(outdata, list) and isinstance(
outdata[0], dict):
customer_id = outdata[0].get("customer_id", "")
customer_name = outdata[0].get("customer_name", "")
outdata = [{k: v
for k, v in d.items() if k not in CUST_KEYS}
for d in outdata]
table_data = tabulate(outdata, headers="keys", tablefmt=tablefmt)
table_data = f"--\n{'Customer ID:':15}{customer_id}\n{'Customer Name:':15} {customer_name}\n--\n{table_data}"
return table_data
def pretty_yaml(data):
if not isinstance(data, basestring):
data = yaml.dump(data)
return highlight(data, lexers.YamlLexer(), formatters.TerminalFormatter())
async def run_command(self, cmd):
try:
result = await cmd.run(self.session)
except Exception as e:
print(e)
if self._output_format in [
Application.OUTPUT_FORMAT_JSON,
Application.OUTPUT_FORMAT_JSON_COLORED
]:
json_str = json.dumps(result.result,
separators=(', ', ': '),
sort_keys=True,
indent=4,
ensure_ascii=False)
if self._output_format == Application.OUTPUT_FORMAT_JSON_COLORED:
console_str = highlight(
json_str, lexers.JsonLexer(),
formatters.Terminal256Formatter(style=self._output_style))
else:
console_str = json_str
elif self._output_format in [
Application.OUTPUT_FORMAT_YAML,
Application.OUTPUT_FORMAT_YAML_COLORED
]:
yaml_str = yaml.safe_dump(result.result)
if self._output_format == Application.OUTPUT_FORMAT_YAML_COLORED:
console_str = highlight(
yaml_str, lexers.YamlLexer(),
formatters.Terminal256Formatter(style=self._output_style))
else:
console_str = yaml_str
elif self._output_format == Application.OUTPUT_FORMAT_PLAIN:
#console_str = u'{}'.format(pprint.pformat(result.result))
console_str = u'{}'.format(result)
else:
# should not arrive here
raise Exception(
'internal error: unprocessed value "{}" for output format'.
format(self._output_format))
# output command metadata (such as runtime)
if self._output_verbosity == Application.OUTPUT_VERBOSITY_SILENT:
pass
else:
# output result of command
click.echo(console_str)
if self._output_verbosity == Application.OUTPUT_VERBOSITY_RESULT_ONLY or self._output_format == Application.OUTPUT_FORMAT_PLAIN:
pass
elif self._output_verbosity == Application.OUTPUT_VERBOSITY_NORMAL:
if result.duration:
click.echo(
style_finished_line(u'Finished in {} ms.'.format(
result.duration)))
else:
click.echo(style_finished_line(u'Finished successfully.'))
elif self._output_verbosity == Application.OUTPUT_VERBOSITY_EXTENDED:
if result.duration:
click.echo(
style_finished_line(u'Finished in {} ms on {}.'.format(
result.duration, localnow())))
else:
click.echo(
style_finished_line(
u'Finished successfully on {}.'.format(
localnow())))
else:
# should not arrive here
raise Exception('internal error')
def format_yaml(obj, prefix='---\n'):
return highlight(prefix + pyaml.dumps(obj, safe=True).decode('utf-8'),
lexers.YamlLexer(), formatters.TerminalFormatter())
def format_yaml_color(obj):
from pygments import highlight, lexers, formatters
return highlight(format_yaml(obj), lexers.YamlLexer(),
formatters.TerminalFormatter()) # pylint: disable=no-member
def _output_result(self, result):
cmd_str = ' '.join(["crossbar", "shell"] + sys.argv[1:])
if self._output_format in [
Application.OUTPUT_FORMAT_JSON,
Application.OUTPUT_FORMAT_JSON_COLORED
]:
json_str = json.dumps(result.result,
separators=(', ', ': '),
sort_keys=False,
indent=4,
ensure_ascii=False)
if self._output_format == Application.OUTPUT_FORMAT_JSON_COLORED:
console_str = highlight(
json_str, lexers.JsonLexer(),
formatters.Terminal256Formatter(style=self._output_style))
else:
console_str = json_str
elif self._output_format in [
Application.OUTPUT_FORMAT_YAML,
Application.OUTPUT_FORMAT_YAML_COLORED
]:
yaml_str = yaml.safe_dump(result.result)
if self._output_format == Application.OUTPUT_FORMAT_YAML_COLORED:
console_str = highlight(
yaml_str, lexers.YamlLexer(),
formatters.Terminal256Formatter(style=self._output_style))
else:
console_str = yaml_str
elif self._output_format == Application.OUTPUT_FORMAT_PLAIN:
console_str = '{}'.format(result)
else:
# should not arrive here
raise Exception(
'internal error: unprocessed value "{}" for output format'.
format(self._output_format))
# output command metadata (such as runtime)
if self._output_verbosity == Application.OUTPUT_VERBOSITY_SILENT:
pass
else:
# output result of command
click.echo(console_str)
if self._output_verbosity == Application.OUTPUT_VERBOSITY_RESULT_ONLY or self._output_format == Application.OUTPUT_FORMAT_PLAIN:
pass
elif self._output_verbosity == Application.OUTPUT_VERBOSITY_NORMAL:
if result.duration:
click.echo(
style_finished_line(
'Finished command in {} ms: {}'.format(
result.duration, cmd_str)))
else:
click.echo(
style_finished_line(
'Finished command successfully: {}'.format(
cmd_str)))
elif self._output_verbosity == Application.OUTPUT_VERBOSITY_EXTENDED:
if result.duration:
click.echo(
style_finished_line(
'Finished command in {} ms on {}: {}'.format(
result.duration, localnow(), cmd_str)))
else:
click.echo(
style_finished_line(
'Finished successfully on {}: {}'.format(
localnow(), cmd_str)))
else:
# should not arrive here
raise Exception('internal error')
def colorizer(content):
return highlight(content, lexers.YamlLexer(),
formatters.TerminalFormatter())
def main(cls, *, args: argparse.Namespace, paths: BuddyPaths) -> None:
# Get experiment name
experiment_name = args.experiment_name
print(experiment_name)
# Generate dynamic-width table
try:
terminal_columns = int(
os.popen("stty size", "r").read().split()[1])
except IndexError:
# stty size fails when run from outside proper terminal (eg in tests)
terminal_columns = 100
table = beautifultable.BeautifulTable(
maxwidth=min(100, terminal_columns),
default_alignment=beautifultable.ALIGN_LEFT,
)
table.set_style(beautifultable.STYLE_BOX_ROUNDED)
def add_table_row(label, value):
table.rows.append(
[termcolor.colored(label, attrs=["bold"]), value])
# Constant for "not applicable" fields
NA = termcolor.colored("N/A", "red")
# Find checkpoint files
checkpoint_paths = paths.find_checkpoints(experiment_name)
# Display size, labels of checkpoints
if len(checkpoint_paths) > 0:
checkpoint_total_size = 0
checkpoint_labels = []
buddy = fannypack.utils.Buddy(experiment_name, verbose=False)
checkpoint_paths, steps = buddy._find_checkpoints(
paths.checkpoint_dir, args.experiment_name)
for checkpoint_path in checkpoint_paths:
prefix = os.path.join(paths.checkpoint_dir,
f"{experiment_name}-")
suffix = ".ckpt"
assert str(checkpoint_path).startswith(prefix)
assert str(checkpoint_path).endswith(suffix)
label = str(checkpoint_path)[len(prefix):-len(suffix)]
checkpoint_labels.append(
f"{label} (steps: {steps[checkpoint_path]})")
checkpoint_total_size += get_size(str(checkpoint_path))
add_table_row(
"Total checkpoint size",
termcolor.colored(format_size(checkpoint_total_size), "green"),
)
add_table_row(
"Average checkpoint size",
termcolor.colored(
format_size(checkpoint_total_size / len(checkpoint_paths)),
"green"),
)
add_table_row("Checkpoint labels", "\n".join(checkpoint_labels))
else:
add_table_row("Total checkpoint size", NA)
add_table_row("Average checkpoint size", NA)
add_table_row("Checkpoint labels", "")
# Display log file size
log_path = paths.get_log_dir(args.experiment_name)
if os.path.exists(log_path):
# _delete(log_path, args.forever)
add_table_row(
"Log size",
termcolor.colored(format_size(get_size(log_path)), "green"))
else:
add_table_row("Log size", NA)
# Display metadata + metadata size
metadata_path = paths.get_metadata_file(args.experiment_name)
if os.path.exists(metadata_path):
add_table_row(
"Metadata size",
termcolor.colored(format_size(get_size(metadata_path)),
"green"),
)
with open(metadata_path, "r") as f:
add_table_row(
"Metadata",
highlight(
f.read().strip(),
lexers.YamlLexer(),
formatters.TerminalFormatter(),
),
)
else:
add_table_row("Metadata size", NA)
add_table_row("Metadata", NA)
# Print table
print(table)