def format_families(families, colorize=True, verbose=True):
if not families:
s = "No algorithm families found."
return s
else:
table = []
table.append([
'Family name',
'interface',
'pattern',
'# found',
'valid',
'filename',
'description',
])
for family in families:
assert isinstance(family, EasyAlgoFamily)
row = []
row.append(family.family_name)
row.append(family.interface)
row.append(family.instances_pattern)
if not family.instances:
row.append('\n(none)')
else:
n_valid = len(
[_ for _ in family.instances.values() if _.valid])
n_invalid = len(family.instances) - n_valid
ss = '%s' % len(family.instances)
if n_invalid:
ss += dtu.make_red(' (%d invalid)' % n_invalid)
row.append(ss)
if family.valid:
ss = 'yes'
else:
ss = 'no: ' + family.error_if_invalid
row.append(ss)
row.append(dtu.friendly_path(family.filename))
if (not family.valid) and colorize:
row = dtu.make_row_red(row)
row.append(family.description.strip())
table.append(row)
if not verbose:
dtu.remove_table_field(table, 'filename')
s = "Found %d algorithm families:\n\n" % len(families)
s += dtu.indent(dtu.format_table_plus(table, colspacing=4), ' ')
return s
def format_logs(logs):
if not logs:
s = "No logs found."
return s
else:
s = "Found %d logs.\n" % len(logs)
table = []
table.append([
'#', 'Log name', 'map', 'description', 'date', 'length',
'vehicle name', 'filename', 'valid', 'topics'
])
for i, (_, log) in enumerate(logs.items()):
row = []
row.append(i)
row.append(log.log_name)
row.append(log.map_name)
row.append(log.description)
row.append(log.date)
if log.length is not None:
l = '%5.1f s' % log.length
else:
l = '(none)'
row.append(l)
row.append(log.vehicle)
if log.filename is None:
row.append('not local')
else:
row.append(friendly_path(log.filename))
if log.valid:
s = 'Yes.'
else:
s = log.error_if_invalid
row.append(s)
if log.bag_info is not None:
info = yaml.dump(log.bag_info['topics'])
else:
info = '(none)'
if not log.valid:
row = make_row_red(row)
row.append(info)
table.append(row)
remove_table_field(table, 'filename')
remove_table_field(table, 'topics')
remove_table_field(table, 'description')
remove_table_field(table, 'map')
s += indent(format_table_plus(table, colspacing=4), '| ')
return s
def config_summary(all_keys, values, origin):
table = []
table.append(['parameter', 'value', 'origin'])
table.append(['-' * len(_) for _ in table[0]])
for k in all_keys:
if k in values:
v = yaml.dump(values[k])
v = v.strip()
if v.endswith('...'):
v = v[:-3]
v = v.strip()
table.append([k, v, dtu.friendly_path(origin[k])])
else:
table.append([k, '(unset)', '(not found)'])
return dtu.format_table_plus(table, 4)
def get_easy_logs_db_cloud():
cloud_file = require_resource('cloud.yaml')
# cloud_file = os.path.join(get_ros_package_path('easy_logs'), 'cloud.yaml')
# if not os.path.exists(cloud_file):
# url = "https://www.dropbox.com/s/vdl1ej8fihggide/duckietown-cloud.yaml?dl=1"
# download_url_to_file(url, cloud_file)
logger.info('Loading cloud DB %s' % friendly_path(cloud_file))
logs = yaml_load_file(cloud_file)
logs = OrderedDict(logs)
logger.info('Loaded cloud DB with %d entries.' % len(logs))
return EasyLogsDB(logs)
def user_config_summary():
db = get_config_db()
def apply_to_lines(f, x):
return "\n".join(f(_) for _ in x.split('\n'))
def red(s):
red_ = lambda _: termcolor.colored(_, 'red')
return apply_to_lines(red_, s)
red = lambda x: x
table = []
table.append([
'package name',
'node name',
'config_name',
'effective',
'extends',
'valid',
'error',
'description',
'filename',
])
for c in db.configs:
assert isinstance(c, ConfigInfo)
d = dtu.truncate_string_right(c.description.replace('\n', ' '), 40)
date = c.date_effective.strftime('%Y-%m-%d')
if c.valid is None:
valid = '?'
valid_error = ''
else:
valid = 'yes' if c.valid else red('no')
valid_error = '' if c.valid else red(c.error_if_invalid)
table.append([
c.package_name, c.node_name, c.config_name, date, c.extends, valid,
valid_error, d,
dtu.friendly_path(c.filename)
])
dtu.remove_table_field(table, 'filename')
s = dtu.format_table_plus(table, colspacing=4)
return s
def get_logs_description_table(logs, color=True):
table = []
table.append([
'#', 'Log name', 'rc', 'description', 'bag size', 'hash bag', 'date',
'length', 'vehicle name', 'filename', 'valid', 'topics'
])
for i, (_, log) in enumerate(logs.items()):
row = []
row.append(i)
row.append(log.log_name)
row.append(len(log.resources))
# row.append(log.map_name)
row.append(log.description)
dtr = DTR.from_yaml(log.resources['bag'])
bag_size_mb = '%8.1f MB' % (dtr.size / (1024.0 * 1024))
row.append(bag_size_mb)
row.append('%s %s \n%s' % (dtr.name, bag_size_mb, dtr.hash['sha1']))
row.append(log.date)
if log.length is not None:
l = '%5.1f s' % log.length
else:
l = '(none)'
row.append(l)
row.append(log.vehicle)
try:
filename = get_local_bag_file(log)
row.append(dtu.friendly_path(filename))
except NotAvailableLocally:
row.append('not local')
if log.valid:
sr = 'Yes.'
else:
sr = log.error_if_invalid
row.append(sr)
if log.bag_info is not None:
info = yaml.dump(log.bag_info['topics'])
else:
info = '(none)'
if color and not log.valid:
row = dtu.make_row_red(row)
row.append(info)
table.append(row)
return table
def get_logs_cloud():
cloud_file = dtu.require_resource('cloud2.yaml')
with dtu.timeit_wall("loading DB"):
dtu.logger.info('Loading cloud DB %s' % dtu.friendly_path(cloud_file))
with dtu.timeit_wall('YAML load file'):
data = dtu.yaml_load_file(cloud_file, plain_yaml=True)
# with dtu.timeit_wall('plain'):
# data = open(cloud_file).read()
# import yaml
# yaml.load(data)
dtu.logger.debug('Conversion')
logs = logs_from_yaml(data)
logs = OrderedDict(logs)
dtu.logger.info('Loaded cloud DB with %d entries.' % len(logs))
return logs
def format_instances(family, colorize, verbose=False):
if not family.instances:
s = ('No instances files found for family "%s" (pattern = %s).\n\n' %
(family.family_name, family.instances_pattern))
return s
else:
s = ('Found %d instances of algorithm family "%s":\n' %
(len(family.instances), family.family_name))
table = []
table.append([
'Instance name', 'constructor', 'parameters', 'description',
'filename'
])
for _ in family.instances.values():
row = []
name = _.instance_name
if (not _.valid) and colorize:
name = dtu.make_red(name)
row.append(name)
row.append(_.constructor)
row.append(dtu.yaml_dump_pretty(_.parameters))
row.append(_.description)
row.append(dtu.friendly_path(_.filename))
table.append(row)
if not verbose:
dtu.remove_table_field(table, 'filename')
dtu.remove_table_field(table, 'description')
s += dtu.indent(dtu.format_table_plus(table, colspacing=4), '| ')
for _ in family.instances.values():
if not _.valid:
msg = _.error_if_invalid
s += dtu.make_red('\n' +
dtu.indent(msg, '', _.instance_name + ': '))
return s
def interpret_config_file(filename):
"""
Returns a ConfigInfo.
"""
try:
basename = os.path.basename(filename)
base = basename.replace(SUFFIX, '')
# now we have something like
# package-node.config_name.date
# or
# package-node.config_name
if not '.' in base:
msg = 'Invalid filename %r.' % filename
raise dtu.DTConfigException(msg)
tokens = base.split('.')
if len(tokens) > 3:
msg = 'Too many periods/tokens (tokens=%s)' % tokens
raise dtu.DTConfigException(msg)
if len(tokens) <= 2:
# package-node.config_name
package_node = tokens[0]
if not '-' in package_node:
msg = 'Expected a "-" in "%s".' % package_node
raise dtu.DTConfigException(msg)
i = package_node.index('-')
package_name = package_node[:i]
node_name = package_node[i + 1:]
config_name = tokens[1]
if len(tokens) == 3:
# package-node.config_name.date
date_effective = tokens[2]
else:
date_effective = '20170101'
from dateutil.parser import parse
try:
date_effective = parse(date_effective)
except:
msg = 'Cannot interpret "%s" as a date.' % date_effective
raise dtu.DTConfigException(msg)
# now read file
contents = open(filename).read()
try:
try:
data = yaml.load(contents)
except YAMLError as e:
dtu.raise_wrapped(dtu.DTConfigException,
e,
'Invalid YAML',
compact=True)
if not isinstance(data, dict):
msg = 'Expected a dictionary inside.'
raise dtu.DTConfigException(msg)
for field in ['description', 'values']:
if not field in data:
msg = 'Missing field "%s".' % field
raise dtu.DTConfigException(msg)
description = data.pop('description')
if not isinstance(description, str):
msg = 'I expected that "description" is a string, obtained %r.' % description
raise dtu.DTConfigException(msg)
extends = data.pop('extends', [])
if not isinstance(extends, list):
msg = 'I expected that "extends" is a list, obtained %r.' % extends
raise dtu.DTConfigException(msg)
values = data.pop('values')
if not isinstance(values, dict):
msg = 'I expected that "values" is a dictionary, obtained %s.' % type(
values)
raise dtu.DTConfigException(msg)
# Freeze the data
extends = tuple(extends)
values = frozendict(values)
except dtu.DTConfigException as e:
msg = 'Could not interpret the contents of the file\n'
msg += ' %s\n' % dtu.friendly_path(filename)
msg += 'Contents:\n' + dtu.indent(contents, ' > ')
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
return ConfigInfo(
filename=filename,
package_name=package_name,
node_name=node_name,
config_name=config_name,
date_effective=date_effective,
extends=extends,
description=description,
values=values,
# not decided
valid=None,
error_if_invalid=None)
except dtu.DTConfigException as e:
msg = 'Invalid file %s' % dtu.friendly_path(filename)
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)