def write_images(bag_filename, topic, basename):
""" Returns the name of the first created file """
dtu.logger.info('reading topic %r from %r' % (topic, bag_filename))
bag = rosbag.Bag(bag_filename)
nfound = bag.get_message_count(topic)
if nfound == 0:
msg = 'Found 0 messages for topic %s' % topic
msg += '\n\n' + dtu.indent(get_summary_of_bag_messages(bag), ' ')
raise ValueError(msg)
res = dtu.d8n_read_all_images_from_bag(bag, topic)
n = len(res)
filenames = []
for i in range(n):
rgb = res[i]['rgb']
data = dtu.png_from_bgr(dtu.bgr_from_rgb(rgb))
if n == 1:
fn = basename + '.png'
else:
fn = basename + '-%02d' % i + '.png'
filenames.append(fn)
dtu.write_data_to_file(data, fn)
bag.close()
return filenames[0]
def go(self):
db = get_easy_algo_db()
extra = self.options.get_extra()
if len(extra) > 1:
msg = 'Only one extra param accepted.'
raise DTUserError(msg)
if len(extra) == 1:
query = extra[0]
else:
query = 'all'
persons = db.query_and_instance('person', query)
if self.options.roster:
out_file = expand_all(self.options.roster)
outd = os.path.dirname(out_file)
roster = create_roster(persons, outd)
write_data_to_file(roster, out_file)
else:
# instance ll
tags = set()
for p in persons.values():
tags.update(p.get_tags())
print(list(persons))
print('tags: %s' % list(tags))
table = table_people(persons)
print(table)
def what_the_duck_stats():
if len(sys.argv) > 1:
output = sys.argv[1]
else:
output = 'what_the_duck_stats.html'
collection = get_upload_collection()
res = list(get_valid_data(collection))
# for r in res:
# del r['_id']
#
data = yaml_dump(res)
write_data_to_file(data, 'last_download.yaml')
hostnames = defaultdict(lambda: 0)
for r in res:
hostnames[r['hostname']] += 1
s = 'Number of tests by hostname: '
for h, n in hostnames.items():
s += '\n- %s: %d tests' % (h, n)
logger.info(s)
html = create_summary(res)
write_data_to_file(html, output)
def do_all_checks():
username = getpass.getuser()
hostname = socket.gethostname()
filename = 'what_the_duck-%s-%s.html' % (hostname, username)
WTD = colored('what_the_duck', 'cyan', attrs=['bold'])
entries = get_checks()
# logger.info("%s checks many things about the Duckiebot configuration" % WTD)
logger.info('%s will run %s tests.\n' % (WTD, len(entries)))
results = run_checks(entries)
o = ""
o += display_results(results) + '\n\n'
o += display_summary(results) + '\n\n'
print(escaped_from_html(o))
write_data_to_file(o, filename)
print('\nNow send the file "%s" to the TA/instructors.' % filename)
print('\nYou can also upload it using the following command: ')
print('\n scp %s [email protected]:%s ' % (filename, filename))
stats = Statistics(results)
if stats.nfailures == 0:
sys.exit(0)
else:
sys.exit(stats.nfailures)
def what_the_duck_stats():
if len(sys.argv) > 1:
output = sys.argv[1]
else:
output = 'what_the_duck_stats.html'
collection = get_upload_collection()
res = list(get_valid_data(collection))
# logger.debug('dumping last YAML')
# import yaml
# data = yaml.dump(res)
# write_data_to_file(data, 'last_download.yaml')
dtu.logger.debug('dumping Pickle')
dtu.safe_pickle_dump(res, 'last_download.pickle')
hostnames = defaultdict(lambda: 0)
for r in res:
hostnames[r['hostname']] += 1
s = 'Number of tests by hostname: '
for h, n in hostnames.items():
s += '\n- %s: %d tests' % (h, n)
dtu.logger.info(s)
html = create_summary(res)
dtu.write_data_to_file(html, output)
def make_shortcuts_main():
if len(sys.argv) != 2:
msg = 'Expected an argument, the output directory.'
raise dtu.DTUserError(msg)
outd = sys.argv[1]
scripts = list(find_cmds_to_shortcut())
for res in scripts:
filename = os.path.join(outd, 'dt-%s-%s' % (res.package, res.script))
script = """
#!/bin/bash
# Autogenerated file - DO NOT EDIT
# This file was generated by the command {command}
# because the file {filename}
# included the tag "{tag}".
rosrun {package} {script} "$@"
"""
d = {}
d['package'] = res.package
d['filename'] = res.filename
d['script'] = res.script
d['command'] = sys.argv[0]
d['tag'] = tag
script = script.format(**d).strip()
dtu.write_data_to_file(script, filename)
make_executable(filename)
def display_check_results(results, out):
s = ""
s += '\n%d results to report' % len(results)
for i, r in enumerate(results):
s += '\n' + dtu.indent(str(r), '', '%d of %d: ' % (i + 1, len(results)))
print(s)
fn = os.path.join(out, 'check_results.txt')
dtu.write_data_to_file(s, fn)
def write_candidate_cloud(logs):
cache_dir = dtu.get_duckietown_cache_dir()
fn = os.path.join(cache_dir, 'candidate_cloud.yaml')
s = yaml_representation_of_phy_logs(logs)
dtu.write_data_to_file(s, fn)
# try reading
print('reading back logs')
logs2 = logs_from_yaml(dtu.yaml_load_plain(s))
print('read back %s' % len(logs2))
def go(self):
robots = get_scuderia_contents()
machines_contents = create_machines(robots)
fn = dtu.get_machines_files_path()
if getattr(self.options, 'print'):
print(machines_contents)
else:
dtu.write_data_to_file(machines_contents, fn)
def compare_faster2():
variables = collections.OrderedDict()
variables['alpha'] = dict(min=-180,
max=180,
description="angle",
resolution=5,
units='deg',
units_display='deg')
variables['r'] = dict(min=3,
max=5,
description="distance",
resolution=0.2,
units='m',
units_display='cm')
# this will fail if precision is float32
gh = GridHelper(variables, precision='float64')
val_fast = gh.create_new()
val_fast.fill(0)
val_slow = gh.create_new()
val_slow.fill(0)
od = dtu.get_output_dir_for_test()
F = 1
alpha0 = 177
# r0 = 4
r0 = 4.1
w0 = 1
value = dict(alpha=alpha0, r=r0)
gh.add_vote(val_slow, value, w0, F)
values = np.zeros((2, 1))
values[0, 0] = alpha0
values[1, 0] = r0
weights = np.zeros(1)
weights[0] = w0
gh.add_vote_faster(val_fast, values, weights, F)
print('sum slow: %s' % np.sum(val_slow))
print('sum fast: %s' % np.sum(val_fast))
d = grid_helper_plot(gh, val_slow)
fn = os.path.join(od, 'compare_faster_slow.jpg')
dtu.write_data_to_file(d.get_png(), fn)
d = grid_helper_plot(gh, val_fast)
fn = os.path.join(od, 'compare_faster_fast.jpg')
dtu.write_data_to_file(d.get_png(), fn)
f = lambda x: ' %5f' % x if x else ' '
print(dtu.indent(array_as_string(val_fast, f), 'val_fast '))
print(dtu.indent(array_as_string(val_slow, f), 'val_slow '))
assert_almost_equal(val_fast, val_slow)
def run_one(cmd):
v = False
cwd = dtu.get_output_dir_for_test()
if not os.path.exists(cwd):
dtu.mkdirs_thread_safe(cwd)
dtu.write_data_to_file('config echo 1', os.path.join(cwd, '.compmake.rc'))
try:
dtu.system_cmd_result(cwd,
cmd,
display_stdout=v,
display_stderr=v,
raise_on_error=True)
finally:
pass
def display_map(id_map, out):
dtu.logger.info('id_map == %s' % id_map)
db = get_easy_algo_db()
smap = db.create_instance(FAMILY_SEGMAPS, id_map)
texture_png = get_texture(smap, dpi=600)
fn = os.path.join(out, '%s-texture.png' % (id_map))
dtu.write_data_to_file(texture_png, fn)
simdata = simulate_camera_view(smap)
fn = os.path.join(out, '%s-view.jpg' % (id_map))
dtu.write_bgr_to_file_as_jpg(simdata.rectified_synthetic_bgr, fn)
fn = os.path.join(out, '%s-segments.jpg' % (id_map))
dtu.write_bgr_to_file_as_jpg(simdata.rectified_segments_bgr, fn)
def save_homography(H, robot_name):
dtu.logger.info('Homography:\n %s' % H)
# Check if specific point in matrix is larger than zero (this would definitly mean we're having a corrupted rotation matrix)
if (H[1][2] > 0):
msg = "WARNING: Homography could be corrupt."
msg += '\n %s' % H
raise Exception(msg)
ob = {'homography': sum(H.reshape(9, 1).tolist(), [])}
import yaml as alt
s = alt.dump(ob)
s += "\n# Calibrated on dtu.format_time_as_YYYY_MM_DD(time.time())"
fn = get_extrinsics_filename(robot_name)
dtu.write_data_to_file(s, fn)
def save_homography(H, robot_name):
dtu.logger.info('Homography:\n %s' % H)
# Check if specific point in matrix is larger than zero (this would definitly mean we're having a corrupted rotation matrix)
if (H[1][2] > 0):
msg = "WARNING: Homography could be corrupt."
msg += '\n %s' % H
raise Exception(msg)
ob = {'homography': sum(H.reshape(9, 1).tolist(), [])}
s = yaml.dump(ob)
s += "\n# Calibrated on "
localTime = "" + time.strftime("%Y-%m-%d @ %H:%M:%S")
s += localTime
fn = get_extrinsics_filename(robot_name)
dtu.write_data_to_file(s, fn)
def create_roster(people, outd):
s = ''
S = '\n\n'
s += '<div style="display:none" id="autogenerated-roster">'
for k, person in people.items():
jpg = get_image_for_person(k, 128)
basename = k + '.small.jpg'
jpg_file = os.path.join(outd, 'roster-images', basename)
write_data_to_file(jpg, jpg_file)
name = person.get_name()
s += '<div id="%s-roster" class="roster-person">' % k
s += '\n <span class="name">%s</span>' % name
s += '\n <img src="%s"/>' % basename
s += '\n' + indent(roster_css, ' ')
s += '\n</div>' + S + S
s += S + '</div>'
return s
def create_report_html(log_name, filenames, out):
html = Tag(name='html')
body = Tag(name='body')
head = Tag(name='head')
link = Tag(name='link')
link.attrs['type'] = 'text/css'
link.attrs['rel'] = 'stylesheet'
link.attrs['href'] = 'style.css'
title = Tag(name='title')
title.append('Report')
head.append(link)
html.append(head)
h = Tag(name='h1')
h.append('Report for %s' % log_name)
body.append(h)
for f in filenames:
f = os.path.realpath(f)
out = os.path.realpath(out)
d = os.path.dirname(out)
rel = os.path.relpath(f, d)
p = Tag(name='p')
if '.jpg' in f or '.png' in f:
img = Tag(name='img')
img.attrs['src'] = rel
p.append(img)
if '.mp4' in f:
v = video_for_source(rel)
p.append(v)
body.append(p)
html.append(body)
s = str(html)
dtu.write_data_to_file(s, out)
def grid_visualization():
variables = collections.OrderedDict()
variables['alpha'] = dict(min=-np.pi / 2,
max=np.pi / 2,
description="angle",
resolution=np.deg2rad(10),
units='rad',
units_display='deg')
variables['r'] = dict(min=3,
max=5,
description="distance",
resolution=0.1,
units='m',
units_display='cm')
gh = GridHelper(variables)
val = gh.create_new()
val.fill(0)
val += np.random.randn(*val.shape)
val[1, 2] = 0
d = grid_helper_plot(gh, val)
od = dtu.get_output_dir_for_test()
fn = os.path.join(od, 'grid_visualization.jpg')
dtu.write_data_to_file(d.get_png(), fn)
def go(self):
extra = self.options.get_extra()
Gallery.deploy_ipfs = self.options.ipfs
if not extra:
query = '*'
else:
query = extra
db = self.get_easy_logs_db()
logs = db.query(query)
logs_valid = OrderedDict()
ninvalid = 0
length_invalid = 0.0
for log_name, log in logs.items():
if log.valid:
logs_valid[log_name] = log
else:
ninvalid += 1
if log.length is not None:
length_invalid += log.length
logs = logs_valid
self.info('Found %d valid logs.' % len(logs))
s = format_logs(logs)
self.info(s)
res = get_report(logs)
out = self.options.destination
fn_html = os.path.join(out, 'index.html')
dtu.write_data_to_file(res, fn_html)
def print_results(analyzers, results_all, out):
base = os.path.join(out, 'statistics')
yaml_data = dtu.yaml_dump_pretty(results_all)
dtu.write_data_to_file(yaml_data, os.path.join(base, 'statistics.yaml'))
print(dtu.indent(yaml_data, 'print_results '))
for a in analyzers:
dtu.write_data_to_file(dtu.yaml_dump_pretty(results_all[a]),
os.path.join(base, '%s.table.yaml' % a))
s = ""
s += '\n' + '-' * 10 + ' Results for %s ' % a + '-' * 10
table = table_for_analyzer(results_all[a])
s += '\n' + dtu.indent(dtu.format_table_plus(table, colspacing=3),
' ')
s += '\n'
dtu.write_data_to_file(s, os.path.join(base, '%s.table.txt' % a))
def write_to_db(rt_name, results_all, out):
rdbe = make_entry(rt_name, results_all)
fn = get_unique_filename(rt_name, rdbe)
s = yaml_from_rdbe(rdbe)
filename = os.path.join(out, fn)
dtu.write_data_to_file(s, filename)
def compare_faster():
variables = collections.OrderedDict()
variables['alpha'] = dict(min=-180,
max=180,
description="angle",
resolution=5,
units='deg',
units_display='deg')
variables['r'] = dict(min=3,
max=5,
description="distance",
resolution=0.1,
units='m',
units_display='cm')
# this will fail if precision is float32
gh = GridHelper(variables, precision='float64')
val_fast = gh.create_new()
val_fast.fill(0)
val_slow = gh.create_new()
val_slow.fill(0)
od = dtu.get_output_dir_for_test()
F = 1
alpha0 = 7
# r0 = 4
r0 = 4.1
w0 = 1.
value = dict(alpha=alpha0, r=r0)
gh.add_vote(val_slow, value, w0, F)
assert_equal(np.sum(val_slow > 0), 9)
values = np.zeros((2, 1))
values[0, 0] = alpha0
values[1, 0] = r0
weights = np.zeros(1)
weights[0] = w0
gh.add_vote_faster(val_fast, values, weights, F)
assert_equal(np.sum(val_fast > 0), 9)
d = grid_helper_plot(gh, val_slow)
fn = os.path.join(od, 'compare_faster_slow.jpg')
dtu.write_data_to_file(d.get_png(), fn)
d = grid_helper_plot(gh, val_fast)
fn = os.path.join(od, 'compare_faster_fast.jpg')
dtu.write_data_to_file(d.get_png(), fn)
D = val_fast - val_slow
diff = np.max(np.abs(D))
print('diff: %r' % diff)
if diff > 1e-8:
print(dtu.indent(array_as_string_sign(val_fast), 'val_fast '))
print(dtu.indent(array_as_string_sign(val_slow), 'val_slow '))
print(dtu.indent(array_as_string_sign(D), 'Diff '))
print('non zero val_fast: %s' % val_fast[val_fast > 0])
print('non zero val_slow: %s' % val_slow[val_slow > 0])
assert_almost_equal(val_fast, val_slow)
body {
font-size: 8pt;
}
td {
overflow-wrap: break-word;
min-width: 4em;
max-width: 4em;
padding-bottom: 3px;
}
td {
border-bottom: solid 1px black;
}
"""
style = Tag(name='style')
style.append(css)
body.append(style)
body.append(table)
return body
if __name__ == '__main__':
filename = sys.argv[1]
mongo_data = yaml_load_file(filename)
html = create_summary(mongo_data)
write_data_to_file(html, 'output.html')
'laptop': u"💻",
'duckiebot': u"🚗"
}
s = cute.get(x, x)
return escaper.substitute_html(s)
if __name__ == '__main__':
filename = sys.argv[1]
if len(sys.argv) >= 3:
output = sys.argv[2]
else:
output = 'output.html'
if filename.endswith('yaml'):
data = open(filename).read()
print('loading data (%d chars)' % (len(data)))
# from ruamel import yaml
import yaml
# mongo_data = yaml.load(data, Loader=yaml.UnsafeLoader)
mongo_data = yaml.load(data)
else:
# data = open(filename).read()
mongo_data = dtu.safe_pickle_load(filename)
# mongo_data = yaml_load(data)
print('creating summary')
html = create_summary(mongo_data)
dtu.write_data_to_file(html, output)
'laptop': u"💻",
'duckiebot': u"🚗"
}
s = cute.get(x, x)
return escaper.substitute_html(s)
if __name__ == '__main__':
filename = sys.argv[1]
if len(sys.argv) >= 3:
output = sys.argv[2]
else:
output = 'output.html'
if filename.endswith('yaml'):
data = open(filename).read()
print('loading data (%d chars)' % (len(data)))
# from ruamel import yaml
import yaml
# mongo_data = yaml.load(data, Loader=yaml.UnsafeLoader)
mongo_data = yaml.load(data)
else:
# data = open(filename).read()
mongo_data = safe_pickle_load(filename)
# mongo_data = yaml_load(data)
print('creating summary')
html = create_summary(mongo_data)
write_data_to_file(html, output)
def voting_kernel1():
resolution = 10.0
variables = collections.OrderedDict()
variables['x'] = dict(min=100,
max=500,
description="x",
resolution=resolution,
units='cm',
units_display='cm')
variables['y'] = dict(min=100,
max=500,
description="y",
resolution=resolution,
units='cm',
units_display='cm')
gh = GridHelper(variables)
votes = gh.create_new()
votes.fill(0)
points = []
estimated = []
estimated_weighted = []
F = 1
N = 25
errors_x = []
errors_x_w = []
for i in range(N):
dx = resolution / N
Dy = 70
Dx = 70
u = (i / 5)
v = i % 5
x = 125 + dx * i + Dx * u
y = 127 + Dy * v
p = dict(x=x, y=y)
points.append(p)
weight = 1
gh.add_vote(votes, p, weight=weight, F=F)
tmp = gh.create_new()
tmp.fill(0)
gh.add_vote(tmp, p, weight=weight, F=F)
assert_almost_equal(tmp.sum(), weight)
estimate = gh.get_max(tmp)
estimated.append(estimate)
estimate_weigthed = gh.get_max_weighted(tmp, F=F)
estimated_weighted.append(estimate_weigthed)
errors_x.append(p['x'] - estimate['x'])
errors_x_w.append(p['x'] - estimate_weigthed['x'])
errors_x = np.array(errors_x)
errors_x_w = np.array(errors_x_w)
dtu.logger.debug('errors_x: %s' % errors_x)
dtu.logger.debug('mean: %s' % np.abs(errors_x).mean())
dtu.logger.debug('errors_x_w: %s' % errors_x_w)
dtu.logger.debug('mean: %s' % np.abs(errors_x_w).mean())
assert (errors_x.max() <= +resolution / 2)
assert (errors_x.min() >= -resolution / 2)
assert (np.abs(errors_x_w).max() <= resolution / 10)
a = dtu.CreateImageFromPylab(dpi=1000)
with a as pylab:
grid_helper_plot_field(gh, votes, pylab)
pylab.axis('equal')
grid_helper_annotate_axes(gh, pylab)
for p in points:
grid_helper_mark_point(gh, pylab, p, color='blue', markersize=4)
for e in estimated:
grid_helper_mark_point(gh, pylab, e, color='red', markersize=3)
for e in estimated_weighted:
grid_helper_mark_point(gh, pylab, e, color='green', markersize=3)
b = dtu.CreateImageFromPylab(dpi=1000)
with b as pylab:
x = np.array([_['x'] for _ in points])
xe = np.array([_['x'] for _ in estimated])
xew = np.array([_['x'] for _ in estimated_weighted])
xe = xe - x
xew = xew - x
x = x * 0
pylab.plot(x, '.', label='x')
pylab.plot(xe, '.', label='x estimated')
pylab.plot(xew, '.', label='x estimated weighted')
pylab.legend()
od = dtu.get_output_dir_for_test()
fn = os.path.join(od, 'voting_kernel1.jpg')
dtu.write_data_to_file(a.get_png(), fn)
fn = os.path.join(od, 'errors.jpg')
dtu.write_data_to_file(b.get_png(), fn)
