def write_images(bag_filename, topic, basename): """ Returns the name of the first created file """ dtu.logger.info('reading topic %r from %r' % (topic, bag_filename)) bag = rosbag.Bag(bag_filename) nfound = bag.get_message_count(topic) if nfound == 0: msg = 'Found 0 messages for topic %s' % topic msg += '\n\n' + dtu.indent(get_summary_of_bag_messages(bag), ' ') raise ValueError(msg) res = dtu.d8n_read_all_images_from_bag(bag, topic) n = len(res) filenames = [] for i in range(n): rgb = res[i]['rgb'] data = dtu.png_from_bgr(dtu.bgr_from_rgb(rgb)) if n == 1: fn = basename + '.png' else: fn = basename + '-%02d' % i + '.png' filenames.append(fn) dtu.write_data_to_file(data, fn) bag.close() return filenames[0]
def go(self): db = get_easy_algo_db() extra = self.options.get_extra() if len(extra) > 1: msg = 'Only one extra param accepted.' raise DTUserError(msg) if len(extra) == 1: query = extra[0] else: query = 'all' persons = db.query_and_instance('person', query) if self.options.roster: out_file = expand_all(self.options.roster) outd = os.path.dirname(out_file) roster = create_roster(persons, outd) write_data_to_file(roster, out_file) else: # instance ll tags = set() for p in persons.values(): tags.update(p.get_tags()) print(list(persons)) print('tags: %s' % list(tags)) table = table_people(persons) print(table)
def what_the_duck_stats(): if len(sys.argv) > 1: output = sys.argv[1] else: output = 'what_the_duck_stats.html' collection = get_upload_collection() res = list(get_valid_data(collection)) # for r in res: # del r['_id'] # data = yaml_dump(res) write_data_to_file(data, 'last_download.yaml') hostnames = defaultdict(lambda: 0) for r in res: hostnames[r['hostname']] += 1 s = 'Number of tests by hostname: ' for h, n in hostnames.items(): s += '\n- %s: %d tests' % (h, n) logger.info(s) html = create_summary(res) write_data_to_file(html, output)
def do_all_checks(): username = getpass.getuser() hostname = socket.gethostname() filename = 'what_the_duck-%s-%s.html' % (hostname, username) WTD = colored('what_the_duck', 'cyan', attrs=['bold']) entries = get_checks() # logger.info("%s checks many things about the Duckiebot configuration" % WTD) logger.info('%s will run %s tests.\n' % (WTD, len(entries))) results = run_checks(entries) o = "" o += display_results(results) + '\n\n' o += display_summary(results) + '\n\n' print(escaped_from_html(o)) write_data_to_file(o, filename) print('\nNow send the file "%s" to the TA/instructors.' % filename) print('\nYou can also upload it using the following command: ') print('\n scp %s [email protected]:%s ' % (filename, filename)) stats = Statistics(results) if stats.nfailures == 0: sys.exit(0) else: sys.exit(stats.nfailures)
def what_the_duck_stats(): if len(sys.argv) > 1: output = sys.argv[1] else: output = 'what_the_duck_stats.html' collection = get_upload_collection() res = list(get_valid_data(collection)) # logger.debug('dumping last YAML') # import yaml # data = yaml.dump(res) # write_data_to_file(data, 'last_download.yaml') dtu.logger.debug('dumping Pickle') dtu.safe_pickle_dump(res, 'last_download.pickle') hostnames = defaultdict(lambda: 0) for r in res: hostnames[r['hostname']] += 1 s = 'Number of tests by hostname: ' for h, n in hostnames.items(): s += '\n- %s: %d tests' % (h, n) dtu.logger.info(s) html = create_summary(res) dtu.write_data_to_file(html, output)
def make_shortcuts_main(): if len(sys.argv) != 2: msg = 'Expected an argument, the output directory.' raise dtu.DTUserError(msg) outd = sys.argv[1] scripts = list(find_cmds_to_shortcut()) for res in scripts: filename = os.path.join(outd, 'dt-%s-%s' % (res.package, res.script)) script = """ #!/bin/bash # Autogenerated file - DO NOT EDIT # This file was generated by the command {command} # because the file {filename} # included the tag "{tag}". rosrun {package} {script} "$@" """ d = {} d['package'] = res.package d['filename'] = res.filename d['script'] = res.script d['command'] = sys.argv[0] d['tag'] = tag script = script.format(**d).strip() dtu.write_data_to_file(script, filename) make_executable(filename)
def display_check_results(results, out): s = "" s += '\n%d results to report' % len(results) for i, r in enumerate(results): s += '\n' + dtu.indent(str(r), '', '%d of %d: ' % (i + 1, len(results))) print(s) fn = os.path.join(out, 'check_results.txt') dtu.write_data_to_file(s, fn)
def write_candidate_cloud(logs): cache_dir = dtu.get_duckietown_cache_dir() fn = os.path.join(cache_dir, 'candidate_cloud.yaml') s = yaml_representation_of_phy_logs(logs) dtu.write_data_to_file(s, fn) # try reading print('reading back logs') logs2 = logs_from_yaml(dtu.yaml_load_plain(s)) print('read back %s' % len(logs2))
def go(self): robots = get_scuderia_contents() machines_contents = create_machines(robots) fn = dtu.get_machines_files_path() if getattr(self.options, 'print'): print(machines_contents) else: dtu.write_data_to_file(machines_contents, fn)
def compare_faster2(): variables = collections.OrderedDict() variables['alpha'] = dict(min=-180, max=180, description="angle", resolution=5, units='deg', units_display='deg') variables['r'] = dict(min=3, max=5, description="distance", resolution=0.2, units='m', units_display='cm') # this will fail if precision is float32 gh = GridHelper(variables, precision='float64') val_fast = gh.create_new() val_fast.fill(0) val_slow = gh.create_new() val_slow.fill(0) od = dtu.get_output_dir_for_test() F = 1 alpha0 = 177 # r0 = 4 r0 = 4.1 w0 = 1 value = dict(alpha=alpha0, r=r0) gh.add_vote(val_slow, value, w0, F) values = np.zeros((2, 1)) values[0, 0] = alpha0 values[1, 0] = r0 weights = np.zeros(1) weights[0] = w0 gh.add_vote_faster(val_fast, values, weights, F) print('sum slow: %s' % np.sum(val_slow)) print('sum fast: %s' % np.sum(val_fast)) d = grid_helper_plot(gh, val_slow) fn = os.path.join(od, 'compare_faster_slow.jpg') dtu.write_data_to_file(d.get_png(), fn) d = grid_helper_plot(gh, val_fast) fn = os.path.join(od, 'compare_faster_fast.jpg') dtu.write_data_to_file(d.get_png(), fn) f = lambda x: ' %5f' % x if x else ' ' print(dtu.indent(array_as_string(val_fast, f), 'val_fast ')) print(dtu.indent(array_as_string(val_slow, f), 'val_slow ')) assert_almost_equal(val_fast, val_slow)
def run_one(cmd): v = False cwd = dtu.get_output_dir_for_test() if not os.path.exists(cwd): dtu.mkdirs_thread_safe(cwd) dtu.write_data_to_file('config echo 1', os.path.join(cwd, '.compmake.rc')) try: dtu.system_cmd_result(cwd, cmd, display_stdout=v, display_stderr=v, raise_on_error=True) finally: pass
def display_map(id_map, out): dtu.logger.info('id_map == %s' % id_map) db = get_easy_algo_db() smap = db.create_instance(FAMILY_SEGMAPS, id_map) texture_png = get_texture(smap, dpi=600) fn = os.path.join(out, '%s-texture.png' % (id_map)) dtu.write_data_to_file(texture_png, fn) simdata = simulate_camera_view(smap) fn = os.path.join(out, '%s-view.jpg' % (id_map)) dtu.write_bgr_to_file_as_jpg(simdata.rectified_synthetic_bgr, fn) fn = os.path.join(out, '%s-segments.jpg' % (id_map)) dtu.write_bgr_to_file_as_jpg(simdata.rectified_segments_bgr, fn)
def save_homography(H, robot_name): dtu.logger.info('Homography:\n %s' % H) # Check if specific point in matrix is larger than zero (this would definitly mean we're having a corrupted rotation matrix) if (H[1][2] > 0): msg = "WARNING: Homography could be corrupt." msg += '\n %s' % H raise Exception(msg) ob = {'homography': sum(H.reshape(9, 1).tolist(), [])} import yaml as alt s = alt.dump(ob) s += "\n# Calibrated on dtu.format_time_as_YYYY_MM_DD(time.time())" fn = get_extrinsics_filename(robot_name) dtu.write_data_to_file(s, fn)
def save_homography(H, robot_name): dtu.logger.info('Homography:\n %s' % H) # Check if specific point in matrix is larger than zero (this would definitly mean we're having a corrupted rotation matrix) if (H[1][2] > 0): msg = "WARNING: Homography could be corrupt." msg += '\n %s' % H raise Exception(msg) ob = {'homography': sum(H.reshape(9, 1).tolist(), [])} s = yaml.dump(ob) s += "\n# Calibrated on " localTime = "" + time.strftime("%Y-%m-%d @ %H:%M:%S") s += localTime fn = get_extrinsics_filename(robot_name) dtu.write_data_to_file(s, fn)
def create_roster(people, outd): s = '' S = '\n\n' s += '<div style="display:none" id="autogenerated-roster">' for k, person in people.items(): jpg = get_image_for_person(k, 128) basename = k + '.small.jpg' jpg_file = os.path.join(outd, 'roster-images', basename) write_data_to_file(jpg, jpg_file) name = person.get_name() s += '<div id="%s-roster" class="roster-person">' % k s += '\n <span class="name">%s</span>' % name s += '\n <img src="%s"/>' % basename s += '\n' + indent(roster_css, ' ') s += '\n</div>' + S + S s += S + '</div>' return s
def create_report_html(log_name, filenames, out): html = Tag(name='html') body = Tag(name='body') head = Tag(name='head') link = Tag(name='link') link.attrs['type'] = 'text/css' link.attrs['rel'] = 'stylesheet' link.attrs['href'] = 'style.css' title = Tag(name='title') title.append('Report') head.append(link) html.append(head) h = Tag(name='h1') h.append('Report for %s' % log_name) body.append(h) for f in filenames: f = os.path.realpath(f) out = os.path.realpath(out) d = os.path.dirname(out) rel = os.path.relpath(f, d) p = Tag(name='p') if '.jpg' in f or '.png' in f: img = Tag(name='img') img.attrs['src'] = rel p.append(img) if '.mp4' in f: v = video_for_source(rel) p.append(v) body.append(p) html.append(body) s = str(html) dtu.write_data_to_file(s, out)
def grid_visualization(): variables = collections.OrderedDict() variables['alpha'] = dict(min=-np.pi / 2, max=np.pi / 2, description="angle", resolution=np.deg2rad(10), units='rad', units_display='deg') variables['r'] = dict(min=3, max=5, description="distance", resolution=0.1, units='m', units_display='cm') gh = GridHelper(variables) val = gh.create_new() val.fill(0) val += np.random.randn(*val.shape) val[1, 2] = 0 d = grid_helper_plot(gh, val) od = dtu.get_output_dir_for_test() fn = os.path.join(od, 'grid_visualization.jpg') dtu.write_data_to_file(d.get_png(), fn)
def go(self): extra = self.options.get_extra() Gallery.deploy_ipfs = self.options.ipfs if not extra: query = '*' else: query = extra db = self.get_easy_logs_db() logs = db.query(query) logs_valid = OrderedDict() ninvalid = 0 length_invalid = 0.0 for log_name, log in logs.items(): if log.valid: logs_valid[log_name] = log else: ninvalid += 1 if log.length is not None: length_invalid += log.length logs = logs_valid self.info('Found %d valid logs.' % len(logs)) s = format_logs(logs) self.info(s) res = get_report(logs) out = self.options.destination fn_html = os.path.join(out, 'index.html') dtu.write_data_to_file(res, fn_html)
def print_results(analyzers, results_all, out): base = os.path.join(out, 'statistics') yaml_data = dtu.yaml_dump_pretty(results_all) dtu.write_data_to_file(yaml_data, os.path.join(base, 'statistics.yaml')) print(dtu.indent(yaml_data, 'print_results ')) for a in analyzers: dtu.write_data_to_file(dtu.yaml_dump_pretty(results_all[a]), os.path.join(base, '%s.table.yaml' % a)) s = "" s += '\n' + '-' * 10 + ' Results for %s ' % a + '-' * 10 table = table_for_analyzer(results_all[a]) s += '\n' + dtu.indent(dtu.format_table_plus(table, colspacing=3), ' ') s += '\n' dtu.write_data_to_file(s, os.path.join(base, '%s.table.txt' % a))
def write_to_db(rt_name, results_all, out): rdbe = make_entry(rt_name, results_all) fn = get_unique_filename(rt_name, rdbe) s = yaml_from_rdbe(rdbe) filename = os.path.join(out, fn) dtu.write_data_to_file(s, filename)
def compare_faster(): variables = collections.OrderedDict() variables['alpha'] = dict(min=-180, max=180, description="angle", resolution=5, units='deg', units_display='deg') variables['r'] = dict(min=3, max=5, description="distance", resolution=0.1, units='m', units_display='cm') # this will fail if precision is float32 gh = GridHelper(variables, precision='float64') val_fast = gh.create_new() val_fast.fill(0) val_slow = gh.create_new() val_slow.fill(0) od = dtu.get_output_dir_for_test() F = 1 alpha0 = 7 # r0 = 4 r0 = 4.1 w0 = 1. value = dict(alpha=alpha0, r=r0) gh.add_vote(val_slow, value, w0, F) assert_equal(np.sum(val_slow > 0), 9) values = np.zeros((2, 1)) values[0, 0] = alpha0 values[1, 0] = r0 weights = np.zeros(1) weights[0] = w0 gh.add_vote_faster(val_fast, values, weights, F) assert_equal(np.sum(val_fast > 0), 9) d = grid_helper_plot(gh, val_slow) fn = os.path.join(od, 'compare_faster_slow.jpg') dtu.write_data_to_file(d.get_png(), fn) d = grid_helper_plot(gh, val_fast) fn = os.path.join(od, 'compare_faster_fast.jpg') dtu.write_data_to_file(d.get_png(), fn) D = val_fast - val_slow diff = np.max(np.abs(D)) print('diff: %r' % diff) if diff > 1e-8: print(dtu.indent(array_as_string_sign(val_fast), 'val_fast ')) print(dtu.indent(array_as_string_sign(val_slow), 'val_slow ')) print(dtu.indent(array_as_string_sign(D), 'Diff ')) print('non zero val_fast: %s' % val_fast[val_fast > 0]) print('non zero val_slow: %s' % val_slow[val_slow > 0]) assert_almost_equal(val_fast, val_slow)
body { font-size: 8pt; } td { overflow-wrap: break-word; min-width: 4em; max-width: 4em; padding-bottom: 3px; } td { border-bottom: solid 1px black; } """ style = Tag(name='style') style.append(css) body.append(style) body.append(table) return body if __name__ == '__main__': filename = sys.argv[1] mongo_data = yaml_load_file(filename) html = create_summary(mongo_data) write_data_to_file(html, 'output.html')
'laptop': u"💻", 'duckiebot': u"🚗" } s = cute.get(x, x) return escaper.substitute_html(s) if __name__ == '__main__': filename = sys.argv[1] if len(sys.argv) >= 3: output = sys.argv[2] else: output = 'output.html' if filename.endswith('yaml'): data = open(filename).read() print('loading data (%d chars)' % (len(data))) # from ruamel import yaml import yaml # mongo_data = yaml.load(data, Loader=yaml.UnsafeLoader) mongo_data = yaml.load(data) else: # data = open(filename).read() mongo_data = dtu.safe_pickle_load(filename) # mongo_data = yaml_load(data) print('creating summary') html = create_summary(mongo_data) dtu.write_data_to_file(html, output)
'laptop': u"💻", 'duckiebot': u"🚗" } s = cute.get(x, x) return escaper.substitute_html(s) if __name__ == '__main__': filename = sys.argv[1] if len(sys.argv) >= 3: output = sys.argv[2] else: output = 'output.html' if filename.endswith('yaml'): data = open(filename).read() print('loading data (%d chars)' % (len(data))) # from ruamel import yaml import yaml # mongo_data = yaml.load(data, Loader=yaml.UnsafeLoader) mongo_data = yaml.load(data) else: # data = open(filename).read() mongo_data = safe_pickle_load(filename) # mongo_data = yaml_load(data) print('creating summary') html = create_summary(mongo_data) write_data_to_file(html, output)
def voting_kernel1(): resolution = 10.0 variables = collections.OrderedDict() variables['x'] = dict(min=100, max=500, description="x", resolution=resolution, units='cm', units_display='cm') variables['y'] = dict(min=100, max=500, description="y", resolution=resolution, units='cm', units_display='cm') gh = GridHelper(variables) votes = gh.create_new() votes.fill(0) points = [] estimated = [] estimated_weighted = [] F = 1 N = 25 errors_x = [] errors_x_w = [] for i in range(N): dx = resolution / N Dy = 70 Dx = 70 u = (i / 5) v = i % 5 x = 125 + dx * i + Dx * u y = 127 + Dy * v p = dict(x=x, y=y) points.append(p) weight = 1 gh.add_vote(votes, p, weight=weight, F=F) tmp = gh.create_new() tmp.fill(0) gh.add_vote(tmp, p, weight=weight, F=F) assert_almost_equal(tmp.sum(), weight) estimate = gh.get_max(tmp) estimated.append(estimate) estimate_weigthed = gh.get_max_weighted(tmp, F=F) estimated_weighted.append(estimate_weigthed) errors_x.append(p['x'] - estimate['x']) errors_x_w.append(p['x'] - estimate_weigthed['x']) errors_x = np.array(errors_x) errors_x_w = np.array(errors_x_w) dtu.logger.debug('errors_x: %s' % errors_x) dtu.logger.debug('mean: %s' % np.abs(errors_x).mean()) dtu.logger.debug('errors_x_w: %s' % errors_x_w) dtu.logger.debug('mean: %s' % np.abs(errors_x_w).mean()) assert (errors_x.max() <= +resolution / 2) assert (errors_x.min() >= -resolution / 2) assert (np.abs(errors_x_w).max() <= resolution / 10) a = dtu.CreateImageFromPylab(dpi=1000) with a as pylab: grid_helper_plot_field(gh, votes, pylab) pylab.axis('equal') grid_helper_annotate_axes(gh, pylab) for p in points: grid_helper_mark_point(gh, pylab, p, color='blue', markersize=4) for e in estimated: grid_helper_mark_point(gh, pylab, e, color='red', markersize=3) for e in estimated_weighted: grid_helper_mark_point(gh, pylab, e, color='green', markersize=3) b = dtu.CreateImageFromPylab(dpi=1000) with b as pylab: x = np.array([_['x'] for _ in points]) xe = np.array([_['x'] for _ in estimated]) xew = np.array([_['x'] for _ in estimated_weighted]) xe = xe - x xew = xew - x x = x * 0 pylab.plot(x, '.', label='x') pylab.plot(xe, '.', label='x estimated') pylab.plot(xew, '.', label='x estimated weighted') pylab.legend() od = dtu.get_output_dir_for_test() fn = os.path.join(od, 'voting_kernel1.jpg') dtu.write_data_to_file(a.get_png(), fn) fn = os.path.join(od, 'errors.jpg') dtu.write_data_to_file(b.get_png(), fn)