def logreader_from_route_or_segment(r): sn = SegmentName(r, allow_route_name=True) route = Route(sn.route_name.canonical_name) if sn.segment_num < 0: return MultiLogIterator(route.log_paths()) else: return LogReader(route.log_paths()[sn.segment_num])
def main(argv): args = get_arg_parser().parse_args(sys.argv[1:]) if not args.data_dir: print('Data directory invalid.') return if not args.route_name: # Extract route name from path args.route_name = os.path.basename(args.data_dir) args.data_dir = os.path.dirname(args.data_dir) route = Route(args.route_name, args.data_dir) lr = MultiLogIterator(route.log_paths()) with open(args.out_path, 'wb') as f: try: done = False i = 0 while not done: msg = next(lr) if not msg: break smsg = msg.as_builder() typ = smsg.which() if typ == 'ubloxRaw': f.write(smsg.to_bytes()) i += 1 except StopIteration: print('All done') print(f'Writed {i} msgs')
def juggle_route(route_name, segment_number, qlog): r = Route(route_name) logs = r.qlog_paths() if qlog else r.log_paths() if segment_number is not None: logs = logs[segment_number:segment_number + 1] if None in logs: fallback_answer = input( "At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : " ) if fallback_answer == 'y': logs = r.qlog_paths() if segment_number is not None: logs = logs[segment_number:segment_number + 1] else: print( f"Please try a different {'segment' if segment_number is not None else 'route'}" ) return all_data = [] pool = multiprocessing.Pool(24) for d in pool.map(load_segment, logs): all_data += d tempfile = NamedTemporaryFile(suffix='.rlog') save_log(tempfile.name, all_data, compress=False) del all_data juggle_file(tempfile.name)
def _process_commands(self, cmd, route, pub_types): seek_to = None if route is None or (isinstance(cmd, SetRoute) and route.name != cmd.name): seek_to = cmd.start_time route = Route(cmd.name, cmd.data_dir) self._lr = MultiLogIterator(route.log_paths(), wraparound=True) if self._frame_reader is not None: self._frame_reader.close() if "frame" in pub_types or "encodeIdx" in pub_types: # reset frames for a route self._frame_id_lookup = {} self._frame_reader = RouteFrameReader( route.camera_paths(), None, self._frame_id_lookup, readahead=True) # always reset this on a seek if isinstance(cmd, SeekRelativeTime): seek_to = self._lr.tell() + cmd.secs elif isinstance(cmd, SeekAbsoluteTime): seek_to = cmd.secs elif isinstance(cmd, StopAndQuit): exit() if seek_to is not None: print("seeking", seek_to) if not self._lr.seek(seek_to): print("Can't seek: time out of bounds") else: next(self._lr) # ignore one return route
def regen_and_save(route, sidx, upload=False, use_route_meta=False): if use_route_meta: r = Route(args.route) lr = LogReader(r.log_paths()[args.seg]) fr = FrameReader(r.camera_paths()[args.seg]) else: lr = LogReader(f"cd:/{route.replace('|', '/')}/{sidx}/rlog.bz2") fr = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/fcamera.hevc") rpath = regen_segment(lr, {'roadCameraState': fr}) # compress raw rlog before uploading with open(os.path.join(rpath, "rlog"), "rb") as f: data = bz2.compress(f.read()) with open(os.path.join(rpath, "rlog.bz2"), "wb") as f: f.write(data) os.remove(os.path.join(rpath, "rlog")) lr = LogReader(os.path.join(rpath, 'rlog.bz2')) controls_state_active = [m.controlsState.active for m in lr if m.which() == 'controlsState'] assert any(controls_state_active), "Segment did not engage" relr = os.path.relpath(rpath) print("\n\n", "*"*30, "\n\n") print("New route:", relr, "\n") if upload: upload_route(relr) return relr
def juggle_route(route_or_segment_name, segment_count, qlog, can, layout): segment_start = 0 if 'cabana' in route_or_segment_name: query = parse_qs(urlparse(route_or_segment_name).query) api = CommaApi(get_token()) logs = api.get( f'v1/route/{query["route"][0]}/log_urls?sig={query["sig"][0]}&exp={query["exp"][0]}' ) elif route_or_segment_name.startswith( "http://") or route_or_segment_name.startswith( "https://") or os.path.isfile(route_or_segment_name): logs = [route_or_segment_name] else: route_or_segment_name = SegmentName(route_or_segment_name, allow_route_name=True) segment_start = max(route_or_segment_name.segment_num, 0) if route_or_segment_name.segment_num != -1 and segment_count is None: segment_count = 1 r = Route(route_or_segment_name.route_name.canonical_name) logs = r.qlog_paths() if qlog else r.log_paths() segment_end = segment_start + segment_count if segment_count else None logs = logs[segment_start:segment_end] if None in logs: ans = input( f"{logs.count(None)}/{len(logs)} of the rlogs in this segment are missing, would you like to fall back to the qlogs? (y/n) " ) if ans == 'y': logs = r.qlog_paths()[segment_start:segment_end] else: print("Please try a different route or segment") return all_data = [] with multiprocessing.Pool(24) as pool: for d in pool.map(load_segment, logs): all_data += d if not can: all_data = [d for d in all_data if d.which() not in ['can', 'sendcan']] # Infer DBC name from logs dbc = None for cp in [m for m in all_data if m.which() == 'carParams']: try: DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values", fromlist=['DBC']).DBC dbc = DBC[cp.carParams.carFingerprint]['pt'] except Exception: pass break with tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) as tmp: save_log(tmp.name, all_data, compress=False) del all_data start_juggler(tmp.name, dbc, layout)
def juggle_segment(route_name, segment_nr): r = Route(route_name) lp = r.log_paths()[segment_nr] if lp is None: print("This segment does not exist, please try a different one") return uf = URLFile(lp) juggle_file(uf.name)
def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): if 'cabana' in route_name: query = parse_qs(urlparse(route_name).query) api = CommaApi(get_token()) logs = api.get( f'v1/route/{query["route"][0]}/log_urls?sig={query["sig"][0]}&exp={query["exp"][0]}' ) elif route_name.startswith("http://") or route_name.startswith( "https://") or os.path.isfile(route_name): logs = [route_name] else: r = Route(route_name) logs = r.qlog_paths() if qlog else r.log_paths() if segment_number is not None: logs = logs[segment_number:segment_number + segment_count] if None in logs: fallback_answer = input( "At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : " ) if fallback_answer == 'y': logs = r.qlog_paths() if segment_number is not None: logs = logs[segment_number:segment_number + segment_count] else: print( f"Please try a different {'segment' if segment_number is not None else 'route'}" ) return all_data = [] with multiprocessing.Pool(24) as pool: for d in pool.map(load_segment, logs): all_data += d if not can: all_data = [d for d in all_data if d.which() not in ['can', 'sendcan']] # Infer DBC name from logs dbc = None for cp in [m for m in all_data if m.which() == 'carParams']: try: DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values", fromlist=['DBC']).DBC dbc = DBC[cp.carParams.carFingerprint]['pt'] except (ImportError, KeyError, AttributeError): pass break tempfile = NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) save_log(tempfile.name, all_data, compress=False) del all_data start_juggler(tempfile.name, dbc, layout)
def juggle_segment(route_name, segment_nr): r = Route(route_name) lp = r.log_paths()[segment_nr] if lp is None: print("This segment does not exist, please try a different one") return uf = URLFile(lp) subprocess.call(f"bin/plotjuggler -d {uf.name}", shell=True)
def juggle_route(route_name, segment_number, qlog, can, layout): if route_name.startswith("http://") or route_name.startswith( "https://") or os.path.isfile(route_name): logs = [route_name] else: r = Route(route_name) logs = r.qlog_paths() if qlog else r.log_paths() if segment_number is not None: logs = logs[segment_number:segment_number + 1] if None in logs: fallback_answer = input( "At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : " ) if fallback_answer == 'y': logs = r.qlog_paths() if segment_number is not None: logs = logs[segment_number:segment_number + 1] else: print( f"Please try a different {'segment' if segment_number is not None else 'route'}" ) return all_data = [] pool = multiprocessing.Pool(24) for d in pool.map(load_segment, logs): all_data += d if not can: all_data = [d for d in all_data if d.which() not in ['can', 'sendcan']] # Infer DBC name from logs dbc = None for cp in [m for m in all_data if m.which() == 'carParams']: try: DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values", fromlist=['DBC']).DBC dbc = DBC[cp.carParams.carFingerprint]['pt'] except (ImportError, KeyError, AttributeError): pass break tempfile = NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) save_log(tempfile.name, all_data, compress=False) del all_data juggle_file(tempfile.name, dbc, layout)
def juggle_route(route_name): r = Route(route_name) all_data = [] pool = multiprocessing.Pool(24) all_data = [] for d in pool.map(load_segment, r.log_paths()): all_data += d tempfile = NamedTemporaryFile(suffix='.rlog') save_log(tempfile.name, all_data, compress=False) del all_data juggle_file(tempfile.name)
def regen_and_save(route, sidx, upload=False, use_route_meta=True): if use_route_meta: r = Route(args.route) lr = LogReader(r.log_paths()[args.seg]) fr = FrameReader(r.camera_paths()[args.seg]) else: lr = LogReader(f"cd:/{route.replace('|', '/')}/{sidx}/rlog.bz2") fr = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/fcamera.hevc") rpath = regen_segment(lr, {'roadCameraState': fr}) relr = os.path.relpath(rpath) print("\n\n", "*"*30, "\n\n") print("New route:", relr, "\n") if upload: upload_route(relr) return relr
import json import sys from tools.lib.route import Route route_name = sys.argv[1] routes = Route(route_name) data_dump = {"camera": routes.camera_paths(), "logs": routes.log_paths()} json.dump(data_dump, open("routes.json", "w"))
parser = argparse.ArgumentParser() parser.add_argument('--level', default='DEBUG') parser.add_argument('--addr', default='127.0.0.1') parser.add_argument("route", type=str, nargs='*', help="route name + segment number for offline usage") args = parser.parse_args() logs = None if len(args.route): r = Route(args.route[0]) logs = [ q_log if r_log is None else r_log for (q_log, r_log) in zip(r.qlog_paths(), r.log_paths()) ] if len(args.route) == 2 and logs: n = int(args.route[1]) logs = [logs[n]] min_level = LEVELS[args.level] if logs: for log in logs: if log: lr = LogReader(log) for m in lr: if m.which() == 'logMessage': print_logmessage(m.logMonoTime, m.logMessage,
if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--level', default='DEBUG') parser.add_argument('--addr', default='127.0.0.1') parser.add_argument("route", type=str, nargs='*', help="route name + segment number for offline usage") args = parser.parse_args() logs = None if len(args.route): r = Route(args.route[0]) logs = r.log_paths() # TODO: switch to qlogs when logs are in there if len(args.route) == 2 and logs: n = int(args.route[1]) logs = [logs[n]] min_level = LEVELS[args.level] if logs: for log in logs: if log: lr = LogReader(log) for m in lr: if m.which() == 'logMessage': print_logmessage(m.logMonoTime, m.logMessage, min_level)
# read also msgs sent by EON on CAN bus 0x80 and filter out the # addr with more than 11 bits if c.src % 0x80 == 0 and c.address < 0x800: msgs[c.address] = len(c.dat) # show CAN fingerprint fingerprint = ', '.join("%d: %d" % v for v in sorted(msgs.items())) print(f"\nfound {len(msgs)} messages. CAN fingerprint:\n") print(fingerprint) # TODO: also print the fw fingerprint merged with the existing ones # show FW fingerprint print("\nFW fingerprint:\n") for f in fw: print( f" (Ecu.{f.ecu}, {hex(f.address)}, {None if f.subAddress == 0 else f.subAddress}): [" ) print(f" {f.fwVersion},") print(" ],") print() if __name__ == "__main__": if len(sys.argv) < 2: print("Usage: ./fingerprint_from_route.py <route>") sys.exit(1) route = Route(sys.argv[1]) lr = MultiLogIterator(route.log_paths()[:5]) get_fingerprint(lr)
if segment_name is None: return [] try: return list(LogReader(segment_name)) except ValueError as e: print(f"Error parsing {segment_name}: {e}") return [] if __name__ == "__main__": route = Route(ROUTE) msgs = [] with multiprocessing.Pool(24) as pool: for d in pool.map(load_segment, route.log_paths()): msgs += d for m in msgs: if m.which() == 'carParams': CP = m.carParams break params = { 'carFingerprint': CP.carFingerprint, 'steerRatio': CP.steerRatio, 'stiffnessFactor': 1.0, 'angleOffsetAverageDeg': 0.0, } for m in msgs:
if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--level', default='DEBUG') parser.add_argument('--addr', default='127.0.0.1') parser.add_argument("route", type=str, nargs='*', help="route name + segment number for offline usage") args = parser.parse_args() logs = None if len(args.route): r = Route(args.route[0]) logs = r.log_paths() if len(args.route) == 2 and logs: n = int(args.route[1]) logs = [logs[n]] min_level = LEVELS[args.level] if logs: for log in logs: if log: lr = LogReader(log) for m in lr: if m.which() == 'logMessage': print_logmessage(m.logMonoTime, m.logMessage, min_level)
# read also msgs sent by EON on CAN bus 0x80 and filter out the # addr with more than 11 bits if c.src % 0x80 == 0 and c.address < 0x800: msgs[c.address] = len(c.dat) # show CAN fingerprint fingerprint = ', '.join("%d: %d" % v for v in sorted(msgs.items())) print(f"\nfound {len(msgs)} messages. CAN fingerprint:\n") print(fingerprint) # TODO: also print the fw fingerprint merged with the existing ones # show FW fingerprint print("\nFW fingerprint:\n") for f in fw: print( f" (Ecu.{f.ecu}, {hex(f.address)}, {None if f.subAddress == 0 else f.subAddress}): [" ) print(f" {f.fwVersion},") print(" ],") print() if __name__ == "__main__": if len(sys.argv) < 2: print("Usage: ./fingerprint_from_route.py <route>") sys.exit(1) route = Route(sys.argv[1]) lr = MultiLogIterator(route.log_paths()[:5], wraparound=False) get_fingerprint(lr)
elif m.address == 0x260 and m.src == 0: eps_torque = to_signed((m.dat[5] << 8) | m.dat[6], 16) if engaged and torque_cmd is not None and eps_torque is not None: cmds.append(torque_cmd) eps.append(eps_torque) else: if len(cmds) > MIN_SAMPLES: break cmds, eps = [], [] if len(cmds) < MIN_SAMPLES: raise Exception("too few samples found in route") lm = linear_model.LinearRegression(fit_intercept=False) lm.fit(np.array(cmds).reshape(-1, 1), eps) scale_factor = 1. / lm.coef_[0] if plot: plt.plot(np.array(eps) * scale_factor) plt.plot(cmds) plt.show() return scale_factor if __name__ == "__main__": r = Route(sys.argv[1]) lr = MultiLogIterator(r.log_paths(), wraparound=False) n = get_eps_factor(lr, plot="--plot" in sys.argv) print("EPS torque factor: ", n)
print(f"[{t / 1e9:.6f}] {source} {msg.pid} {msg.tag} - {m}") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--level', default='DEBUG') parser.add_argument('--addr', default='127.0.0.1') parser.add_argument("route", type=str, nargs='*', help="route name + segment number for offline usage") args = parser.parse_args() logs = None if len(args.route): r = Route(args.route[0]) logs = [q if r is None else r for (q, r) in zip(r.qlog_paths(), r.log_paths())] if len(args.route) == 2 and logs: n = int(args.route[1]) logs = [logs[n]] min_level = LEVELS[args.level] if logs: for log in logs: if log: lr = LogReader(log) for m in lr: if m.which() == 'logMessage': print_logmessage(m.logMonoTime, m.logMessage, min_level) elif m.which() == 'androidLog':
#!/usr/bin/env python3 import json import sys from tools.lib.route import Route route_name = sys.argv[1] routes = Route(route_name) data_dump = { "camera": routes.camera_paths(), "logs": routes.log_paths() } json.dump(data_dump, open("routes.json", "w"))
from tools.lib.logreader import LogReader from tools.lib.route import Route import argparse if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("route", help="The route name") parser.add_argument("segment", type=int, help="The index of the segment") args = parser.parse_args() out_path = os.path.join("jpegs", f"{args.route.replace('|', '_')}_{args.segment}") mkdirs_exists_ok(out_path) r = Route(args.route) path = r.log_paths()[args.segment] or r.qlog_paths()[args.segment] lr = list(LogReader(path)) for msg in tqdm(lr): if msg.which() == 'thumbnail': with open(os.path.join(out_path, f"{msg.thumbnail.frameId}.jpg"), 'wb') as f: f.write(msg.thumbnail.thumbnail) elif msg.which() == 'navThumbnail': with open( os.path.join(out_path, f"nav_{msg.navThumbnail.frameId}.jpg"), 'wb') as f: f.write(msg.navThumbnail.thumbnail)
labels, sizes = zip(*sizes_large) plt.figure() plt.title(f"{typ}") plt.pie(sizes, labels=labels, autopct='%1.1f%%') if __name__ == "__main__": parser = argparse.ArgumentParser( description='Check qlog size based on a rlog') parser.add_argument('route', help='route to use') parser.add_argument('segment', type=int, help='segment number to use') args = parser.parse_args() r = Route(args.route) rlog = r.log_paths()[args.segment] msgs = list(LogReader(rlog)) msgs_by_type = defaultdict(list) for m in msgs: msgs_by_type[m.which()].append(m.as_builder().to_bytes()) qlog_by_type = defaultdict(list) for name, service in service_list.items(): if service.decimation is None: continue for i, msg in enumerate(msgs_by_type[name]): if i % service.decimation == 0: qlog_by_type[name].append(msg)
if not recv: rx_invalid += 1 invalid_addrs.add(canmsg.address) rx_tot += 1 print("\nRX") print("total rx msgs:", rx_tot) print("invalid rx msgs:", rx_invalid) print("invalid addrs:", invalid_addrs) print("\nTX") print("total openpilot msgs:", tx_tot) print("total msgs with controls allowed:", tx_controls) print("blocked msgs:", tx_blocked) print("blocked with controls allowed:", tx_controls_blocked) print("blocked addrs:", blocked_addrs) return tx_controls_blocked == 0 and rx_invalid == 0 if __name__ == "__main__": from tools.lib.route import Route from tools.lib.logreader import MultiLogIterator # pylint: disable=import-error mode = int(sys.argv[2]) param = 0 if len(sys.argv) < 4 else int(sys.argv[3]) r = Route(sys.argv[1]) lr = MultiLogIterator(r.log_paths()) print("replaying drive %s with safety mode %d and param %d" % (sys.argv[1], mode, param)) replay_drive(lr, mode, param)