def logreader_from_route_or_segment(r):
sn = SegmentName(r, allow_route_name=True)
route = Route(sn.route_name.canonical_name)
if sn.segment_num < 0:
return MultiLogIterator(route.log_paths())
else:
return LogReader(route.log_paths()[sn.segment_num])
def main(argv):
args = get_arg_parser().parse_args(sys.argv[1:])
if not args.data_dir:
print('Data directory invalid.')
return
if not args.route_name:
# Extract route name from path
args.route_name = os.path.basename(args.data_dir)
args.data_dir = os.path.dirname(args.data_dir)
route = Route(args.route_name, args.data_dir)
lr = MultiLogIterator(route.log_paths())
with open(args.out_path, 'wb') as f:
try:
done = False
i = 0
while not done:
msg = next(lr)
if not msg:
break
smsg = msg.as_builder()
typ = smsg.which()
if typ == 'ubloxRaw':
f.write(smsg.to_bytes())
i += 1
except StopIteration:
print('All done')
print(f'Writed {i} msgs')
def juggle_route(route_name, segment_number, qlog):
r = Route(route_name)
logs = r.qlog_paths() if qlog else r.log_paths()
if segment_number is not None:
logs = logs[segment_number:segment_number + 1]
if None in logs:
fallback_answer = input(
"At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : "
)
if fallback_answer == 'y':
logs = r.qlog_paths()
if segment_number is not None:
logs = logs[segment_number:segment_number + 1]
else:
print(
f"Please try a different {'segment' if segment_number is not None else 'route'}"
)
return
all_data = []
pool = multiprocessing.Pool(24)
for d in pool.map(load_segment, logs):
all_data += d
tempfile = NamedTemporaryFile(suffix='.rlog')
save_log(tempfile.name, all_data, compress=False)
del all_data
juggle_file(tempfile.name)
def _process_commands(self, cmd, route, pub_types):
seek_to = None
if route is None or (isinstance(cmd, SetRoute) and route.name != cmd.name):
seek_to = cmd.start_time
route = Route(cmd.name, cmd.data_dir)
self._lr = MultiLogIterator(route.log_paths(), wraparound=True)
if self._frame_reader is not None:
self._frame_reader.close()
if "frame" in pub_types or "encodeIdx" in pub_types:
# reset frames for a route
self._frame_id_lookup = {}
self._frame_reader = RouteFrameReader(
route.camera_paths(), None, self._frame_id_lookup, readahead=True)
# always reset this on a seek
if isinstance(cmd, SeekRelativeTime):
seek_to = self._lr.tell() + cmd.secs
elif isinstance(cmd, SeekAbsoluteTime):
seek_to = cmd.secs
elif isinstance(cmd, StopAndQuit):
exit()
if seek_to is not None:
print("seeking", seek_to)
if not self._lr.seek(seek_to):
print("Can't seek: time out of bounds")
else:
next(self._lr) # ignore one
return route
def regen_and_save(route, sidx, upload=False, use_route_meta=False):
if use_route_meta:
r = Route(args.route)
lr = LogReader(r.log_paths()[args.seg])
fr = FrameReader(r.camera_paths()[args.seg])
else:
lr = LogReader(f"cd:/{route.replace('|', '/')}/{sidx}/rlog.bz2")
fr = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/fcamera.hevc")
rpath = regen_segment(lr, {'roadCameraState': fr})
# compress raw rlog before uploading
with open(os.path.join(rpath, "rlog"), "rb") as f:
data = bz2.compress(f.read())
with open(os.path.join(rpath, "rlog.bz2"), "wb") as f:
f.write(data)
os.remove(os.path.join(rpath, "rlog"))
lr = LogReader(os.path.join(rpath, 'rlog.bz2'))
controls_state_active = [m.controlsState.active for m in lr if m.which() == 'controlsState']
assert any(controls_state_active), "Segment did not engage"
relr = os.path.relpath(rpath)
print("\n\n", "*"*30, "\n\n")
print("New route:", relr, "\n")
if upload:
upload_route(relr)
return relr
def juggle_route(route_or_segment_name, segment_count, qlog, can, layout):
segment_start = 0
if 'cabana' in route_or_segment_name:
query = parse_qs(urlparse(route_or_segment_name).query)
api = CommaApi(get_token())
logs = api.get(
f'v1/route/{query["route"][0]}/log_urls?sig={query["sig"][0]}&exp={query["exp"][0]}'
)
elif route_or_segment_name.startswith(
"http://") or route_or_segment_name.startswith(
"https://") or os.path.isfile(route_or_segment_name):
logs = [route_or_segment_name]
else:
route_or_segment_name = SegmentName(route_or_segment_name,
allow_route_name=True)
segment_start = max(route_or_segment_name.segment_num, 0)
if route_or_segment_name.segment_num != -1 and segment_count is None:
segment_count = 1
r = Route(route_or_segment_name.route_name.canonical_name)
logs = r.qlog_paths() if qlog else r.log_paths()
segment_end = segment_start + segment_count if segment_count else None
logs = logs[segment_start:segment_end]
if None in logs:
ans = input(
f"{logs.count(None)}/{len(logs)} of the rlogs in this segment are missing, would you like to fall back to the qlogs? (y/n) "
)
if ans == 'y':
logs = r.qlog_paths()[segment_start:segment_end]
else:
print("Please try a different route or segment")
return
all_data = []
with multiprocessing.Pool(24) as pool:
for d in pool.map(load_segment, logs):
all_data += d
if not can:
all_data = [d for d in all_data if d.which() not in ['can', 'sendcan']]
# Infer DBC name from logs
dbc = None
for cp in [m for m in all_data if m.which() == 'carParams']:
try:
DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values",
fromlist=['DBC']).DBC
dbc = DBC[cp.carParams.carFingerprint]['pt']
except Exception:
pass
break
with tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) as tmp:
save_log(tmp.name, all_data, compress=False)
del all_data
start_juggler(tmp.name, dbc, layout)
def juggle_segment(route_name, segment_nr):
r = Route(route_name)
lp = r.log_paths()[segment_nr]
if lp is None:
print("This segment does not exist, please try a different one")
return
uf = URLFile(lp)
juggle_file(uf.name)
def juggle_route(route_name, segment_number, segment_count, qlog, can, layout):
if 'cabana' in route_name:
query = parse_qs(urlparse(route_name).query)
api = CommaApi(get_token())
logs = api.get(
f'v1/route/{query["route"][0]}/log_urls?sig={query["sig"][0]}&exp={query["exp"][0]}'
)
elif route_name.startswith("http://") or route_name.startswith(
"https://") or os.path.isfile(route_name):
logs = [route_name]
else:
r = Route(route_name)
logs = r.qlog_paths() if qlog else r.log_paths()
if segment_number is not None:
logs = logs[segment_number:segment_number + segment_count]
if None in logs:
fallback_answer = input(
"At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : "
)
if fallback_answer == 'y':
logs = r.qlog_paths()
if segment_number is not None:
logs = logs[segment_number:segment_number + segment_count]
else:
print(
f"Please try a different {'segment' if segment_number is not None else 'route'}"
)
return
all_data = []
with multiprocessing.Pool(24) as pool:
for d in pool.map(load_segment, logs):
all_data += d
if not can:
all_data = [d for d in all_data if d.which() not in ['can', 'sendcan']]
# Infer DBC name from logs
dbc = None
for cp in [m for m in all_data if m.which() == 'carParams']:
try:
DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values",
fromlist=['DBC']).DBC
dbc = DBC[cp.carParams.carFingerprint]['pt']
except (ImportError, KeyError, AttributeError):
pass
break
tempfile = NamedTemporaryFile(suffix='.rlog', dir=juggle_dir)
save_log(tempfile.name, all_data, compress=False)
del all_data
start_juggler(tempfile.name, dbc, layout)
def juggle_segment(route_name, segment_nr):
r = Route(route_name)
lp = r.log_paths()[segment_nr]
if lp is None:
print("This segment does not exist, please try a different one")
return
uf = URLFile(lp)
subprocess.call(f"bin/plotjuggler -d {uf.name}", shell=True)
def juggle_route(route_name, segment_number, qlog, can, layout):
if route_name.startswith("http://") or route_name.startswith(
"https://") or os.path.isfile(route_name):
logs = [route_name]
else:
r = Route(route_name)
logs = r.qlog_paths() if qlog else r.log_paths()
if segment_number is not None:
logs = logs[segment_number:segment_number + 1]
if None in logs:
fallback_answer = input(
"At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : "
)
if fallback_answer == 'y':
logs = r.qlog_paths()
if segment_number is not None:
logs = logs[segment_number:segment_number + 1]
else:
print(
f"Please try a different {'segment' if segment_number is not None else 'route'}"
)
return
all_data = []
pool = multiprocessing.Pool(24)
for d in pool.map(load_segment, logs):
all_data += d
if not can:
all_data = [d for d in all_data if d.which() not in ['can', 'sendcan']]
# Infer DBC name from logs
dbc = None
for cp in [m for m in all_data if m.which() == 'carParams']:
try:
DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values",
fromlist=['DBC']).DBC
dbc = DBC[cp.carParams.carFingerprint]['pt']
except (ImportError, KeyError, AttributeError):
pass
break
tempfile = NamedTemporaryFile(suffix='.rlog', dir=juggle_dir)
save_log(tempfile.name, all_data, compress=False)
del all_data
juggle_file(tempfile.name, dbc, layout)
def juggle_route(route_name):
r = Route(route_name)
all_data = []
pool = multiprocessing.Pool(24)
all_data = []
for d in pool.map(load_segment, r.log_paths()):
all_data += d
tempfile = NamedTemporaryFile(suffix='.rlog')
save_log(tempfile.name, all_data, compress=False)
del all_data
juggle_file(tempfile.name)
def regen_and_save(route, sidx, upload=False, use_route_meta=True):
if use_route_meta:
r = Route(args.route)
lr = LogReader(r.log_paths()[args.seg])
fr = FrameReader(r.camera_paths()[args.seg])
else:
lr = LogReader(f"cd:/{route.replace('|', '/')}/{sidx}/rlog.bz2")
fr = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/fcamera.hevc")
rpath = regen_segment(lr, {'roadCameraState': fr})
relr = os.path.relpath(rpath)
print("\n\n", "*"*30, "\n\n")
print("New route:", relr, "\n")
if upload:
upload_route(relr)
return relr
import json
import sys
from tools.lib.route import Route
route_name = sys.argv[1]
routes = Route(route_name)
data_dump = {"camera": routes.camera_paths(), "logs": routes.log_paths()}
json.dump(data_dump, open("routes.json", "w"))
parser = argparse.ArgumentParser()
parser.add_argument('--level', default='DEBUG')
parser.add_argument('--addr', default='127.0.0.1')
parser.add_argument("route",
type=str,
nargs='*',
help="route name + segment number for offline usage")
args = parser.parse_args()
logs = None
if len(args.route):
r = Route(args.route[0])
logs = [
q_log if r_log is None else r_log
for (q_log, r_log) in zip(r.qlog_paths(), r.log_paths())
]
if len(args.route) == 2 and logs:
n = int(args.route[1])
logs = [logs[n]]
min_level = LEVELS[args.level]
if logs:
for log in logs:
if log:
lr = LogReader(log)
for m in lr:
if m.which() == 'logMessage':
print_logmessage(m.logMonoTime, m.logMessage,
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--level', default='DEBUG')
parser.add_argument('--addr', default='127.0.0.1')
parser.add_argument("route",
type=str,
nargs='*',
help="route name + segment number for offline usage")
args = parser.parse_args()
logs = None
if len(args.route):
r = Route(args.route[0])
logs = r.log_paths() # TODO: switch to qlogs when logs are in there
if len(args.route) == 2 and logs:
n = int(args.route[1])
logs = [logs[n]]
min_level = LEVELS[args.level]
if logs:
for log in logs:
if log:
lr = LogReader(log)
for m in lr:
if m.which() == 'logMessage':
print_logmessage(m.logMonoTime, m.logMessage,
min_level)
# read also msgs sent by EON on CAN bus 0x80 and filter out the
# addr with more than 11 bits
if c.src % 0x80 == 0 and c.address < 0x800:
msgs[c.address] = len(c.dat)
# show CAN fingerprint
fingerprint = ', '.join("%d: %d" % v for v in sorted(msgs.items()))
print(f"\nfound {len(msgs)} messages. CAN fingerprint:\n")
print(fingerprint)
# TODO: also print the fw fingerprint merged with the existing ones
# show FW fingerprint
print("\nFW fingerprint:\n")
for f in fw:
print(
f" (Ecu.{f.ecu}, {hex(f.address)}, {None if f.subAddress == 0 else f.subAddress}): ["
)
print(f" {f.fwVersion},")
print(" ],")
print()
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage: ./fingerprint_from_route.py <route>")
sys.exit(1)
route = Route(sys.argv[1])
lr = MultiLogIterator(route.log_paths()[:5])
get_fingerprint(lr)
if segment_name is None:
return []
try:
return list(LogReader(segment_name))
except ValueError as e:
print(f"Error parsing {segment_name}: {e}")
return []
if __name__ == "__main__":
route = Route(ROUTE)
msgs = []
with multiprocessing.Pool(24) as pool:
for d in pool.map(load_segment, route.log_paths()):
msgs += d
for m in msgs:
if m.which() == 'carParams':
CP = m.carParams
break
params = {
'carFingerprint': CP.carFingerprint,
'steerRatio': CP.steerRatio,
'stiffnessFactor': 1.0,
'angleOffsetAverageDeg': 0.0,
}
for m in msgs:
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--level', default='DEBUG')
parser.add_argument('--addr', default='127.0.0.1')
parser.add_argument("route",
type=str,
nargs='*',
help="route name + segment number for offline usage")
args = parser.parse_args()
logs = None
if len(args.route):
r = Route(args.route[0])
logs = r.log_paths()
if len(args.route) == 2 and logs:
n = int(args.route[1])
logs = [logs[n]]
min_level = LEVELS[args.level]
if logs:
for log in logs:
if log:
lr = LogReader(log)
for m in lr:
if m.which() == 'logMessage':
print_logmessage(m.logMonoTime, m.logMessage,
min_level)
# read also msgs sent by EON on CAN bus 0x80 and filter out the
# addr with more than 11 bits
if c.src % 0x80 == 0 and c.address < 0x800:
msgs[c.address] = len(c.dat)
# show CAN fingerprint
fingerprint = ', '.join("%d: %d" % v for v in sorted(msgs.items()))
print(f"\nfound {len(msgs)} messages. CAN fingerprint:\n")
print(fingerprint)
# TODO: also print the fw fingerprint merged with the existing ones
# show FW fingerprint
print("\nFW fingerprint:\n")
for f in fw:
print(
f" (Ecu.{f.ecu}, {hex(f.address)}, {None if f.subAddress == 0 else f.subAddress}): ["
)
print(f" {f.fwVersion},")
print(" ],")
print()
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage: ./fingerprint_from_route.py <route>")
sys.exit(1)
route = Route(sys.argv[1])
lr = MultiLogIterator(route.log_paths()[:5], wraparound=False)
get_fingerprint(lr)
elif m.address == 0x260 and m.src == 0:
eps_torque = to_signed((m.dat[5] << 8) | m.dat[6], 16)
if engaged and torque_cmd is not None and eps_torque is not None:
cmds.append(torque_cmd)
eps.append(eps_torque)
else:
if len(cmds) > MIN_SAMPLES:
break
cmds, eps = [], []
if len(cmds) < MIN_SAMPLES:
raise Exception("too few samples found in route")
lm = linear_model.LinearRegression(fit_intercept=False)
lm.fit(np.array(cmds).reshape(-1, 1), eps)
scale_factor = 1. / lm.coef_[0]
if plot:
plt.plot(np.array(eps) * scale_factor)
plt.plot(cmds)
plt.show()
return scale_factor
if __name__ == "__main__":
r = Route(sys.argv[1])
lr = MultiLogIterator(r.log_paths(), wraparound=False)
n = get_eps_factor(lr, plot="--plot" in sys.argv)
print("EPS torque factor: ", n)
print(f"[{t / 1e9:.6f}] {source} {msg.pid} {msg.tag} - {m}")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--level', default='DEBUG')
parser.add_argument('--addr', default='127.0.0.1')
parser.add_argument("route", type=str, nargs='*', help="route name + segment number for offline usage")
args = parser.parse_args()
logs = None
if len(args.route):
r = Route(args.route[0])
logs = [q if r is None else r for (q, r) in zip(r.qlog_paths(), r.log_paths())]
if len(args.route) == 2 and logs:
n = int(args.route[1])
logs = [logs[n]]
min_level = LEVELS[args.level]
if logs:
for log in logs:
if log:
lr = LogReader(log)
for m in lr:
if m.which() == 'logMessage':
print_logmessage(m.logMonoTime, m.logMessage, min_level)
elif m.which() == 'androidLog':
#!/usr/bin/env python3
import json
import sys
from tools.lib.route import Route
route_name = sys.argv[1]
routes = Route(route_name)
data_dump = {
"camera": routes.camera_paths(),
"logs": routes.log_paths()
}
json.dump(data_dump, open("routes.json", "w"))
from tools.lib.logreader import LogReader
from tools.lib.route import Route
import argparse
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("route", help="The route name")
parser.add_argument("segment", type=int, help="The index of the segment")
args = parser.parse_args()
out_path = os.path.join("jpegs",
f"{args.route.replace('|', '_')}_{args.segment}")
mkdirs_exists_ok(out_path)
r = Route(args.route)
path = r.log_paths()[args.segment] or r.qlog_paths()[args.segment]
lr = list(LogReader(path))
for msg in tqdm(lr):
if msg.which() == 'thumbnail':
with open(os.path.join(out_path, f"{msg.thumbnail.frameId}.jpg"),
'wb') as f:
f.write(msg.thumbnail.thumbnail)
elif msg.which() == 'navThumbnail':
with open(
os.path.join(out_path,
f"nav_{msg.navThumbnail.frameId}.jpg"),
'wb') as f:
f.write(msg.navThumbnail.thumbnail)
labels, sizes = zip(*sizes_large)
plt.figure()
plt.title(f"{typ}")
plt.pie(sizes, labels=labels, autopct='%1.1f%%')
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Check qlog size based on a rlog')
parser.add_argument('route', help='route to use')
parser.add_argument('segment', type=int, help='segment number to use')
args = parser.parse_args()
r = Route(args.route)
rlog = r.log_paths()[args.segment]
msgs = list(LogReader(rlog))
msgs_by_type = defaultdict(list)
for m in msgs:
msgs_by_type[m.which()].append(m.as_builder().to_bytes())
qlog_by_type = defaultdict(list)
for name, service in service_list.items():
if service.decimation is None:
continue
for i, msg in enumerate(msgs_by_type[name]):
if i % service.decimation == 0:
qlog_by_type[name].append(msg)
if not recv:
rx_invalid += 1
invalid_addrs.add(canmsg.address)
rx_tot += 1
print("\nRX")
print("total rx msgs:", rx_tot)
print("invalid rx msgs:", rx_invalid)
print("invalid addrs:", invalid_addrs)
print("\nTX")
print("total openpilot msgs:", tx_tot)
print("total msgs with controls allowed:", tx_controls)
print("blocked msgs:", tx_blocked)
print("blocked with controls allowed:", tx_controls_blocked)
print("blocked addrs:", blocked_addrs)
return tx_controls_blocked == 0 and rx_invalid == 0
if __name__ == "__main__":
from tools.lib.route import Route
from tools.lib.logreader import MultiLogIterator # pylint: disable=import-error
mode = int(sys.argv[2])
param = 0 if len(sys.argv) < 4 else int(sys.argv[3])
r = Route(sys.argv[1])
lr = MultiLogIterator(r.log_paths())
print("replaying drive %s with safety mode %d and param %d" % (sys.argv[1], mode, param))
replay_drive(lr, mode, param)
