def cpp_replay_process(cfg, lr):
sub_sockets = [s for _, sub in cfg.pub_sub.items()
for s in sub] # We get responses here
pm = messaging.PubMaster(cfg.pub_sub.keys())
sockets = {s: messaging.sub_sock(s, timeout=1000) for s in sub_sockets}
all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
pub_msgs = [
msg for msg in all_msgs if msg.which() in list(cfg.pub_sub.keys())
]
manager.prepare_managed_process(cfg.proc_name)
manager.start_managed_process(cfg.proc_name)
time.sleep(1) # We give the process time to start
log_msgs = []
for s in sub_sockets:
messaging.recv_one_or_none(sockets[s])
for msg in tqdm(pub_msgs):
pm.send(msg.which(), msg.as_builder())
resp_sockets = sub_sockets if cfg.should_recv_callback is None else cfg.should_recv_callback(
msg)
for s in resp_sockets:
response = messaging.recv_one(sockets[s])
if response is not None:
log_msgs.append(response)
manager.kill_managed_process(cfg.proc_name)
return log_msgs
def test_rlog(self):
services = random.sample(CEREAL_SERVICES, random.randint(5, 10))
pm = messaging.PubMaster(services)
# sleep enough for the first poll to time out
# TOOD: fix loggerd bug dropping the msgs from the first poll
manager.start_managed_process("loggerd")
time.sleep(2)
sent_msgs = defaultdict(list)
for _ in range(random.randint(2, 10) * 100):
for s in services:
try:
m = messaging.new_message(s)
except Exception:
m = messaging.new_message(s, random.randint(2, 10))
pm.send(s, m)
sent_msgs[s].append(m)
time.sleep(0.01)
time.sleep(1)
manager.kill_managed_process("loggerd")
lr = list(LogReader(os.path.join(self._get_latest_log_dir(), "rlog.bz2")))
# check initData and sentinel
self._check_init_data(lr)
self._check_sentinel(lr, True)
# check all messages were logged and in order
lr = lr[2:-1] # slice off initData and both sentinels
for m in lr:
sent = sent_msgs[m.which()].pop(0)
sent.clear_write_flag()
self.assertEqual(sent.to_bytes(), m.as_builder().to_bytes())
def test_rotation(self):
os.environ["LOGGERD_TEST"] = "1"
Params().put("RecordFront", "1")
expected_files = {
"rlog.bz2", "qlog.bz2", "qcamera.ts", "fcamera.hevc",
"dcamera.hevc"
}
if TICI:
expected_files.add("ecamera.hevc")
for _ in range(5):
num_segs = random.randint(1, 10)
length = random.randint(2, 5)
os.environ["LOGGERD_SEGMENT_LENGTH"] = str(length)
manager.start_managed_process("loggerd")
time.sleep(num_segs * length)
manager.kill_managed_process("loggerd")
route_path = str(self._get_latest_log_dir()).rsplit("--", 1)[0]
for n in range(num_segs):
p = Path(f"{route_path}--{n}")
logged = set([f.name for f in p.iterdir() if f.is_file()])
diff = logged ^ expected_files
self.assertEqual(len(diff), 0)
def fingerprint(msgs, pub_socks, sub_socks):
print "start fingerprinting"
manager.prepare_managed_process("logmessaged")
manager.start_managed_process("logmessaged")
can = pub_socks["can"]
logMessage = messaging.sub_sock(service_list["logMessage"].port)
time.sleep(1)
messaging.drain_sock(logMessage)
# controlsd waits for a health packet before fingerprinting
msg = messaging.new_message()
msg.init("health")
pub_socks["health"].send(msg.to_bytes())
canmsgs = filter(lambda msg: msg.which() == "can", msgs)
for msg in canmsgs[:200]:
can.send(msg.as_builder().to_bytes())
time.sleep(0.005)
log = messaging.recv_one_or_none(logMessage)
if log is not None and "fingerprinted" in log.logMessage:
break
manager.kill_managed_process("logmessaged")
print "finished fingerprinting"
def test_qlog(self):
qlog_services = [
s for s in CEREAL_SERVICES
if service_list[s].decimation is not None
]
no_qlog_services = [
s for s in CEREAL_SERVICES if service_list[s].decimation is None
]
services = random.sample(qlog_services, random.randint(2, 10)) + \
random.sample(no_qlog_services, random.randint(2, 10))
pm = messaging.PubMaster(services)
# sleep enough for the first poll to time out
# TOOD: fix loggerd bug dropping the msgs from the first poll
manager.start_managed_process("loggerd")
time.sleep(2)
sent_msgs = defaultdict(list)
for _ in range(random.randint(2, 10) * 100):
for s in services:
try:
m = messaging.new_message(s)
except Exception:
m = messaging.new_message(s, random.randint(2, 10))
pm.send(s, m)
sent_msgs[s].append(m)
time.sleep(0.01)
time.sleep(1)
manager.kill_managed_process("loggerd")
qlog_path = os.path.join(self._get_latest_log_dir(), "qlog.bz2")
lr = list(LogReader(qlog_path))
# check initData and sentinel
self._check_init_data(lr)
self._check_sentinel(lr, True)
recv_msgs = defaultdict(list)
for m in lr:
recv_msgs[m.which()].append(m)
for s, msgs in sent_msgs.items():
recv_cnt = len(recv_msgs[s])
if s in no_qlog_services:
# check services with no specific decimation aren't in qlog
self.assertEqual(recv_cnt, 0,
f"got {recv_cnt} {s} msgs in qlog")
else:
# check logged message count matches decimation
expected_cnt = len(msgs) // service_list[s].decimation
self.assertEqual(
recv_cnt, expected_cnt,
f"expected {expected_cnt} msgs for {s}, got {recv_cnt}")
def camera_replay(lr, fr):
spinner = Spinner()
pm = messaging.PubMaster(['frame', 'liveCalibration'])
sm = messaging.SubMaster(['model'])
# TODO: add dmonitoringmodeld
print("preparing procs")
manager.prepare_managed_process("camerad")
manager.prepare_managed_process("modeld")
try:
print("starting procs")
manager.start_managed_process("camerad")
manager.start_managed_process("modeld")
time.sleep(5)
print("procs started")
cal = [msg for msg in lr if msg.which() == "liveCalibration"]
for msg in cal[:5]:
pm.send(msg.which(), msg.as_builder())
log_msgs = []
frame_idx = 0
for msg in tqdm(lr):
if msg.which() == "liveCalibrationd":
pm.send(msg.which(), msg.as_builder())
elif msg.which() == "frame":
f = msg.as_builder()
img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, ::, -1]
f.frame.image = img.flatten().tobytes()
frame_idx += 1
pm.send(msg.which(), f)
with Timeout(seconds=15):
log_msgs.append(messaging.recv_one(sm.sock['model']))
spinner.update("modeld replay %d/%d" %
(frame_idx, fr.frame_count))
if frame_idx >= fr.frame_count:
break
except KeyboardInterrupt:
pass
print("replay done")
spinner.close()
manager.kill_managed_process('modeld')
time.sleep(2)
manager.kill_managed_process('camerad')
return log_msgs
def replay_process(cfg, lr):
gc.disable() # gc can occasionally cause canparser to timeout
pub_socks, sub_socks = {}, {}
for pub, sub in cfg.pub_sub.iteritems():
pub_socks[pub] = messaging.pub_sock(service_list[pub].port)
for s in sub:
sub_socks[s] = messaging.sub_sock(service_list[s].port)
all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
pub_msgs = filter(lambda msg: msg.which() in pub_socks.keys(), all_msgs)
params = Params()
params.manager_start()
params.put("Passive", "0")
manager.gctx = {}
manager.prepare_managed_process(cfg.proc_name)
manager.start_managed_process(cfg.proc_name)
time.sleep(3) # Wait for started process to be ready
if cfg.init_callback is not None:
cfg.init_callback(all_msgs, pub_socks, sub_socks)
CP = car.CarParams.from_bytes(params.get("CarParams", block=True))
log_msgs = []
for msg in tqdm(pub_msgs):
if cfg.should_recv_callback is not None:
recv_socks = cfg.should_recv_callback(msg, CP)
else:
recv_socks = cfg.pub_sub[msg.which()]
pub_socks[msg.which()].send(msg.as_builder().to_bytes())
if len(recv_socks):
# TODO: add timeout
for sock in recv_socks:
m = messaging.recv_one(sub_socks[sock])
# make these values fixed for faster comparison
m_builder = m.as_builder()
m_builder.logMonoTime = 0
m_builder.valid = True
log_msgs.append(m_builder.as_reader())
gc.enable()
manager.kill_managed_process(cfg.proc_name)
return log_msgs
def wrap(*args, **kwargs):
# start and assert started
for n, p in enumerate(processes):
start_managed_process(p)
if n < len(processes)-1:
time.sleep(init_time)
assert all(get_running()[name].exitcode is None for name in processes)
# call the function
try:
func(*args, **kwargs)
# assert processes are still started
assert all(get_running()[name].exitcode is None for name in processes)
finally:
# kill and assert all stopped
for p in processes:
kill_managed_process(p)
assert len(get_running()) == 0
def wrap():
# start and assert started
[start_managed_process(p) for p in processes]
assert all(get_running()[name].exitcode is None
for name in processes)
# call the function
try:
func()
# assert processes are still started
assert all(get_running()[name].exitcode is None
for name in processes)
finally:
# kill and assert all stopped
[kill_managed_process(p) for p in processes]
assert len(get_running()) == 0
def wrap():
if not DID_INIT:
test_manager_prepare()
# start and assert started
[start_managed_process(p) for p in processes]
assert all(get_running()[name].exitcode is None for name in processes)
# call the function
func()
# assert processes are still started
assert all(get_running()[name].exitcode is None for name in processes)
# kill and assert all stopped
[kill_managed_process(p) for p in processes]
assert len(get_running()) == 0
def test_clean_exit(self):
manager.manager_prepare()
for p in ALL_PROCESSES:
manager.start_managed_process(p)
time.sleep(10)
for p in reversed(ALL_PROCESSES):
exit_code = manager.kill_managed_process(p, retry=False)
if not EON and (p == 'ui' or p == 'loggerd'):
# TODO: make Qt UI exit gracefully and fix OMX encoder exiting
continue
# TODO: interrupted blocking read exits with 1 in cereal. use a more unique return code
exit_codes = [0, 1]
if p in manager.kill_processes:
exit_codes = [-signal.SIGKILL]
assert exit_code in exit_codes, f"{p} died with {exit_code}"
def inject_model(msgs, segment_name):
if segment_name.count('--') == 2:
segment_name = rreplace(segment_name, '--', '/', 1)
frame_reader = FrameReader('cd:/' + segment_name.replace("|", "/") +
"/fcamera.hevc")
manager.start_managed_process('camerad')
manager.start_managed_process('modeld')
# TODO do better than just wait for modeld to boot
time.sleep(5)
pm = PubMaster(['liveCalibration', 'roadCameraState'])
model_sock = sub_sock('model')
try:
out_msgs = regen_model(msgs, pm, frame_reader, model_sock)
except (KeyboardInterrupt, SystemExit, Exception) as e:
manager.kill_managed_process('modeld')
time.sleep(2)
manager.kill_managed_process('camerad')
raise e
manager.kill_managed_process('modeld')
time.sleep(2)
manager.kill_managed_process('camerad')
new_msgs = []
midx = 0
for msg in msgs:
if (msg.which() == 'model') and (midx < len(out_msgs)):
model = out_msgs[midx].as_builder()
model.logMonoTime = msg.logMonoTime
model = model.as_reader()
new_msgs.append(model)
midx += 1
else:
new_msgs.append(msg)
print(len(new_msgs), len(list(msgs)))
assert abs(len(new_msgs) - len(list(msgs))) < 2
return new_msgs
def run(self):
print(man.title)
valid = False
for retries in range(3):
manager.start_managed_process('radard')
manager.start_managed_process('controlsd')
manager.start_managed_process('plannerd')
manager.start_managed_process('dmonitoringd')
plot, valid = man.evaluate()
plot.write_plot(output_dir, "maneuver" + str(k + 1).zfill(2))
manager.kill_managed_process('radard')
manager.kill_managed_process('controlsd')
manager.kill_managed_process('plannerd')
manager.kill_managed_process('dmonitoringd')
if valid:
break
self.assertTrue(valid)
def run_maneuver_worker(k):
output_dir = os.path.join(os.getcwd(), 'out/longitudinal')
for i, man in enumerate(maneuvers[k::WORKERS]):
manager.start_managed_process('radard')
manager.start_managed_process('controlsd')
manager.start_managed_process('plannerd')
score, plot = man.evaluate()
plot.write_plot(output_dir,
"maneuver" + str(WORKERS * i + k + 1).zfill(2))
manager.kill_managed_process('radard')
manager.kill_managed_process('controlsd')
manager.kill_managed_process('plannerd')
time.sleep(5)
def run(self):
print(man.title)
manager.start_managed_process('radard')
manager.start_managed_process('controlsd')
manager.start_managed_process('plannerd')
plot, valid = man.evaluate()
plot.write_plot(output_dir, "maneuver" + str(k+1).zfill(2))
manager.kill_managed_process('radard')
manager.kill_managed_process('controlsd')
manager.kill_managed_process('plannerd')
time.sleep(5)
self.assertTrue(valid)
N = int(os.getenv("N", "5"))
TIME = int(os.getenv("TIME", "30"))
if __name__ == "__main__":
sock = messaging.sub_sock('modelV2', conflate=False, timeout=1000)
execution_times = []
for _ in range(N):
os.environ['LOGPRINT'] = 'debug'
manager.start_managed_process('modeld')
time.sleep(5)
t = []
start = time.monotonic()
while time.monotonic() - start < TIME:
msgs = messaging.drain_sock(sock, wait_for_one=True)
for m in msgs:
t.append(m.modelV2.modelExecutionTime)
execution_times.append(np.array(t[10:]) * 1000)
manager.kill_managed_process('modeld')
print("\n\n")
print(f"ran modeld {N} times for {TIME}s each")
for n, t in enumerate(execution_times):
print(
f"\tavg: {sum(t)/len(t):0.2f}ms, min: {min(t):0.2f}ms, max: {max(t):0.2f}ms"
)
print("\n\n")
cnt = 0
best = 100
for i in range(100):
ret = messaging.drain_sock(logmessage, True)
for r in ret:
if 'dmonitoring process' in r.logMessage:
cnt += 1
done = r.logMessage
ms = float(done.split('dmonitoring process: ')[1].split("ms")[0])
best = min(ms, best)
if cnt >= 5:
break
print(ln, best, done)
#if best > 16:
if best > 4:
print("HIT HIT HIT")
hitcount += 1
hits.append(best)
#start_managed_process("modeld")
kill_managed_process("dmonitoringmodeld")
#kill_managed_process("modeld")
kill_managed_process("camerad")
ln += 1
if hitcount >= 1:
break
def tearDownClass(cls):
manager.kill_managed_process('radard')
manager.kill_managed_process('controlsd')
time.sleep(5)
def camera_replay(lr, fr, desire=None, calib=None):
spinner = Spinner()
spinner.update("starting model replay")
pm = messaging.PubMaster(['frame', 'liveCalibration', 'lateralPlan'])
sm = messaging.SubMaster(['modelV2'])
# TODO: add dmonitoringmodeld
print("preparing procs")
manager.prepare_managed_process("camerad")
manager.prepare_managed_process("modeld")
try:
print("starting procs")
manager.start_managed_process("camerad")
manager.start_managed_process("modeld")
time.sleep(5)
sm.update(1000)
print("procs started")
desires_by_index = {
v: k
for k, v in log.LateralPlan.Desire.schema.enumerants.items()
}
cal = [msg for msg in lr if msg.which() == "liveCalibration"]
for msg in cal[:5]:
pm.send(msg.which(), replace_calib(msg, calib))
log_msgs = []
frame_idx = 0
for msg in tqdm(lr):
if msg.which() == "liveCalibration":
pm.send(msg.which(), replace_calib(msg, calib))
elif msg.which() == "frame":
if desire is not None:
for i in desire[frame_idx].nonzero()[0]:
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = desires_by_index[i]
pm.send('lateralPlan', dat)
f = msg.as_builder()
img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, :, ::-1]
f.frame.image = img.flatten().tobytes()
frame_idx += 1
pm.send(msg.which(), f)
with Timeout(seconds=15):
log_msgs.append(messaging.recv_one(sm.sock['modelV2']))
spinner.update("modeld replay %d/%d" %
(frame_idx, fr.frame_count))
if frame_idx >= fr.frame_count:
break
except KeyboardInterrupt:
pass
print("replay done")
spinner.close()
manager.kill_managed_process('modeld')
time.sleep(2)
manager.kill_managed_process('camerad')
return log_msgs
route not in forced_dashcam_routes) and has_camera:
extra_socks.append("sendcan")
if route not in passive_routes:
extra_socks.append("pathPlan")
recvd_socks = wait_for_sockets(tested_socks + extra_socks, timeout=30)
failures = [
s for s in tested_socks + extra_socks if s not in recvd_socks
]
print("Check if everything is running")
running = manager.get_running()
for p in tested_procs:
if not running[p].is_alive:
failures.append(p)
manager.kill_managed_process(p)
os.killpg(os.getpgid(unlogger.pid), signal.SIGTERM)
sockets_ok = len(failures) == 0
params_ok = True
try:
car_params = car.CarParams.from_bytes(params.get("CarParams"))
for k, v in checks.items():
if not v == getattr(car_params, k):
params_ok = False
failures.append(k)
except Exception:
params_ok = False
if sockets_ok and params_ok:
plan_result = wait_for_socket('plan', timeout=30)
if route not in passive_routes: # TODO The passive routes have very flaky models
path_plan_result = wait_for_socket('pathPlan', timeout=30)
else:
path_plan_result = True
carstate_result = wait_for_socket('carState', timeout=30)
print "Check if everything is running"
running = manager.get_running()
controlsd_running = running['controlsd'].is_alive()
radard_running = running['radard'].is_alive()
plannerd_running = running['plannerd'].is_alive()
manager.kill_managed_process("controlsd")
manager.kill_managed_process("radard")
manager.kill_managed_process("plannerd")
os.killpg(os.getpgid(unlogger.pid), signal.SIGTERM)
sockets_ok = all([
controls_state_result, radarstate_result, plan_result,
path_plan_result, carstate_result, controlsd_running,
radard_running, plannerd_running
])
params_ok = True
failures = []
if not controlsd_running:
failures.append('controlsd')
if not radard_running:
#!/usr/bin/env python3
import time
import cereal.messaging as messaging
from selfdrive.manager import start_managed_process, kill_managed_process
services = ['controlsState', 'deviceState', 'radarState'] # the services needed to be spoofed to start ui offroad
procs = ['camerad', 'ui', 'modeld', 'calibrationd']
[start_managed_process(p) for p in procs] # start needed processes
pm = messaging.PubMaster(services)
dat_cs, dat_deviceState, dat_radar = [messaging.new_message(s) for s in services]
dat_cs.controlsState.rearViewCam = False # ui checks for these two messages
dat_deviceState.deviceState.started = True
try:
while True:
pm.send('controlsState', dat_cs)
pm.send('deviceState', dat_deviceState)
pm.send('radarState', dat_radar)
time.sleep(1 / 100) # continually send, rate doesn't matter for deviceState
except KeyboardInterrupt:
[kill_managed_process(p) for p in procs]