def setUpClass(cls):
if "DEBUG" in os.environ:
segs = filter(
lambda x: os.path.exists(os.path.join(x, "rlog.bz2")),
Path(ROOT).iterdir())
segs = sorted(segs, key=lambda x: x.stat().st_mtime)
cls.lr = list(LogReader(os.path.join(segs[-1], "rlog.bz2")))
return
os.environ['SKIP_FW_QUERY'] = "1"
os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019"
set_params_enabled()
# Make sure athena isn't running
Params().delete("DongleId")
Params().delete("AthenadPid")
os.system("pkill -9 -f athena")
logger_root = Path(ROOT)
initial_segments = set()
if logger_root.exists():
initial_segments = set(Path(ROOT).iterdir())
# start manager and run openpilot for a minute
try:
manager_path = os.path.join(BASEDIR,
"selfdrive/manager/manager.py")
proc = subprocess.Popen(["python", manager_path])
sm = messaging.SubMaster(['carState'])
with Timeout(150, "controls didn't start"):
while sm.rcv_frame['carState'] < 0:
sm.update(1000)
# make sure we get at least two full segments
cls.segments = []
with Timeout(300, "timed out waiting for logs"):
while len(cls.segments) < 3:
new_paths = set()
if logger_root.exists():
new_paths = set(
logger_root.iterdir()) - initial_segments
segs = [p for p in new_paths if "--" in str(p)]
cls.segments = sorted(
segs, key=lambda s: int(str(s).rsplit('--')[-1]))
time.sleep(5)
finally:
proc.terminate()
if proc.wait(60) is None:
proc.kill()
cls.lrs = [
list(LogReader(os.path.join(str(s), "rlog.bz2")))
for s in cls.segments
]
# use the second segment by default as it's the first full segment
cls.lr = list(LogReader(os.path.join(str(cls.segments[1]),
"rlog.bz2")))
def setUpClass(cls):
os.environ['SKIP_FW_QUERY'] = "1"
os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019"
set_params_enabled()
initial_segments = set(Path(ROOT).iterdir())
# start manager and run openpilot for a minute
try:
manager_path = os.path.join(BASEDIR,
"selfdrive/manager/manager.py")
proc = subprocess.Popen(["python", manager_path])
sm = messaging.SubMaster(['carState'])
with Timeout(150, "controls didn't start"):
while sm.rcv_frame['carState'] < 0:
sm.update(1000)
# make sure we get at least two full segments
cls.segments = []
with Timeout(300, "timed out waiting for logs"):
while len(cls.segments) < 3:
new_paths = set(Path(ROOT).iterdir()) - initial_segments
segs = [p for p in new_paths if "--" in str(p)]
cls.segments = sorted(
segs, key=lambda s: int(str(s).rsplit('--')[-1]))
time.sleep(5)
finally:
proc.terminate()
if proc.wait(60) is None:
proc.kill()
cls.lr = list(LogReader(os.path.join(str(cls.segments[1]),
"rlog.bz2")))
def inner(*args, **kwargs):
with Timeout(2, 'HTTP Server did not start'):
p = None
host = '127.0.0.1'
while p is None or p.exitcode is not None:
port = random.randrange(40000, 50000)
p = Process(target=http.server.test,
kwargs={
'port': port,
'HandlerClass': HTTPRequestHandler,
'bind': host
})
p.start()
time.sleep(0.1)
with Timeout(2, 'HTTP Server seeding failed'):
while True:
try:
requests.put(f'http://{host}:{port}/qlog.bz2', data='')
break
except requests.exceptions.ConnectionError:
time.sleep(0.1)
try:
return func(*args, f'http://{host}:{port}', **kwargs)
finally:
p.terminate()
def test_log_rotation(self, record_front):
Params().put("RecordFront", str(int(record_front)))
num_segments = random.randint(80, 150)
if "CI" in os.environ:
num_segments = random.randint(15,
20) # ffprobe is slow on comma two
# wait for loggerd to make the dir for first segment
route_prefix_path = None
with Timeout(int(SEGMENT_LENGTH * 3)):
while route_prefix_path is None:
try:
route_prefix_path = self._get_latest_segment_path().rsplit(
"--", 1)[0]
except Exception:
time.sleep(0.1)
def check_seg(i):
# check each camera file size
for camera, fps, size in CAMERAS:
if not record_front and "dcamera" in camera:
continue
file_path = f"{route_prefix_path}--{i}/{camera}"
# check file size
self.assertTrue(os.path.exists(file_path))
file_size = os.path.getsize(file_path)
self.assertTrue(
math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE))
# TODO: this ffprobe call is really slow
# check frame count
cmd = f"ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames \
-of default=nokey=1:noprint_wrappers=1 {file_path}"
expected_frames = fps * SEGMENT_LENGTH
frame_tolerance = 1 if (EON
and camera == 'dcamera.hevc') else 0
probe = subprocess.check_output(cmd,
shell=True,
encoding='utf8')
frame_count = int(probe.split('\n')[0].strip())
self.assertTrue(
abs(expected_frames - frame_count) <= frame_tolerance,
f"{camera} failed frame count check: expected {expected_frames}, got {frame_count}"
)
shutil.rmtree(f"{route_prefix_path}--{i}")
for i in trange(num_segments):
# poll for next segment
with Timeout(int(SEGMENT_LENGTH * 2),
error_msg=f"timed out waiting for segment {i}"):
while int(self._get_latest_segment_path().rsplit("--",
1)[1]) <= i:
time.sleep(0.1)
check_seg(i)
def setUpClass(cls):
if "DEBUG" in os.environ:
segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog")), Path(ROOT).iterdir())
segs = sorted(segs, key=lambda x: x.stat().st_mtime)
cls.lr = list(LogReader(os.path.join(segs[-1], "rlog")))
return
# setup env
os.environ['REPLAY'] = "1"
os.environ['SKIP_FW_QUERY'] = "1"
os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019"
params = Params()
params.clear_all()
set_params_enabled()
# Make sure athena isn't running
os.system("pkill -9 -f athena")
# start manager and run openpilot for a minute
proc = None
try:
manager_path = os.path.join(BASEDIR, "selfdrive/manager/manager.py")
proc = subprocess.Popen(["python", manager_path])
sm = messaging.SubMaster(['carState'])
with Timeout(150, "controls didn't start"):
while sm.rcv_frame['carState'] < 0:
sm.update(1000)
# make sure we get at least two full segments
route = None
cls.segments = []
with Timeout(300, "timed out waiting for logs"):
while route is None:
route = params.get("CurrentRoute", encoding="utf-8")
time.sleep(0.1)
while len(cls.segments) < 3:
segs = set()
if Path(ROOT).exists():
segs = set(Path(ROOT).glob(f"{route}--*"))
cls.segments = sorted(segs, key=lambda s: int(str(s).rsplit('--')[-1]))
time.sleep(2)
# chop off last, incomplete segment
cls.segments = cls.segments[:-1]
finally:
if proc is not None:
proc.terminate()
if proc.wait(60) is None:
proc.kill()
cls.lrs = [list(LogReader(os.path.join(str(s), "rlog"))) for s in cls.segments]
# use the second segment by default as it's the first full segment
cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog")))
def process_frame(msg, pm, sm, log_msgs, vipc_server, spinner, frs, frame_idxs,
last_desire):
if msg.which() == "roadCameraState" and last_desire is not None:
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = last_desire
pm.send('lateralPlan', dat)
f = msg.as_builder()
pm.send(msg.which(), f)
img = frs[msg.which()].get(frame_idxs[msg.which()], pix_fmt="yuv420p")[0]
if msg.which == "roadCameraState":
vipc_server.send(VisionStreamType.VISION_STREAM_YUV_BACK,
img.flatten().tobytes(), f.roadCameraState.frameId,
f.roadCameraState.timestampSof,
f.roadCameraState.timestampEof)
else:
vipc_server.send(VisionStreamType.VISION_STREAM_YUV_FRONT,
img.flatten().tobytes(), f.driverCameraState.frameId,
f.driverCameraState.timestampSof,
f.driverCameraState.timestampEof)
with Timeout(seconds=15):
log_msgs.append(
messaging.recv_one(sm.sock[packet_from_camera[msg.which()]]))
frame_idxs[msg.which()] += 1
if frame_idxs[msg.which()] >= frs[msg.which()].frame_count:
return None
update_spinner(spinner, frame_idxs['roadCameraState'],
frs['roadCameraState'].frame_count,
frame_idxs['driverCameraState'],
frs['driverCameraState'].frame_count)
return 0
def setUpClass(cls):
os.environ['SKIP_FW_QUERY'] = "1"
os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019"
set_params_enabled()
Panda().reset()
initial_segments = set(Path(ROOT).iterdir())
# start manager and run openpilot for a minute
try:
manager_path = os.path.join(BASEDIR, "selfdrive/manager.py")
proc = subprocess.Popen(["python", manager_path])
sm = messaging.SubMaster(['carState'])
with Timeout(150, "controls didn't start"):
while sm.rcv_frame['carState'] < 0:
sm.update(1000)
time.sleep(60)
finally:
proc.terminate()
if proc.wait(20) is None:
proc.kill()
new_segments = set(Path(ROOT).iterdir()) - initial_segments
segments = [p for p in new_segments if len(list(p.iterdir())) > 1]
cls.segment = [s for s in segments if str(s).endswith("--0")][0]
cls.lr = list(LogReader(os.path.join(str(cls.segment), "rlog.bz2")))
def camera_replay(lr, fr, desire=None, calib=None):
spinner = Spinner()
spinner.update("starting model replay")
pm = messaging.PubMaster(['roadCameraState', 'liveCalibration', 'lateralPlan'])
sm = messaging.SubMaster(['modelV2'])
# TODO: add dmonitoringmodeld
print("preparing procs")
managed_processes['camerad'].prepare()
managed_processes['modeld'].prepare()
try:
print("starting procs")
managed_processes['camerad'].start()
managed_processes['modeld'].start()
time.sleep(5)
sm.update(1000)
print("procs started")
desires_by_index = {v:k for k,v in log.LateralPlan.Desire.schema.enumerants.items()}
cal = [msg for msg in lr if msg.which() == "liveCalibration"]
for msg in cal[:5]:
pm.send(msg.which(), replace_calib(msg, calib))
log_msgs = []
frame_idx = 0
for msg in tqdm(lr):
if msg.which() == "liveCalibration":
pm.send(msg.which(), replace_calib(msg, calib))
elif msg.which() == "roadCameraState":
if desire is not None:
for i in desire[frame_idx].nonzero()[0]:
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = desires_by_index[i]
pm.send('lateralPlan', dat)
f = msg.as_builder()
img = fr.get(frame_idx, pix_fmt="rgb24")[0][:,:,::-1]
f.roadCameraState.image = img.flatten().tobytes()
frame_idx += 1
pm.send(msg.which(), f)
with Timeout(seconds=15):
log_msgs.append(messaging.recv_one(sm.sock['modelV2']))
spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count))
if frame_idx >= fr.frame_count:
break
except KeyboardInterrupt:
pass
print("replay done")
spinner.close()
managed_processes['modeld'].stop()
time.sleep(2)
managed_processes['camerad'].stop()
return log_msgs
def heartbeat():
work_dir = '/data/openpilot'
while True:
try:
with open(os.path.join(work_dir, "selfdrive", "common", "version.h")) as _versionf:
version = _versionf.read().split('"')[1]
tmux = ""
# try:
# tmux = os.popen('tail -n 100 /tmp/tmux_out').read()
# except Exception:
# pass
params = Params()
msg = {
'version': version,
'dongle_id': params.get("DongleId").rstrip().decode('utf8'),
'remote': subprocess.check_output(["git", "config", "--get", "remote.origin.url"], cwd=work_dir).decode('utf8').rstrip(),
'revision': subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=work_dir).decode('utf8').rstrip(),
'serial': subprocess.check_output(["getprop", "ro.boot.serialno"]).decode('utf8').rstrip(),
'tmux': tmux,
}
with Timeout(10):
requests.post('http://%s/eon/heartbeat/' % HOST, json=msg, timeout=5.0)
except Exception as e:
print("Unable to send heartbeat", e)
time.sleep(5)
def test_log_rotation(self):
# wait for first seg to start being written
time.sleep(5)
route_prefix_path = self._get_latest_segment_path().rsplit("--", 1)[0]
num_segments = random.randint(80, 150)
for i in range(num_segments):
if i < num_segments - 1:
with Timeout(self.segment_length * 3,
error_msg=f"timed out waiting for segment {i}"):
while True:
seg_num = int(self._get_latest_segment_path().rsplit(
"--", 1)[1])
if seg_num > i:
break
time.sleep(0.1)
else:
time.sleep(self.segment_length + 2)
# check each camera file size
for camera, size in CAMERAS.items():
f = f"{route_prefix_path}--{i}/{camera}.hevc"
self.assertTrue(os.path.exists(f), f"couldn't find {f}")
file_size = os.path.getsize(f)
self.assertTrue(
math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE),
f"{camera} failed size check: expected {size}, got {file_size}"
)
def cpp_replay_process(cfg, lr, fingerprint=None):
sub_sockets = [s for _, sub in cfg.pub_sub.items()
for s in sub] # We get responses here
pm = messaging.PubMaster(cfg.pub_sub.keys())
all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
pub_msgs = [
msg for msg in all_msgs if msg.which() in list(cfg.pub_sub.keys())
]
log_msgs = []
# We need to fake SubMaster alive since we can't inject a fake clock
setup_env(simulation=True)
managed_processes[cfg.proc_name].prepare()
managed_processes[cfg.proc_name].start()
try:
with Timeout(TIMEOUT):
while not all(
pm.all_readers_updated(s) for s in cfg.pub_sub.keys()):
time.sleep(0)
# Make sure all subscribers are connected
sockets = {
s: messaging.sub_sock(s, timeout=2000)
for s in sub_sockets
}
for s in sub_sockets:
messaging.recv_one_or_none(sockets[s])
for i, msg in enumerate(tqdm(pub_msgs, disable=False)):
pm.send(msg.which(), msg.as_builder())
resp_sockets = cfg.pub_sub[msg.which(
)] if cfg.should_recv_callback is None else cfg.should_recv_callback(
msg)
for s in resp_sockets:
response = messaging.recv_one(sockets[s])
if response is None:
print(f"Warning, no response received {i}")
else:
response = response.as_builder()
response.logMonoTime = msg.logMonoTime
response = response.as_reader()
log_msgs.append(response)
if not len(
resp_sockets
): # We only need to wait if we didn't already wait for a response
while not pm.all_readers_updated(msg.which()):
time.sleep(0)
finally:
managed_processes[cfg.proc_name].signal(signal.SIGKILL)
managed_processes[cfg.proc_name].stop()
return log_msgs
def test_boardd_loopback():
# wait for boardd to init
spinner = Spinner()
time.sleep(2)
with Timeout(60, "boardd didn't start"):
sm = messaging.SubMaster(['pandaState'])
while sm.rcv_frame['pandaState'] < 1:
sm.update(1000)
# boardd blocks on CarVin and CarParams
cp = car.CarParams.new_message()
cp.safetyModel = car.CarParams.SafetyModel.allOutput
Params().put("CarVin", b"0" * 17)
Params().put_bool("ControlsReady", True)
Params().put("CarParams", cp.to_bytes())
sendcan = messaging.pub_sock('sendcan')
can = messaging.sub_sock('can', conflate=False, timeout=100)
time.sleep(1)
n = 1000
for i in range(n):
spinner.update(f"boardd loopback {i}/{n}")
sent_msgs = defaultdict(set)
for _ in range(random.randrange(10)):
to_send = []
for __ in range(random.randrange(100)):
bus = random.randrange(3)
addr = random.randrange(1, 1 << 29)
dat = bytes([
random.getrandbits(8)
for _ in range(random.randrange(1, 9))
])
sent_msgs[bus].add((addr, dat))
to_send.append(make_can_msg(addr, dat, bus))
sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan'))
max_recv = 10
while max_recv > 0 and any(len(sent_msgs[bus]) for bus in range(3)):
recvd = messaging.drain_sock(can, wait_for_one=True)
for msg in recvd:
for m in msg.can:
if m.src >= 128:
k = (m.address, m.dat)
assert k in sent_msgs[m.src - 128]
sent_msgs[m.src - 128].discard(k)
max_recv -= 1
# if a set isn't empty, messages got dropped
for bus in range(3):
assert not len(
sent_msgs[bus]
), f"loop {i}: bus {bus} missing {len(sent_msgs[bus])} messages"
spinner.close()
def _gen_bootlog(self):
with Timeout(5):
out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8')
# check existence
d = self._get_log_dir(out)
path = Path(os.path.join(d, "bootlog.bz2"))
assert path.is_file(), "failed to create bootlog file"
return path
def _gen_bootlog(self):
with Timeout(5):
out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8')
log_fn = self._get_log_fn(out)
# check existence
assert log_fn is not None
return log_fn
def test_delete(self):
f_path = self.make_file_with_data(self.seg_dir, self.f_type, 1)
self.start_thread()
with Timeout(5, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path):
time.sleep(0.01)
self.join_thread()
self.assertFalse(os.path.exists(f_path), "File not deleted")
def heartbeat():
work_dir = get_workdir()
env = {
"LD_LIBRARY_PATH": "",
"ANDROID_DATA": "/data",
"ANDROID_ROOT": "/system",
}
while True:
try:
with open(
os.path.join(work_dir, "selfdrive", "common",
"version.h")) as _versionf:
version = _versionf.read().split('"')[1]
# subprocess.check_output(["/system/bin/screencap", "-p", "/tmp/screen.png"], cwd=work_dir, env=env)
# screenshot = base64.b64encode(open('/tmp/screen.png').read())
tmux = ""
try:
tmux = os.popen('tail -n 100 /tmp/tmux_out').read()
except:
pass
params = Params()
msg = {
'version':
version,
'dongle_id':
params.get("DongleId").rstrip().decode('utf8'),
'remote':
subprocess.check_output(
["git", "config", "--get", "remote.origin.url"],
cwd=work_dir).decode('utf8').rstrip(),
'revision':
subprocess.check_output(["git", "rev-parse", "HEAD"],
cwd=work_dir).decode('utf8').rstrip(),
'serial':
subprocess.check_output(["getprop", "ro.boot.serialno"
]).decode('utf8').rstrip(),
# 'screenshot': screenshot,
'tmux':
tmux,
}
with Timeout(10):
requests.post('http://%s/eon/heartbeat/' % MASTER_HOST,
json=msg,
timeout=5.0)
except Exception as e:
print("Unable to send heartbeat", e)
time.sleep(5)
def camera_replay(lr, fr):
spinner = Spinner()
pm = messaging.PubMaster(['frame', 'liveCalibration'])
sm = messaging.SubMaster(['model'])
# TODO: add dmonitoringmodeld
print("preparing procs")
manager.prepare_managed_process("camerad")
manager.prepare_managed_process("modeld")
try:
print("starting procs")
manager.start_managed_process("camerad")
manager.start_managed_process("modeld")
time.sleep(5)
print("procs started")
cal = [msg for msg in lr if msg.which() == "liveCalibration"]
for msg in cal[:5]:
pm.send(msg.which(), msg.as_builder())
log_msgs = []
frame_idx = 0
for msg in tqdm(lr):
if msg.which() == "liveCalibrationd":
pm.send(msg.which(), msg.as_builder())
elif msg.which() == "frame":
f = msg.as_builder()
img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, ::, -1]
f.frame.image = img.flatten().tobytes()
frame_idx += 1
pm.send(msg.which(), f)
with Timeout(seconds=15):
log_msgs.append(messaging.recv_one(sm.sock['model']))
spinner.update("modeld replay %d/%d" %
(frame_idx, fr.frame_count))
if frame_idx >= fr.frame_count:
break
except KeyboardInterrupt:
pass
print("replay done")
spinner.close()
manager.kill_managed_process('modeld')
time.sleep(2)
manager.kill_managed_process('camerad')
return log_msgs
def test_upload(self):
f_paths = self.gen_files(lock=False)
self.start_thread()
with Timeout(5, "Timeout waiting for file to be uploaded"):
while len(os.listdir(self.root)):
time.sleep(0.01)
self.join_thread()
for f_path in f_paths:
self.assertFalse(os.path.exists(f_path), "All files not uploaded")
exp_order = self.gen_order([self.seg_num], [])
self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
def test_no_delete_with_lock_file(self):
f_path = self.make_file_with_data(self.seg_dir, self.f_type, lock=True)
self.start_thread()
try:
with Timeout(2, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path):
time.sleep(0.01)
except TimeoutException:
pass
finally:
self.join_thread()
self.assertTrue(os.path.exists(f_path), "File deleted when locked")
def test_log_rotation(self):
# wait for first seg to start being written
time.sleep(5)
route_prefix_path = self._get_latest_segment_path().rsplit("--", 1)[0]
num_segments = random.randint(80, 150)
for i in range(num_segments):
if i < num_segments - 1:
with Timeout(self.segment_length * 3,
error_msg=f"timed out waiting for segment {i}"):
while True:
seg_num = int(self._get_latest_segment_path().rsplit(
"--", 1)[1])
if seg_num > i:
break
time.sleep(0.1)
else:
time.sleep(self.segment_length + 2)
# check each camera file size
for camera, size in CAMERAS.items():
file_path = f"{route_prefix_path}--{i}/{camera}.hevc"
# check file size
self.assertTrue(os.path.exists(file_path),
f"couldn't find {file_path}")
file_size = os.path.getsize(file_path)
self.assertTrue(
math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE),
f"{camera} failed size check: expected {size}, got {file_size}"
)
# check frame count
cmd = f"ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames \
-of default=nokey=1:noprint_wrappers=1 {file_path}"
expected_frames = self.segment_length * CAMERA_FPS
frame_count = int(
subprocess.check_output(cmd, shell=True,
encoding='utf8').strip())
self.assertTrue(
abs(expected_frames - frame_count) <= FRAME_TOLERANCE,
f"{camera} failed frame count check: expected {expected_frames}, got {frame_count}"
)
def test_no_delete_when_available_space(self):
global fake_stats
f_path = self.make_file_with_data(self.seg_dir, self.f_type)
fake_stats = Stats(f_bavail=10, f_blocks=10)
self.start_thread()
try:
with Timeout(2, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path):
time.sleep(0.01)
except TimeoutException:
pass
finally:
self.join_thread()
self.assertTrue(os.path.exists(f_path),
"File deleted with available space")
def test_delete_files_in_create_order(self):
f_path_1 = self.make_file_with_data(self.seg_dir, self.f_type)
time.sleep(1)
self.seg_num += 1
self.seg_dir = self.seg_format.format(self.seg_num)
f_path_2 = self.make_file_with_data(self.seg_dir, self.f_type)
self.start_thread()
with Timeout(5, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path_1) and os.path.exists(f_path_2):
time.sleep(0.01)
self.join_thread()
self.assertFalse(os.path.exists(f_path_1), "Older file not deleted")
self.assertTrue(os.path.exists(f_path_2),
"Newer file deleted before older file")
def test_run(self):
test_url = get_url("ffccc77938ddbc44|2021-01-04--16-55-41", 0)
# Launch PlotJuggler with the executable in the bin directory
os.environ["PLOTJUGGLER_PATH"] = f'{os.path.join(BASEDIR, "tools/plotjuggler/bin/plotjuggler")}'
p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")} \
"{test_url}"', stderr=subprocess.PIPE, shell=True,
start_new_session=True)
# Wait max 60 seconds for the "Done reading Rlog data" signal from the plugin
output = "\n"
with Timeout(120, error_msg=output):
while output.splitlines()[-1] != "Done reading Rlog data":
output += p.stderr.readline().decode("utf-8")
# ensure plotjuggler didn't crash after exiting the plugin
time.sleep(15)
self.assertEqual(p.poll(), None)
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
def test_demo(self):
install()
pj = os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")
p = subprocess.Popen(
f'QT_QPA_PLATFORM=offscreen {pj} --demo None 1 --qlog',
stderr=subprocess.PIPE,
shell=True,
start_new_session=True)
# Wait for "Done reading Rlog data" signal from the plugin
output = "\n"
with Timeout(180, error_msg=output):
while output.splitlines()[-1] != "Done reading Rlog data":
output += p.stderr.readline().decode("utf-8")
# ensure plotjuggler didn't crash after exiting the plugin
time.sleep(15)
self.assertEqual(p.poll(), None)
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
def test_no_delete_when_available_space(self):
f_path = self.make_file_with_data(self.seg_dir, self.f_type)
block_size = 4096
available = (10 * 1024 * 1024 * 1024) / block_size # 10GB free
self.fake_stats = Stats(f_bavail=available,
f_blocks=10,
f_frsize=block_size)
self.start_thread()
try:
with Timeout(2, "Timeout waiting for file to be deleted"):
while os.path.exists(f_path):
time.sleep(0.01)
except TimeoutException:
pass
finally:
self.join_thread()
self.assertTrue(os.path.exists(f_path),
"File deleted with available space")
def test_upload_files_in_create_order(self):
f_paths = list()
seg1_nums = [0,1,2,10,20]
for i in seg1_nums:
self.seg_dir = self.seg_format.format(i)
f_paths += self.gen_files()
seg2_nums = [5,50,51]
for i in seg2_nums:
self.seg_dir = self.seg_format2.format(i)
f_paths += self.gen_files()
self.start_thread()
with Timeout(5, "Timeout waiting for file to be upload"):
while len(os.listdir(self.root)):
time.sleep(0.01)
self.join_thread()
for f_path in f_paths:
self.assertFalse(os.path.exists(f_path), "All files not uploaded")
exp_order = self.gen_order(seg1_nums, seg2_nums)
self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
def test_log_rotation(self, record_front):
Params().put_bool("RecordFront", record_front)
managed_processes['sensord'].start()
managed_processes['loggerd'].start()
managed_processes['encoderd'].start()
time.sleep(1.0)
managed_processes['camerad'].start()
num_segments = int(os.getenv("SEGMENTS", random.randint(10, 15)))
# wait for loggerd to make the dir for first segment
route_prefix_path = None
with Timeout(int(SEGMENT_LENGTH * 3)):
while route_prefix_path is None:
try:
route_prefix_path = self._get_latest_segment_path().rsplit(
"--", 1)[0]
except Exception:
time.sleep(0.1)
def check_seg(i):
# check each camera file size
counts = []
first_frames = []
for camera, fps, size, encode_idx_name in CAMERAS:
if not record_front and "dcamera" in camera:
continue
file_path = f"{route_prefix_path}--{i}/{camera}"
# check file exists
self.assertTrue(os.path.exists(file_path),
f"segment #{i}: '{file_path}' missing")
# TODO: this ffprobe call is really slow
# check frame count
cmd = f"ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets -of csv=p=0 {file_path}"
if TICI:
cmd = "LD_LIBRARY_PATH=/usr/local/lib " + cmd
expected_frames = fps * SEGMENT_LENGTH
probe = subprocess.check_output(cmd,
shell=True,
encoding='utf8')
frame_count = int(probe.split('\n')[0].strip())
counts.append(frame_count)
self.assertEqual(
frame_count, expected_frames,
f"segment #{i}: {camera} failed frame count check: expected {expected_frames}, got {frame_count}"
)
# sanity check file size
file_size = os.path.getsize(file_path)
self.assertTrue(
math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE),
f"{file_path} size {file_size} isn't close to target size {size}"
)
# Check encodeIdx
if encode_idx_name is not None:
rlog_path = f"{route_prefix_path}--{i}/rlog"
msgs = [
m for m in LogReader(rlog_path)
if m.which() == encode_idx_name
]
encode_msgs = [getattr(m, encode_idx_name) for m in msgs]
valid = [m.valid for m in msgs]
segment_idxs = [m.segmentId for m in encode_msgs]
encode_idxs = [m.encodeId for m in encode_msgs]
frame_idxs = [m.frameId for m in encode_msgs]
# Check frame count
self.assertEqual(frame_count, len(segment_idxs))
self.assertEqual(frame_count, len(encode_idxs))
# Check for duplicates or skips
self.assertEqual(0, segment_idxs[0])
self.assertEqual(len(set(segment_idxs)), len(segment_idxs))
self.assertTrue(all(valid))
self.assertEqual(expected_frames * i, encode_idxs[0])
first_frames.append(frame_idxs[0])
self.assertEqual(len(set(encode_idxs)), len(encode_idxs))
self.assertEqual(1, len(set(first_frames)))
if TICI:
expected_frames = fps * SEGMENT_LENGTH
self.assertEqual(min(counts), expected_frames)
shutil.rmtree(f"{route_prefix_path}--{i}")
try:
for i in trange(num_segments):
# poll for next segment
with Timeout(int(SEGMENT_LENGTH * 10),
error_msg=f"timed out waiting for segment {i}"):
while Path(f"{route_prefix_path}--{i+1}") not in Path(
ROOT).iterdir():
time.sleep(0.1)
check_seg(i)
finally:
managed_processes['loggerd'].stop()
managed_processes['encoderd'].stop()
managed_processes['camerad'].stop()
managed_processes['sensord'].stop()
def test_loopback(self):
# wait for boardd to init
time.sleep(2)
with Timeout(60, "boardd didn't start"):
sm = messaging.SubMaster(['pandaStates'])
while sm.rcv_frame['pandaStates'] < 1 and len(
sm['pandaStates']) == 0:
sm.update(1000)
num_pandas = len(sm['pandaStates'])
if TICI:
self.assertGreater(num_pandas, 1,
"connect another panda for multipanda tests")
# boardd blocks on CarVin and CarParams
cp = car.CarParams.new_message()
safety_config = car.CarParams.SafetyConfig.new_message()
safety_config.safetyModel = car.CarParams.SafetyModel.allOutput
cp.safetyConfigs = [safety_config] * num_pandas
params = Params()
params.put("CarVin", b"0" * 17)
params.put_bool("ControlsReady", True)
params.put("CarParams", cp.to_bytes())
sendcan = messaging.pub_sock('sendcan')
can = messaging.sub_sock('can', conflate=False, timeout=100)
time.sleep(0.2)
n = 200
for i in range(n):
self.spinner.update(f"boardd loopback {i}/{n}")
sent_msgs = defaultdict(set)
for _ in range(random.randrange(10)):
to_send = []
for __ in range(random.randrange(100)):
bus = random.choice(
[b for b in range(3 * num_pandas) if b % 4 != 3])
addr = random.randrange(1, 1 << 29)
dat = bytes([
random.getrandbits(8)
for _ in range(random.randrange(1, 9))
])
sent_msgs[bus].add((addr, dat))
to_send.append(make_can_msg(addr, dat, bus))
sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan'))
for _ in range(100 * 2):
recvd = messaging.drain_sock(can, wait_for_one=True)
for msg in recvd:
for m in msg.can:
if m.src >= 128:
key = (m.address, m.dat)
assert key in sent_msgs[
m.src -
128], f"got unexpected msg: {m.src=} {m.address=} {m.dat=}"
sent_msgs[m.src - 128].discard(key)
if all(len(v) == 0 for v in sent_msgs.values()):
break
# if a set isn't empty, messages got dropped
for bus in sent_msgs.keys():
assert not len(
sent_msgs[bus]
), f"loop {i}: bus {bus} missing {len(sent_msgs[bus])} messages"
def model_replay(lr, frs):
spinner = Spinner()
spinner.update("starting model replay")
vipc_server = VisionIpcServer("camerad")
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_ROAD, 40, False, *(tici_f_frame_size if TICI else eon_f_frame_size))
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_DRIVER, 40, False, *(tici_d_frame_size if TICI else eon_d_frame_size))
vipc_server.start_listener()
sm = messaging.SubMaster(['modelV2', 'driverState'])
pm = messaging.PubMaster(['roadCameraState', 'driverCameraState', 'liveCalibration', 'lateralPlan'])
try:
managed_processes['modeld'].start()
managed_processes['dmonitoringmodeld'].start()
time.sleep(2)
sm.update(1000)
log_msgs = []
last_desire = None
frame_idxs = defaultdict(lambda: 0)
# init modeld with valid calibration
cal_msgs = [msg for msg in lr if msg.which() == "liveCalibration"]
for _ in range(5):
pm.send(cal_msgs[0].which(), cal_msgs[0].as_builder())
time.sleep(0.1)
for msg in tqdm(lr):
if SEND_EXTRA_INPUTS:
if msg.which() == "liveCalibration":
last_calib = list(msg.liveCalibration.rpyCalib)
pm.send(msg.which(), replace_calib(msg, last_calib))
elif msg.which() == "lateralPlan":
last_desire = msg.lateralPlan.desire
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = last_desire
pm.send('lateralPlan', dat)
if msg.which() in ["roadCameraState", "driverCameraState"]:
camera_state = getattr(msg, msg.which())
stream = VisionStreamType.VISION_STREAM_ROAD if msg.which() == "roadCameraState" else VisionStreamType.VISION_STREAM_DRIVER
img = frs[msg.which()].get(frame_idxs[msg.which()], pix_fmt="yuv420p")[0]
# send camera state and frame
pm.send(msg.which(), msg.as_builder())
vipc_server.send(stream, img.flatten().tobytes(), camera_state.frameId,
camera_state.timestampSof, camera_state.timestampEof)
# wait for a response
with Timeout(seconds=15):
packet_from_camera = {"roadCameraState": "modelV2", "driverCameraState": "driverState"}
log_msgs.append(messaging.recv_one(sm.sock[packet_from_camera[msg.which()]]))
frame_idxs[msg.which()] += 1
if frame_idxs[msg.which()] >= frs[msg.which()].frame_count:
break
spinner.update("replaying models: road %d/%d, driver %d/%d" % (frame_idxs['roadCameraState'],
frs['roadCameraState'].frame_count, frame_idxs['driverCameraState'], frs['driverCameraState'].frame_count))
finally:
spinner.close()
managed_processes['modeld'].stop()
managed_processes['dmonitoringmodeld'].stop()
return log_msgs
def model_replay(lr, frs):
spinner = Spinner()
spinner.update("starting model replay")
vipc_server = VisionIpcServer("camerad")
vipc_server.create_buffers(
VisionStreamType.VISION_STREAM_ROAD, 40, False,
*(tici_f_frame_size if TICI else eon_f_frame_size))
vipc_server.create_buffers(
VisionStreamType.VISION_STREAM_DRIVER, 40, False,
*(tici_d_frame_size if TICI else eon_d_frame_size))
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 40,
False, *(tici_f_frame_size))
vipc_server.start_listener()
sm = messaging.SubMaster(['modelV2', 'driverState'])
pm = messaging.PubMaster([
'roadCameraState', 'wideRoadCameraState', 'driverCameraState',
'liveCalibration', 'lateralPlan'
])
try:
managed_processes['modeld'].start()
managed_processes['dmonitoringmodeld'].start()
time.sleep(5)
sm.update(1000)
log_msgs = []
last_desire = None
recv_cnt = defaultdict(int)
frame_idxs = defaultdict(int)
# init modeld with valid calibration
cal_msgs = [msg for msg in lr if msg.which() == "liveCalibration"]
for _ in range(5):
pm.send(cal_msgs[0].which(), cal_msgs[0].as_builder())
time.sleep(0.1)
msgs = defaultdict(list)
for msg in lr:
msgs[msg.which()].append(msg)
for cam_msgs in zip_longest(msgs['roadCameraState'],
msgs['wideRoadCameraState'],
msgs['driverCameraState']):
# need a pair of road/wide msgs
if TICI and None in (cam_msgs[0], cam_msgs[1]):
break
for msg in cam_msgs:
if msg is None:
continue
if SEND_EXTRA_INPUTS:
if msg.which() == "liveCalibration":
last_calib = list(msg.liveCalibration.rpyCalib)
pm.send(msg.which(), replace_calib(msg, last_calib))
elif msg.which() == "lateralPlan":
last_desire = msg.lateralPlan.desire
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = last_desire
pm.send('lateralPlan', dat)
if msg.which() in VIPC_STREAM:
msg = msg.as_builder()
camera_state = getattr(msg, msg.which())
img = frs[msg.which()].get(frame_idxs[msg.which()],
pix_fmt="nv12")[0]
frame_idxs[msg.which()] += 1
# send camera state and frame
camera_state.frameId = frame_idxs[msg.which()]
pm.send(msg.which(), msg)
vipc_server.send(VIPC_STREAM[msg.which()],
img.flatten().tobytes(),
camera_state.frameId,
camera_state.timestampSof,
camera_state.timestampEof)
recv = None
if msg.which() in ('roadCameraState',
'wideRoadCameraState'):
if not TICI or min(frame_idxs['roadCameraState'],
frame_idxs['wideRoadCameraState']
) > recv_cnt['modelV2']:
recv = "modelV2"
elif msg.which() == 'driverCameraState':
recv = "driverState"
# wait for a response
with Timeout(15, f"timed out waiting for {recv}"):
if recv:
recv_cnt[recv] += 1
log_msgs.append(messaging.recv_one(sm.sock[recv]))
spinner.update(
"replaying models: road %d/%d, driver %d/%d" %
(frame_idxs['roadCameraState'],
frs['roadCameraState'].frame_count,
frame_idxs['driverCameraState'],
frs['driverCameraState'].frame_count))
if any(frame_idxs[c] >= frs[c].frame_count
for c in frame_idxs.keys()):
break
finally:
spinner.close()
managed_processes['modeld'].stop()
managed_processes['dmonitoringmodeld'].stop()
return log_msgs
