def setUpClass(cls): if "DEBUG" in os.environ: segs = filter( lambda x: os.path.exists(os.path.join(x, "rlog.bz2")), Path(ROOT).iterdir()) segs = sorted(segs, key=lambda x: x.stat().st_mtime) cls.lr = list(LogReader(os.path.join(segs[-1], "rlog.bz2"))) return os.environ['SKIP_FW_QUERY'] = "1" os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019" set_params_enabled() # Make sure athena isn't running Params().delete("DongleId") Params().delete("AthenadPid") os.system("pkill -9 -f athena") logger_root = Path(ROOT) initial_segments = set() if logger_root.exists(): initial_segments = set(Path(ROOT).iterdir()) # start manager and run openpilot for a minute try: manager_path = os.path.join(BASEDIR, "selfdrive/manager/manager.py") proc = subprocess.Popen(["python", manager_path]) sm = messaging.SubMaster(['carState']) with Timeout(150, "controls didn't start"): while sm.rcv_frame['carState'] < 0: sm.update(1000) # make sure we get at least two full segments cls.segments = [] with Timeout(300, "timed out waiting for logs"): while len(cls.segments) < 3: new_paths = set() if logger_root.exists(): new_paths = set( logger_root.iterdir()) - initial_segments segs = [p for p in new_paths if "--" in str(p)] cls.segments = sorted( segs, key=lambda s: int(str(s).rsplit('--')[-1])) time.sleep(5) finally: proc.terminate() if proc.wait(60) is None: proc.kill() cls.lrs = [ list(LogReader(os.path.join(str(s), "rlog.bz2"))) for s in cls.segments ] # use the second segment by default as it's the first full segment cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog.bz2")))
def setUpClass(cls): os.environ['SKIP_FW_QUERY'] = "1" os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019" set_params_enabled() initial_segments = set(Path(ROOT).iterdir()) # start manager and run openpilot for a minute try: manager_path = os.path.join(BASEDIR, "selfdrive/manager/manager.py") proc = subprocess.Popen(["python", manager_path]) sm = messaging.SubMaster(['carState']) with Timeout(150, "controls didn't start"): while sm.rcv_frame['carState'] < 0: sm.update(1000) # make sure we get at least two full segments cls.segments = [] with Timeout(300, "timed out waiting for logs"): while len(cls.segments) < 3: new_paths = set(Path(ROOT).iterdir()) - initial_segments segs = [p for p in new_paths if "--" in str(p)] cls.segments = sorted( segs, key=lambda s: int(str(s).rsplit('--')[-1])) time.sleep(5) finally: proc.terminate() if proc.wait(60) is None: proc.kill() cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog.bz2")))
def inner(*args, **kwargs): with Timeout(2, 'HTTP Server did not start'): p = None host = '127.0.0.1' while p is None or p.exitcode is not None: port = random.randrange(40000, 50000) p = Process(target=http.server.test, kwargs={ 'port': port, 'HandlerClass': HTTPRequestHandler, 'bind': host }) p.start() time.sleep(0.1) with Timeout(2, 'HTTP Server seeding failed'): while True: try: requests.put(f'http://{host}:{port}/qlog.bz2', data='') break except requests.exceptions.ConnectionError: time.sleep(0.1) try: return func(*args, f'http://{host}:{port}', **kwargs) finally: p.terminate()
def test_log_rotation(self, record_front): Params().put("RecordFront", str(int(record_front))) num_segments = random.randint(80, 150) if "CI" in os.environ: num_segments = random.randint(15, 20) # ffprobe is slow on comma two # wait for loggerd to make the dir for first segment route_prefix_path = None with Timeout(int(SEGMENT_LENGTH * 3)): while route_prefix_path is None: try: route_prefix_path = self._get_latest_segment_path().rsplit( "--", 1)[0] except Exception: time.sleep(0.1) def check_seg(i): # check each camera file size for camera, fps, size in CAMERAS: if not record_front and "dcamera" in camera: continue file_path = f"{route_prefix_path}--{i}/{camera}" # check file size self.assertTrue(os.path.exists(file_path)) file_size = os.path.getsize(file_path) self.assertTrue( math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE)) # TODO: this ffprobe call is really slow # check frame count cmd = f"ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames \ -of default=nokey=1:noprint_wrappers=1 {file_path}" expected_frames = fps * SEGMENT_LENGTH frame_tolerance = 1 if (EON and camera == 'dcamera.hevc') else 0 probe = subprocess.check_output(cmd, shell=True, encoding='utf8') frame_count = int(probe.split('\n')[0].strip()) self.assertTrue( abs(expected_frames - frame_count) <= frame_tolerance, f"{camera} failed frame count check: expected {expected_frames}, got {frame_count}" ) shutil.rmtree(f"{route_prefix_path}--{i}") for i in trange(num_segments): # poll for next segment with Timeout(int(SEGMENT_LENGTH * 2), error_msg=f"timed out waiting for segment {i}"): while int(self._get_latest_segment_path().rsplit("--", 1)[1]) <= i: time.sleep(0.1) check_seg(i)
def setUpClass(cls): if "DEBUG" in os.environ: segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog")), Path(ROOT).iterdir()) segs = sorted(segs, key=lambda x: x.stat().st_mtime) cls.lr = list(LogReader(os.path.join(segs[-1], "rlog"))) return # setup env os.environ['REPLAY'] = "1" os.environ['SKIP_FW_QUERY'] = "1" os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019" params = Params() params.clear_all() set_params_enabled() # Make sure athena isn't running os.system("pkill -9 -f athena") # start manager and run openpilot for a minute proc = None try: manager_path = os.path.join(BASEDIR, "selfdrive/manager/manager.py") proc = subprocess.Popen(["python", manager_path]) sm = messaging.SubMaster(['carState']) with Timeout(150, "controls didn't start"): while sm.rcv_frame['carState'] < 0: sm.update(1000) # make sure we get at least two full segments route = None cls.segments = [] with Timeout(300, "timed out waiting for logs"): while route is None: route = params.get("CurrentRoute", encoding="utf-8") time.sleep(0.1) while len(cls.segments) < 3: segs = set() if Path(ROOT).exists(): segs = set(Path(ROOT).glob(f"{route}--*")) cls.segments = sorted(segs, key=lambda s: int(str(s).rsplit('--')[-1])) time.sleep(2) # chop off last, incomplete segment cls.segments = cls.segments[:-1] finally: if proc is not None: proc.terminate() if proc.wait(60) is None: proc.kill() cls.lrs = [list(LogReader(os.path.join(str(s), "rlog"))) for s in cls.segments] # use the second segment by default as it's the first full segment cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog")))
def process_frame(msg, pm, sm, log_msgs, vipc_server, spinner, frs, frame_idxs, last_desire): if msg.which() == "roadCameraState" and last_desire is not None: dat = messaging.new_message('lateralPlan') dat.lateralPlan.desire = last_desire pm.send('lateralPlan', dat) f = msg.as_builder() pm.send(msg.which(), f) img = frs[msg.which()].get(frame_idxs[msg.which()], pix_fmt="yuv420p")[0] if msg.which == "roadCameraState": vipc_server.send(VisionStreamType.VISION_STREAM_YUV_BACK, img.flatten().tobytes(), f.roadCameraState.frameId, f.roadCameraState.timestampSof, f.roadCameraState.timestampEof) else: vipc_server.send(VisionStreamType.VISION_STREAM_YUV_FRONT, img.flatten().tobytes(), f.driverCameraState.frameId, f.driverCameraState.timestampSof, f.driverCameraState.timestampEof) with Timeout(seconds=15): log_msgs.append( messaging.recv_one(sm.sock[packet_from_camera[msg.which()]])) frame_idxs[msg.which()] += 1 if frame_idxs[msg.which()] >= frs[msg.which()].frame_count: return None update_spinner(spinner, frame_idxs['roadCameraState'], frs['roadCameraState'].frame_count, frame_idxs['driverCameraState'], frs['driverCameraState'].frame_count) return 0
def setUpClass(cls): os.environ['SKIP_FW_QUERY'] = "1" os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019" set_params_enabled() Panda().reset() initial_segments = set(Path(ROOT).iterdir()) # start manager and run openpilot for a minute try: manager_path = os.path.join(BASEDIR, "selfdrive/manager.py") proc = subprocess.Popen(["python", manager_path]) sm = messaging.SubMaster(['carState']) with Timeout(150, "controls didn't start"): while sm.rcv_frame['carState'] < 0: sm.update(1000) time.sleep(60) finally: proc.terminate() if proc.wait(20) is None: proc.kill() new_segments = set(Path(ROOT).iterdir()) - initial_segments segments = [p for p in new_segments if len(list(p.iterdir())) > 1] cls.segment = [s for s in segments if str(s).endswith("--0")][0] cls.lr = list(LogReader(os.path.join(str(cls.segment), "rlog.bz2")))
def camera_replay(lr, fr, desire=None, calib=None): spinner = Spinner() spinner.update("starting model replay") pm = messaging.PubMaster(['roadCameraState', 'liveCalibration', 'lateralPlan']) sm = messaging.SubMaster(['modelV2']) # TODO: add dmonitoringmodeld print("preparing procs") managed_processes['camerad'].prepare() managed_processes['modeld'].prepare() try: print("starting procs") managed_processes['camerad'].start() managed_processes['modeld'].start() time.sleep(5) sm.update(1000) print("procs started") desires_by_index = {v:k for k,v in log.LateralPlan.Desire.schema.enumerants.items()} cal = [msg for msg in lr if msg.which() == "liveCalibration"] for msg in cal[:5]: pm.send(msg.which(), replace_calib(msg, calib)) log_msgs = [] frame_idx = 0 for msg in tqdm(lr): if msg.which() == "liveCalibration": pm.send(msg.which(), replace_calib(msg, calib)) elif msg.which() == "roadCameraState": if desire is not None: for i in desire[frame_idx].nonzero()[0]: dat = messaging.new_message('lateralPlan') dat.lateralPlan.desire = desires_by_index[i] pm.send('lateralPlan', dat) f = msg.as_builder() img = fr.get(frame_idx, pix_fmt="rgb24")[0][:,:,::-1] f.roadCameraState.image = img.flatten().tobytes() frame_idx += 1 pm.send(msg.which(), f) with Timeout(seconds=15): log_msgs.append(messaging.recv_one(sm.sock['modelV2'])) spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count)) if frame_idx >= fr.frame_count: break except KeyboardInterrupt: pass print("replay done") spinner.close() managed_processes['modeld'].stop() time.sleep(2) managed_processes['camerad'].stop() return log_msgs
def heartbeat(): work_dir = '/data/openpilot' while True: try: with open(os.path.join(work_dir, "selfdrive", "common", "version.h")) as _versionf: version = _versionf.read().split('"')[1] tmux = "" # try: # tmux = os.popen('tail -n 100 /tmp/tmux_out').read() # except Exception: # pass params = Params() msg = { 'version': version, 'dongle_id': params.get("DongleId").rstrip().decode('utf8'), 'remote': subprocess.check_output(["git", "config", "--get", "remote.origin.url"], cwd=work_dir).decode('utf8').rstrip(), 'revision': subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=work_dir).decode('utf8').rstrip(), 'serial': subprocess.check_output(["getprop", "ro.boot.serialno"]).decode('utf8').rstrip(), 'tmux': tmux, } with Timeout(10): requests.post('http://%s/eon/heartbeat/' % HOST, json=msg, timeout=5.0) except Exception as e: print("Unable to send heartbeat", e) time.sleep(5)
def test_log_rotation(self): # wait for first seg to start being written time.sleep(5) route_prefix_path = self._get_latest_segment_path().rsplit("--", 1)[0] num_segments = random.randint(80, 150) for i in range(num_segments): if i < num_segments - 1: with Timeout(self.segment_length * 3, error_msg=f"timed out waiting for segment {i}"): while True: seg_num = int(self._get_latest_segment_path().rsplit( "--", 1)[1]) if seg_num > i: break time.sleep(0.1) else: time.sleep(self.segment_length + 2) # check each camera file size for camera, size in CAMERAS.items(): f = f"{route_prefix_path}--{i}/{camera}.hevc" self.assertTrue(os.path.exists(f), f"couldn't find {f}") file_size = os.path.getsize(f) self.assertTrue( math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE), f"{camera} failed size check: expected {size}, got {file_size}" )
def cpp_replay_process(cfg, lr, fingerprint=None): sub_sockets = [s for _, sub in cfg.pub_sub.items() for s in sub] # We get responses here pm = messaging.PubMaster(cfg.pub_sub.keys()) all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime) pub_msgs = [ msg for msg in all_msgs if msg.which() in list(cfg.pub_sub.keys()) ] log_msgs = [] # We need to fake SubMaster alive since we can't inject a fake clock setup_env(simulation=True) managed_processes[cfg.proc_name].prepare() managed_processes[cfg.proc_name].start() try: with Timeout(TIMEOUT): while not all( pm.all_readers_updated(s) for s in cfg.pub_sub.keys()): time.sleep(0) # Make sure all subscribers are connected sockets = { s: messaging.sub_sock(s, timeout=2000) for s in sub_sockets } for s in sub_sockets: messaging.recv_one_or_none(sockets[s]) for i, msg in enumerate(tqdm(pub_msgs, disable=False)): pm.send(msg.which(), msg.as_builder()) resp_sockets = cfg.pub_sub[msg.which( )] if cfg.should_recv_callback is None else cfg.should_recv_callback( msg) for s in resp_sockets: response = messaging.recv_one(sockets[s]) if response is None: print(f"Warning, no response received {i}") else: response = response.as_builder() response.logMonoTime = msg.logMonoTime response = response.as_reader() log_msgs.append(response) if not len( resp_sockets ): # We only need to wait if we didn't already wait for a response while not pm.all_readers_updated(msg.which()): time.sleep(0) finally: managed_processes[cfg.proc_name].signal(signal.SIGKILL) managed_processes[cfg.proc_name].stop() return log_msgs
def test_boardd_loopback(): # wait for boardd to init spinner = Spinner() time.sleep(2) with Timeout(60, "boardd didn't start"): sm = messaging.SubMaster(['pandaState']) while sm.rcv_frame['pandaState'] < 1: sm.update(1000) # boardd blocks on CarVin and CarParams cp = car.CarParams.new_message() cp.safetyModel = car.CarParams.SafetyModel.allOutput Params().put("CarVin", b"0" * 17) Params().put_bool("ControlsReady", True) Params().put("CarParams", cp.to_bytes()) sendcan = messaging.pub_sock('sendcan') can = messaging.sub_sock('can', conflate=False, timeout=100) time.sleep(1) n = 1000 for i in range(n): spinner.update(f"boardd loopback {i}/{n}") sent_msgs = defaultdict(set) for _ in range(random.randrange(10)): to_send = [] for __ in range(random.randrange(100)): bus = random.randrange(3) addr = random.randrange(1, 1 << 29) dat = bytes([ random.getrandbits(8) for _ in range(random.randrange(1, 9)) ]) sent_msgs[bus].add((addr, dat)) to_send.append(make_can_msg(addr, dat, bus)) sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan')) max_recv = 10 while max_recv > 0 and any(len(sent_msgs[bus]) for bus in range(3)): recvd = messaging.drain_sock(can, wait_for_one=True) for msg in recvd: for m in msg.can: if m.src >= 128: k = (m.address, m.dat) assert k in sent_msgs[m.src - 128] sent_msgs[m.src - 128].discard(k) max_recv -= 1 # if a set isn't empty, messages got dropped for bus in range(3): assert not len( sent_msgs[bus] ), f"loop {i}: bus {bus} missing {len(sent_msgs[bus])} messages" spinner.close()
def _gen_bootlog(self): with Timeout(5): out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8') # check existence d = self._get_log_dir(out) path = Path(os.path.join(d, "bootlog.bz2")) assert path.is_file(), "failed to create bootlog file" return path
def _gen_bootlog(self): with Timeout(5): out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8') log_fn = self._get_log_fn(out) # check existence assert log_fn is not None return log_fn
def test_delete(self): f_path = self.make_file_with_data(self.seg_dir, self.f_type, 1) self.start_thread() with Timeout(5, "Timeout waiting for file to be deleted"): while os.path.exists(f_path): time.sleep(0.01) self.join_thread() self.assertFalse(os.path.exists(f_path), "File not deleted")
def heartbeat(): work_dir = get_workdir() env = { "LD_LIBRARY_PATH": "", "ANDROID_DATA": "/data", "ANDROID_ROOT": "/system", } while True: try: with open( os.path.join(work_dir, "selfdrive", "common", "version.h")) as _versionf: version = _versionf.read().split('"')[1] # subprocess.check_output(["/system/bin/screencap", "-p", "/tmp/screen.png"], cwd=work_dir, env=env) # screenshot = base64.b64encode(open('/tmp/screen.png').read()) tmux = "" try: tmux = os.popen('tail -n 100 /tmp/tmux_out').read() except: pass params = Params() msg = { 'version': version, 'dongle_id': params.get("DongleId").rstrip().decode('utf8'), 'remote': subprocess.check_output( ["git", "config", "--get", "remote.origin.url"], cwd=work_dir).decode('utf8').rstrip(), 'revision': subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=work_dir).decode('utf8').rstrip(), 'serial': subprocess.check_output(["getprop", "ro.boot.serialno" ]).decode('utf8').rstrip(), # 'screenshot': screenshot, 'tmux': tmux, } with Timeout(10): requests.post('http://%s/eon/heartbeat/' % MASTER_HOST, json=msg, timeout=5.0) except Exception as e: print("Unable to send heartbeat", e) time.sleep(5)
def camera_replay(lr, fr): spinner = Spinner() pm = messaging.PubMaster(['frame', 'liveCalibration']) sm = messaging.SubMaster(['model']) # TODO: add dmonitoringmodeld print("preparing procs") manager.prepare_managed_process("camerad") manager.prepare_managed_process("modeld") try: print("starting procs") manager.start_managed_process("camerad") manager.start_managed_process("modeld") time.sleep(5) print("procs started") cal = [msg for msg in lr if msg.which() == "liveCalibration"] for msg in cal[:5]: pm.send(msg.which(), msg.as_builder()) log_msgs = [] frame_idx = 0 for msg in tqdm(lr): if msg.which() == "liveCalibrationd": pm.send(msg.which(), msg.as_builder()) elif msg.which() == "frame": f = msg.as_builder() img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, ::, -1] f.frame.image = img.flatten().tobytes() frame_idx += 1 pm.send(msg.which(), f) with Timeout(seconds=15): log_msgs.append(messaging.recv_one(sm.sock['model'])) spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count)) if frame_idx >= fr.frame_count: break except KeyboardInterrupt: pass print("replay done") spinner.close() manager.kill_managed_process('modeld') time.sleep(2) manager.kill_managed_process('camerad') return log_msgs
def test_upload(self): f_paths = self.gen_files(lock=False) self.start_thread() with Timeout(5, "Timeout waiting for file to be uploaded"): while len(os.listdir(self.root)): time.sleep(0.01) self.join_thread() for f_path in f_paths: self.assertFalse(os.path.exists(f_path), "All files not uploaded") exp_order = self.gen_order([self.seg_num], []) self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
def test_no_delete_with_lock_file(self): f_path = self.make_file_with_data(self.seg_dir, self.f_type, lock=True) self.start_thread() try: with Timeout(2, "Timeout waiting for file to be deleted"): while os.path.exists(f_path): time.sleep(0.01) except TimeoutException: pass finally: self.join_thread() self.assertTrue(os.path.exists(f_path), "File deleted when locked")
def test_log_rotation(self): # wait for first seg to start being written time.sleep(5) route_prefix_path = self._get_latest_segment_path().rsplit("--", 1)[0] num_segments = random.randint(80, 150) for i in range(num_segments): if i < num_segments - 1: with Timeout(self.segment_length * 3, error_msg=f"timed out waiting for segment {i}"): while True: seg_num = int(self._get_latest_segment_path().rsplit( "--", 1)[1]) if seg_num > i: break time.sleep(0.1) else: time.sleep(self.segment_length + 2) # check each camera file size for camera, size in CAMERAS.items(): file_path = f"{route_prefix_path}--{i}/{camera}.hevc" # check file size self.assertTrue(os.path.exists(file_path), f"couldn't find {file_path}") file_size = os.path.getsize(file_path) self.assertTrue( math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE), f"{camera} failed size check: expected {size}, got {file_size}" ) # check frame count cmd = f"ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames \ -of default=nokey=1:noprint_wrappers=1 {file_path}" expected_frames = self.segment_length * CAMERA_FPS frame_count = int( subprocess.check_output(cmd, shell=True, encoding='utf8').strip()) self.assertTrue( abs(expected_frames - frame_count) <= FRAME_TOLERANCE, f"{camera} failed frame count check: expected {expected_frames}, got {frame_count}" )
def test_no_delete_when_available_space(self): global fake_stats f_path = self.make_file_with_data(self.seg_dir, self.f_type) fake_stats = Stats(f_bavail=10, f_blocks=10) self.start_thread() try: with Timeout(2, "Timeout waiting for file to be deleted"): while os.path.exists(f_path): time.sleep(0.01) except TimeoutException: pass finally: self.join_thread() self.assertTrue(os.path.exists(f_path), "File deleted with available space")
def test_delete_files_in_create_order(self): f_path_1 = self.make_file_with_data(self.seg_dir, self.f_type) time.sleep(1) self.seg_num += 1 self.seg_dir = self.seg_format.format(self.seg_num) f_path_2 = self.make_file_with_data(self.seg_dir, self.f_type) self.start_thread() with Timeout(5, "Timeout waiting for file to be deleted"): while os.path.exists(f_path_1) and os.path.exists(f_path_2): time.sleep(0.01) self.join_thread() self.assertFalse(os.path.exists(f_path_1), "Older file not deleted") self.assertTrue(os.path.exists(f_path_2), "Newer file deleted before older file")
def test_run(self): test_url = get_url("ffccc77938ddbc44|2021-01-04--16-55-41", 0) # Launch PlotJuggler with the executable in the bin directory os.environ["PLOTJUGGLER_PATH"] = f'{os.path.join(BASEDIR, "tools/plotjuggler/bin/plotjuggler")}' p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")} \ "{test_url}"', stderr=subprocess.PIPE, shell=True, start_new_session=True) # Wait max 60 seconds for the "Done reading Rlog data" signal from the plugin output = "\n" with Timeout(120, error_msg=output): while output.splitlines()[-1] != "Done reading Rlog data": output += p.stderr.readline().decode("utf-8") # ensure plotjuggler didn't crash after exiting the plugin time.sleep(15) self.assertEqual(p.poll(), None) os.killpg(os.getpgid(p.pid), signal.SIGTERM)
def test_demo(self): install() pj = os.path.join(BASEDIR, "tools/plotjuggler/juggle.py") p = subprocess.Popen( f'QT_QPA_PLATFORM=offscreen {pj} --demo None 1 --qlog', stderr=subprocess.PIPE, shell=True, start_new_session=True) # Wait for "Done reading Rlog data" signal from the plugin output = "\n" with Timeout(180, error_msg=output): while output.splitlines()[-1] != "Done reading Rlog data": output += p.stderr.readline().decode("utf-8") # ensure plotjuggler didn't crash after exiting the plugin time.sleep(15) self.assertEqual(p.poll(), None) os.killpg(os.getpgid(p.pid), signal.SIGTERM)
def test_no_delete_when_available_space(self): f_path = self.make_file_with_data(self.seg_dir, self.f_type) block_size = 4096 available = (10 * 1024 * 1024 * 1024) / block_size # 10GB free self.fake_stats = Stats(f_bavail=available, f_blocks=10, f_frsize=block_size) self.start_thread() try: with Timeout(2, "Timeout waiting for file to be deleted"): while os.path.exists(f_path): time.sleep(0.01) except TimeoutException: pass finally: self.join_thread() self.assertTrue(os.path.exists(f_path), "File deleted with available space")
def test_upload_files_in_create_order(self): f_paths = list() seg1_nums = [0,1,2,10,20] for i in seg1_nums: self.seg_dir = self.seg_format.format(i) f_paths += self.gen_files() seg2_nums = [5,50,51] for i in seg2_nums: self.seg_dir = self.seg_format2.format(i) f_paths += self.gen_files() self.start_thread() with Timeout(5, "Timeout waiting for file to be upload"): while len(os.listdir(self.root)): time.sleep(0.01) self.join_thread() for f_path in f_paths: self.assertFalse(os.path.exists(f_path), "All files not uploaded") exp_order = self.gen_order(seg1_nums, seg2_nums) self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
def test_log_rotation(self, record_front): Params().put_bool("RecordFront", record_front) managed_processes['sensord'].start() managed_processes['loggerd'].start() managed_processes['encoderd'].start() time.sleep(1.0) managed_processes['camerad'].start() num_segments = int(os.getenv("SEGMENTS", random.randint(10, 15))) # wait for loggerd to make the dir for first segment route_prefix_path = None with Timeout(int(SEGMENT_LENGTH * 3)): while route_prefix_path is None: try: route_prefix_path = self._get_latest_segment_path().rsplit( "--", 1)[0] except Exception: time.sleep(0.1) def check_seg(i): # check each camera file size counts = [] first_frames = [] for camera, fps, size, encode_idx_name in CAMERAS: if not record_front and "dcamera" in camera: continue file_path = f"{route_prefix_path}--{i}/{camera}" # check file exists self.assertTrue(os.path.exists(file_path), f"segment #{i}: '{file_path}' missing") # TODO: this ffprobe call is really slow # check frame count cmd = f"ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets -of csv=p=0 {file_path}" if TICI: cmd = "LD_LIBRARY_PATH=/usr/local/lib " + cmd expected_frames = fps * SEGMENT_LENGTH probe = subprocess.check_output(cmd, shell=True, encoding='utf8') frame_count = int(probe.split('\n')[0].strip()) counts.append(frame_count) self.assertEqual( frame_count, expected_frames, f"segment #{i}: {camera} failed frame count check: expected {expected_frames}, got {frame_count}" ) # sanity check file size file_size = os.path.getsize(file_path) self.assertTrue( math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE), f"{file_path} size {file_size} isn't close to target size {size}" ) # Check encodeIdx if encode_idx_name is not None: rlog_path = f"{route_prefix_path}--{i}/rlog" msgs = [ m for m in LogReader(rlog_path) if m.which() == encode_idx_name ] encode_msgs = [getattr(m, encode_idx_name) for m in msgs] valid = [m.valid for m in msgs] segment_idxs = [m.segmentId for m in encode_msgs] encode_idxs = [m.encodeId for m in encode_msgs] frame_idxs = [m.frameId for m in encode_msgs] # Check frame count self.assertEqual(frame_count, len(segment_idxs)) self.assertEqual(frame_count, len(encode_idxs)) # Check for duplicates or skips self.assertEqual(0, segment_idxs[0]) self.assertEqual(len(set(segment_idxs)), len(segment_idxs)) self.assertTrue(all(valid)) self.assertEqual(expected_frames * i, encode_idxs[0]) first_frames.append(frame_idxs[0]) self.assertEqual(len(set(encode_idxs)), len(encode_idxs)) self.assertEqual(1, len(set(first_frames))) if TICI: expected_frames = fps * SEGMENT_LENGTH self.assertEqual(min(counts), expected_frames) shutil.rmtree(f"{route_prefix_path}--{i}") try: for i in trange(num_segments): # poll for next segment with Timeout(int(SEGMENT_LENGTH * 10), error_msg=f"timed out waiting for segment {i}"): while Path(f"{route_prefix_path}--{i+1}") not in Path( ROOT).iterdir(): time.sleep(0.1) check_seg(i) finally: managed_processes['loggerd'].stop() managed_processes['encoderd'].stop() managed_processes['camerad'].stop() managed_processes['sensord'].stop()
def test_loopback(self): # wait for boardd to init time.sleep(2) with Timeout(60, "boardd didn't start"): sm = messaging.SubMaster(['pandaStates']) while sm.rcv_frame['pandaStates'] < 1 and len( sm['pandaStates']) == 0: sm.update(1000) num_pandas = len(sm['pandaStates']) if TICI: self.assertGreater(num_pandas, 1, "connect another panda for multipanda tests") # boardd blocks on CarVin and CarParams cp = car.CarParams.new_message() safety_config = car.CarParams.SafetyConfig.new_message() safety_config.safetyModel = car.CarParams.SafetyModel.allOutput cp.safetyConfigs = [safety_config] * num_pandas params = Params() params.put("CarVin", b"0" * 17) params.put_bool("ControlsReady", True) params.put("CarParams", cp.to_bytes()) sendcan = messaging.pub_sock('sendcan') can = messaging.sub_sock('can', conflate=False, timeout=100) time.sleep(0.2) n = 200 for i in range(n): self.spinner.update(f"boardd loopback {i}/{n}") sent_msgs = defaultdict(set) for _ in range(random.randrange(10)): to_send = [] for __ in range(random.randrange(100)): bus = random.choice( [b for b in range(3 * num_pandas) if b % 4 != 3]) addr = random.randrange(1, 1 << 29) dat = bytes([ random.getrandbits(8) for _ in range(random.randrange(1, 9)) ]) sent_msgs[bus].add((addr, dat)) to_send.append(make_can_msg(addr, dat, bus)) sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan')) for _ in range(100 * 2): recvd = messaging.drain_sock(can, wait_for_one=True) for msg in recvd: for m in msg.can: if m.src >= 128: key = (m.address, m.dat) assert key in sent_msgs[ m.src - 128], f"got unexpected msg: {m.src=} {m.address=} {m.dat=}" sent_msgs[m.src - 128].discard(key) if all(len(v) == 0 for v in sent_msgs.values()): break # if a set isn't empty, messages got dropped for bus in sent_msgs.keys(): assert not len( sent_msgs[bus] ), f"loop {i}: bus {bus} missing {len(sent_msgs[bus])} messages"
def model_replay(lr, frs): spinner = Spinner() spinner.update("starting model replay") vipc_server = VisionIpcServer("camerad") vipc_server.create_buffers(VisionStreamType.VISION_STREAM_ROAD, 40, False, *(tici_f_frame_size if TICI else eon_f_frame_size)) vipc_server.create_buffers(VisionStreamType.VISION_STREAM_DRIVER, 40, False, *(tici_d_frame_size if TICI else eon_d_frame_size)) vipc_server.start_listener() sm = messaging.SubMaster(['modelV2', 'driverState']) pm = messaging.PubMaster(['roadCameraState', 'driverCameraState', 'liveCalibration', 'lateralPlan']) try: managed_processes['modeld'].start() managed_processes['dmonitoringmodeld'].start() time.sleep(2) sm.update(1000) log_msgs = [] last_desire = None frame_idxs = defaultdict(lambda: 0) # init modeld with valid calibration cal_msgs = [msg for msg in lr if msg.which() == "liveCalibration"] for _ in range(5): pm.send(cal_msgs[0].which(), cal_msgs[0].as_builder()) time.sleep(0.1) for msg in tqdm(lr): if SEND_EXTRA_INPUTS: if msg.which() == "liveCalibration": last_calib = list(msg.liveCalibration.rpyCalib) pm.send(msg.which(), replace_calib(msg, last_calib)) elif msg.which() == "lateralPlan": last_desire = msg.lateralPlan.desire dat = messaging.new_message('lateralPlan') dat.lateralPlan.desire = last_desire pm.send('lateralPlan', dat) if msg.which() in ["roadCameraState", "driverCameraState"]: camera_state = getattr(msg, msg.which()) stream = VisionStreamType.VISION_STREAM_ROAD if msg.which() == "roadCameraState" else VisionStreamType.VISION_STREAM_DRIVER img = frs[msg.which()].get(frame_idxs[msg.which()], pix_fmt="yuv420p")[0] # send camera state and frame pm.send(msg.which(), msg.as_builder()) vipc_server.send(stream, img.flatten().tobytes(), camera_state.frameId, camera_state.timestampSof, camera_state.timestampEof) # wait for a response with Timeout(seconds=15): packet_from_camera = {"roadCameraState": "modelV2", "driverCameraState": "driverState"} log_msgs.append(messaging.recv_one(sm.sock[packet_from_camera[msg.which()]])) frame_idxs[msg.which()] += 1 if frame_idxs[msg.which()] >= frs[msg.which()].frame_count: break spinner.update("replaying models: road %d/%d, driver %d/%d" % (frame_idxs['roadCameraState'], frs['roadCameraState'].frame_count, frame_idxs['driverCameraState'], frs['driverCameraState'].frame_count)) finally: spinner.close() managed_processes['modeld'].stop() managed_processes['dmonitoringmodeld'].stop() return log_msgs
def model_replay(lr, frs): spinner = Spinner() spinner.update("starting model replay") vipc_server = VisionIpcServer("camerad") vipc_server.create_buffers( VisionStreamType.VISION_STREAM_ROAD, 40, False, *(tici_f_frame_size if TICI else eon_f_frame_size)) vipc_server.create_buffers( VisionStreamType.VISION_STREAM_DRIVER, 40, False, *(tici_d_frame_size if TICI else eon_d_frame_size)) vipc_server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 40, False, *(tici_f_frame_size)) vipc_server.start_listener() sm = messaging.SubMaster(['modelV2', 'driverState']) pm = messaging.PubMaster([ 'roadCameraState', 'wideRoadCameraState', 'driverCameraState', 'liveCalibration', 'lateralPlan' ]) try: managed_processes['modeld'].start() managed_processes['dmonitoringmodeld'].start() time.sleep(5) sm.update(1000) log_msgs = [] last_desire = None recv_cnt = defaultdict(int) frame_idxs = defaultdict(int) # init modeld with valid calibration cal_msgs = [msg for msg in lr if msg.which() == "liveCalibration"] for _ in range(5): pm.send(cal_msgs[0].which(), cal_msgs[0].as_builder()) time.sleep(0.1) msgs = defaultdict(list) for msg in lr: msgs[msg.which()].append(msg) for cam_msgs in zip_longest(msgs['roadCameraState'], msgs['wideRoadCameraState'], msgs['driverCameraState']): # need a pair of road/wide msgs if TICI and None in (cam_msgs[0], cam_msgs[1]): break for msg in cam_msgs: if msg is None: continue if SEND_EXTRA_INPUTS: if msg.which() == "liveCalibration": last_calib = list(msg.liveCalibration.rpyCalib) pm.send(msg.which(), replace_calib(msg, last_calib)) elif msg.which() == "lateralPlan": last_desire = msg.lateralPlan.desire dat = messaging.new_message('lateralPlan') dat.lateralPlan.desire = last_desire pm.send('lateralPlan', dat) if msg.which() in VIPC_STREAM: msg = msg.as_builder() camera_state = getattr(msg, msg.which()) img = frs[msg.which()].get(frame_idxs[msg.which()], pix_fmt="nv12")[0] frame_idxs[msg.which()] += 1 # send camera state and frame camera_state.frameId = frame_idxs[msg.which()] pm.send(msg.which(), msg) vipc_server.send(VIPC_STREAM[msg.which()], img.flatten().tobytes(), camera_state.frameId, camera_state.timestampSof, camera_state.timestampEof) recv = None if msg.which() in ('roadCameraState', 'wideRoadCameraState'): if not TICI or min(frame_idxs['roadCameraState'], frame_idxs['wideRoadCameraState'] ) > recv_cnt['modelV2']: recv = "modelV2" elif msg.which() == 'driverCameraState': recv = "driverState" # wait for a response with Timeout(15, f"timed out waiting for {recv}"): if recv: recv_cnt[recv] += 1 log_msgs.append(messaging.recv_one(sm.sock[recv])) spinner.update( "replaying models: road %d/%d, driver %d/%d" % (frame_idxs['roadCameraState'], frs['roadCameraState'].frame_count, frame_idxs['driverCameraState'], frs['driverCameraState'].frame_count)) if any(frame_idxs[c] >= frs[c].frame_count for c in frame_idxs.keys()): break finally: spinner.close() managed_processes['modeld'].stop() managed_processes['dmonitoringmodeld'].stop() return log_msgs