def test_poll_and_create_many_subscribers(self):
context = messaging.Context()
pub = messaging.PubSocket()
pub.connect(context, 'controlsState')
with concurrent.futures.ThreadPoolExecutor() as e:
poll = e.submit(poller)
time.sleep(0.1) # Slow joiner syndrome
c = messaging.Context()
for _ in range(10):
messaging.SubSocket().connect(c, 'controlsState')
time.sleep(0.1)
# Send message
pub.send("a")
# Wait for poll result
result = poll.result()
del pub
context.term()
self.assertEqual(result, [b"a"])
def test_multiple_messages(self):
context = messaging.Context()
pub = messaging.PubSocket()
pub.connect(context, 'controlsState')
sub = messaging.SubSocket()
sub.connect(context, 'controlsState')
time.sleep(0.1) # Slow joiner
for i in range(100):
pub.send(str(i))
msg_seen = False
i = 0
while True:
r = sub.receive(non_blocking=True)
if r is not None:
self.assertEqual(str(i), r.decode('utf8'))
msg_seen = True
i += 1
if r is None and msg_seen: # ZMQ sometimes receives nothing on the first receive
break
del pub
del sub
context.term()
def writer(fn, addr, sock_name):
import cereal.messaging as messaging
HEADER = b"\x00\x00\x00\x01\x40\x01\x0c\x01\xff\xff\x01\x60\x00\x00\x03\x00\xb0\x00\x00\x03\x00\x00\x03\x00\x96\xac\x09\x00\x00\x00\x01\x42\x01\x01\x01\x60\x00\x00\x03\x00\xb0\x00\x00\x03\x00\x00\x03\x00\x96\xa0\x03\xd0\x80\x13\x07\x1b\x2e\x5a\xee\x4c\x92\xea\x00\xbb\x42\x84\xa0\x00\x00\x00\x01\x44\x01\xc0\xe2\x4f\x09\xc1\x80\xc6\x08\x40\x00"
fifo_file = open(fn, "wb")
fifo_file.write(HEADER)
fifo_file.flush()
os.environ["ZMQ"] = "1"
messaging.context = messaging.Context()
sock = messaging.sub_sock(sock_name, None, addr=addr, conflate=False)
last_idx = -1
seen_iframe = False
while 1:
msgs = messaging.drain_sock(sock, wait_for_one=True)
for evt in msgs:
evta = getattr(evt, evt.which())
lat = ((evt.logMonoTime / 1e9) - (evta.timestampEof / 1e6)) * 1000
print(
"%2d %4d %.3f %.3f latency %.2f ms" %
(len(msgs), evta.idx, evt.logMonoTime / 1e9,
evta.timestampEof / 1e6, lat), len(evta.data), sock_name)
if evta.idx != 0 and evta.idx != (last_idx + 1):
print("DROP!")
last_idx = evta.idx
if len(evta.data) > 4 and evta.data[4] == 0x26:
seen_iframe = True
if not seen_iframe:
print("waiting for iframe")
continue
fifo_file.write(evta.data)
fifo_file.flush()
def send_deviceState():
messaging.context = messaging.Context()
pub_sock = messaging.pub_sock("deviceState")
start = time.time()
while time.time() - start < 1:
msg = messaging.new_message('deviceState')
pub_sock.send(msg.to_bytes())
time.sleep(0.01)
def send_thermal():
messaging.context = messaging.Context()
pub_sock = messaging.pub_sock("thermal")
start = time.time()
while time.time() - start < 1:
msg = messaging.new_message('thermal')
pub_sock.send(msg.to_bytes())
time.sleep(0.01)
def poller():
context = messaging.Context()
p = messaging.Poller()
sub = messaging.SubSocket()
sub.connect(context, 'controlsState')
p.registerSocket(sub)
socks = p.poll(10000)
r = [s.receive(non_blocking=True) for s in socks]
return r
def test_multiple_publishers_exception(self):
context = messaging.Context()
with self.assertRaises(messaging.MultiplePublishersError):
pub1 = messaging.PubSocket()
pub1.connect(context, 'controlsState')
pub2 = messaging.PubSocket()
pub2.connect(context, 'controlsState')
pub1.send("a")
del pub1
del pub2
context.term()
def test_conflate(self):
context = messaging.Context()
pub = messaging.PubSocket()
pub.connect(context, 'controlsState')
sub = messaging.SubSocket()
sub.connect(context, 'controlsState', conflate=True)
time.sleep(0.1) # Slow joiner
pub.send('a')
pub.send('b')
self.assertEqual(b'b', sub.receive())
del pub
del sub
context.term()
def launcher(proc):
try:
# import the process
mod = importlib.import_module(proc)
# rename the process
setproctitle(proc)
# create new context since we forked
messaging.context = messaging.Context()
# exec the process
mod.main()
except KeyboardInterrupt:
cloudlog.warning("child %s got SIGINT" % proc)
except Exception:
# can't install the crash handler becuase sys.excepthook doesn't play nice
# with threads, so catch it here.
crash.capture_exception()
raise
def test_poll_once(self):
context = messaging.Context()
pub = messaging.PubSocket()
pub.connect(context, 'controlsState')
with concurrent.futures.ThreadPoolExecutor() as e:
poll = e.submit(poller)
time.sleep(0.1) # Slow joiner syndrome
# Send message
pub.send(b"a")
# Wait for poll result
result = poll.result()
del pub
context.term()
self.assertEqual(result, [b"a"])
def launcher(proc: str, name: str) -> None:
try:
# import the process
mod = importlib.import_module(proc)
# rename the process
setproctitle(proc)
# create new context since we forked
messaging.context = messaging.Context()
# add daemon name to cloudlog ctx
cloudlog.bind(daemon=name)
# exec the process
getattr(mod, 'main')()
except KeyboardInterrupt:
cloudlog.warning(f"child {proc} got SIGINT")
except Exception:
# can't install the crash handler because sys.excepthook doesn't play nice
# with threads, so catch it here.
sentry.capture_exception()
raise
print("handler!")
exit(0)
signal.signal(signal.SIGINT, sigint_handler)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Sniff a communication socket')
parser.add_argument('--addr', default='127.0.0.1')
args = parser.parse_args()
if args.addr != "127.0.0.1":
os.environ["ZMQ"] = "1"
messaging.context = messaging.Context()
carControl = messaging.sub_sock('carControl',
addr=args.addr,
conflate=True)
sm = messaging.SubMaster(['carState', 'carControl', 'controlsState'],
addr=args.addr)
msg_cnt = 0
stats = defaultdict(lambda: {'err': 0, "cnt": 0, "=": 0, "+": 0, "-": 0})
cnt = 0
total_error = 0
while messaging.recv_one(carControl):
sm.update()
msg_cnt += 1
def decoder(addr, sock_name, vipc_server, vst, nvidia):
print("start decoder for %s" % sock_name)
if nvidia:
os.environ[
"NV_LOW_LATENCY"] = "3" # both bLowLatency and CUVID_PKT_ENDOFPICTURE
sys.path += os.environ["LD_LIBRARY_PATH"].split(":")
import PyNvCodec as nvc # pylint: disable=import-error
nvDec = nvc.PyNvDecoder(W, H, nvc.PixelFormat.NV12,
nvc.CudaVideoCodec.HEVC, 0)
cc1 = nvc.ColorspaceConversionContext(nvc.ColorSpace.BT_709,
nvc.ColorRange.JPEG)
conv_yuv = nvc.PySurfaceConverter(W, H, nvc.PixelFormat.NV12,
nvc.PixelFormat.YUV420, 0)
nvDwn_yuv = nvc.PySurfaceDownloader(W, H, nvc.PixelFormat.YUV420, 0)
img_yuv = np.ndarray((H * W // 2 * 3), dtype=np.uint8)
else:
codec = av.CodecContext.create("hevc", "r")
os.environ["ZMQ"] = "1"
messaging.context = messaging.Context()
sock = messaging.sub_sock(sock_name, None, addr=addr, conflate=False)
cnt = 0
last_idx = -1
seen_iframe = False
time_q = []
while 1:
msgs = messaging.drain_sock(sock, wait_for_one=True)
for evt in msgs:
evta = getattr(evt, evt.which())
if evta.idx.encodeId != 0 and evta.idx.encodeId != (last_idx + 1):
print("DROP PACKET!")
last_idx = evta.idx.encodeId
if not seen_iframe and not (evta.idx.flags
& V4L2_BUF_FLAG_KEYFRAME):
print("waiting for iframe")
continue
time_q.append(time.monotonic())
network_latency = (int(time.time() * 1e9) -
evta.unixTimestampNanos) / 1e6
frame_latency = ((evta.idx.timestampEof / 1e9) -
(evta.idx.timestampSof / 1e9)) * 1000
process_latency = ((evt.logMonoTime / 1e9) -
(evta.idx.timestampEof / 1e9)) * 1000
# put in header (first)
if not seen_iframe:
if nvidia:
nvDec.DecodeSurfaceFromPacket(
np.frombuffer(evta.header, dtype=np.uint8))
else:
codec.decode(av.packet.Packet(evta.header))
seen_iframe = True
if nvidia:
rawSurface = nvDec.DecodeSurfaceFromPacket(
np.frombuffer(evta.data, dtype=np.uint8))
if rawSurface.Empty():
print("DROP SURFACE")
continue
convSurface = conv_yuv.Execute(rawSurface, cc1)
nvDwn_yuv.DownloadSingleSurface(convSurface, img_yuv)
else:
frames = codec.decode(av.packet.Packet(evta.data))
if len(frames) == 0:
print("DROP SURFACE")
continue
assert len(frames) == 1
img_yuv = frames[0].to_ndarray(
format=av.video.format.VideoFormat('yuv420p')).flatten()
uv_offset = H * W
y = img_yuv[:uv_offset]
uv = img_yuv[uv_offset:].reshape(2, -1).ravel('F')
img_yuv = np.hstack((y, uv))
vipc_server.send(vst, img_yuv.data, cnt, int(time_q[0] * 1e9),
int(time.monotonic() * 1e9))
cnt += 1
pc_latency = (time.monotonic() - time_q[0]) * 1000
time_q = time_q[1:]
print(
"%2d %4d %.3f %.3f roll %6.2f ms latency %6.2f ms + %6.2f ms + %6.2f ms = %6.2f ms"
% (len(msgs), evta.idx.encodeId, evt.logMonoTime / 1e9,
evta.idx.timestampEof / 1e6, frame_latency, process_latency,
network_latency, pc_latency,
process_latency + network_latency + pc_latency),
len(evta.data), sock_name)
