def test_injection_async(config_topic): injected = {} recved = [] with MessageWriter(SERVICE, topic=config_topic + "-W", value_type=TEXT, data_encryption=True) as fw: fw.debug_last_msg_bytes = True for msg in msgs: fw.publish(msg) injected[msg] = fw.debug_last_msg_bytes def recv_msg(msg): recved.append(msg) with AsyncMessageReader(SERVICE, topic=config_topic + "-R", value_type=TEXT) as fr: fr.on_message = recv_msg for b in injected.values(): fr.debug_inject_msg_bytes(b, config_topic + "-R", None) time.sleep(1) # wait to receive. assert len(recved) == len(msgs) for rec in recved: assert rec.value in injected
def test_no_auth_write_no_ack(setup_config): params = { 'sasl_plain_username': KAFKA_READ_USER, 'sasl_plain_password': KAFKA_READ_PASSWD, } with MessageWriter(SERVICE, consistency=AT_MOST_ONCE, **params) as f: f.publish(b'message-001')
def drain(): try: os.mkdir(data_dir.joinpath("python")) except FileExistsError: pass for service in service_list: show(f"drain {service}") with MessageWriter(service) as writer: writer.debug_last_msg_bytes = True for i, msg in enumerate(msg_list, 1): result = None comment = "" try: writer.publish(msg) result = "OK" except Exception as e: show(str(e)) result = "NG" comment = "#" + str(e) show( f"TEST:RESULT: python encode {service} {i} {result} {comment}" ) with open(data_dir.joinpath("python", f"{service}-{i}.in"), "wb") as fin: fin.write(msg) with open(data_dir.joinpath("python", f"{service}-{i}.out"), "wb") as fout: fout.write(writer.debug_last_msg_bytes)
def writer(msgs, *args, **kwargs): ev.wait() logger.debug("start message writing") with MessageWriter(*args, **kwargs) as f: for msg in msgs: logger.info(f"write message: {msg}") f.publish(msg)
def test_sync_thru(config_topic): with MessageWriter(SERVICE, value_type=TEXT) as fw: m0 = fw.metrics logger.info(f"writer.metrics: {m0}") for msg in msgs: fw.publish(msg) m = fw.metrics m2 = fw.metrics fw.reset_metrics() m3 = fw.metrics logger.info(f"writer.metrics: {m}") assert_metrics(m) assert equal_metrics(m, m2) assert m3.msg_count_total == 0 with MessageReader(SERVICE, value_type=TEXT) as fr: for expected, msg in zip(msgs, fr): assert msg.topic == config_topic assert msg.value == expected m = fr.metrics m2 = fr.metrics fr.reset_metrics() m3 = fr.metrics logger.info(f"reader.metrics: {m}") assert_metrics(m) assert equal_metrics(m, m2) assert m3.msg_count_total == 0
def do_sync(service, output_count, num_samples, payload_size): reader = None def consumer(service, event): nonlocal reader with MessageReader(service, value_type=BYTE_ARRAY) as reader_thread: reader = reader_thread event.set() for message in reader_thread: pass event = threading.Event() # for waiting to create consumer # run consumer on thread thread = threading.Thread(target=consumer, args=(service, event)) thread.daemon = True thread.start() results = [] with MessageWriter(service, value_type=BYTE_ARRAY) as writer: event.wait() # wait to create reader for i_output in range(output_count): writer.reset_metrics() reader.reset_metrics() for i_samples in range(num_samples): message = get_message(payload_size) writer.publish(message) results.append({ 'writer': writer.metrics, 'reader': reader.metrics, }) return results
def test_read_bad_type(): with MessageWriter(SERVICE, value_type='text') as fw: fw.publish('messsage-001') with MessageReader(SERVICE, value_type='image') as fr: with pytest.raises(InvalidArgumentError): for _ in fr: pass
def test_no_auth_write(setup_config): params = { 'sasl_plain_username': KAFKA_READ_USER, 'sasl_plain_password': KAFKA_READ_PASSWD, } with pytest.raises(AuthorizationError): with MessageWriter(SERVICE, consistency=AT_LEAST_ONCE, **params) as f: f.publish(b'message-001')
def test_text(config_topic): with MessageWriter(SERVICE, value_type=TEXT) as fw: for msg in tmsgs: fw.publish(msg) with MessageReader(SERVICE, value_type=TEXT) as fr: for expected, msg in zip(tmsgs, fr): assert msg.topic == config_topic assert msg.value == expected
def test_thru(): with MessageWriter(SERVICE, value_type=TEXT) as fw: for msg in msgs: fw.publish(msg) with MessageReader(SERVICE, value_type=TEXT) as fr: for expected, msg in zip(msgs, fr): assert msg.topic == TOPIC assert msg.value == expected
def test_byte_array(config_topic): with MessageWriter(SERVICE, value_type=BYTE_ARRAY) as fw: for msg in bmsgs: fw.publish(msg) with MessageReader(SERVICE, value_type=BYTE_ARRAY) as fr: for expected, msg in zip(bmsgs, fr): assert msg.topic == config_topic assert msg.value == expected
def producer(service, video, preview=False): with MessageWriter(service, value_type='image') as writer: image = next_frame(video) while image is not None: writer.publish(image) if preview and show_preview(image): break image = next_frame(video)
def test_enc_text(): with MessageWriter(SERVICE, value_type=TEXT, data_encryption=True) as fw: for msg in msgs: fw.publish(msg) with MessageReader(SERVICE, value_type=TEXT, data_encryption=True) as fr: for expected, msg in zip(msgs, fr): assert msg.topic == TOPIC assert msg.value == expected
def test_timestamp(): with MessageWriter(SERVICE) as fw: for msg in msgs: fw.publish(msg) with MessageReader(SERVICE) as fr: for msg, orig in zip(fr, msgs): assert msg.topic == TOPIC assert msg.value == orig assert msg.timestamp != 0
def test_enc_bin(config_topic): with MessageWriter(SERVICE, value_type=BYTE_ARRAY, data_encryption=True) as fw: for msg in bmsgs: fw.publish(msg) with MessageReader(SERVICE, value_type=BYTE_ARRAY, data_encryption=True) as fr: for expected, msg in zip(bmsgs, fr): assert msg.topic == config_topic assert msg.value == expected
def test_timestamp(config_topic): with MessageWriter(SERVICE) as fw: for msg in msgs: fw.publish(msg) with MessageReader(SERVICE, receive_timeout_ms=100) as fr: for msg, orig in zip(fr, msgs): assert msg.topic == config_topic assert msg.value == orig assert msg.timestamp != 0
def create_topic(): cwd = Path.cwd().absolute() with TemporaryDirectory() as work_dir: try: os.chdir(str(work_dir)) create_config_file(brokers=[BROKER]) with MessageWriter(SERVICE, TOPIC) as f: logger.debug(f"create topic: {TOPIC}") f.publish(b"message 000") finally: os.chdir(str(cwd))
def test_image(image_object): msgs = [image_object, image_object] with MessageWriter(SERVICE, value_type='image') as fw: for msg in msgs: fw.publish(msg) with MessageReader(SERVICE, value_type='image') as fr: for expected, msg in zip(msgs, fr): assert msg.topic == TOPIC assert np.array_equal(msg.value, expected)
def test_async_read_err(config_topic): cv = Condition() expected = set(msgs) called = 0 err = 0 def assert_messages(message): nonlocal expected nonlocal called with cv: assert message.topic == config_topic assert message.value in expected expected.remove(message.value) called += 1 # print(f"XXX assert_messages: called={called} ++, message={message}") cv.notify_all() def failed(e, traceback=None): nonlocal called nonlocal err with cv: called += 1 err += 1 # print(f"XXX failed: called={called} ++, e={e}, traceback={traceback}") cv.notify_all() global test_qread_failure test_qread_failure += [1, 2, 3] with AsyncMessageReader(SERVICE, value_type=TEXT) as reader: reader.on_message = assert_messages reader.on_failure = failed with MessageWriter(SERVICE, value_type=TEXT) as writer: for msg in msgs: writer.publish(msg) import time deadline = time.time() + 5 with cv: while called != nmsg: # print(f"XXX AsyncMessageReader: called={called} nmsg={nmsg}") if time.time() >= deadline: raise Exception("TIMEOUT") cv.wait(1) # print(f"XXX AsyncMessageReader: FIN called={called} nmsg={nmsg}") assert called == nmsg m = reader.metrics logger.info(f"reader.metrics: {m}") assert err == 1 assert m.msg_count_total == 2 assert m.error_count_total == 1
def test_sync_write_err(config_topic): global test_qwrite_failure test_qwrite_failure += [1, 2] with MessageWriter(SERVICE, value_type=TEXT) as fw: for msg in msgs: try: fw.publish(msg) except Exception: logger.info("caught") m = fw.metrics logger.info(f"writer.metrics: {m}") assert m.error_count_total == 1
def test_enc_mode(crypto, config_topic): with MessageWriter(SERVICE, value_type=TEXT, data_encryption=True, crypto=crypto) as fw: for msg in msgs: fw.publish(msg) with MessageReader(SERVICE, value_type=TEXT, data_encryption=True, crypto=crypto) as fr: for expected, msg in zip(msgs, fr): assert msg.topic == config_topic assert msg.value == expected
def test_metrics0_writer(config_topic): with MessageWriter(SERVICE, topic=config_topic) as f: m = f.metrics assert isinstance(m, Metrics) assert m.raw is None assert m.start_time > 0 assert m.end_time > 0 assert m.end_time >= m.start_time assert m.msg_count_total == 0 assert m.msg_bytes_total >= 0 assert m.msg_size_min is None assert m.msg_size_max is None assert m.error_count_total == 0 m2 = f.metrics assert isinstance(m2, Metrics) assert m2.raw is None
def test_async_thru(config_topic): cv = Condition() expected = set(msgs) called = 0 def assert_messages(message): nonlocal expected nonlocal called with cv: assert message.topic == config_topic assert message.value in expected expected.remove(message.value) called += 1 cv.notify_all() assert False # XXX assertion failure in callback is ignored... with AsyncMessageReader(SERVICE, value_type=TEXT) as reader: reader.on_message = assert_messages with MessageWriter(SERVICE, value_type=TEXT) as writer: for msg in msgs: writer.publish(msg) m = writer.metrics m2 = writer.metrics writer.reset_metrics() m3 = writer.metrics logger.info(f"writer.metrics: {m}") assert_metrics(m) assert equal_metrics(m, m2) assert m3.msg_count_total == 0 with cv: while called != nmsg: cv.wait(1) assert called == nmsg m = reader.metrics m2 = reader.metrics reader.reset_metrics() m3 = reader.metrics logger.info(f"reader.metrics: {m}") assert_metrics(m) assert equal_metrics(m, m2) assert m3.msg_count_total == 0
def test_metrics(setup_messages, config_topic): config_topic = TOPIC nmsg = len(setup_messages) with MessageReader(SERVICE, config_topic, value_type=TEXT) as reader: with MessageWriter(SERVICE, config_topic, value_type=TEXT) as writer: for msg in setup_messages: writer.publish(msg) writer_metrics = writer.metrics logger.info(f"writer.metrics: {writer_metrics}") count = 0 logger.info("OK") reader.seek_to_beginning() for msg in reader: count += 1 if count >= nmsg: break reader_metrics = reader.metrics logger.info(f"reader.metrics: {reader_metrics}") min_avro_overhead = 2 + 8 + 8 + 1 # marker + fingerprint + timestamp + body_len msg_sizes = [len(msg) + min_avro_overhead for msg in setup_messages] msg_bytes_total = sum(msg_sizes) msg_size_min = min(msg_sizes) msg_size_max = max(msg_sizes) assert isinstance(writer_metrics, Metrics) assert isinstance(writer_metrics.raw, dict) assert writer_metrics.start_time > 0 assert writer_metrics.end_time > 0 assert writer_metrics.end_time >= writer_metrics.start_time assert writer_metrics.msg_count_total == nmsg assert writer_metrics.msg_bytes_total >= msg_bytes_total assert writer_metrics.msg_size_min >= msg_size_min assert writer_metrics.msg_size_max >= msg_size_max assert writer_metrics.error_count_total == 0 assert isinstance(reader_metrics, Metrics) assert isinstance(reader_metrics.raw, dict) assert reader_metrics.start_time > 0 assert reader_metrics.end_time > 0 assert reader_metrics.end_time >= reader_metrics.start_time assert reader_metrics.msg_count_total == nmsg assert reader_metrics.msg_bytes_total >= msg_bytes_total assert reader_metrics.msg_size_min >= msg_size_min assert reader_metrics.msg_size_max >= msg_size_max assert reader_metrics.error_count_total == 0
def test_sync_read_err(config_topic): with MessageWriter(SERVICE, value_type=TEXT) as fw: for msg in msgs: fw.publish(msg) global test_qread_failure test_qread_failure += [1, 2, 3] with MessageReader(SERVICE, value_type=TEXT) as fr: try: for expected, msg in zip(msgs, fr): assert msg.topic == config_topic assert msg.value == expected except Exception: logger.info("caught") m = fr.metrics logger.info(f"reader.metrics: {m}") assert m.msg_count_total == 2 assert m.error_count_total == 1
def producer(service, video, preview=False): global n_frame with MessageWriter(service, value_type='image') as writer: # capture frames from the camera for frame in camera.capture_continuous(video, format="bgr", use_video_port=True): # grab the raw NumPy array representing the image, then initialize the timestamp # and occupied/unoccupied text image = frame.array n_frame += 1 print(f"frame: {n_frame}") writer.publish(image) # show the frame if preview and show_preview(image): break # clear the stream in preparation for the next frame video.truncate(0)
def test_injection(config_topic): injected = [] with MessageWriter(SERVICE, topic=config_topic + "-W", value_type=TEXT, data_encryption=True) as fw: fw.debug_last_msg_bytes = True for msg in msgs: fw.publish(msg) injected.append((msg, fw.debug_last_msg_bytes)) with MessageReader(SERVICE, topic=config_topic + "-R", value_type=TEXT, data_encryption=True) as fr: for expected, b in injected: fr.debug_inject_msg_bytes = (b, config_topic, None) msg = next(fr) assert msg.topic == config_topic assert msg.value == expected
def test_on_message(config_topic): messages = ["message-1", "message-2"] called = [] def assert_messages(message): expected = messages[len(called)] assert message.topic == config_topic assert message.value == expected called.append(1) with AsyncMessageReader(SERVICE, value_type=TEXT) as reader: reader.on_message = assert_messages with MessageWriter(SERVICE, value_type=TEXT) as writer: for msg in messages: writer.publish(msg) for _ in range(5): if len(called) == len(messages): break time.sleep(1) assert reader.on_message == assert_messages assert len(called) == len(messages)
def test_key(param, config_topic): crypto = {} crypto["key"] = param["key"] crypto["password"] = param["password"] try: with MessageWriter(SERVICE, value_type=TEXT, data_encryption=True, crypto=crypto) as fw: for msg in msgs: fw.publish(msg) with MessageReader(SERVICE, value_type=TEXT, data_encryption=True, crypto=crypto) as fr: for expected, msg in zip(msgs, fr): assert msg.topic == config_topic assert msg.value == expected assert param["ex"] is None except Exception as ex: assert type(ex) == param["ex"]
def test_publish_result(): with MessageWriter(SERVICE, TOPIC) as f: ret = f.publish(b'message') assert ret is not None