def test_injection_async(config_topic):
injected = {}
recved = []
with MessageWriter(SERVICE,
topic=config_topic + "-W",
value_type=TEXT,
data_encryption=True) as fw:
fw.debug_last_msg_bytes = True
for msg in msgs:
fw.publish(msg)
injected[msg] = fw.debug_last_msg_bytes
def recv_msg(msg):
recved.append(msg)
with AsyncMessageReader(SERVICE,
topic=config_topic + "-R",
value_type=TEXT) as fr:
fr.on_message = recv_msg
for b in injected.values():
fr.debug_inject_msg_bytes(b, config_topic + "-R", None)
time.sleep(1) # wait to receive.
assert len(recved) == len(msgs)
for rec in recved:
assert rec.value in injected
def test_no_auth_write_no_ack(setup_config):
params = {
'sasl_plain_username': KAFKA_READ_USER,
'sasl_plain_password': KAFKA_READ_PASSWD,
}
with MessageWriter(SERVICE, consistency=AT_MOST_ONCE, **params) as f:
f.publish(b'message-001')
def drain():
try:
os.mkdir(data_dir.joinpath("python"))
except FileExistsError:
pass
for service in service_list:
show(f"drain {service}")
with MessageWriter(service) as writer:
writer.debug_last_msg_bytes = True
for i, msg in enumerate(msg_list, 1):
result = None
comment = ""
try:
writer.publish(msg)
result = "OK"
except Exception as e:
show(str(e))
result = "NG"
comment = "#" + str(e)
show(
f"TEST:RESULT: python encode {service} {i} {result} {comment}"
)
with open(data_dir.joinpath("python", f"{service}-{i}.in"),
"wb") as fin:
fin.write(msg)
with open(data_dir.joinpath("python", f"{service}-{i}.out"),
"wb") as fout:
fout.write(writer.debug_last_msg_bytes)
def writer(msgs, *args, **kwargs):
ev.wait()
logger.debug("start message writing")
with MessageWriter(*args, **kwargs) as f:
for msg in msgs:
logger.info(f"write message: {msg}")
f.publish(msg)
def test_sync_thru(config_topic):
with MessageWriter(SERVICE, value_type=TEXT) as fw:
m0 = fw.metrics
logger.info(f"writer.metrics: {m0}")
for msg in msgs:
fw.publish(msg)
m = fw.metrics
m2 = fw.metrics
fw.reset_metrics()
m3 = fw.metrics
logger.info(f"writer.metrics: {m}")
assert_metrics(m)
assert equal_metrics(m, m2)
assert m3.msg_count_total == 0
with MessageReader(SERVICE, value_type=TEXT) as fr:
for expected, msg in zip(msgs, fr):
assert msg.topic == config_topic
assert msg.value == expected
m = fr.metrics
m2 = fr.metrics
fr.reset_metrics()
m3 = fr.metrics
logger.info(f"reader.metrics: {m}")
assert_metrics(m)
assert equal_metrics(m, m2)
assert m3.msg_count_total == 0
def do_sync(service, output_count, num_samples, payload_size):
reader = None
def consumer(service, event):
nonlocal reader
with MessageReader(service, value_type=BYTE_ARRAY) as reader_thread:
reader = reader_thread
event.set()
for message in reader_thread:
pass
event = threading.Event() # for waiting to create consumer
# run consumer on thread
thread = threading.Thread(target=consumer, args=(service, event))
thread.daemon = True
thread.start()
results = []
with MessageWriter(service, value_type=BYTE_ARRAY) as writer:
event.wait() # wait to create reader
for i_output in range(output_count):
writer.reset_metrics()
reader.reset_metrics()
for i_samples in range(num_samples):
message = get_message(payload_size)
writer.publish(message)
results.append({
'writer': writer.metrics,
'reader': reader.metrics,
})
return results
def test_read_bad_type():
with MessageWriter(SERVICE, value_type='text') as fw:
fw.publish('messsage-001')
with MessageReader(SERVICE, value_type='image') as fr:
with pytest.raises(InvalidArgumentError):
for _ in fr:
pass
def test_no_auth_write(setup_config):
params = {
'sasl_plain_username': KAFKA_READ_USER,
'sasl_plain_password': KAFKA_READ_PASSWD,
}
with pytest.raises(AuthorizationError):
with MessageWriter(SERVICE, consistency=AT_LEAST_ONCE, **params) as f:
f.publish(b'message-001')
def test_text(config_topic):
with MessageWriter(SERVICE, value_type=TEXT) as fw:
for msg in tmsgs:
fw.publish(msg)
with MessageReader(SERVICE, value_type=TEXT) as fr:
for expected, msg in zip(tmsgs, fr):
assert msg.topic == config_topic
assert msg.value == expected
def test_thru():
with MessageWriter(SERVICE, value_type=TEXT) as fw:
for msg in msgs:
fw.publish(msg)
with MessageReader(SERVICE, value_type=TEXT) as fr:
for expected, msg in zip(msgs, fr):
assert msg.topic == TOPIC
assert msg.value == expected
def test_byte_array(config_topic):
with MessageWriter(SERVICE, value_type=BYTE_ARRAY) as fw:
for msg in bmsgs:
fw.publish(msg)
with MessageReader(SERVICE, value_type=BYTE_ARRAY) as fr:
for expected, msg in zip(bmsgs, fr):
assert msg.topic == config_topic
assert msg.value == expected
def producer(service, video, preview=False):
with MessageWriter(service, value_type='image') as writer:
image = next_frame(video)
while image is not None:
writer.publish(image)
if preview and show_preview(image):
break
image = next_frame(video)
def test_enc_text():
with MessageWriter(SERVICE, value_type=TEXT, data_encryption=True) as fw:
for msg in msgs:
fw.publish(msg)
with MessageReader(SERVICE, value_type=TEXT, data_encryption=True) as fr:
for expected, msg in zip(msgs, fr):
assert msg.topic == TOPIC
assert msg.value == expected
def test_timestamp():
with MessageWriter(SERVICE) as fw:
for msg in msgs:
fw.publish(msg)
with MessageReader(SERVICE) as fr:
for msg, orig in zip(fr, msgs):
assert msg.topic == TOPIC
assert msg.value == orig
assert msg.timestamp != 0
def test_enc_bin(config_topic):
with MessageWriter(SERVICE, value_type=BYTE_ARRAY,
data_encryption=True) as fw:
for msg in bmsgs:
fw.publish(msg)
with MessageReader(SERVICE, value_type=BYTE_ARRAY,
data_encryption=True) as fr:
for expected, msg in zip(bmsgs, fr):
assert msg.topic == config_topic
assert msg.value == expected
def test_timestamp(config_topic):
with MessageWriter(SERVICE) as fw:
for msg in msgs:
fw.publish(msg)
with MessageReader(SERVICE, receive_timeout_ms=100) as fr:
for msg, orig in zip(fr, msgs):
assert msg.topic == config_topic
assert msg.value == orig
assert msg.timestamp != 0
def create_topic():
cwd = Path.cwd().absolute()
with TemporaryDirectory() as work_dir:
try:
os.chdir(str(work_dir))
create_config_file(brokers=[BROKER])
with MessageWriter(SERVICE, TOPIC) as f:
logger.debug(f"create topic: {TOPIC}")
f.publish(b"message 000")
finally:
os.chdir(str(cwd))
def test_image(image_object):
msgs = [image_object, image_object]
with MessageWriter(SERVICE, value_type='image') as fw:
for msg in msgs:
fw.publish(msg)
with MessageReader(SERVICE, value_type='image') as fr:
for expected, msg in zip(msgs, fr):
assert msg.topic == TOPIC
assert np.array_equal(msg.value, expected)
def test_async_read_err(config_topic):
cv = Condition()
expected = set(msgs)
called = 0
err = 0
def assert_messages(message):
nonlocal expected
nonlocal called
with cv:
assert message.topic == config_topic
assert message.value in expected
expected.remove(message.value)
called += 1
# print(f"XXX assert_messages: called={called} ++, message={message}")
cv.notify_all()
def failed(e, traceback=None):
nonlocal called
nonlocal err
with cv:
called += 1
err += 1
# print(f"XXX failed: called={called} ++, e={e}, traceback={traceback}")
cv.notify_all()
global test_qread_failure
test_qread_failure += [1, 2, 3]
with AsyncMessageReader(SERVICE, value_type=TEXT) as reader:
reader.on_message = assert_messages
reader.on_failure = failed
with MessageWriter(SERVICE, value_type=TEXT) as writer:
for msg in msgs:
writer.publish(msg)
import time
deadline = time.time() + 5
with cv:
while called != nmsg:
# print(f"XXX AsyncMessageReader: called={called} nmsg={nmsg}")
if time.time() >= deadline:
raise Exception("TIMEOUT")
cv.wait(1)
# print(f"XXX AsyncMessageReader: FIN called={called} nmsg={nmsg}")
assert called == nmsg
m = reader.metrics
logger.info(f"reader.metrics: {m}")
assert err == 1
assert m.msg_count_total == 2
assert m.error_count_total == 1
def test_sync_write_err(config_topic):
global test_qwrite_failure
test_qwrite_failure += [1, 2]
with MessageWriter(SERVICE, value_type=TEXT) as fw:
for msg in msgs:
try:
fw.publish(msg)
except Exception:
logger.info("caught")
m = fw.metrics
logger.info(f"writer.metrics: {m}")
assert m.error_count_total == 1
def test_enc_mode(crypto, config_topic):
with MessageWriter(SERVICE,
value_type=TEXT,
data_encryption=True,
crypto=crypto) as fw:
for msg in msgs:
fw.publish(msg)
with MessageReader(SERVICE,
value_type=TEXT,
data_encryption=True,
crypto=crypto) as fr:
for expected, msg in zip(msgs, fr):
assert msg.topic == config_topic
assert msg.value == expected
def test_metrics0_writer(config_topic):
with MessageWriter(SERVICE, topic=config_topic) as f:
m = f.metrics
assert isinstance(m, Metrics)
assert m.raw is None
assert m.start_time > 0
assert m.end_time > 0
assert m.end_time >= m.start_time
assert m.msg_count_total == 0
assert m.msg_bytes_total >= 0
assert m.msg_size_min is None
assert m.msg_size_max is None
assert m.error_count_total == 0
m2 = f.metrics
assert isinstance(m2, Metrics)
assert m2.raw is None
def test_async_thru(config_topic):
cv = Condition()
expected = set(msgs)
called = 0
def assert_messages(message):
nonlocal expected
nonlocal called
with cv:
assert message.topic == config_topic
assert message.value in expected
expected.remove(message.value)
called += 1
cv.notify_all()
assert False # XXX assertion failure in callback is ignored...
with AsyncMessageReader(SERVICE, value_type=TEXT) as reader:
reader.on_message = assert_messages
with MessageWriter(SERVICE, value_type=TEXT) as writer:
for msg in msgs:
writer.publish(msg)
m = writer.metrics
m2 = writer.metrics
writer.reset_metrics()
m3 = writer.metrics
logger.info(f"writer.metrics: {m}")
assert_metrics(m)
assert equal_metrics(m, m2)
assert m3.msg_count_total == 0
with cv:
while called != nmsg:
cv.wait(1)
assert called == nmsg
m = reader.metrics
m2 = reader.metrics
reader.reset_metrics()
m3 = reader.metrics
logger.info(f"reader.metrics: {m}")
assert_metrics(m)
assert equal_metrics(m, m2)
assert m3.msg_count_total == 0
def test_metrics(setup_messages, config_topic):
config_topic = TOPIC
nmsg = len(setup_messages)
with MessageReader(SERVICE, config_topic, value_type=TEXT) as reader:
with MessageWriter(SERVICE, config_topic, value_type=TEXT) as writer:
for msg in setup_messages:
writer.publish(msg)
writer_metrics = writer.metrics
logger.info(f"writer.metrics: {writer_metrics}")
count = 0
logger.info("OK")
reader.seek_to_beginning()
for msg in reader:
count += 1
if count >= nmsg:
break
reader_metrics = reader.metrics
logger.info(f"reader.metrics: {reader_metrics}")
min_avro_overhead = 2 + 8 + 8 + 1 # marker + fingerprint + timestamp + body_len
msg_sizes = [len(msg) + min_avro_overhead for msg in setup_messages]
msg_bytes_total = sum(msg_sizes)
msg_size_min = min(msg_sizes)
msg_size_max = max(msg_sizes)
assert isinstance(writer_metrics, Metrics)
assert isinstance(writer_metrics.raw, dict)
assert writer_metrics.start_time > 0
assert writer_metrics.end_time > 0
assert writer_metrics.end_time >= writer_metrics.start_time
assert writer_metrics.msg_count_total == nmsg
assert writer_metrics.msg_bytes_total >= msg_bytes_total
assert writer_metrics.msg_size_min >= msg_size_min
assert writer_metrics.msg_size_max >= msg_size_max
assert writer_metrics.error_count_total == 0
assert isinstance(reader_metrics, Metrics)
assert isinstance(reader_metrics.raw, dict)
assert reader_metrics.start_time > 0
assert reader_metrics.end_time > 0
assert reader_metrics.end_time >= reader_metrics.start_time
assert reader_metrics.msg_count_total == nmsg
assert reader_metrics.msg_bytes_total >= msg_bytes_total
assert reader_metrics.msg_size_min >= msg_size_min
assert reader_metrics.msg_size_max >= msg_size_max
assert reader_metrics.error_count_total == 0
def test_sync_read_err(config_topic):
with MessageWriter(SERVICE, value_type=TEXT) as fw:
for msg in msgs:
fw.publish(msg)
global test_qread_failure
test_qread_failure += [1, 2, 3]
with MessageReader(SERVICE, value_type=TEXT) as fr:
try:
for expected, msg in zip(msgs, fr):
assert msg.topic == config_topic
assert msg.value == expected
except Exception:
logger.info("caught")
m = fr.metrics
logger.info(f"reader.metrics: {m}")
assert m.msg_count_total == 2
assert m.error_count_total == 1
def producer(service, video, preview=False):
global n_frame
with MessageWriter(service, value_type='image') as writer:
# capture frames from the camera
for frame in camera.capture_continuous(video,
format="bgr",
use_video_port=True):
# grab the raw NumPy array representing the image, then initialize the timestamp
# and occupied/unoccupied text
image = frame.array
n_frame += 1
print(f"frame: {n_frame}")
writer.publish(image)
# show the frame
if preview and show_preview(image):
break
# clear the stream in preparation for the next frame
video.truncate(0)
def test_injection(config_topic):
injected = []
with MessageWriter(SERVICE,
topic=config_topic + "-W",
value_type=TEXT,
data_encryption=True) as fw:
fw.debug_last_msg_bytes = True
for msg in msgs:
fw.publish(msg)
injected.append((msg, fw.debug_last_msg_bytes))
with MessageReader(SERVICE,
topic=config_topic + "-R",
value_type=TEXT,
data_encryption=True) as fr:
for expected, b in injected:
fr.debug_inject_msg_bytes = (b, config_topic, None)
msg = next(fr)
assert msg.topic == config_topic
assert msg.value == expected
def test_on_message(config_topic):
messages = ["message-1", "message-2"]
called = []
def assert_messages(message):
expected = messages[len(called)]
assert message.topic == config_topic
assert message.value == expected
called.append(1)
with AsyncMessageReader(SERVICE, value_type=TEXT) as reader:
reader.on_message = assert_messages
with MessageWriter(SERVICE, value_type=TEXT) as writer:
for msg in messages:
writer.publish(msg)
for _ in range(5):
if len(called) == len(messages):
break
time.sleep(1)
assert reader.on_message == assert_messages
assert len(called) == len(messages)
def test_key(param, config_topic):
crypto = {}
crypto["key"] = param["key"]
crypto["password"] = param["password"]
try:
with MessageWriter(SERVICE,
value_type=TEXT,
data_encryption=True,
crypto=crypto) as fw:
for msg in msgs:
fw.publish(msg)
with MessageReader(SERVICE,
value_type=TEXT,
data_encryption=True,
crypto=crypto) as fr:
for expected, msg in zip(msgs, fr):
assert msg.topic == config_topic
assert msg.value == expected
assert param["ex"] is None
except Exception as ex:
assert type(ex) == param["ex"]
def test_publish_result():
with MessageWriter(SERVICE, TOPIC) as f:
ret = f.publish(b'message')
assert ret is not None