def test_injection_async(config_topic):
injected = {}
recved = []
with MessageWriter(SERVICE,
topic=config_topic + "-W",
value_type=TEXT,
data_encryption=True) as fw:
fw.debug_last_msg_bytes = True
for msg in msgs:
fw.publish(msg)
injected[msg] = fw.debug_last_msg_bytes
def recv_msg(msg):
recved.append(msg)
with AsyncMessageReader(SERVICE,
topic=config_topic + "-R",
value_type=TEXT) as fr:
fr.on_message = recv_msg
for b in injected.values():
fr.debug_inject_msg_bytes(b, config_topic + "-R", None)
time.sleep(1) # wait to receive.
assert len(recved) == len(msgs)
for rec in recved:
assert rec.value in injected
def reader_test(topics, cons, messages, on_message, wait_on_messages):
reader = AsyncMessageReader(
SERVICE, config_topics, value_type=TEXT, consistency=consistency)
reader.on_message = on_message
reader.open()
write_messages(messages, topics)
wait_on_messages()
reader.close()
def test_async_read_err(config_topic):
cv = Condition()
expected = set(msgs)
called = 0
err = 0
def assert_messages(message):
nonlocal expected
nonlocal called
with cv:
assert message.topic == config_topic
assert message.value in expected
expected.remove(message.value)
called += 1
# print(f"XXX assert_messages: called={called} ++, message={message}")
cv.notify_all()
def failed(e, traceback=None):
nonlocal called
nonlocal err
with cv:
called += 1
err += 1
# print(f"XXX failed: called={called} ++, e={e}, traceback={traceback}")
cv.notify_all()
global test_qread_failure
test_qread_failure += [1, 2, 3]
with AsyncMessageReader(SERVICE, value_type=TEXT) as reader:
reader.on_message = assert_messages
reader.on_failure = failed
with MessageWriter(SERVICE, value_type=TEXT) as writer:
for msg in msgs:
writer.publish(msg)
import time
deadline = time.time() + 5
with cv:
while called != nmsg:
# print(f"XXX AsyncMessageReader: called={called} nmsg={nmsg}")
if time.time() >= deadline:
raise Exception("TIMEOUT")
cv.wait(1)
# print(f"XXX AsyncMessageReader: FIN called={called} nmsg={nmsg}")
assert called == nmsg
m = reader.metrics
logger.info(f"reader.metrics: {m}")
assert err == 1
assert m.msg_count_total == 2
assert m.error_count_total == 1
def test_async_thru(config_topic):
cv = Condition()
expected = set(msgs)
called = 0
def assert_messages(message):
nonlocal expected
nonlocal called
with cv:
assert message.topic == config_topic
assert message.value in expected
expected.remove(message.value)
called += 1
cv.notify_all()
assert False # XXX assertion failure in callback is ignored...
with AsyncMessageReader(SERVICE, value_type=TEXT) as reader:
reader.on_message = assert_messages
with MessageWriter(SERVICE, value_type=TEXT) as writer:
for msg in msgs:
writer.publish(msg)
m = writer.metrics
m2 = writer.metrics
writer.reset_metrics()
m3 = writer.metrics
logger.info(f"writer.metrics: {m}")
assert_metrics(m)
assert equal_metrics(m, m2)
assert m3.msg_count_total == 0
with cv:
while called != nmsg:
cv.wait(1)
assert called == nmsg
m = reader.metrics
m2 = reader.metrics
reader.reset_metrics()
m3 = reader.metrics
logger.info(f"reader.metrics: {m}")
assert_metrics(m)
assert equal_metrics(m, m2)
assert m3.msg_count_total == 0
def do_async(service, output_count, num_samples, payload_size):
def on_message(message):
pass
results = []
for i in range(output_count):
with AsyncMessageReader(service, value_type=BYTE_ARRAY) as reader:
reader.on_message = on_message
with AsyncMessageWriter(service, value_type=BYTE_ARRAY) as writer:
for i in range(num_samples):
message = get_message(payload_size)
writer.publish(message)
results.append({
'writer': writer.metrics,
'reader': reader.metrics,
})
return results
def test_on_message(setup_messages, consistency):
count = len(setup_messages)
cv = Condition()
def assert_messages(message):
nonlocal count
expected = setup_messages[len(setup_messages) - count]
assert message.topic == TOPIC
assert message.value == expected
with cv:
count -= 1
cv.notify_all()
with AsyncMessageReader(SERVICE, value_type=TEXT,
consistency=consistency) as reader:
reader.on_message = assert_messages
write_messages(setup_messages, consistency)
with cv:
while count > 0:
cv.wait()
def test_on_message(config_topic):
messages = ["message-1", "message-2"]
called = []
def assert_messages(message):
expected = messages[len(called)]
assert message.topic == config_topic
assert message.value == expected
called.append(1)
with AsyncMessageReader(SERVICE, value_type=TEXT) as reader:
reader.on_message = assert_messages
with MessageWriter(SERVICE, value_type=TEXT) as writer:
for msg in messages:
writer.publish(msg)
for _ in range(5):
if len(called) == len(messages):
break
time.sleep(1)
assert reader.on_message == assert_messages
assert len(called) == len(messages)
def test_no_auth_async_read_with_stmt(setup_config):
params = {
'sasl_plain_username': KAFKA_WRITE_USER,
'sasl_plain_password': KAFKA_WRITE_PASSWD,
}
error = 0
countdown = 1
cv = Condition()
def error_handler(e, traceback=None):
nonlocal countdown, error
with cv:
countdown -= 1
if isinstance(e, AuthorizationError):
error += 1
cv.notify_all()
with AsyncMessageReader(SERVICE, **params) as reader:
reader.on_failure = error_handler
with cv:
cv.wait(10)
assert error == 1
def test_reader_deser():
with AsyncMessageReader(SERVICE, value_deserializer=(lambda x: x)) as _:
pass
def reader_test(topics, cons, messages, on_message, wait_on_messages):
with AsyncMessageReader(
SERVICE, topics, value_type=TEXT, consistency=consistency) as reader:
reader.on_message = on_message
write_messages(messages, topics)
wait_on_messages()
def test_reader_timeout():
with AsyncMessageReader(SERVICE, receive_timeout_ms=3000) as _:
pass
def test_reader_client_id_set():
cid = "oreore"
with AsyncMessageReader(SERVICE, client_id=cid) as f:
assert f.client_id == cid
def test_close_twice():
f = AsyncMessageReader(SERVICE).open()
f.close()
f.close()
def test_reader_topics_list_in_config_file():
with AsyncMessageReader(SERVICE) as f:
assert f.topics == [TOPIC, TOPIC2]
def test_reader_client_id_empty():
with AsyncMessageReader(SERVICE, client_id="") as f:
assert f.client_id is not None and f.client_id != ""
def test_reader_consistency_in_config_file(config_params):
with AsyncMessageReader(SERVICE) as f:
consistency = config_params['consistency']
assert eval(consistency) == f.consistency
def test_reader_bad_consistency(consistency):
with pytest.raises(InvalidArgumentError):
with AsyncMessageReader(SERVICE, consistency=consistency) as _:
pass
def test_reader_topic_in_config_file(config_topic):
with AsyncMessageReader(SERVICE) as f:
assert f.topics == config_topic
def test_reader_consistency(consistency):
with AsyncMessageReader(SERVICE, consistency=consistency) as f:
assert consistency == f.consistency
def test_reader_topic(topics):
with AsyncMessageReader(SERVICE, topics) as _:
pass
def test_reader_topic_in_config_file_and_arg():
with AsyncMessageReader(SERVICE, TOPIC2) as f:
assert f.topics == TOPIC2
def test_service():
with AsyncMessageReader(SERVICE) as f:
assert f.service == SERVICE
def consumer(service):
reader = AsyncMessageReader(service)
reader.on_message = show_message
reader.open()
wait_for_keyboard_interrupt()
reader.close()
def test_reader_client_id_default():
with AsyncMessageReader(SERVICE) as f:
assert f.client_id is not None and f.client_id != ""
def test_reader_bad_topics():
with pytest.raises(InvalidArgumentError):
with AsyncMessageReader(SERVICE) as _:
pass
def test_reader_topics_in_config_file_and_kwargs():
with AsyncMessageReader(service=SERVICE, topic=TOPIC2) as f:
assert f.topics == TOPIC2