def test_async_write():
count = []
with AsyncMessageWriter(SERVICE, TOPIC, value_type=TEXT) as f:
for msg in ['message-1', 'message-2']:
ret = f.publish(msg).then(lambda _: count.append(1))
assert isinstance(ret, Promise)
assert 2 == len(count)
def test_no_auth_async_write(setup_config):
params = {
'sasl_plain_username': KAFKA_READ_USER,
'sasl_plain_password': KAFKA_READ_PASSWD,
}
error = 0
countdown = 1
cv = Condition()
def on_error(e):
nonlocal countdown, error
with cv:
countdown -= 1
if isinstance(e, AuthorizationError):
error += 1
cv.notify_all()
def on_success(_):
nonlocal countdown
with cv:
countdown -= 1
cv.notify_all()
def wait_done():
with cv:
while countdown > 0:
cv.wait()
with AsyncMessageWriter(SERVICE, consistency=AT_LEAST_ONCE, **params) as f:
f.publish(b'message-001').catch(on_error).then(on_success)
wait_done()
assert error == 1
def test_async_write_err(config_topic):
test_err = [1, 2]
global test_qwrite_failure
test_qwrite_failure += test_err
count = []
err = []
with AsyncMessageWriter(SERVICE, value_type=TEXT) as fw:
for msg in msgs:
ret = fw.publish(msg).then(lambda _: count.append(1),
lambda _: err.append(1))
from promise import Promise
assert isinstance(ret, Promise)
m = fw.metrics
logger.info(f"writer.metrics: {m}")
assert len(count) == len(test_err)
assert len(err) == 1
assert m.error_count_total == 1
def do_async(service, output_count, num_samples, payload_size):
def on_message(message):
pass
results = []
for i in range(output_count):
with AsyncMessageReader(service, value_type=BYTE_ARRAY) as reader:
reader.on_message = on_message
with AsyncMessageWriter(service, value_type=BYTE_ARRAY) as writer:
for i in range(num_samples):
message = get_message(payload_size)
writer.publish(message)
results.append({
'writer': writer.metrics,
'reader': reader.metrics,
})
return results
def test_write_message(setup_messages, consistency, config_topic):
count = len(setup_messages)
check = 0
cv = Condition()
def wait_on_messages():
nonlocal count
with cv:
cv.wait_for(lambda: count == 0, timeout=15)
def on_success(r):
nonlocal count, check
with cv:
count -= 1
check += 1
cv.notify_all()
def on_failure(e, traceback=None):
nonlocal count
logger.error(f"failure: {e}")
with cv:
count -= 1
cv.notify_all()
with AsyncMessageWriter(service=SERVICE,
topic=config_topic,
value_type=TEXT,
consistency=consistency) as writer:
for msg in setup_messages:
writer.publish(msg).then(on_success, on_failure)
wait_on_messages()
if consistency != AT_MOST_ONCE:
assert len(setup_messages) == check
else:
assert count == 0
def test_writer_consistency(consistency):
with AsyncMessageWriter(SERVICE, consistency=consistency) as f:
assert consistency == f.consistency
def producer(service):
with AsyncMessageWriter(service) as writer:
while True:
message = get_message()
writer.publish(message)
def test_writer_deser():
with AsyncMessageWriter(SERVICE, value_serializer=(lambda x: x)) as _:
pass
def test_writer_client_id_default():
with AsyncMessageWriter(SERVICE) as f:
assert f.client_id is not None and f.client_id != ""
def test_writer_client_id_set():
cid = "oreore"
with AsyncMessageWriter(SERVICE, client_id=cid) as f:
assert f.client_id == cid
def test_writer_consistency_in_config_file(config_params):
with AsyncMessageWriter(SERVICE) as f:
consistency = config_params['consistency']
assert eval(consistency) == f.consistency
def test_writer_bad_consistency(consistency):
with pytest.raises(InvalidArgumentError):
with AsyncMessageWriter(SERVICE, consistency=consistency) as _:
pass
def test_writer_bad_topics():
with pytest.raises(InvalidArgumentError):
with AsyncMessageWriter(SERVICE) as _:
pass
def test_close_twice():
f = AsyncMessageWriter(SERVICE).open()
f.close()
f.close()
def test_writer_topic(topics):
with AsyncMessageWriter(SERVICE, topics) as _:
pass
def test_writer_topic_list_one_item_in_config_file():
with AsyncMessageWriter(SERVICE) as f:
assert f.topic == TOPIC
def test_writer_topic_in_config_file_and_kwarg():
with AsyncMessageWriter(topic=TOPIC2, service=SERVICE) as f:
assert f.topic == TOPIC2
def test_writer_topic_in_config_file_and_arg():
with AsyncMessageWriter(SERVICE, TOPIC2) as f:
assert f.topic == TOPIC2
def test_writer_topic_list_in_config_file():
with pytest.raises(InvalidArgumentError):
with AsyncMessageWriter(SERVICE) as _:
pass
