def test_injection_async(config_topic): injected = {} recved = [] with MessageWriter(SERVICE, topic=config_topic + "-W", value_type=TEXT, data_encryption=True) as fw: fw.debug_last_msg_bytes = True for msg in msgs: fw.publish(msg) injected[msg] = fw.debug_last_msg_bytes def recv_msg(msg): recved.append(msg) with AsyncMessageReader(SERVICE, topic=config_topic + "-R", value_type=TEXT) as fr: fr.on_message = recv_msg for b in injected.values(): fr.debug_inject_msg_bytes(b, config_topic + "-R", None) time.sleep(1) # wait to receive. assert len(recved) == len(msgs) for rec in recved: assert rec.value in injected
def reader_test(topics, cons, messages, on_message, wait_on_messages): reader = AsyncMessageReader( SERVICE, config_topics, value_type=TEXT, consistency=consistency) reader.on_message = on_message reader.open() write_messages(messages, topics) wait_on_messages() reader.close()
def test_async_read_err(config_topic): cv = Condition() expected = set(msgs) called = 0 err = 0 def assert_messages(message): nonlocal expected nonlocal called with cv: assert message.topic == config_topic assert message.value in expected expected.remove(message.value) called += 1 # print(f"XXX assert_messages: called={called} ++, message={message}") cv.notify_all() def failed(e, traceback=None): nonlocal called nonlocal err with cv: called += 1 err += 1 # print(f"XXX failed: called={called} ++, e={e}, traceback={traceback}") cv.notify_all() global test_qread_failure test_qread_failure += [1, 2, 3] with AsyncMessageReader(SERVICE, value_type=TEXT) as reader: reader.on_message = assert_messages reader.on_failure = failed with MessageWriter(SERVICE, value_type=TEXT) as writer: for msg in msgs: writer.publish(msg) import time deadline = time.time() + 5 with cv: while called != nmsg: # print(f"XXX AsyncMessageReader: called={called} nmsg={nmsg}") if time.time() >= deadline: raise Exception("TIMEOUT") cv.wait(1) # print(f"XXX AsyncMessageReader: FIN called={called} nmsg={nmsg}") assert called == nmsg m = reader.metrics logger.info(f"reader.metrics: {m}") assert err == 1 assert m.msg_count_total == 2 assert m.error_count_total == 1
def test_async_thru(config_topic): cv = Condition() expected = set(msgs) called = 0 def assert_messages(message): nonlocal expected nonlocal called with cv: assert message.topic == config_topic assert message.value in expected expected.remove(message.value) called += 1 cv.notify_all() assert False # XXX assertion failure in callback is ignored... with AsyncMessageReader(SERVICE, value_type=TEXT) as reader: reader.on_message = assert_messages with MessageWriter(SERVICE, value_type=TEXT) as writer: for msg in msgs: writer.publish(msg) m = writer.metrics m2 = writer.metrics writer.reset_metrics() m3 = writer.metrics logger.info(f"writer.metrics: {m}") assert_metrics(m) assert equal_metrics(m, m2) assert m3.msg_count_total == 0 with cv: while called != nmsg: cv.wait(1) assert called == nmsg m = reader.metrics m2 = reader.metrics reader.reset_metrics() m3 = reader.metrics logger.info(f"reader.metrics: {m}") assert_metrics(m) assert equal_metrics(m, m2) assert m3.msg_count_total == 0
def do_async(service, output_count, num_samples, payload_size): def on_message(message): pass results = [] for i in range(output_count): with AsyncMessageReader(service, value_type=BYTE_ARRAY) as reader: reader.on_message = on_message with AsyncMessageWriter(service, value_type=BYTE_ARRAY) as writer: for i in range(num_samples): message = get_message(payload_size) writer.publish(message) results.append({ 'writer': writer.metrics, 'reader': reader.metrics, }) return results
def test_on_message(setup_messages, consistency): count = len(setup_messages) cv = Condition() def assert_messages(message): nonlocal count expected = setup_messages[len(setup_messages) - count] assert message.topic == TOPIC assert message.value == expected with cv: count -= 1 cv.notify_all() with AsyncMessageReader(SERVICE, value_type=TEXT, consistency=consistency) as reader: reader.on_message = assert_messages write_messages(setup_messages, consistency) with cv: while count > 0: cv.wait()
def test_on_message(config_topic): messages = ["message-1", "message-2"] called = [] def assert_messages(message): expected = messages[len(called)] assert message.topic == config_topic assert message.value == expected called.append(1) with AsyncMessageReader(SERVICE, value_type=TEXT) as reader: reader.on_message = assert_messages with MessageWriter(SERVICE, value_type=TEXT) as writer: for msg in messages: writer.publish(msg) for _ in range(5): if len(called) == len(messages): break time.sleep(1) assert reader.on_message == assert_messages assert len(called) == len(messages)
def test_no_auth_async_read_with_stmt(setup_config): params = { 'sasl_plain_username': KAFKA_WRITE_USER, 'sasl_plain_password': KAFKA_WRITE_PASSWD, } error = 0 countdown = 1 cv = Condition() def error_handler(e, traceback=None): nonlocal countdown, error with cv: countdown -= 1 if isinstance(e, AuthorizationError): error += 1 cv.notify_all() with AsyncMessageReader(SERVICE, **params) as reader: reader.on_failure = error_handler with cv: cv.wait(10) assert error == 1
def test_reader_deser(): with AsyncMessageReader(SERVICE, value_deserializer=(lambda x: x)) as _: pass
def reader_test(topics, cons, messages, on_message, wait_on_messages): with AsyncMessageReader( SERVICE, topics, value_type=TEXT, consistency=consistency) as reader: reader.on_message = on_message write_messages(messages, topics) wait_on_messages()
def test_reader_timeout(): with AsyncMessageReader(SERVICE, receive_timeout_ms=3000) as _: pass
def test_reader_client_id_set(): cid = "oreore" with AsyncMessageReader(SERVICE, client_id=cid) as f: assert f.client_id == cid
def test_close_twice(): f = AsyncMessageReader(SERVICE).open() f.close() f.close()
def test_reader_topics_list_in_config_file(): with AsyncMessageReader(SERVICE) as f: assert f.topics == [TOPIC, TOPIC2]
def test_reader_client_id_empty(): with AsyncMessageReader(SERVICE, client_id="") as f: assert f.client_id is not None and f.client_id != ""
def test_reader_consistency_in_config_file(config_params): with AsyncMessageReader(SERVICE) as f: consistency = config_params['consistency'] assert eval(consistency) == f.consistency
def test_reader_bad_consistency(consistency): with pytest.raises(InvalidArgumentError): with AsyncMessageReader(SERVICE, consistency=consistency) as _: pass
def test_reader_topic_in_config_file(config_topic): with AsyncMessageReader(SERVICE) as f: assert f.topics == config_topic
def test_reader_consistency(consistency): with AsyncMessageReader(SERVICE, consistency=consistency) as f: assert consistency == f.consistency
def test_reader_topic(topics): with AsyncMessageReader(SERVICE, topics) as _: pass
def test_reader_topic_in_config_file_and_arg(): with AsyncMessageReader(SERVICE, TOPIC2) as f: assert f.topics == TOPIC2
def test_service(): with AsyncMessageReader(SERVICE) as f: assert f.service == SERVICE
def consumer(service): reader = AsyncMessageReader(service) reader.on_message = show_message reader.open() wait_for_keyboard_interrupt() reader.close()
def test_reader_client_id_default(): with AsyncMessageReader(SERVICE) as f: assert f.client_id is not None and f.client_id != ""
def test_reader_bad_topics(): with pytest.raises(InvalidArgumentError): with AsyncMessageReader(SERVICE) as _: pass
def test_reader_topics_in_config_file_and_kwargs(): with AsyncMessageReader(service=SERVICE, topic=TOPIC2) as f: assert f.topics == TOPIC2