def thread_function(host_port: str, topic: str, in_queue: Queue,
out_queue: Queue):
"""
Background thread for consuming Kafka messages.
:param host_port: The host + port of the Kafka broker that we are using.
:param topic: The Kafka topic that we are listening to.
:param in_queue: A queue for sending "exit" messages to the thread.
.. note:: The queue will exit upon the reception of the string "exit" on this queue.
:param out_queue: The queue to which status updates are published.
"""
status_tracker = InThreadStatusTracker(out_queue)
while True:
try:
consumer = KafkaConsumer(
topic,
bootstrap_servers=host_port,
fetch_max_bytes=52428800 * 6,
consumer_timeout_ms=100,
) # Roughly 300MB
break
except NoBrokersAvailable:
pass # Do not fail if the broker is not immediately available.
if not in_queue.empty():
new_msg = in_queue.get()
if new_msg == "exit":
return
while True:
for message in consumer:
status_tracker.process_message(message.value)
status_tracker.check_for_lost_connections()
if not in_queue.empty():
new_msg = in_queue.get()
if new_msg == "exit":
break
consumer.close(True)
def test_process_msg_has_stopped():
stopped_msg = serialise_done("service id", "job id", True, "file name")
status_queue = Queue()
under_test = InThreadStatusTracker(status_queue)
under_test.process_stopped = Mock()
under_test.process_message(stopped_msg)
under_test.process_stopped.assert_called_once_with(
deserialise_done(stopped_msg))
def test_process_msg_stop():
stop_msg = serialise_stop("job id")
status_queue = Queue()
under_test = InThreadStatusTracker(status_queue)
under_test.process_set_stop_time = Mock()
under_test.process_message(stop_msg)
under_test.process_set_stop_time.assert_called_once_with(
deserialise_stop(stop_msg))
def test_process_msg_start():
start_msg = serialise_start("job id", "file name")
status_queue = Queue()
under_test = InThreadStatusTracker(status_queue)
under_test.process_start = Mock()
under_test.process_message(start_msg)
under_test.process_start.assert_called_once_with(
deserialise_start(start_msg))
def test_process_msg_status():
status_msg = serialise_status("sw",
"v1",
"service_id",
"host",
12,
142,
status_json="{}")
status_queue = Queue()
under_test = InThreadStatusTracker(status_queue)
under_test.process_status = Mock()
under_test.process_message(status_msg)
under_test.process_status.assert_called_once_with(
deserialise_status(status_msg))
def test_process_msg_answer():
answ_msg = serialise_answer(
"service id",
"job id",
"command id",
ActionType.StartJob,
ActionOutcome.Success,
"some message",
0,
datetime.now(),
)
status_queue = Queue()
under_test = InThreadStatusTracker(status_queue)
under_test.process_answer = Mock()
under_test.process_message(answ_msg)
under_test.process_answer.assert_called_once_with(
deserialise_answer(answ_msg))