def thread_function(host_port: str, topic: str, in_queue: Queue, out_queue: Queue): """ Background thread for consuming Kafka messages. :param host_port: The host + port of the Kafka broker that we are using. :param topic: The Kafka topic that we are listening to. :param in_queue: A queue for sending "exit" messages to the thread. .. note:: The queue will exit upon the reception of the string "exit" on this queue. :param out_queue: The queue to which status updates are published. """ status_tracker = InThreadStatusTracker(out_queue) while True: try: consumer = KafkaConsumer( topic, bootstrap_servers=host_port, fetch_max_bytes=52428800 * 6, consumer_timeout_ms=100, ) # Roughly 300MB break except NoBrokersAvailable: pass # Do not fail if the broker is not immediately available. if not in_queue.empty(): new_msg = in_queue.get() if new_msg == "exit": return while True: for message in consumer: status_tracker.process_message(message.value) status_tracker.check_for_lost_connections() if not in_queue.empty(): new_msg = in_queue.get() if new_msg == "exit": break consumer.close(True)
def test_process_msg_has_stopped(): stopped_msg = serialise_done("service id", "job id", True, "file name") status_queue = Queue() under_test = InThreadStatusTracker(status_queue) under_test.process_stopped = Mock() under_test.process_message(stopped_msg) under_test.process_stopped.assert_called_once_with( deserialise_done(stopped_msg))
def test_process_msg_stop(): stop_msg = serialise_stop("job id") status_queue = Queue() under_test = InThreadStatusTracker(status_queue) under_test.process_set_stop_time = Mock() under_test.process_message(stop_msg) under_test.process_set_stop_time.assert_called_once_with( deserialise_stop(stop_msg))
def test_process_msg_start(): start_msg = serialise_start("job id", "file name") status_queue = Queue() under_test = InThreadStatusTracker(status_queue) under_test.process_start = Mock() under_test.process_message(start_msg) under_test.process_start.assert_called_once_with( deserialise_start(start_msg))
def test_process_msg_status(): status_msg = serialise_status("sw", "v1", "service_id", "host", 12, 142, status_json="{}") status_queue = Queue() under_test = InThreadStatusTracker(status_queue) under_test.process_status = Mock() under_test.process_message(status_msg) under_test.process_status.assert_called_once_with( deserialise_status(status_msg))
def test_process_msg_answer(): answ_msg = serialise_answer( "service id", "job id", "command id", ActionType.StartJob, ActionOutcome.Success, "some message", 0, datetime.now(), ) status_queue = Queue() under_test = InThreadStatusTracker(status_queue) under_test.process_answer = Mock() under_test.process_message(answ_msg) under_test.process_answer.assert_called_once_with( deserialise_answer(answ_msg))