def test_message_send_after_interval_expired_is_published(self): last_pulse_times = (12345, ) sums = (1999, ) diffs = (678, ) mock_process = mock.create_autospec(HistogramProcess) mock_process.get_stats.return_value = [ generate_stats_message(last_pulse_times[0] * 10**9, sums[0], diffs[0]) ] histogram_processes = [mock_process] # First message is always published current_time_ms = time_in_ns() // 1_000_000 self.publisher.publish_histogram_stats(histogram_processes, current_time_ms=current_time_ms) # Increase time by a full interval current_time_ms = self.publisher.next_publish_time_ms self.publisher.publish_histogram_stats(histogram_processes, current_time_ms=current_time_ms) # Messages sent for both publish assert self.sender.send.call_count == 4
def generate_data(source, message_id, num_points): tofs, dets = generate_fake_data(TOF_RANGE, DET_RANGE, num_points) time_stamp = time_in_ns() data = serialise_ev42(source, message_id, time_stamp, tofs, dets) return time_stamp, data
def test_after_first_message_publish_if_interval_has_passed(self): current_time_ms = time_in_ns() // 1_000_000 # Ignore first message self.publisher.publish(current_time_ms) current_time_ms = self.publisher.next_time_to_publish self.publisher.publish(current_time_ms) assert len(self.producer.messages) == 2
def test_message_contents_are_correct(self): current_time_ms = time_in_ns() // 1_000_000 self.publisher.publish(current_time_ms) _, msg = self.producer.messages[0] msg = json.loads(msg) assert msg["message"] == current_time_ms assert msg["message_interval"] == self.update_interval
def generate_and_send_data(self, msg_id): time_stamp = time_in_ns() # Generate a random number of events so we can be sure the correct data matches # up at the end. num_events = random.randint(500, 1500) data = generate_data(msg_id, time_stamp, num_events) self.send_message(self.data_topic_name, data) # Need timestamp in ms self.time_stamps.append(time_stamp // 1_000_000) self.num_events_per_msg.append(num_events)
def run_processing( msg_queue, stats_queue, configuration, start, stop, publish_interval, simulation=False, ): """ The target to run in a multi-processing instance for histogramming. Note: passing objects into a process requires the classes to be pickleable. The histogrammer and event source classes are not pickleable, so need to be created within the process. In effect, this is the 'main' for a process so all the dependencies etc. are set up here. :param msg_queue: The message queue for communicating with the process. :param stats_queue: The queue to send statistics to. :param configuration: The histogramming configuration. :param start: The start time. :param stop: The stop time. :param publish_interval: How often to publish histograms and stats in milliseconds. :param simulation: Whether to run in simulation. """ histogrammer = None try: # Setting up histogrammer = create_histogrammer(configuration, start, stop) if simulation: event_source = create_simulated_event_source( configuration, start, stop) else: event_source = create_event_source(configuration, start, stop) processor = Processor(histogrammer, event_source, msg_queue, stats_queue, publish_interval) # Start up the processing while not processor.processing_finished: processor.run_processing() time.sleep(0.01) except Exception as error: logging.error("Histogram process failed: %s", error) if histogrammer: try: # Try to send failure message histogrammer.send_failure_message(time_in_ns(), str(error)) except Exception as send_error: logging.error("Could not send failure message: %s", send_error)
def generate_dethist_data(source, message_id, num_points): dets = [] for h in range(DET_HEIGHT): _, new_dets = generate_fake_data(TOF_RANGE, (0, DET_WIDTH), num_points) for det in new_dets: dets.append(h * DET_WIDTH + det) time_stamp = time_in_ns() data = serialise_ev42(source, message_id, time_stamp, [], dets) return time_stamp, data
def run_processing(self): """ Run the processing chain once. """ if not self.msg_queue.empty(): self.processing_finished |= self.process_command_message() event_buffer = self.event_source.get_new_data() self.processing_finished |= self.stop_time_exceeded(time_in_ns()) if event_buffer: # Even if the stop time has been exceeded there still may be data # in the buffer to add. self.histogrammer.add_data(event_buffer) if self.processing_finished: self.histogrammer.set_finished() # Only publish at specified rate or if the process is stopping. curr_time = time_in_ns() if curr_time // 1_000_000 > self.time_to_publish or self.processing_finished: self.publish_data(curr_time) self.time_to_publish = curr_time // 1_000_000 + self.publish_interval self.time_to_publish -= self.time_to_publish % self.publish_interval
def prepare(self): # Create unique topics for each test conf = {"bootstrap.servers": BROKERS[0], "api.version.request": True} admin_client = AdminClient(conf) uid = time_in_ns() // 1000 self.hist_topic_name = f"hist_{uid}" self.data_topic_name = f"data_{uid}" hist_topic = NewTopic(self.hist_topic_name, 1, 1) data_topic = NewTopic(self.data_topic_name, 2, 1) admin_client.create_topics([hist_topic, data_topic]) self.producer = KafkaProducer(bootstrap_servers=BROKERS) time.sleep(1) self.consumer, topic_partitions = create_consumer(self.hist_topic_name) # Only one partition for histogram topic self.topic_part = topic_partitions[0] self.initial_offset = self.consumer.position(self.topic_part) self.time_stamps = [] self.num_events_per_msg = []
def __init__(self, histogrammer, event_source, msg_queue, stats_queue, publish_interval): """ Constructor. :param histogrammer: The histogrammer for this process. :param event_source: The event source for this process. :param msg_queue: The queue for receiving messages from outside the process :param stats_queue: The queue for publishing stats to the "outside". :param publish_interval: How often to publish histograms and stats in milliseconds. """ assert publish_interval > 0 self.time_to_publish = 0 self.histogrammer = histogrammer self.event_source = event_source self.msg_queue = msg_queue self.stats_queue = stats_queue self.publish_interval = publish_interval self.processing_finished = False # Publish initial empty histograms and stats. self.publish_data(time_in_ns())
def run(self): """ The main loop for listening to messages and handling them. """ if self.simulation: logging.warning("RUNNING IN SIMULATION MODE!") # Blocks until can connect to the config topic. self.create_config_listener() self.create_publishers() while True: # Handle configuration messages if self.initial_config or self.config_listener.check_for_messages( ): if self.initial_config: # If initial configuration supplied, use it only once. msg = self.initial_config self.initial_config = None else: msg = self.config_listener.consume_message() logging.warning("New command received") logging.warning("%s", msg) self.command_actioner.handle_command_message( msg, self.hist_processes) # Publishing of statistics and heartbeat curr_time_ms = time_in_ns() // 1_000_000 if self.stats_publisher: self.stats_publisher.publish_histogram_stats( self.hist_processes, curr_time_ms) if self.heartbeat_publisher: self.heartbeat_publisher.publish(curr_time_ms) time.sleep(0.1)
def test_first_message_published_immediately(self): time_way_in_the_future = time_in_ns() * 10 self.publisher.publish(time_way_in_the_future) assert len(self.producer.messages) == 1