def test_raises_not_implemented_error_when_trying_to_serialise_numpy_complex_number_type( self, ): complex_log = { "source_name": "some_source", "value": np.complex(3, 4), "timestamp_unix_ns": 1585332414000000000, } with pytest.raises(NotImplementedError): serialise_f142(**complex_log)
async def test_data_returned_if_multiple_slow_metadata_msgs_exceed_buffer( queues): data_queue, worker_instruction_queue, test_message_queue = queues run_info_topic = "fake_topic" test_instrument_name = "DATA_STREAM_TEST" # The Kafka topics to get metadata from are recorded as "stream" objects in # the nexus_structure field of the run start message f142_source_name = "f142_source" f142_log_name = "f142_log" streams = [ Stream(f"/entry/{f142_log_name}", "f142_topic", f142_source_name, "f142", "double", "m"), ] first_f142_value = 26.1236 f142_timestamp = 123456 # ns after epoch first_message = serialise_f142(first_f142_value, f142_source_name, f142_timestamp) second_f142_value = 62.721 second_message = serialise_f142(second_f142_value, f142_source_name, f142_timestamp) test_stream_args = TEST_STREAM_ARGS.copy() test_stream_args["slow_metadata_buffer_size"] = 1 test_stream_args["topics"] = None n_chunks = 0 reached_asserts = False async for data in _data_stream(data_queue, worker_instruction_queue, run_info_topic=run_info_topic, query_consumer=FakeQueryConsumer( test_instrument_name, streams=streams), halt_after_n_data_chunks=3, **test_stream_args, test_message_queue=test_message_queue): # n_chunks == 0 zeroth chunk contains data # from run start message if n_chunks == 0: test_message_queue.put(FakeMessage(first_message)) test_message_queue.put(FakeMessage(second_message)) elif n_chunks == 1: # Contains data from first message assert isclose(data.attrs[f142_source_name].value.values[0], first_f142_value) elif n_chunks == 2: # Contains data from second message assert isclose(data.attrs[f142_source_name].value.values[0], second_f142_value) reached_asserts = True n_chunks += 1 assert reached_asserts
async def test_data_stream_returns_data_from_multiple_slow_metadata_messages( queues): data_queue, worker_instruction_queue, test_message_queue = queues run_info_topic = "fake_topic" test_instrument_name = "DATA_STREAM_TEST" # The Kafka topics to get metadata from are recorded as "stream" objects in # the nexus_structure field of the run start message f142_source_name = "f142_source" f142_log_name = "f142_log" streams = [ Stream(f"/entry/{f142_log_name}", "f142_topic", f142_source_name, "f142", "double", "m"), ] test_stream_args = TEST_STREAM_ARGS.copy() test_stream_args["topics"] = None n_chunks = 0 async for data in _data_stream(data_queue, worker_instruction_queue, run_info_topic=run_info_topic, query_consumer=FakeQueryConsumer( test_instrument_name, streams=streams), halt_after_n_data_chunks=2, **test_stream_args, test_message_queue=test_message_queue): data_from_stream = data if n_chunks == 0: # Fake receiving a Kafka message for each metadata schema # Do this after the run start message has been parsed, so that # a metadata buffer will have been created for each data source # described in the start message. f142_value_1 = 26.1236 f142_timestamp_1 = 123456 # ns after epoch f142_test_message = serialise_f142(f142_value_1, f142_source_name, f142_timestamp_1) test_message_queue.put(FakeMessage(f142_test_message)) f142_value_2 = 2.725 f142_timestamp_2 = 234567 # ns after epoch f142_test_message = serialise_f142(f142_value_2, f142_source_name, f142_timestamp_2) test_message_queue.put(FakeMessage(f142_test_message)) n_chunks += 1 assert np.allclose(data_from_stream.attrs[f142_source_name].value.values, np.array([f142_value_1, f142_value_2])) assert np.array_equal( data_from_stream.attrs[f142_source_name].value.coords['time'].values, np.array([f142_timestamp_1, f142_timestamp_2], dtype=np.dtype('datetime64[ns]')))
def publish_f142_message( producer: Producer, topic: str, kafka_timestamp: Optional[int] = None, source_name: Optional[str] = None, alarm_status: Optional[int] = None, alarm_severity: Optional[int] = None, ): """ Publish an f142 message to a given topic. Optionally set the timestamp in the kafka header to allow, for example, fake "historical" data. :param producer: Producer to publish the message with :param topic: Name of topic to publish to :param kafka_timestamp: Timestamp to set in the Kafka header (milliseconds after unix epoch) :param source_name: Name of the source in the f142 message :param alarm_status: EPICS alarm status, use enum-like class from streaming_data_types.fbschemas.logdata_f142.AlarmStatus :param alarm_severity: EPICS alarm severity, use enum-like class from streaming_data_types.fbschemas.logdata_f142.AlarmSeverity """ if source_name is None: source_name = "fw-test-helpers" value = 42 f142_message = serialise_f142( value, source_name, _millseconds_to_nanoseconds(kafka_timestamp), alarm_status, alarm_severity, ) producer.produce(topic, f142_message, timestamp=kafka_timestamp) producer.poll(0) producer.flush()
def test_if_buffer_has_wrong_id_then_throws(self): buf = serialise_f142(**self.original_entry) # Manually hack the id buf = bytearray(buf) buf[4:8] = b"1234" with pytest.raises(RuntimeError): deserialise_f142(buf)
def test_serialises_and_deserialises_integer_f142_message_correctly(self): buf = serialise_f142(**self.original_entry) deserialised_tuple = deserialise_f142(buf) assert deserialised_tuple.source_name == self.original_entry["source_name"] assert deserialised_tuple.value == self.original_entry["value"] assert ( deserialised_tuple.timestamp_unix_ns == self.original_entry["timestamp_unix_ns"] )
def test_epics_alarms_default_to_no_change_when_not_provided_to_serialiser(self): float_log = { "source_name": "some_source", "value": 1.234, "timestamp_unix_ns": 1585332414000000000, } buf = serialise_f142(**float_log) deserialised_tuple = deserialise_f142(buf) assert deserialised_tuple.alarm_status == AlarmStatus.NO_CHANGE assert deserialised_tuple.alarm_severity == AlarmSeverity.NO_CHANGE
def to_f142(dev_name, dev_value, dev_severity, timestamp_ns): """Convert the device information in to an f142 FlatBuffer. :param dev_name: the device name :param dev_value: the device's value :param dev_severity: the device's status :param timestamp_ns: the associated timestamp in nanoseconds :return: FlatBuffer representation of data """ # Alarm status is not relevant for NICOS but we have to send something return serialise_f142(dev_value, dev_name, timestamp_ns, AlarmStatus.NO_ALARM, dev_severity)
def test_serialises_and_deserialises_numpy_array_integers_correctly(self): array_log = { "source_name": "some_source", "value": np.array([1, 2, 3]), "timestamp_unix_ns": 1585332414000000000, } buf = serialise_f142(**array_log) deserialised_tuple = deserialise_f142(buf) assert deserialised_tuple.source_name == array_log["source_name"] assert np.array_equal(deserialised_tuple.value, array_log["value"]) assert deserialised_tuple.timestamp_unix_ns == array_log["timestamp_unix_ns"]
def test_serialises_and_deserialises_string_f142_message_correctly(self): string_log = { "source_name": "some_source", "value": "some_string", "timestamp_unix_ns": 1585332414000000000, } buf = serialise_f142(**string_log) deserialised_tuple = deserialise_f142(buf) assert deserialised_tuple.source_name == string_log["source_name"] assert deserialised_tuple.value == string_log["value"] assert deserialised_tuple.timestamp_unix_ns == string_log["timestamp_unix_ns"]
def test_serialises_and_deserialises_scalar_ndarray_f142_message_correctly(self): numpy_log = { "source_name": "some_source", "value": np.array(42), "timestamp_unix_ns": 1585332414000000000, } buf = serialise_f142(**numpy_log) deserialised_tuple = deserialise_f142(buf) assert deserialised_tuple.source_name == numpy_log["source_name"] assert deserialised_tuple.value == np.array(numpy_log["value"]) assert deserialised_tuple.timestamp_unix_ns == numpy_log["timestamp_unix_ns"]
def test_serialises_and_deserialises_epics_alarms_correctly(self): float_log = { "source_name": "some_source", "value": 1.234, "timestamp_unix_ns": 1585332414000000000, "alarm_status": AlarmStatus.HIHI, "alarm_severity": AlarmSeverity.MAJOR, } buf = serialise_f142(**float_log) deserialised_tuple = deserialise_f142(buf) assert deserialised_tuple.alarm_status == float_log["alarm_status"] assert deserialised_tuple.alarm_severity == float_log["alarm_severity"]
def test_serialises_and_deserialises_native_list_correctly(self): list_log = { "source_name": "some_source", "value": [1, 2, 3], "timestamp_unix_ns": 1585332414000000000, } buf = serialise_f142(**list_log) deserialised_tuple = deserialise_f142(buf) assert deserialised_tuple.source_name == list_log["source_name"] # Array values are output as numpy array assert np.array_equal(deserialised_tuple.value, np.array(list_log["value"])) assert deserialised_tuple.timestamp_unix_ns == list_log["timestamp_unix_ns"]
def test_serialises_and_deserialises_numpy_array_preserves_integer_type_correctly( self, ): array_log = { "source_name": "some_source", "value": np.array([1, 2, 3], dtype=np.uint16), "timestamp_unix_ns": 1585332414000000000, } buf = serialise_f142(**array_log) deserialised_tuple = deserialise_f142(buf) assert np.array_equal(deserialised_tuple.value, array_log["value"]) assert deserialised_tuple.value.dtype == array_log["value"].dtype
def publish_f142_message( producer: KafkaProducer, topic: str, data: np.array, source_name: str, timestamp_ns: int, alarm_status: Optional[AlarmStatus] = None, alarm_severity: Optional[AlarmSeverity] = None, ): """ Publish an f142 message to a given topic. :param producer: Kafka producer to publish update with :param topic: Name of topic to publish to :param data: Value of the PV update :param source_name: Name of the PV :param timestamp_ns: Timestamp for value (nanoseconds after unix epoch) :param alarm_status: :param alarm_severity: """ if alarm_status is None: f142_message = serialise_f142( value=data, source_name=source_name, timestamp_unix_ns=timestamp_ns, ) else: f142_message = serialise_f142( value=data, source_name=source_name, timestamp_unix_ns=timestamp_ns, alarm_status=alarm_status, alarm_severity=alarm_severity, ) producer.produce( topic, f142_message, key=source_name, timestamp_ms=_nanoseconds_to_milliseconds(timestamp_ns), )
def create_f142_message(self, timestamp, tag, value): timestamp_ns = timestamp_to_nsecs(timestamp) if type(value) == str: # overrride the current implementation as it appears wrong, # it converts a ubyte array to a ushort array bvalue = np.array(list(value.encode('utf-8')), dtype=np.ubyte) builder, source = _setup_builder(tag) _serialise_ubytearray(builder, bvalue, source) buff = _complete_buffer( builder, timestamp_ns, ) else: buff = serialise_f142(value, tag, timestamp_ns) return buff
from streaming_data_types.logdata_f142 import serialise_f142 from datetime import datetime import time projection = 0 flat_field = 1 dark_field = 2 invalid = 3 if __name__ == '__main__': kafka_producer = KafkaProducer(bootstrap_servers='localhost:9092') idx = 150 for i in range(1, 201): idx += 1 nbr_str = str(i) nbr_zeros = 4 - len(nbr_str) first_zeros = '0' * nbr_zeros path_to_image = os.path.join('..', 'Lego1', f'tomo_{first_zeros}{nbr_str}.tif') image = Image.open(path_to_image) img_array = np.array(image) serialized_output = serialise_ADAr('image_source', idx, datetime.now(), img_array) rotation_angle = serialise_f142(i, 'rotation_angle', time.time_ns()) image_key = serialise_f142(projection, 'image_key', time.time_ns()) kafka_producer.send('odin_topic', serialized_output) kafka_producer.send('odin_topic', rotation_angle) kafka_producer.send('odin_topic', image_key) time.sleep(0.1)
async def test_data_stream_returns_metadata(queues): data_queue, worker_instruction_queue, test_message_queue = queues run_info_topic = "fake_topic" test_instrument_name = "DATA_STREAM_TEST" # The Kafka topics to get metadata from are recorded as "stream" objects in # the nexus_structure field of the run start message # There are currently 3 schemas for metadata, they have flatbuffer ids # f142, senv and tdct f142_source_name = "f142_source" f142_log_name = "f142_log" senv_source_name = "senv_source" senv_log_name = "senv_log" tdct_source_name = "tdct_source" tdct_log_name = "tdct_log" streams = [ Stream(f"/entry/{f142_log_name}", "f142_topic", f142_source_name, "f142", "double", "m"), Stream(f"/entry/{senv_log_name}", "senv_topic", senv_source_name, "senv", "double", "m"), Stream(f"/entry/{tdct_log_name}", "tdct_topic", tdct_source_name, "tdct") ] test_stream_args = TEST_STREAM_ARGS.copy() test_stream_args["topics"] = None n_chunks = 0 async for data in _data_stream(data_queue, worker_instruction_queue, run_info_topic=run_info_topic, query_consumer=FakeQueryConsumer( test_instrument_name, streams=streams), halt_after_n_data_chunks=2, **test_stream_args, test_message_queue=test_message_queue): data_from_stream = data if n_chunks == 0: # Fake receiving a Kafka message for each metadata schema # Do this after the run start message has been parsed, so that # a metadata buffer will have been created for each data source # described in the start message. f142_value = 26.1236 f142_timestamp = 123456 # ns after epoch f142_test_message = serialise_f142(f142_value, f142_source_name, f142_timestamp) test_message_queue.put(FakeMessage(f142_test_message)) senv_values = np.array([26, 127, 52]) senv_timestamp_ns = 123000 # ns after epoch senv_timestamp = datetime.datetime.fromtimestamp( senv_timestamp_ns * 1e-9, datetime.timezone.utc) senv_time_between_samples = 100 # ns senv_test_message = serialise_senv(senv_source_name, -1, senv_timestamp, senv_time_between_samples, 0, senv_values, Location.Start) test_message_queue.put(FakeMessage(senv_test_message)) tdct_timestamps = np.array([1234, 2345, 3456]) # ns tdct_test_message = serialise_tdct(tdct_source_name, tdct_timestamps) test_message_queue.put(FakeMessage(tdct_test_message)) n_chunks += 1 assert isclose(data_from_stream.attrs[f142_source_name].value.values[0], f142_value) assert data_from_stream.attrs[f142_source_name].value.coords[ 'time'].values[0] == np.array(f142_timestamp, dtype=np.dtype('datetime64[ns]')) assert np.array_equal( data_from_stream.attrs[senv_source_name].value.values, senv_values) senv_expected_timestamps = np.array([ senv_timestamp_ns, senv_timestamp_ns + senv_time_between_samples, senv_timestamp_ns + (2 * senv_time_between_samples) ], dtype=np.dtype('datetime64[ns]')) assert np.array_equal( data_from_stream.attrs[senv_source_name].value.coords['time'].values, senv_expected_timestamps) assert np.array_equal( data_from_stream.attrs[tdct_source_name].value.values, tdct_timestamps)
async def test_stream_loop_exits_if_stop_time_reached_and_later_message_seen( queues): data_queue, worker_instruction_queue, test_message_queue = queues run_info_topic = "fake_topic" test_instrument_name = "DATA_STREAM_TEST" # The Kafka topics to get metadata from are recorded as "stream" objects in # the nexus_structure field of the run start message f142_source_name = "f142_source" f142_log_name = "f142_log" streams = [ Stream(f"/entry/{f142_log_name}", "f142_topic", f142_source_name, "f142", "double", "m"), ] test_stream_args = TEST_STREAM_ARGS.copy() test_stream_args["topics"] = None # System time is already after this stop time so the stream will stop # as soon as it sees the end of partition or a message with a # timestamp after the stop time stop_time_in_past = datetime.datetime(2017, 11, 28, 23, 55, 59, 342380) n_chunks = 0 async for data in _data_stream(data_queue, worker_instruction_queue, run_info_topic=run_info_topic, query_consumer=FakeQueryConsumer( test_instrument_name, stop_time=stop_time_in_past, streams=streams), **test_stream_args, test_message_queue=test_message_queue, end_at=StopTime.END_OF_RUN): if n_chunks == 0: # Publish a message with a timestamp before the stop time f142_value_1 = 26.1236 timestamp_before_stop_dt = datetime.datetime( 2017, 11, 28, 23, 55, 50, 0) # Convert to integer nanoseconds # (for timestamp in message payload) timestamp_before_stop_ns = int( timestamp_before_stop_dt.timestamp() * 1_000_000_000) # Convert to integer milliseconds # (for Kafka message header) timestamp_before_stop_ms = int( timestamp_before_stop_dt.timestamp() * 1_000) f142_test_message = serialise_f142(f142_value_1, f142_source_name, timestamp_before_stop_ns) test_message_queue.put( FakeMessage(f142_test_message, timestamp=timestamp_before_stop_ms)) elif n_chunks == 1: # The data from the first message will be returned assert np.allclose(data.attrs[f142_source_name].value.values, np.array([f142_value_1])) assert np.array_equal( data.attrs[f142_source_name].value.coords['time'].values, np.array([timestamp_before_stop_ns], dtype=np.dtype('datetime64[ns]'))) # Publish message with timestamp after stop time, this will trigger # the consumer to stop and data_stream to exit. # A TimeoutError would occur if the functionality is broken. f142_value_2 = 2.725 timestamp_after_stop_dt = datetime.datetime( 2017, 11, 28, 23, 56, 50, 0) timestamp_after_stop_ns = int(timestamp_after_stop_dt.timestamp() * 1_000_000_000) timestamp_after_stop_ms = int(timestamp_after_stop_dt.timestamp() * 1_000) f142_test_message = serialise_f142(f142_value_2, f142_source_name, timestamp_after_stop_ns) test_message_queue.put( FakeMessage(f142_test_message, timestamp=timestamp_after_stop_ms)) n_chunks += 1
def create_f142_buffer(value, source_name='mypv'): return serialise_f142(value, source_name)