def test_forwarder_sends_pv_updates_single_pv_string(docker_compose): """ Test the forwarder pushes new PV value when the value is updated. :param docker_compose: Test fixture :return: None """ data_topic = "TEST_forwarderData_string_pv_update" pvs = [PVSTR] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) # Wait for config to be pushed sleep(2) cons = create_consumer() # Update value change_pv_value(PVSTR, "stop") # Wait for PV to be updated sleep(5) cons.subscribe([data_topic]) # Poll for empty update - initial value of PVSTR is nothing poll_for_valid_message(cons) # Poll for message which should contain forwarded PV update data_msg = poll_for_valid_message(cons) check_expected_values(data_msg, Value.String, PVSTR, b'stop') cons.close()
def test_forwarder_sends_pv_updates_single_pv_enum(docker_compose): """ Test the forwarder pushes new PV value when the value is updated. NOTE: Enums are converted to Ints in the forwarder. :param docker_compose: Test fixture :return: None """ data_topic = "TEST_forwarderData_enum_pv_update" pvs = [PVENUM] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) # Wait for config to be pushed sleep(5) cons = create_consumer() # Update value change_pv_value(PVENUM, "START") # Wait for PV to be updated sleep(5) cons.subscribe([data_topic]) first_msg = poll_for_valid_message(cons) check_expected_values(first_msg, Value.Int, PVENUM, 0) second_msg = poll_for_valid_message(cons) check_expected_values(second_msg, Value.Int, PVENUM, 1) cons.close()
def test_forwarder_sends_pv_updates_single_pv_double(docker_compose): """ Test the forwarder pushes new PV value when the value is updated. :param docker_compose: Test fixture :return: None """ data_topic = "TEST_forwarderData_double_pv_update" pvs = [PVDOUBLE] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) # Wait for config to be pushed sleep(2) cons = create_consumer() cons.subscribe([data_topic]) # Update value change_pv_value(PVDOUBLE, 5) # Wait for PV to be updated sleep(5) first_msg = poll_for_valid_message(cons) check_expected_values(first_msg, Value.Double, PVDOUBLE, 0.0) second_msg = poll_for_valid_message(cons) check_expected_values(second_msg, Value.Double, PVDOUBLE, 5.0) cons.close()
def test_forwarder_updates_pv_when_config_changed_from_two_pvs(docker_compose): data_topic = "TEST_forwarderData_change_config" pvs = [PVSTR, PVLONG] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) sleep(2) prod.add_config([PVDOUBLE]) sleep(2) cons = create_consumer() sleep(2) cons.subscribe([data_topic]) sleep(2) poll_for_valid_message(cons) poll_for_valid_message(cons) expected_values = { PVSTR: (Value.String, b''), PVLONG: (Value.Int, 0), PVDOUBLE: (Value.Double, 0.0) } messages = [ poll_for_valid_message(cons), poll_for_valid_message(cons), poll_for_valid_message(cons) ] check_multiple_expected_values(messages, expected_values) cons.close()
def test_forwarder_sends_pv_updates_single_floatarray( docker_compose_no_command): """ GIVEN PV of enum type is configured to be forwarded WHEN PV value is updated THEN Forwarder publishes the update to Kafka """ data_topic = "TEST_forwarderData_floatarray_pv_update" pvs = [PVFLOATARRAY] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) # Wait for config to be pushed sleep(5) cons = create_consumer() # Wait for PV to be updated cons.subscribe([data_topic]) sleep(5) first_msg, _ = poll_for_valid_message(cons) expectedarray = [1.1, 2.2, 3.3] check_expected_array_values(first_msg, Value.ArrayFloat, PVFLOATARRAY, expectedarray) cons.close()
def test_forwarder_sends_pv_updates_single_pv_long(docker_compose): """ Test the forwarder pushes new PV value when the value is updated. NOTE: longs are converted to ints in the forwarder as they will fit in a 32 bit integer :param docker_compose: Test fixture :return: None """ data_topic = "TEST_forwarderData_long_pv_update" pvs = [PVLONG] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) # Wait for config to be pushed sleep(2) cons = create_consumer() # Set initial PV value change_pv_value(PVLONG, 0) sleep(2) # Update value change_pv_value(PVLONG, 5) # Wait for PV to be updated sleep(2) cons.subscribe([data_topic]) first_msg = poll_for_valid_message(cons) check_expected_values(first_msg, Value.Int, PVLONG, 0) second_msg = poll_for_valid_message(cons) check_expected_values(second_msg, Value.Int, PVLONG, 5) cons.close()
def test_forwarder_sends_pv_updates_single_pv_enum(docker_compose_no_command, start_ioc): """ GIVEN PV of enum type is configured to be forwarded WHEN PV value is updated THEN Forwarder publishes the update to Kafka NOTE: Enums are converted to Ints in the forwarder. """ data_topic = "TEST_forwarderData_enum_pv_update" pvs = [PVENUM] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) # Wait for config to be pushed sleep(5) cons = create_consumer() # Update value change_pv_value(PVENUM, "START") # Wait for PV to be updated cons.subscribe([data_topic]) sleep(5) first_msg, _ = poll_for_valid_message(cons) check_expected_values(first_msg, Value.Int, PVENUM, 0) second_msg, _ = poll_for_valid_message(cons) check_expected_values(second_msg, Value.Int, PVENUM, 1) cons.close()
def test_connection_status_messages(docker_compose_no_command): """ GIVEN PV is configured to be forwarded WHEN Connection status changes THEN Forwarder publishes ep00 message with connection status NOTE: Enums are converted to Ints in the forwarder. """ data_topic = "TEST_forwarderData_connection_status" pvs = [PVENUM] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) # Wait for config to be pushed sleep(5) cons = create_consumer() # Update value change_pv_value(PVENUM, "START") # Wait for PV to be updated sleep(5) cons.subscribe([data_topic]) first_msg = poll_for_connection_status_message(cons) check_expected_connection_status_values(first_msg, EventType.CONNECTED) cons.close()
def test_forwarder_updates_pv_when_config_change_add_two_pvs( docker_compose_no_command): """ GIVEN A PV (double type) is already being forwarded WHEN A message configures two additional PV (str and long types) to be forwarded THEN Forwarder publishes initial values for added PVs """ data_topic = "TEST_forwarderData_change_config" pvs = [PVSTR, PVLONG] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) sleep(2) cons = create_consumer() sleep(2) cons.subscribe([data_topic]) sleep(2) poll_for_valid_message(cons) poll_for_valid_message(cons) expected_values = {PVSTR: (Value.String, b""), PVLONG: (Value.Int, 0)} messages = [ poll_for_valid_message(cons)[0], poll_for_valid_message(cons)[0] ] check_multiple_expected_values(messages, expected_values) cons.close()
def test_forwarder_updates_multiple_pvs(docker_compose_no_command): """ GIVEN multiple PVs (string and long types) are configured to be forwarded WHEN PV value is updated THEN Forwarder publishes the updates to Kafka """ data_topic = "TEST_forwarderData_multiple" pvs = [PVSTR, PVLONG] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) sleep(2) cons = create_consumer() sleep(2) cons.subscribe([data_topic]) sleep(4) expected_values = {PVSTR: (Value.String, b""), PVLONG: (Value.Int, 0)} first_msg, _ = poll_for_valid_message(cons) second_msg, _ = poll_for_valid_message(cons) messages = [first_msg, second_msg] check_multiple_expected_values(messages, expected_values) cons.close()
def test_forwarder_sends_idle_pv_updates(docker_compose_idle_updates): consumer = create_consumer() data_topic = "TEST_forwarderData_idle_updates" consumer.subscribe([data_topic]) sleep(10) for i in range(3): msg = poll_for_valid_message(consumer) check_expected_values(msg, Value.Value.Double, PVDOUBLE, 0) consumer.close()
def test_forwarder_sends_fake_pv_updates(docker_compose_fake_epics): # A fake PV is defined in the config json file with channel name "FakePV" consumer = create_consumer() data_topic = "TEST_forward_fake_generated_pvs" consumer.subscribe([data_topic]) sleep(5) msg = poll_for_valid_message(consumer) # We should see PV updates in Kafka despite there being no IOC running check_expected_values(msg, Value.Value.Double, "FakePV")
def test_forwarder_does_not_send_pv_update_more_than_once_when_periodic_update_is_used( docker_compose_idle_updates_long_period): consumer = create_consumer() data_topic = "TEST_forwarderData_long_idle_updates" consumer.subscribe([data_topic]) sleep(3) msg = poll_for_valid_message(consumer) check_expected_values(msg, Value.Value.Double, PVDOUBLE, 0) with raises(MsgErrorException): # AssertionError because there are no more messages to poll poll_for_valid_message(consumer)
def test_ignores_commands_with_incorrect_id(docker_compose_multiple_instances): producer = create_producer() sleep(20) send_writer_command( os.path.join("filewriter_tests", "commands", "add-command-never-ends.json"), producer, ) send_writer_command( os.path.join("filewriter_tests", "commands", "add-command-never-ends2.json"), producer, ) sleep(10) send_writer_command( os.path.join("filewriter_tests", "commands", "writer-stop-single.json"), producer, ) consumer = create_consumer() consumer.subscribe(["TEST_writerStatus2"]) # poll a few times on the status topic to see if the filewriter2 has stopped writing files. stopped = False for i in range(30): msg = consumer.poll() if b'"files":{}' in msg.value(): # filewriter2 is not currently writing a file - stop command has been processed. stopped = True break sleep(1) assert stopped sleep(5) consumer.unsubscribe() consumer.subscribe(["TEST_writerStatus1"]) writer1msg = consumer.poll() # Check filewriter1's job queue is not empty assert b'"files":{}' not in writer1msg.value()
def test_long_run(docker_compose_lr, start_ioc): """ Test that the channel defined in the config file is created. :param docker_compose: Test fixture :return: None """ # Set up consumer now and subscribe from earliest offset on data topic cons = create_consumer("earliest") cons.subscribe(["TEST_forwarderDataLR"]) with open(os.path.join("logs", "forwarder_lr_stats.log"), "w+") as stats_file: with open(os.path.join("logs", "forwarder_lr_missedupdates.log"), "w+") as file: for i in range(5150): # minimum 12 hours with 4 second sleep time # Change pv value now change_pv_value(PVDOUBLE, i) # Wait for the forwarder to push the update sleep(3) try: msg, _ = poll_for_valid_message(cons) except MsgErrorException: sleep(3) msg, _ = poll_for_valid_message(cons) try: check_expected_values(msg, Value.Double, PVDOUBLE, float(i)) except AssertionError: # Message is either incorrect or empty - log expected value to file file.write(str(i) + "\n") container = False # Report stats every 10th iteration if i % 10 == 0: client = docker.from_env() for item in client.containers.list(): if "forwarder" in item.name: container = item break if container: stats_file.write("{}\t{}\n".format( datetime.now(), container.stats( stream=False)["memory_stats"]["usage"], ))
def test_forwarder_status_shows_added_pvs(docker_compose_no_command): """ GIVEN A PV (double type) is already being forwarded WHEN A message configures two additional PV (str and long types) to be forwarded THEN Forwarder status message lists new PVs """ data_topic = "TEST_forwarderData_change_config" status_topic = "TEST_forwarderStatus" pvs = [PVSTR, PVLONG] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) sleep(5) cons = create_consumer() sleep(2) cons.assign([TopicPartition(status_topic, partition=0)]) sleep(2) # Get the last available status message partitions = cons.assignment() _, hi = cons.get_watermark_offsets(partitions[0], cached=False, timeout=2.0) last_msg_offset = hi - 1 cons.assign( [TopicPartition(status_topic, partition=0, offset=last_msg_offset)]) status_msg, _ = poll_for_valid_message(cons, expected_file_identifier=None) status_json = json.loads(status_msg) names_of_channels_being_forwarded = { stream["channel_name"] for stream in status_json["streams"] } expected_names_of_channels_being_forwarded = {PVSTR, PVLONG} assert ( expected_names_of_channels_being_forwarded == names_of_channels_being_forwarded ), (f"Expect these channels to be configured as forwarded: {expected_names_of_channels_being_forwarded}, " f"but status message report these as forwarded: {names_of_channels_being_forwarded}" ) cons.close()
def test_forwarder_updates_pv_when_config_changed_from_one_pv(docker_compose): data_topic = "TEST_forwarderData_change_config" prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config([PVLONG]) prod.add_config([PVDOUBLE]) sleep(2) cons = create_consumer() sleep(2) cons.subscribe([data_topic]) sleep(2) expected_values = {PVLONG: (Value.Int, 0), PVDOUBLE: (Value.Double, 0.0)} first_msg = poll_for_valid_message(cons) second_msg = poll_for_valid_message(cons) messages = [first_msg, second_msg] check_multiple_expected_values(messages, expected_values) cons.close()
def test_forwarder_updates_multiple_pvs(docker_compose): data_topic = "TEST_forwarderData_multiple" pvs = [PVSTR, PVLONG] prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic) prod.add_config(pvs) sleep(2) cons = create_consumer() sleep(2) cons.subscribe([data_topic]) sleep(4) expected_values = {PVSTR: (Value.String, b''), PVLONG: (Value.Int, 0)} first_msg = poll_for_valid_message(cons) second_msg = poll_for_valid_message(cons) messages = [first_msg, second_msg] check_multiple_expected_values(messages, expected_values) cons.close()
def test_config_file_channel_created_correctly(docker_compose): """ Test that the channel defined in the config file is created. :param docker_compose: Test fixture :return: None """ cons = create_consumer() cons.subscribe(['TEST_forwarderData_pv_from_config']) sleep(5) # Change the PV value, so something is forwarded change_pv_value(PVDOUBLE, 10) # Wait for PV to be updated sleep(5) # Check the initial value is forwarded first_msg = poll_for_valid_message(cons) check_expected_values(first_msg, Value.Double, PVDOUBLE, 0.0) # Check the new value is forwarded second_msg = poll_for_valid_message(cons) check_expected_values(second_msg, Value.Double, PVDOUBLE, 10.0) cons.close()