def test_forwarder_sends_pv_updates_single_pv_string(docker_compose):
"""
Test the forwarder pushes new PV value when the value is updated.
:param docker_compose: Test fixture
:return: None
"""
data_topic = "TEST_forwarderData_string_pv_update"
pvs = [PVSTR]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
# Wait for config to be pushed
sleep(2)
cons = create_consumer()
# Update value
change_pv_value(PVSTR, "stop")
# Wait for PV to be updated
sleep(5)
cons.subscribe([data_topic])
# Poll for empty update - initial value of PVSTR is nothing
poll_for_valid_message(cons)
# Poll for message which should contain forwarded PV update
data_msg = poll_for_valid_message(cons)
check_expected_values(data_msg, Value.String, PVSTR, b'stop')
cons.close()
def test_forwarder_sends_pv_updates_single_pv_enum(docker_compose):
"""
Test the forwarder pushes new PV value when the value is updated.
NOTE: Enums are converted to Ints in the forwarder.
:param docker_compose: Test fixture
:return: None
"""
data_topic = "TEST_forwarderData_enum_pv_update"
pvs = [PVENUM]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
# Wait for config to be pushed
sleep(5)
cons = create_consumer()
# Update value
change_pv_value(PVENUM, "START")
# Wait for PV to be updated
sleep(5)
cons.subscribe([data_topic])
first_msg = poll_for_valid_message(cons)
check_expected_values(first_msg, Value.Int, PVENUM, 0)
second_msg = poll_for_valid_message(cons)
check_expected_values(second_msg, Value.Int, PVENUM, 1)
cons.close()
def test_forwarder_sends_pv_updates_single_pv_double(docker_compose):
"""
Test the forwarder pushes new PV value when the value is updated.
:param docker_compose: Test fixture
:return: None
"""
data_topic = "TEST_forwarderData_double_pv_update"
pvs = [PVDOUBLE]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
# Wait for config to be pushed
sleep(2)
cons = create_consumer()
cons.subscribe([data_topic])
# Update value
change_pv_value(PVDOUBLE, 5)
# Wait for PV to be updated
sleep(5)
first_msg = poll_for_valid_message(cons)
check_expected_values(first_msg, Value.Double, PVDOUBLE, 0.0)
second_msg = poll_for_valid_message(cons)
check_expected_values(second_msg, Value.Double, PVDOUBLE, 5.0)
cons.close()
def test_forwarder_updates_pv_when_config_changed_from_two_pvs(docker_compose):
data_topic = "TEST_forwarderData_change_config"
pvs = [PVSTR, PVLONG]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
sleep(2)
prod.add_config([PVDOUBLE])
sleep(2)
cons = create_consumer()
sleep(2)
cons.subscribe([data_topic])
sleep(2)
poll_for_valid_message(cons)
poll_for_valid_message(cons)
expected_values = {
PVSTR: (Value.String, b''),
PVLONG: (Value.Int, 0),
PVDOUBLE: (Value.Double, 0.0)
}
messages = [
poll_for_valid_message(cons),
poll_for_valid_message(cons),
poll_for_valid_message(cons)
]
check_multiple_expected_values(messages, expected_values)
cons.close()
def test_forwarder_sends_pv_updates_single_floatarray(
docker_compose_no_command):
"""
GIVEN PV of enum type is configured to be forwarded
WHEN PV value is updated
THEN Forwarder publishes the update to Kafka
"""
data_topic = "TEST_forwarderData_floatarray_pv_update"
pvs = [PVFLOATARRAY]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
# Wait for config to be pushed
sleep(5)
cons = create_consumer()
# Wait for PV to be updated
cons.subscribe([data_topic])
sleep(5)
first_msg, _ = poll_for_valid_message(cons)
expectedarray = [1.1, 2.2, 3.3]
check_expected_array_values(first_msg, Value.ArrayFloat, PVFLOATARRAY,
expectedarray)
cons.close()
def test_forwarder_sends_pv_updates_single_pv_long(docker_compose):
"""
Test the forwarder pushes new PV value when the value is updated.
NOTE: longs are converted to ints in the forwarder as they will fit in a 32 bit integer
:param docker_compose: Test fixture
:return: None
"""
data_topic = "TEST_forwarderData_long_pv_update"
pvs = [PVLONG]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
# Wait for config to be pushed
sleep(2)
cons = create_consumer()
# Set initial PV value
change_pv_value(PVLONG, 0)
sleep(2)
# Update value
change_pv_value(PVLONG, 5)
# Wait for PV to be updated
sleep(2)
cons.subscribe([data_topic])
first_msg = poll_for_valid_message(cons)
check_expected_values(first_msg, Value.Int, PVLONG, 0)
second_msg = poll_for_valid_message(cons)
check_expected_values(second_msg, Value.Int, PVLONG, 5)
cons.close()
def test_forwarder_sends_pv_updates_single_pv_enum(docker_compose_no_command,
start_ioc):
"""
GIVEN PV of enum type is configured to be forwarded
WHEN PV value is updated
THEN Forwarder publishes the update to Kafka
NOTE: Enums are converted to Ints in the forwarder.
"""
data_topic = "TEST_forwarderData_enum_pv_update"
pvs = [PVENUM]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
# Wait for config to be pushed
sleep(5)
cons = create_consumer()
# Update value
change_pv_value(PVENUM, "START")
# Wait for PV to be updated
cons.subscribe([data_topic])
sleep(5)
first_msg, _ = poll_for_valid_message(cons)
check_expected_values(first_msg, Value.Int, PVENUM, 0)
second_msg, _ = poll_for_valid_message(cons)
check_expected_values(second_msg, Value.Int, PVENUM, 1)
cons.close()
def test_connection_status_messages(docker_compose_no_command):
"""
GIVEN PV is configured to be forwarded
WHEN Connection status changes
THEN Forwarder publishes ep00 message with connection status
NOTE: Enums are converted to Ints in the forwarder.
"""
data_topic = "TEST_forwarderData_connection_status"
pvs = [PVENUM]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
# Wait for config to be pushed
sleep(5)
cons = create_consumer()
# Update value
change_pv_value(PVENUM, "START")
# Wait for PV to be updated
sleep(5)
cons.subscribe([data_topic])
first_msg = poll_for_connection_status_message(cons)
check_expected_connection_status_values(first_msg, EventType.CONNECTED)
cons.close()
def test_forwarder_updates_pv_when_config_change_add_two_pvs(
docker_compose_no_command):
"""
GIVEN A PV (double type) is already being forwarded
WHEN A message configures two additional PV (str and long types) to be forwarded
THEN Forwarder publishes initial values for added PVs
"""
data_topic = "TEST_forwarderData_change_config"
pvs = [PVSTR, PVLONG]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
sleep(2)
cons = create_consumer()
sleep(2)
cons.subscribe([data_topic])
sleep(2)
poll_for_valid_message(cons)
poll_for_valid_message(cons)
expected_values = {PVSTR: (Value.String, b""), PVLONG: (Value.Int, 0)}
messages = [
poll_for_valid_message(cons)[0],
poll_for_valid_message(cons)[0]
]
check_multiple_expected_values(messages, expected_values)
cons.close()
def test_forwarder_updates_multiple_pvs(docker_compose_no_command):
"""
GIVEN multiple PVs (string and long types) are configured to be forwarded
WHEN PV value is updated
THEN Forwarder publishes the updates to Kafka
"""
data_topic = "TEST_forwarderData_multiple"
pvs = [PVSTR, PVLONG]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
sleep(2)
cons = create_consumer()
sleep(2)
cons.subscribe([data_topic])
sleep(4)
expected_values = {PVSTR: (Value.String, b""), PVLONG: (Value.Int, 0)}
first_msg, _ = poll_for_valid_message(cons)
second_msg, _ = poll_for_valid_message(cons)
messages = [first_msg, second_msg]
check_multiple_expected_values(messages, expected_values)
cons.close()
def test_forwarder_sends_idle_pv_updates(docker_compose_idle_updates):
consumer = create_consumer()
data_topic = "TEST_forwarderData_idle_updates"
consumer.subscribe([data_topic])
sleep(10)
for i in range(3):
msg = poll_for_valid_message(consumer)
check_expected_values(msg, Value.Value.Double, PVDOUBLE, 0)
consumer.close()
def test_forwarder_sends_fake_pv_updates(docker_compose_fake_epics):
# A fake PV is defined in the config json file with channel name "FakePV"
consumer = create_consumer()
data_topic = "TEST_forward_fake_generated_pvs"
consumer.subscribe([data_topic])
sleep(5)
msg = poll_for_valid_message(consumer)
# We should see PV updates in Kafka despite there being no IOC running
check_expected_values(msg, Value.Value.Double, "FakePV")
def test_forwarder_does_not_send_pv_update_more_than_once_when_periodic_update_is_used(
docker_compose_idle_updates_long_period):
consumer = create_consumer()
data_topic = "TEST_forwarderData_long_idle_updates"
consumer.subscribe([data_topic])
sleep(3)
msg = poll_for_valid_message(consumer)
check_expected_values(msg, Value.Value.Double, PVDOUBLE, 0)
with raises(MsgErrorException):
# AssertionError because there are no more messages to poll
poll_for_valid_message(consumer)
def test_ignores_commands_with_incorrect_id(docker_compose_multiple_instances):
producer = create_producer()
sleep(20)
send_writer_command(
os.path.join("filewriter_tests", "commands",
"add-command-never-ends.json"),
producer,
)
send_writer_command(
os.path.join("filewriter_tests", "commands",
"add-command-never-ends2.json"),
producer,
)
sleep(10)
send_writer_command(
os.path.join("filewriter_tests", "commands",
"writer-stop-single.json"),
producer,
)
consumer = create_consumer()
consumer.subscribe(["TEST_writerStatus2"])
# poll a few times on the status topic to see if the filewriter2 has stopped writing files.
stopped = False
for i in range(30):
msg = consumer.poll()
if b'"files":{}' in msg.value():
# filewriter2 is not currently writing a file - stop command has been processed.
stopped = True
break
sleep(1)
assert stopped
sleep(5)
consumer.unsubscribe()
consumer.subscribe(["TEST_writerStatus1"])
writer1msg = consumer.poll()
# Check filewriter1's job queue is not empty
assert b'"files":{}' not in writer1msg.value()
def test_long_run(docker_compose_lr, start_ioc):
"""
Test that the channel defined in the config file is created.
:param docker_compose: Test fixture
:return: None
"""
# Set up consumer now and subscribe from earliest offset on data topic
cons = create_consumer("earliest")
cons.subscribe(["TEST_forwarderDataLR"])
with open(os.path.join("logs", "forwarder_lr_stats.log"),
"w+") as stats_file:
with open(os.path.join("logs", "forwarder_lr_missedupdates.log"),
"w+") as file:
for i in range(5150): # minimum 12 hours with 4 second sleep time
# Change pv value now
change_pv_value(PVDOUBLE, i)
# Wait for the forwarder to push the update
sleep(3)
try:
msg, _ = poll_for_valid_message(cons)
except MsgErrorException:
sleep(3)
msg, _ = poll_for_valid_message(cons)
try:
check_expected_values(msg, Value.Double, PVDOUBLE,
float(i))
except AssertionError:
# Message is either incorrect or empty - log expected value to file
file.write(str(i) + "\n")
container = False
# Report stats every 10th iteration
if i % 10 == 0:
client = docker.from_env()
for item in client.containers.list():
if "forwarder" in item.name:
container = item
break
if container:
stats_file.write("{}\t{}\n".format(
datetime.now(),
container.stats(
stream=False)["memory_stats"]["usage"],
))
def test_forwarder_status_shows_added_pvs(docker_compose_no_command):
"""
GIVEN A PV (double type) is already being forwarded
WHEN A message configures two additional PV (str and long types) to be forwarded
THEN Forwarder status message lists new PVs
"""
data_topic = "TEST_forwarderData_change_config"
status_topic = "TEST_forwarderStatus"
pvs = [PVSTR, PVLONG]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
sleep(5)
cons = create_consumer()
sleep(2)
cons.assign([TopicPartition(status_topic, partition=0)])
sleep(2)
# Get the last available status message
partitions = cons.assignment()
_, hi = cons.get_watermark_offsets(partitions[0],
cached=False,
timeout=2.0)
last_msg_offset = hi - 1
cons.assign(
[TopicPartition(status_topic, partition=0, offset=last_msg_offset)])
status_msg, _ = poll_for_valid_message(cons, expected_file_identifier=None)
status_json = json.loads(status_msg)
names_of_channels_being_forwarded = {
stream["channel_name"]
for stream in status_json["streams"]
}
expected_names_of_channels_being_forwarded = {PVSTR, PVLONG}
assert (
expected_names_of_channels_being_forwarded ==
names_of_channels_being_forwarded
), (f"Expect these channels to be configured as forwarded: {expected_names_of_channels_being_forwarded}, "
f"but status message report these as forwarded: {names_of_channels_being_forwarded}"
)
cons.close()
def test_forwarder_updates_pv_when_config_changed_from_one_pv(docker_compose):
data_topic = "TEST_forwarderData_change_config"
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config([PVLONG])
prod.add_config([PVDOUBLE])
sleep(2)
cons = create_consumer()
sleep(2)
cons.subscribe([data_topic])
sleep(2)
expected_values = {PVLONG: (Value.Int, 0), PVDOUBLE: (Value.Double, 0.0)}
first_msg = poll_for_valid_message(cons)
second_msg = poll_for_valid_message(cons)
messages = [first_msg, second_msg]
check_multiple_expected_values(messages, expected_values)
cons.close()
def test_forwarder_updates_multiple_pvs(docker_compose):
data_topic = "TEST_forwarderData_multiple"
pvs = [PVSTR, PVLONG]
prod = ProducerWrapper("localhost:9092", CONFIG_TOPIC, data_topic)
prod.add_config(pvs)
sleep(2)
cons = create_consumer()
sleep(2)
cons.subscribe([data_topic])
sleep(4)
expected_values = {PVSTR: (Value.String, b''), PVLONG: (Value.Int, 0)}
first_msg = poll_for_valid_message(cons)
second_msg = poll_for_valid_message(cons)
messages = [first_msg, second_msg]
check_multiple_expected_values(messages, expected_values)
cons.close()
def test_config_file_channel_created_correctly(docker_compose):
"""
Test that the channel defined in the config file is created.
:param docker_compose: Test fixture
:return: None
"""
cons = create_consumer()
cons.subscribe(['TEST_forwarderData_pv_from_config'])
sleep(5)
# Change the PV value, so something is forwarded
change_pv_value(PVDOUBLE, 10)
# Wait for PV to be updated
sleep(5)
# Check the initial value is forwarded
first_msg = poll_for_valid_message(cons)
check_expected_values(first_msg, Value.Double, PVDOUBLE, 0.0)
# Check the new value is forwarded
second_msg = poll_for_valid_message(cons)
check_expected_values(second_msg, Value.Double, PVDOUBLE, 10.0)
cons.close()
