def test_start_missing_header(self):
with sender(port=self.STREAM_PORT) as output_stream:
with source(host="localhost", port=self.STREAM_PORT) as input_stream:
for index in range(10):
data = {"fast_source": index,
"slow_source": None}
output_stream.send(data=data)
self.writer.write_message(input_stream.receive(handler=self.handler.receive))
for index in range(10, 20):
data = {"fast_source": index,
"slow_source": index}
output_stream.send(data=data)
self.writer.write_message(input_stream.receive(handler=self.handler.receive))
self.writer.close()
file = h5py.File(self.OUTPUT_FILE)
fast_source = file["/data/fast_source/data"]
slow_source = file["/data/slow_source/data"]
self.assertIsNotNone(fast_source)
self.assertIsNotNone(slow_source)
self.assertListEqual(list(fast_source), list(range(20)))
self.assertListEqual(list(slow_source[:10]), [0] * 10)
self.assertListEqual(list(slow_source[10:]), list(range(10, 20)))
file.close()
def test_classic_interaction(self):
response = requests.get(self.rest_url + "status").json()
self.assertEqual(response["state"], "ok")
self.assertEqual(response["status"], "waiting")
response = requests.post(self.rest_url + "parameters", json={}).json()
self.assertEqual(response["state"], 'error')
self.assertTrue("Missing mandatory" in response["status"])
parameters = {
"general/created": "today",
"general/user": "******",
"general/process": "dia",
"general/instrument": "jungfrau"
}
response = requests.post(self.rest_url + "parameters",
json=parameters).json()
self.assertEqual(response["state"], "ok")
self.assertEqual(response["status"], "waiting")
self.assertEqual(response["parameters"], parameters)
requests.put(self.rest_url + "start_pulse_id/3")
response = requests.get(self.rest_url + "statistics").json()
self.assertEqual(response["state"], "ok")
self.assertEqual(response["statistics"]["start_pulse_id"], 3)
self.assertEqual(response["statistics"]["stop_pulse_id"], None)
timestamp = 0
timestamp_offset = 0
data = {"device1": 0, "device2": 0, "device3": None}
with sender(port=self.stream_port, mode=PUSH,
queue_size=1) as output_stream:
for index in range(10):
pulse_id = index
data["device1"] = index + 100
data["device2"] = index + 200
output_stream.send(timestamp=(timestamp, timestamp_offset),
pulse_id=pulse_id,
data=data)
# Somewhere before the end of the acquisition.
if pulse_id == 6:
requests.put(self.rest_url + "stop_pulse_id/8")
response = requests.get(self.rest_url +
"statistics").json()
self.assertEqual(response["state"], "ok")
self.assertEqual(response["statistics"]["start_pulse_id"],
3)
self.assertEqual(response["statistics"]["stop_pulse_id"],
8)
sleep(0.5)
self.assertFalse(self.writer_process.is_alive())
def test_change_header(self):
with sender(port=self.STREAM_PORT) as output_stream:
with source(host="localhost", port=self.STREAM_PORT) as input_stream:
for index in range(10):
data = {"scalar_source": index,
"changing_source": index}
output_stream.send(data=data)
self.writer.write_message(input_stream.receive(handler=self.handler.receive))
for index in range(10, 20):
data = {"scalar_source": index,
"changing_source": [index, index, index]}
output_stream.send(data=data)
self.writer.write_message(input_stream.receive(handler=self.handler.receive))
self.writer.close()
file = h5py.File(self.OUTPUT_FILE)
scalar_source = file["/data/scalar_source/data"]
changing_source_1 = file["/data/changing_source/data(1)"]
changing_source_2 = file["/data/changing_source/data"]
self.assertIsNotNone(scalar_source)
self.assertIsNotNone(changing_source_1)
self.assertIsNotNone(changing_source_2)
self.assertEqual(len(scalar_source), 20)
self.assertEqual(len(changing_source_1), 10)
self.assertEqual(len(changing_source_2), 20)
self.assertListEqual(list(scalar_source), list(range(20)))
self.assertListEqual(list(changing_source_1[:10]), list(range(10)))
for index in range(10):
self.assertListEqual(list(changing_source_2[index]), [0] * 3)
for index in range(10, 20):
self.assertListEqual(list(changing_source_2[index]), [index] * 3)
file.close()
def test_classic_interaction(self):
data = {}
with sender(port=self.stream_port, mode=PUSH,
queue_size=1) as output_stream:
for index in range(10):
pulse_id = index
data["device1"] = index + 100
data["device2"] = index + 200
data["device3"] = None
output_stream.send(pulse_id=pulse_id, data=data)
self.runningEvent.clear()
sleep(1.5)
self.assertFalse(self.buffer_process.is_alive())
def generate_stream(output_port=8888, n_images=1000, delay=0.1):
"""
Open a stream and generate the simulated messages. This call is blocking.
:param output_port: Port to bind the output stream to.
:param n_images: Number of images to send (-1 == infinite)
:param delay: How much time in seconds to wait between each message. 0.1 == 10Hz.
"""
with sender(port=output_port) as output_stream:
if n_images == -1:
while True:
generate_and_send_message(output_stream)
sleep(delay)
else:
for _ in range(n_images):
generate_and_send_message(output_stream)
sleep(delay)
def test_change_roi_while_running(self):
client = PsssProcessingClient("http://localhost:10000/")
roi = [0, 1024, 0, 1024]
client.set_roi(roi)
processed_data = []
client.start()
with source(host="localhost", port=12000, mode=PULL) as input_stream:
# First pulse_id comes before the source connects.
for index in range(self.n_images - 1):
message = input_stream.receive()
processed_data.append(message)
updated_roi = [100, 200, 100, 200]
client.set_roi(updated_roi)
data_to_send = {
self.pv_name_prefix + config.EPICS_PV_SUFFIX_IMAGE: self.image
}
with sender(port=10001) as output_stream:
for x in range(self.n_images):
output_stream.send(data=data_to_send)
with source(host="localhost", port=12000, mode=PULL) as input_stream:
for index in range(self.n_images):
message = input_stream.receive()
processed_data.append(message)
client.stop()
processing_parameters_name = self.pv_name_prefix + config.EPICS_PV_SUFFIX_IMAGE + ".processing_parameters"
start_processing_parameters = json.loads(
processed_data[0].data.data[processing_parameters_name].value)
end_processing_parameters = json.loads(
processed_data[8].data.data[processing_parameters_name].value)
self.assertListEqual(roi, start_processing_parameters["roi"])
self.assertListEqual(updated_roi, end_processing_parameters["roi"])
def test_receive_timeout(self):
from bsread import source
from bsread.sender import sender
with source(host="localhost", port=9999,
receive_timeout=10) as in_stream:
with sender(queue_size=10) as stream:
# Send Data
# stream.send(one=1, two=2)
# stream.send(one=3, two=4)
# Receive and check data
# If timeout is not working - this will hang forever - and therefore the test will fail
message = in_stream.receive()
if message:
print(message.data.data['one'].value)
self.assertIsNone(message)
def send_bsread_message(output_port,
message_buffer,
running_event,
mode=PUSH,
buffer_timeout=0.01):
_logger.info("Output stream binding to port '%s'.", output_port)
try:
with sender(port=output_port, mode=mode,
queue_size=1) as output_stream:
while running_event.is_set():
if len(message_buffer) == 0:
sleep(buffer_timeout)
continue
message, message_timestamp = message_buffer.popleft()
data = {}
for value_name, bsread_value in message.data.data.items():
data[value_name] = bsread_value.value
_logger.debug("Sending message with pulse_id '%s'.",
message.data.pulse_id)
output_stream.send(timestamp=message_timestamp,
pulse_id=message.data.pulse_id,
data=data,
check_data=True)
_logger.debug("Message with pulse_id '%s' forwarded.",
message.data.pulse_id)
except Exception as e:
running_event.clear()
_logger.error("Exception happened in sending thread. Stopping buffer.",
e)
def test_receive_filter(self):
from bsread import source
from bsread.sender import sender
def filter_method(m):
print(m.data.data['two'].value)
return m.data.data['two'].value <= 4
with source(host="localhost", port=9999) as in_stream:
with sender(queue_size=10) as stream:
# Send Data
stream.send(one=1, two=12.0)
stream.send(one=2, two=4)
stream.send(one=3, two=10)
stream.send(one=4, two=9.5)
# Receive and check data
# If timeout is not working - this will hang forever - and therefore the test will fail
message = in_stream.receive(
filter=lambda m: m.data.data['two'].value <= 4)
# message = in_stream.receive(filter=filter_method)
print(message.data.data['one'].value)
simulated_camera = CameraSimulation(camera_config=CameraConfig("simulation"),
size_x=image_size_x, size_y=image_size_y)
x_axis, y_axis = simulated_camera.get_x_y_axis()
x_size, y_size = simulated_camera.get_geometry()
# Documentation: https://github.com/datastreaming/cam_server#pipeline_configuration
pipeline_parameters = {
"camera_name": "simulation"
}
pipeline_config = PipelineConfig("test_pipeline", pipeline_parameters)
parameters = pipeline_config.get_configuration()
image_number = 0
with sender(port=output_stream_port, mode=output_stream_mode) as output_stream:
# Get simulated image.
image = simulated_camera.get_image()
# Generate timestamp.
timestamp = time.time()
# Pass data to processing pipeline.
processed_data = process_image(image, 0, timestamp, x_axis, y_axis, pipeline_parameters)
# Set height and width.
processed_data["width"] = processed_data["image"].shape[1]
processed_data["height"] = processed_data["image"].shape[0]
print("Sending image number: ", image_number)
image_number += 1
def send_data():
sleep(1)
with sender(port=10000) as output_stream:
for x in range(n_images):
output_stream.send(data=data_to_send)
def send_data():
with sender(port=10000, queue_size=100) as output_stream:
for x in range(n_images):
output_stream.send(data=data_to_send)
def send_data():
with sender(port=11000) as output_stream:
for x in range(self.n_images):
output_stream.send(data=data_to_send)
def stream_processor(running_flag, roi, parameters, statistics):
try:
running_flag.set()
_logger.info("Connecting to input_stream_host %s and input_stream_port %s.",
input_stream_host, input_stream_port)
_logger.info("Sending out data on stream port %s.", output_stream_port)
if output_pv_name:
_logger.info("Sending out data on EPICS PV %s.", output_pv_name)
epics.ca.clear_cache()
output_pv = PV(output_pv_name)
else:
_logger.warning("Output EPICS PV not specified. Only bsread will be sent out.")
with source(host=input_stream_host, port=input_stream_port, mode=PULL,
queue_size=config.INPUT_STREAM_QUEUE_SIZE,
receive_timeout=config.INPUT_STREAM_RECEIVE_TIMEOUT) as input_stream:
with sender(port=output_stream_port, send_timeout=config.OUTPUT_STREAM_SEND_TIMEOUT,
block=False) as output_stream:
statistics["processing_start_time"] = str(datetime.datetime.now())
statistics["last_sent_pulse_id"] = None
statistics["last_sent_time"] = None
statistics["last_calculated_spectrum"] = None
statistics["n_processed_images"] = 0
image_property_name = epics_pv_name_prefix + config.EPICS_PV_SUFFIX_IMAGE
_logger.info("Using image_to_process property name '%s'.", image_property_name)
while running_flag.is_set():
message = input_stream.receive()
if message is None:
continue
pulse_id = message.data.pulse_id
timestamp = (message.data.global_timestamp, message.data.global_timestamp_offset)
_logger.debug("Received message with pulse_id %s", pulse_id)
image_to_process = message.data.data[image_property_name].value
processed_data = process_image(image_to_process,
image_property_name,
roi,
parameters["min_threshold"],
parameters["max_threshold"],
parameters["rotation"])
try:
output_stream.send(pulse_id=pulse_id,
timestamp=timestamp,
data=processed_data)
_logger.debug("Sent message with pulse_id %s", pulse_id)
statistics["last_sent_pulse_id"] = pulse_id
statistics["last_sent_time"] = str(datetime.datetime.now())
except Again:
pass
statistics["last_calculated_spectrum"] = processed_data[image_property_name + ".spectrum"]
statistics["n_processed_images"] = statistics.get("n_processed_images", 0) + 1
if output_pv_name:
output_pv.put(processed_data[image_property_name + ".spectrum"])
_logger.debug("caput on %s for pulse_id %s", output_pv, pulse_id)
except Exception as e:
_logger.error("Error while processing the stream. Exiting. Error: ", e)
running_flag.clear()
raise
except KeyboardInterrupt:
_logger.warning("Terminating processing due to user request.")
running_flag.clear()
raise
def stream_processor(running_flag, roi_signal, roi_background, statistics):
try:
running_flag.set()
_logger.info(
"Connecting to input_stream_host %s and input_stream_port %s.",
input_stream_host, input_stream_port)
_logger.info("Sending out data on stream port %s.",
data_output_stream_port)
with source(host=input_stream_host,
port=input_stream_port,
mode=PULL,
queue_size=config.INPUT_STREAM_QUEUE_SIZE,
receive_timeout=config.INPUT_STREAM_RECEIVE_TIMEOUT
) as input_stream:
with sender(port=data_output_stream_port,
send_timeout=config.DATA_OUTPUT_STREAM_SEND_TIMEOUT
) as data_output_stream:
with sender(
port=image_output_stream_port,
block=False,
queue_size=config.IMAGE_OUTPUT_STREAM_QUEUE_SIZE
) as image_output_stream:
statistics["processing_start_time"] = str(
datetime.datetime.now())
statistics["last_sent_pulse_id"] = None
statistics["last_sent_time"] = None
statistics["n_processed_images"] = 0
image_property_name = epics_pv_name_prefix + config.EPICS_PV_SUFFIX_IMAGE
_logger.info("Using image property name '%s'.",
image_property_name)
while running_flag.is_set():
message = input_stream.receive()
if message is None:
continue
pulse_id = message.data.pulse_id
timestamp = (message.data.global_timestamp,
message.data.global_timestamp_offset)
_logger.debug("Received message with pulse_id %s",
pulse_id)
image = message.data.data[
image_property_name].value
processed_data = process_image(
pulse_id, image, image_property_name,
roi_signal, roi_background)
# Send out processed data.
try:
data_output_stream.send(pulse_id=pulse_id,
timestamp=timestamp,
data=processed_data)
_logger.debug(
"Sent data message with pulse_id %s",
pulse_id)
statistics["last_sent_pulse_id"] = pulse_id
statistics["last_sent_time"] = str(
datetime.datetime.now())
statistics[
"n_processed_images"] = statistics.get(
"n_processed_images", 0) + 1
except Again:
pass
# Send out image.
try:
image_output_stream.send(
pulse_id=pulse_id,
timestamp=timestamp,
data={image_property_name: image})
_logger.debug(
"Sent image message with pulse_id %s",
pulse_id)
except Again:
pass
except Exception as e:
_logger.error(
"Error while processing the stream. Exiting. Error: ", e)
running_flag.clear()
raise
except KeyboardInterrupt:
_logger.warning("Terminating processing due to user request.")
running_flag.clear()
raise
def test_buffer_writer_integration(self):
with sender(port=self.buffer_stream_port, queue_size=100,
mode=PUSH) as source_stream:
self.buffer_process.start()
sleep(0.5)
for index in range(0, 10):
data = {
"normal_source": index,
"slow_source": None,
"changing_source": index
}
source_stream.send(data=data)
for index in range(10, 20):
data = {
"normal_source": index,
"slow_source": index,
"changing_source": [index] * 3
}
source_stream.send(data=data)
# 31, because the writer stops AFTER receiving the first message it should not write.
for index in range(20, 31):
data = {
"normal_source": index,
"slow_source": None,
"changing_source": [index] * 3
}
source_stream.send(data=data)
self.writer_process.start()
sleep(0.5)
response = requests.get(self.writer_rest_url + "status").json()
self.assertEqual(response["state"], "ok")
self.assertEqual(response["status"], "waiting")
parameters = {
"general/created": "today",
"general/user": "******",
"general/process": "dia",
"general/instrument": "jungfrau"
}
response = requests.post(self.writer_rest_url + "parameters",
json=parameters).json()
self.assertEqual(response["state"], "ok")
self.assertEqual(response["status"], "waiting")
self.assertEqual(response["parameters"], parameters)
requests.put(self.writer_rest_url + "start_pulse_id/0")
response = requests.get(self.writer_rest_url + "statistics").json()
self.assertEqual(response["state"], "ok")
self.assertEqual(response["statistics"]["start_pulse_id"], 0)
self.assertEqual(response["statistics"]["stop_pulse_id"], None)
requests.put(self.writer_rest_url + "stop_pulse_id/29")
# Wait for the writer to terminate.
sleep(1.5)
self.assertFalse(self.writer_process.is_alive(),
"Writer process should be dead by now.")
self.assertTrue(self.buffer_process.is_alive(),
"Buffer process should be still alive.")
file = h5py.File(self.writer_output_file)
normal_source = file["/data/normal_source/data"]
slow_source = file["/data/slow_source/data"]
changing_source_1 = file["/data/changing_source/data(1)"]
changing_source_2 = file["/data/changing_source/data"]
self.assertEqual(len(normal_source), 30)
self.assertEqual(len(slow_source), 30)
self.assertEqual(len(changing_source_1), 10)
self.assertEqual(len(changing_source_2), 30)
def send_data():
with sender(port=10001, queue_size=100) as output_stream:
sleep(1)
for x in range(self.n_images):
output_stream.send(data=data_to_send)
def stream_processor(running_flag, parameters, statistics):
try:
running_flag.set()
_logger.info("Connecting to input_stream_host %s and input_stream_port %s.",
input_stream_host, input_stream_port)
_logger.info("Sending out data on stream port %s.", data_output_stream_port)
_logger.info("Sending out images on stream port %s.", image_output_stream_port)
if output_pv_name:
_logger.info("Sending out spectrum data on EPICS PV %s.", output_pv_name)
epics.ca.clear_cache()
output_pv = epics.PV(output_pv_name)
else:
_logger.warning("Output EPICS PV not specified. Only bsread will be sent out.")
if center_pv_name:
_logger.info("Sending out spectrum center on EPICS PV %s.", center_pv_name)
center_pv = epics.PV(center_pv_name)
else:
_logger.warning("Output EPICS PV not specified. Only bsread will be sent out.")
if fwhm_pv_name:
_logger.info("Sending out spectrum fwhm on EPICS PV %s.", fwhm_pv_name)
fwhm_pv = epics.PV(fwhm_pv_name)
else:
_logger.warning("Output EPICS PV not specified. Only bsread will be sent out.")
# EPICS PV for vertical ROI
if ymin_pv_name:
ymin_pv = epics.PV(ymin_pv_name)
ymin_pv.wait_for_connection()
if ymax_pv_name:
ymax_pv = epics.PV(ymax_pv_name)
ymax_pv.wait_for_connection()
if axis_pv_name:
axis_pv = epics.PV(axis_pv_name)
axis_pv.wait_for_connection()
roi = [0, 0]
with source(host=input_stream_host, port=input_stream_port, mode=PULL,
queue_size=config.INPUT_STREAM_QUEUE_SIZE,
receive_timeout=config.INPUT_STREAM_RECEIVE_TIMEOUT) as input_stream:
with sender(port=data_output_stream_port, send_timeout=config.OUTPUT_STREAM_SEND_TIMEOUT,
block=False) as data_output_stream:
with sender(port=image_output_stream_port, send_timeout=config.OUTPUT_STREAM_SEND_TIMEOUT,
block=False, queue_size=config.IMAGE_OUTPUT_STREAM_QUEUE_SIZE) as image_output_stream:
# use zmq zero-copy for image data
image_output_stream.stream.zmq_copy = False
image_output_stream.stream.zmq_track = True
statistics["processing_start_time"] = str(datetime.datetime.now())
statistics["last_sent_pulse_id"] = None
statistics["last_sent_time"] = None
statistics["last_calculated_spectrum"] = None
statistics["n_processed_images"] = 0
image_property_name = epics_pv_name_prefix + config.EPICS_PV_SUFFIX_IMAGE
_logger.info("Using image_to_process property name '%s'.", image_property_name)
while running_flag.is_set():
try:
message = input_stream.receive()
except:
_logger.exception("input stream receiving error")
continue
if message is None:
continue
start_time = time.time()
pulse_id = message.data.pulse_id
timestamp = (message.data.global_timestamp, message.data.global_timestamp_offset)
image_to_process = message.data.data[image_property_name].value
if image_to_process is None:
continue
_logger.debug("Received message with pulse_id %s", pulse_id)
image_data = {image_property_name: image_to_process}
if ymin_pv_name and ymin_pv.connected:
roi[0] = ymin_pv.value
if ymax_pv_name and ymax_pv.connected:
roi[1] = ymax_pv.value
if axis_pv_name and axis_pv.connected:
axis = axis_pv.value
else:
axis = None
if axis is None or len(axis) != image_to_process.shape[1]:
_logger.warn("Invalid energy axis")
continue
processed_data = process_image(image_to_process,
axis,
epics_pv_name_prefix,
roi,
parameters)
try:
data_output_stream.send(pulse_id=pulse_id,
timestamp=timestamp,
data=processed_data)
_logger.debug("Sent data message with pulse_id %s", pulse_id)
statistics["last_sent_pulse_id"] = pulse_id
statistics["last_sent_time"] = str(datetime.datetime.now())
except zmq.Again:
pass
try:
image_output_stream.send(pulse_id=pulse_id,
timestamp=timestamp,
data=image_data)
_logger.debug("Sent image message with pulse_id %s", pulse_id)
except zmq.Again:
pass
statistics["last_calculated_spectrum"] = processed_data[epics_pv_name_prefix +
":SPECTRUM_Y"]
statistics["n_processed_images"] = statistics.get("n_processed_images", 0) + 1
if output_pv_name and output_pv.connected:
output_pv.put(processed_data[epics_pv_name_prefix + ":SPECTRUM_Y"])
_logger.debug("caput on %s for pulse_id %s", output_pv, pulse_id)
if center_pv_name and center_pv.connected:
center_pv.put(processed_data[epics_pv_name_prefix + ":SPECTRUM_CENTER"])
if fwhm_pv_name and fwhm_pv.connected:
fwhm_pv.put(processed_data[epics_pv_name_prefix + ":SPECTRUM_FWHM"])
duration = (time.time() - start_time) * 1000
statistics["last_processing_duration_ms"] = duration
except Exception as e:
_logger.error("Error while processing the stream. Exiting. Error: ", e)
running_flag.clear()
raise
except KeyboardInterrupt:
_logger.warning("Terminating processing due to user request.")
running_flag.clear()
raise