class StreamRequestSender(object):
def __init__(self, output_port, queue_length, send_timeout, mode,
epics_writer_url):
self.output_port = output_port
self.queue_length = queue_length
self.send_timeout = send_timeout
self.mode = mode
self.epics_writer_url = epics_writer_url
_logger.info(
"Starting stream request sender with output_port=%s, queue_length=%s, send_timeout=%s, mode=%s "
"and epics_writer_url=%s" %
(self.output_port, self.queue_length, self.send_timeout, self.mode,
self.epics_writer_url))
self.output_stream = Sender(port=self.output_port,
queue_size=self.queue_length,
send_timeout=self.send_timeout,
mode=self.mode)
self.output_stream.open()
def send(self, write_request, sendto_epics_writer=True):
_logger.info("Sending write write_request: %s" % write_request)
self.output_stream.send(data=write_request)
if self.epics_writer_url and sendto_epics_writer:
def send_epics_request():
try:
epics_writer_request = {
"range":
json.loads(write_request["data_api_request"])["range"],
"parameters":
json.loads(write_request["parameters"])
}
_logger.info("Sending epics writer request %s" %
epics_writer_request)
requests.put(url=self.epics_writer_url,
json=epics_writer_request)
except Exception as e:
_logger.error(
"Error while trying to forward the write request to the epics writer.",
e)
Thread(target=send_epics_request).start()
def start_sender():
# Start a mock sender stream.
generator = Sender(block=False)
generator.add_channel('CAMERA1:X', lambda x: x, metadata={'type': 'int32'})
generator.add_channel('CAMERA1:Y', lambda x: x, metadata={'type': 'int32'})
generator.add_channel('CAMERA1:VALID',
lambda x: 10,
metadata={'type': 'int32'})
generator.open()
while bs_sending:
generator.send()
time.sleep(0.05)
generator.close()
def start_sender():
# Start a mock sender stream.
generator = Sender(block=False)
generator.add_channel('CAMERA1:X', lambda x: x, metadata={'type': 'int32'})
generator.add_channel('CAMERA1:Y', lambda x: x, metadata={'type': 'int32'})
generator.add_channel('CAMERA1:VALID',
lambda x: 10,
metadata={'type': 'int32'})
generator.add_channel('BEAM_OK',
lambda x: 1 if x % 20 == 0 else 0,
metadata={'type': 'int32'})
generator.open()
while bs_sending:
generator.send()
time.sleep(MOCK_SENDER_INTERVAL)
generator.close()
def run(stop_event,
statistics,
parameter_queue,
cam_client,
pipeline_config,
output_stream_port,
background_manager,
user_scripts_manager=None):
def no_client_action():
nonlocal parameters
if parameters["no_client_timeout"] > 0:
_logger.warning(
"No client connected to the pipeline stream for %d seconds. Closing instance. %s"
% (config.MFLOW_NO_CLIENTS_TIMEOUT, log_tag))
stop_event.set()
source = None
sender = None
set_log_tag("store_pipeline")
exit_code = 0
parameters = init_pipeline_parameters(pipeline_config, parameter_queue,
user_scripts_manager)
if parameters.get("no_client_timeout") is None:
parameters["no_client_timeout"] = config.MFLOW_NO_CLIENTS_TIMEOUT
modulo = parameters.get("modulo", None)
try:
init_statistics(statistics)
camera_name = pipeline_config.get_camera_name()
stream_image_name = camera_name + config.EPICS_PV_SUFFIX_IMAGE
set_log_tag(" [" + str(camera_name) + " | " +
str(pipeline_config.get_name()) + ":" +
str(output_stream_port) + "]")
source = connect_to_camera(cam_client)
_logger.debug("Opening output stream on port %d. %s",
output_stream_port, log_tag)
sender = Sender(port=output_stream_port,
mode=PUSH,
data_header_compression=config.
CAMERA_BSREAD_DATA_HEADER_COMPRESSION,
block=False)
sender.open(no_client_action=no_client_action,
no_client_timeout=parameters["no_client_timeout"]
if parameters["no_client_timeout"] > 0 else sys.maxsize)
init_sender(sender, parameters)
# TODO: Register proper channels.
# Indicate that the startup was successful.
stop_event.clear()
_logger.debug("Transceiver started. %s" % log_tag)
counter = 1
while not stop_event.is_set():
try:
data = source.receive()
update_statistics(
sender,
data.statistics.total_bytes_received if data else 0,
1 if data else 0)
if modulo:
if counter < modulo:
counter = counter + 1
continue
else:
counter = 1
# In case of receiving error or timeout, the returned data is None.
if data is None:
continue
forward_data = {
stream_image_name: data.data.data["image"].value
}
pulse_id = data.data.pulse_id
timestamp = (data.data.global_timestamp,
data.data.global_timestamp_offset)
send(sender, forward_data, timestamp, pulse_id)
except:
_logger.exception("Could not process message. %s" % log_tag)
stop_event.set()
_logger.info("Stopping transceiver. %s" % log_tag)
except:
_logger.exception(
"Exception while trying to start the receive and process thread. %s"
% log_tag)
exit_code = 1
raise
finally:
_logger.info("Stopping transceiver. %s" % log_tag)
if source:
source.disconnect()
if sender:
try:
sender.close()
except:
pass
sys.exit(exit_code)
def process_bsread_camera(stop_event, statistics, parameter_queue, camera,
port):
"""
Start the camera stream and receive the incoming bsread streams. This function blocks until stop_event is set.
:param stop_event: Event when to stop the process.
:param statistics: Statistics namespace.
:param parameter_queue: Parameters queue to be passed to the pipeline.
:param camera: Camera instance to get the stream from.
:param port: Port to use to bind the output stream.
"""
sender = None
camera_stream = None
try:
# If there is no client for some time, disconnect.
def no_client_timeout():
_logger.info(
"No client connected to the '%s' stream for %d seconds. Closing instance."
% (camera.get_name(), config.MFLOW_NO_CLIENTS_TIMEOUT))
stop_event.set()
def process_parameters():
nonlocal x_size, y_size, x_axis, y_axis
x_axis, y_axis = camera.get_x_y_axis()
x_size, y_size = camera.get_geometry()
# TODO: Use to register proper channels. But be aware that the size and dtype can change during the running.
# def register_image_channel(size_x, size_y, dtype):
# sender.add_channel("image", metadata={"compression": config.CAMERA_BSREAD_IMAGE_COMPRESSION,
# "shape": [size_x, size_y],
# "type": dtype})
x_size = y_size = x_axis = y_axis = None
sender = Sender(port=port,
mode=PUB,
data_header_compression=config.
CAMERA_BSREAD_DATA_HEADER_COMPRESSION)
sender.open(no_client_action=no_client_timeout,
no_client_timeout=config.MFLOW_NO_CLIENTS_TIMEOUT)
camera_name = camera.get_name()
camera_stream = camera.get_stream()
_logger.info("Connecting to camera '%s' over bsread.", camera_name)
process_parameters()
# register_image_channel(x_size, y_size, dtype)
statistics.counter = 0
camera_stream.connect()
# This signals that the camera has successfully started.
stop_event.clear()
while not stop_event.is_set():
try:
data = camera_stream.receive()
# In case of receiving error or timeout, the returned data is None.
if data is None:
continue
image = data.data.data[camera_name +
config.EPICS_PV_SUFFIX_IMAGE].value
# Rotate and mirror the image if needed - this is done in the epics:_get_image for epics cameras.
image = transform_image(image, camera.camera_config)
# Numpy is slowest dimension first, but bsread is fastest dimension first.
height, width = image.shape
pulse_id = data.data.pulse_id
timestamp_s = data.data.global_timestamp
timestamp_ns = data.data.global_timestamp_offset
timestamp = timestamp_s + (timestamp_ns / 1e9)
data = {
"image": image,
"height": height,
"width": width,
"x_axis": x_axis,
"y_axis": y_axis,
"timestamp": timestamp
}
sender.send(data=data,
pulse_id=pulse_id,
timestamp=timestamp,
check_data=True)
while not parameter_queue.empty():
new_parameters = parameter_queue.get()
camera.camera_config.set_configuration(new_parameters)
process_parameters()
except Exception as e:
_logger.exception("Could not process message.", e)
stop_event.set()
_logger.info("Stopping transceiver.")
except Exception as e:
_logger.exception("Error while processing camera stream.", e)
finally:
# Wait for termination / update configuration / etc.
stop_event.wait()
if camera_stream:
camera.disconnect()
if sender:
sender.close()
def process_epics_camera(stop_event, statistics, parameter_queue, camera,
port):
"""
Start the camera stream and listen for image monitors. This function blocks until stop_event is set.
:param stop_event: Event when to stop the process.
:param statistics: Statistics namespace.
:param parameter_queue: Parameters queue to be passed to the pipeline.
:param camera: Camera instance to get the images from.
:param port: Port to use to bind the output stream.
"""
sender = None
try:
# If there is no client for some time, disconnect.
def no_client_timeout():
_logger.info(
"No client connected to the '%s' stream for %d seconds. Closing instance."
% (camera.get_name(), config.MFLOW_NO_CLIENTS_TIMEOUT))
stop_event.set()
def process_parameters():
nonlocal x_size, y_size, x_axis, y_axis
x_size, y_size = camera.get_geometry()
x_axis, y_axis = camera.get_x_y_axis()
sender.add_channel("image",
metadata={
"compression":
config.CAMERA_BSREAD_IMAGE_COMPRESSION,
"shape": [x_size, y_size],
"type": "uint16"
})
x_size = y_size = x_axis = y_axis = None
camera.connect()
sender = Sender(port=port,
mode=PUB,
data_header_compression=config.
CAMERA_BSREAD_DATA_HEADER_COMPRESSION)
# Register the bsread channels - compress only the image.
sender.add_channel("width",
metadata={
"compression":
config.CAMERA_BSREAD_SCALAR_COMPRESSION,
"type": "int64"
})
sender.add_channel("height",
metadata={
"compression":
config.CAMERA_BSREAD_SCALAR_COMPRESSION,
"type": "int64"
})
sender.add_channel("timestamp",
metadata={
"compression":
config.CAMERA_BSREAD_SCALAR_COMPRESSION,
"type": "float64"
})
sender.add_channel("x_axis",
metadata={
"compression":
config.CAMERA_BSREAD_SCALAR_COMPRESSION,
"type": "float32"
})
sender.add_channel("y_axis",
metadata={
"compression":
config.CAMERA_BSREAD_SCALAR_COMPRESSION,
"type": "float32"
})
sender.open(no_client_action=no_client_timeout,
no_client_timeout=config.MFLOW_NO_CLIENTS_TIMEOUT)
process_parameters()
statistics.counter = 0
def collect_and_send(image, timestamp):
nonlocal x_size, y_size, x_axis, y_axis
# Data to be sent over the stream.
data = {
"image": image,
"timestamp": timestamp,
"width": x_size,
"height": y_size,
"x_axis": x_axis,
"y_axis": y_axis
}
try:
sender.send(data=data, timestamp=timestamp, check_data=False)
except Again:
_logger.warning(
"Send timeout. Lost image with timestamp '%s'." %
timestamp)
while not parameter_queue.empty():
new_parameters = parameter_queue.get()
camera.camera_config.set_configuration(new_parameters)
process_parameters()
camera.add_callback(collect_and_send)
# This signals that the camera has successfully started.
stop_event.clear()
except:
_logger.exception("Error while processing camera stream.")
finally:
# Wait for termination / update configuration / etc.
stop_event.wait()
camera.disconnect()
if sender:
sender.close()
def store_pipeline(stop_event, statistics, parameter_queue, cam_client,
pipeline_config, output_stream_port, background_manager):
# TODO: Implement statistics: n_clients, input_throughput
def no_client_timeout():
_logger.warning(
"No client connected to the pipeline stream for %d seconds. Closing instance."
% config.MFLOW_NO_CLIENTS_TIMEOUT)
stop_event.set()
source = None
sender = None
try:
camera_stream_address = cam_client.get_camera_stream(
pipeline_config.get_camera_name())
camera_name = pipeline_config.get_camera_name()
_logger.debug("Connecting to camera %s on stream address %s.",
camera_name, camera_stream_address)
source_host, source_port = get_host_port_from_stream_address(
camera_stream_address)
source = Source(host=source_host,
port=source_port,
receive_timeout=config.PIPELINE_RECEIVE_TIMEOUT,
mode=SUB)
source.connect()
_logger.debug("Opening output stream on port %d.", output_stream_port)
sender = Sender(port=output_stream_port,
mode=PUSH,
data_header_compression=config.
CAMERA_BSREAD_DATA_HEADER_COMPRESSION,
block=False)
sender.open(no_client_action=no_client_timeout,
no_client_timeout=config.MFLOW_NO_CLIENTS_TIMEOUT)
# TODO: Register proper channels.
# Indicate that the startup was successful.
stop_event.clear()
_logger.debug("Transceiver started.")
while not stop_event.is_set():
try:
data = source.receive()
# In case of receiving error or timeout, the returned data is None.
if data is None:
continue
forward_data = {camera_name: data.data.data["image"].value}
pulse_id = data.data.pulse_id
timestamp = (data.data.global_timestamp,
data.data.global_timestamp_offset)
sender.send(data=forward_data,
pulse_id=pulse_id,
timestamp=timestamp)
except:
_logger.exception("Could not process message.")
stop_event.set()
_logger.info("Stopping transceiver.")
except:
_logger.exception(
"Exception while trying to start the receive and process thread.")
raise
finally:
if source:
source.disconnect()
if sender:
sender.close()
def processing_pipeline(stop_event, statistics, parameter_queue, cam_client,
pipeline_config, output_stream_port,
background_manager):
# TODO: Implement statistics: n_clients, input_throughput
def no_client_timeout():
_logger.warning(
"No client connected to the pipeline stream for %d seconds. Closing instance."
% config.MFLOW_NO_CLIENTS_TIMEOUT)
stop_event.set()
def process_pipeline_parameters():
parameters = pipeline_config.get_configuration()
_logger.debug("Processing pipeline parameters %s.", parameters)
background_array = None
if parameters.get("image_background_enable"):
background_id = pipeline_config.get_background_id()
_logger.debug("Image background enabled. Using background_id %s.",
background_id)
background_array = background_manager.get_background(background_id)
size_x, size_y = cam_client.get_camera_geometry(
pipeline_config.get_camera_name())
image_region_of_interest = parameters.get("image_region_of_interest")
if image_region_of_interest:
_, size_x, _, size_y = image_region_of_interest
_logger.debug("Image width %d and height %d.", size_x, size_y)
return parameters, background_array
source = None
sender = None
try:
pipeline_parameters, image_background_array = process_pipeline_parameters(
)
camera_stream_address = cam_client.get_camera_stream(
pipeline_config.get_camera_name())
_logger.debug("Connecting to camera stream address %s.",
camera_stream_address)
source_host, source_port = get_host_port_from_stream_address(
camera_stream_address)
source = Source(host=source_host,
port=source_port,
receive_timeout=config.PIPELINE_RECEIVE_TIMEOUT,
mode=SUB)
source.connect()
_logger.debug("Opening output stream on port %d.", output_stream_port)
sender = Sender(port=output_stream_port,
mode=PUB,
data_header_compression=config.
CAMERA_BSREAD_DATA_HEADER_COMPRESSION)
sender.open(no_client_action=no_client_timeout,
no_client_timeout=config.MFLOW_NO_CLIENTS_TIMEOUT)
# TODO: Register proper channels.
# Indicate that the startup was successful.
stop_event.clear()
_logger.debug("Transceiver started.")
while not stop_event.is_set():
try:
while not parameter_queue.empty():
new_parameters = parameter_queue.get()
pipeline_config.set_configuration(new_parameters)
pipeline_parameters, image_background_array = process_pipeline_parameters(
)
data = source.receive()
# In case of receiving error or timeout, the returned data is None.
if data is None:
continue
image = data.data.data["image"].value
x_axis = data.data.data["x_axis"].value
y_axis = data.data.data["y_axis"].value
processing_timestamp = data.data.data["timestamp"].value
processed_data = process_image(image, processing_timestamp,
x_axis, y_axis,
pipeline_parameters,
image_background_array)
processed_data["width"] = processed_data["image"].shape[1]
processed_data["height"] = processed_data["image"].shape[0]
pulse_id = data.data.pulse_id
timestamp = (data.data.global_timestamp,
data.data.global_timestamp_offset)
sender.send(data=processed_data,
timestamp=timestamp,
pulse_id=pulse_id)
except:
_logger.exception("Could not process message.")
stop_event.set()
_logger.info("Stopping transceiver.")
except:
_logger.exception(
"Exception while trying to start the receive and process thread.")
raise
finally:
if source:
source.disconnect()
if sender:
sender.close()