def setUp(self):
self.host = "0.0.0.0"
self.cam_port = 8888
self.pipeline_port = 8889
self.cam_proxy_port = 8898
self.pipeline_proxy_port = 8899
for port in self.cam_port, self.pipeline_port, self.cam_proxy_port, self.pipeline_proxy_port:
print("Port ", port, "avilable: ", is_port_available(port))
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_proxy_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_proxy_port)
self.cam_process = Process(target=start_camera_server,
args=(self.host, self.cam_port,
self.cam_config_folder,
self.user_scripts_folder))
self.cam_process.start()
self.cam_proxy_process = Process(target=start_camera_proxy,
args=(self.host, self.cam_proxy_port,
cam_server_address,
self.cam_config_folder))
self.cam_proxy_process.start()
self.pipeline_process = Process(
target=start_pipeline_server,
args=(self.host, self.pipeline_port, self.pipeline_config_folder,
self.background_config_folder, self.user_scripts_folder,
cam_server_proxy_address))
self.pipeline_process.start()
self.pipeline_proxy_process = Process(
target=start_pipeline_proxy,
args=(self.host, self.pipeline_proxy_port, pipeline_server_address,
self.pipeline_config_folder, self.background_config_folder,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
self.cam_client = CamClient(cam_server_proxy_address)
self.pipeline_client = PipelineClient(pipeline_server_proxy_address)
# Give it some time to start.
sleep(1.0) # Give it some time to start.
def setUp(self):
self.host = "0.0.0.0"
self.cam_port = 8888
self.pipeline_port = 8889
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
self.cam_process = Process(target=start_camera_server,
args=(self.host, self.cam_port,
self.cam_config_folder))
self.cam_process.start()
self.pipeline_process = Process(
target=start_pipeline_server,
args=(self.host, self.pipeline_port, self.pipeline_config_folder,
self.background_config_folder, cam_server_address))
self.pipeline_process.start()
# Give it some time to start.
sleep(1)
self.cam_client = CamClient(cam_server_address)
self.pipeline_client = PipelineClient(pipeline_server_address)
def __init__(
self,
name=None,
reduction_client_address="http://sf-daqsync-02:12003/",
delay_stages={"spatial_tt": "SLAAR21-LMOT-M522:MOTOR_1"},
pipeline_id="SARES20-CAMS142-M4_psen_db1",
):
self.name = name
self.alias = Alias(name)
# append_object_to_object(self,MotorRecord,pvname+":MOTOR_X1",name='x_target')
# append_object_to_object(self,MotorRecord,pvname+":MOTOR_Y1",name='y_target')
if delay_stages:
for key, pv in delay_stages.items():
tname = "delay_" + key + "_stg"
append_object_to_object(self, MotorRecord, pv, name=tname)
append_object_to_object(self,
DelayTime,
self.__dict__[tname],
name="delay_" + key)
# self.delay = MotorRecord(self.Id + "-M424:MOT")
# self.delayTime = DelayStage(self.delay)
# self.data_reduction_client = PsenProcessingClient(address=reduction_client_address)
self._camera_server_client = PipelineClient()
self._camera_server_pipeline_id = pipeline_id
def get_images(screen, n_images, beamline='Aramis', dry_run=None):
print('Start get_images for screen %s, %i images, beamline %s' %
(screen, n_images, beamline))
meta_dict_1 = get_meta_data(screen)
positioner = pyscan.BsreadPositioner(n_messages=n_images)
settings = pyscan.scan_settings(settling_time=0.01,
measurement_interval=0.2,
n_measurements=1)
pipeline_client = PipelineClient("http://sf-daqsync-01:8889/")
cam_instance_name = str(screen) + "_sp1"
stream_address = pipeline_client.get_instance_stream(cam_instance_name)
stream_host, stream_port = get_host_port_from_stream_address(
stream_address)
# Configure bsread
pyscan.config.bs_default_host = stream_host
pyscan.config.bs_default_port = stream_port
logging.getLogger("mflow.mflow").setLevel(logging.ERROR)
readables = get_readables(beamline)
raw_output = pyscan.scan(positioner=positioner,
readables=readables,
settings=settings)
output = [[x] for x in raw_output]
result_dict = pyscan_result_to_dict(readables, output, scrap_bs=True)
for ax in ['x_axis', 'y_axis']:
arr = result_dict[ax] * 1e-6 # convert to m
if len(arr.shape) == 3:
result_dict[ax + '_m'] = arr[0, 0, :]
elif len(arr.shape) == 2:
result_dict[ax + '_m'] = arr[0, :]
else:
raise ValueError('Unexpected', len(arr.shape))
meta_dict_2 = get_meta_data(screen)
output_dict = {
'pyscan_result': result_dict,
'meta_data_begin': meta_dict_1,
'meta_data_end': meta_dict_2,
}
print('End get_images')
return output_dict
def test_create_pipeline_with_background(self):
from cam_server import PipelineClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Change to match your pipeline server
server_address = "http://0.0.0.0:8889"
camera_name = "simulation"
# Initialize the client.
pipeline_client = PipelineClient(server_address)
# Collect the background for the given camera.
background_id = pipeline_client.collect_background(camera_name)
# Setup the pipeline config. Use the simulation camera as the pipeline source, and the collected background.
pipeline_config = {
"camera_name": camera_name,
"background_id": background_id
}
# Create a new pipeline with the provided configuration. Stream address in format tcp://hostname:port.
instance_id, pipeline_stream_address = pipeline_client.create_instance_from_config(
pipeline_config)
# Extract the stream hostname and port from the stream address.
pipeline_host, pipeline_port = get_host_port_from_stream_address(
pipeline_stream_address)
# Subscribe to the stream.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
# Receive next message.
data = stream.receive()
image_width = data.data.data["width"].value
image_height = data.data.data["height"].value
image_bytes = data.data.data["image"].value
print("Image size: %d x %d" % (image_width, image_height))
print("Image data: %s" % image_bytes)
x_size, y_size = get_simulated_camera().get_geometry()
self.assertIsNotNone(data)
self.assertEqual(image_width, x_size)
self.assertIsNotNone(image_height, y_size)
def get_image(camName, ROI, numImg, angle):
pipeline_client = PipelineClient()
pipeline_config = {"camera_name": camName, "image_region_of_interest": ROI}
pipeline_config = {"camera_name": camName,"rotation":angle}
instance_id, pipeline_stream_address = pipeline_client.create_instance_from_config(pipeline_config)
pipeline_host, pipeline_port = get_host_port_from_stream_address(pipeline_stream_address)
img = []
x_profile = []
y_profile = []
x_fwhm = []
y_fwhm = []
x_center_of_mass = []
y_center_of_mass = []
intensity = []
max_value = []
width = []
height = []
with source(host=pipeline_host, port=pipeline_port, mode=SUB) as stream:
for i in range(0,numImg):
data = stream.receive()
img.append(data.data.data["image"].value)
x_profile.append(data.data.data["x_profile"].value)
y_profile.append(data.data.data["y_profile"].value)
x_fwhm.append(data.data.data["x_fwhm"].value)
y_fwhm.append(data.data.data["y_fwhm"].value)
x_center_of_mass.append(data.data.data["x_center_of_mass"].value)
intensity.append(data.data.data["intensity"].value)
width.append(data.data.data["width"].value)
height.append(data.data.data["height"].value)
img = np.asarray(img)
dataout = {
"mean": img.mean(axis=0),
"image":img,
"x_profile":x_profile,
"y_profile":y_profile,
"x_fwhm":x_fwhm,
"y_fwhm":y_fwhm,
"x_center_of_mass":x_center_of_mass,
"y_center_of_mass":y_center_of_mass,
"intensity":intensity,
"max_value":max_value,
"width":width,
"height":height
}
return dataout
def test_a_quick_start(self):
from cam_server import PipelineClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Create a pipeline client.
client = PipelineClient()
# ADDITIONAL, FOR THE TEST
camera_name = "simulation"
client = self.pipeline_client
client.create_instance_from_config(
configuration={"camera_name": camera_name},
instance_id=camera_name + "_sp1")
# Define the camera name you want to read. This should be the same camera you are streaming in screen panel.
camera_name = "simulation"
# Format of the instance id when screen_panel creates a pipeline.
pipeline_instance_id = camera_name + "_sp1"
# Get the stream for the pipelie instance.
stream_address = client.get_instance_stream(pipeline_instance_id)
# Extract the stream host and port from the stream_address.
stream_host, stream_port = get_host_port_from_stream_address(
stream_address)
# Open connection to the stream. When exiting the 'with' section, the source disconnects by itself.
with source(host=stream_host, port=stream_port,
mode=SUB) as input_stream:
input_stream.connect()
# Read one message.
message = input_stream.receive()
# Print out the received stream data - dictionary.
print("Dictionary with data:\n", message.data.data)
# Print out the X center of mass.
print("X center of mass: ",
message.data.data["x_center_of_mass"].value)
def test_save_camera_stream_to_h5(self):
from bsread import h5, SUB
camera_name = "simulation"
file_name = "testing_dump.h5"
n_messages = 10
# Change to match your camera server
server_address = "http://0.0.0.0:8889"
client = PipelineClient(server_address)
instance_id, stream_address = client.create_instance_from_config(
{"camera_name": camera_name})
# The output file 'output.h5' has 10 images from the simulation camera stream.
h5.receive(source=stream_address,
file_name=file_name,
mode=SUB,
n_messages=n_messages)
self.assertTrue(os.path.exists(file_name))
def test_modify_pipeline_config(self):
self.pipeline_client.save_pipeline_config(
"test_pipeline",
self.pipeline_client.get_pipeline_config("pipeline_example_1"))
from cam_server import PipelineClient
# Initialize the pipeline client.
pipeline_client = PipelineClient()
pipeline_client = self.pipeline_client
# Print the list of available pipelines.
print(pipeline_client.get_pipelines())
# Put the name of the pipeline you want to modify.
pipeline_to_modify = "test_pipeline"
# Retrieve the camera config.
pipeline_config = pipeline_client.get_pipeline_config(
pipeline_to_modify)
# Change the image threshold.
pipeline_config["image_threshold"] = 0.5
# Change the image region of interest.
pipeline_config["image_region_of_interest"] = [0, 100, 0, 100]
# Save the camera configuration.
pipeline_client.save_pipeline_config(pipeline_to_modify,
pipeline_config)
# You can also save the same (or another) config under a different camera name.
pipeline_client.save_pipeline_config("pipeline_to_delete",
pipeline_config)
# And also delete camera configs.
pipeline_client.delete_pipeline_config("pipeline_to_delete")
def data_streaker_offset(streaker,
offset_range,
screen,
n_images,
dry_run,
beamline='Aramis'):
print(
'Start data_streaker_offset for streaker %s, screen %s, beamline %s, dry_run %s'
% (streaker, screen, beamline, dry_run))
meta_dict_1 = get_meta_data(screen)
pipeline_client = PipelineClient('http://sf-daqsync-01:8889/')
offset_pv = streaker + ':CENTER'
current_val = caget(offset_pv + '.RBV')
# Start from closer edge of scan
if abs(current_val - offset_range[0]) > abs(current_val -
offset_range[-1]):
offset_range = offset_range[::-1]
if dry_run:
screen = 'simulation'
writables = None
positioner = pyscan.TimePositioner(time_interval=(1.1 * n_images),
n_intervals=len(offset_range))
else:
positions = offset_range * 1e3 # convert to mm
positioner = pyscan.VectorPositioner(positions=positions.tolist())
writables = [
pyscan.epics_pv(pv_name=offset_pv,
readback_pv_name=offset_pv + '.RBV',
tolerance=0.005)
]
cam_instance_name = screen + '_sp1'
stream_address = pipeline_client.get_instance_stream(cam_instance_name)
stream_host, stream_port = get_host_port_from_stream_address(
stream_address)
# Configure bsread
pyscan.config.bs_default_host = stream_host
pyscan.config.bs_default_port = stream_port
logging.getLogger('mflow.mflow').setLevel(logging.ERROR)
settings = pyscan.scan_settings(settling_time=1,
n_measurements=n_images,
write_timeout=60)
readables = get_readables(beamline)
raw_output = pyscan.scan(positioner=positioner,
readables=readables,
settings=settings,
writables=writables)
result_dict = pyscan_result_to_dict(readables, raw_output, scrap_bs=True)
#import pdb; pdb.set_trace()
for ax in ['x_axis', 'y_axis']:
arr = result_dict[ax] * 1e-6
if len(arr.shape) == 3:
result_dict[ax + '_m'] = arr[0][0]
elif len(arr.shape) == 2:
result_dict[ax + '_m'] = arr[0]
else:
raise ValueError('Unexpected', len(arr.shape))
meta_dict_2 = get_meta_data(screen)
output = {
'pyscan_result': result_dict,
'streaker_offsets': offset_range,
'screen': screen,
'n_images': n_images,
'dry_run': dry_run,
'streaker': streaker,
'meta_data_begin': meta_dict_1,
'meta_data_end': meta_dict_2,
}
print('End data_streaker_offset')
return output
class CameraClientProxyTest(unittest.TestCase):
def setUp(self):
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.temp_folder = os.path.join(test_base_dir, "temp/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
require_folder(self.temp_folder)
self.host = "localhost"
self.cam_server_ports = [8880, 8881]
self.cam_manager_port = 8888
self.pipeline_server_ports = [8890, 8891]
self.pipeline_manager_port = 8889
self.cam_server_address = []
self.process_camserver = []
port_range = config.CAMERA_STREAM_PORT_RANGE
for p in self.cam_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.cam_server_address.append("http://%s:%s" % (self.host, p))
process = Process(target=start_camera_worker,
args=(self.host, p, self.user_scripts_folder,
None, port_range))
self.process_camserver.append(process)
process.start()
self.cam_proxy_host = "0.0.0.0"
self.process_camproxy = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port,
",".join(self.cam_server_address), self.cam_config_folder,
self.user_scripts_folder))
self.process_camproxy.start()
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_manager_port)
self.pipeline_server_address = []
self.process_pipelineserver = []
port_range = config.PIPELINE_STREAM_PORT_RANGE
for p in self.pipeline_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.pipeline_server_address.append("http://%s:%s" %
(self.host, p))
process = Process(target=start_pipeline_worker,
args=(self.host, p, self.temp_folder,
self.temp_folder, cam_server_proxy_address,
None, port_range))
self.process_pipelineserver.append(process)
process.start()
cfg = """{
"http://localhost:8890": {
"expanding": true
}, "http://localhost:8891": {
"cameras": [
"simulation3"
],
"expanding": false,
"instances": [
"simulation_sp:10123"
]
}
}"""
self.pipeline_proxy_process = Process(
target=start_pipeline_manager,
args=(self.host, self.pipeline_manager_port, cfg,
self.pipeline_config_folder, self.background_config_folder,
config.DEFAULT_BACKGROUND_FILES_DAYS_TO_LIVE,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
sleep(1.0) # Give it some time to start.
cam_server_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
self.cam_client = CamClient(cam_server_address)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_manager_port)
self.pipeline_client = PipelineClient(pipeline_server_address)
self.cam_proxy_client = ProxyClient(cam_server_address)
self.pipeline_proxy_client = ProxyClient(pipeline_server_address)
def tearDown(self):
test_cleanup([self.cam_client, self.pipeline_client], [
self.pipeline_proxy_process,
] + self.process_pipelineserver + [
self.process_camproxy,
] + self.process_camserver, [
self.background_config_folder, self.user_scripts_folder,
self.temp_folder
])
def test_manager(self):
#Creating instances from config
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name(
"simulation_sp")
pipeline_host_1, pipeline_port_1 = get_host_port_from_stream_address(
instance_stream_1)
self.assertEqual(pipeline_port_1, 10123)
self.pipeline_client.stop_instance(instance_id_1)
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_name(
"simulation_sp")
pipeline_host_2, pipeline_port_2 = get_host_port_from_stream_address(
instance_stream_2)
self.assertEqual(pipeline_host_1, pipeline_host_2)
self.assertEqual(pipeline_port_1, pipeline_port_2)
class TimetoolBerninaUSD(Assembly):
def __init__(
self,
name=None,
processing_pipeline="SARES20-CAMS142-M5_psen_db",
processing_instance="SARES20-CAMS142-M5_psen_db1",
spectrometer_camera_channel="SARES20-CAMS142-M5:FPICTURE",
delaystage_PV="SLAAR21-LMOT-M524:MOTOR_1",
pvname_mirror="SARES23-LIC9",
pvname_zoom="SARES20-MF1:MOT_8",
mirror_in=15,
mirror_out=-5,
):
super().__init__(name=name)
self.mirror_in_position = mirror_in
self.mirror_out_position = mirror_out
# Table 1, Benrina hutch
self._append(MotorRecord,
delaystage_PV,
name="delaystage_tt_usd",
is_setting=True)
self._append(DelayTime,
self.delaystage_tt_usd,
name="delay",
is_setting=True)
self.proc_client = PipelineClient()
self.proc_pipeline = processing_pipeline
self.proc_instance = processing_instance
self.spectrometer_camera_channel = spectrometer_camera_channel
self._append(
Target_xyz,
pvname_x="SARES20-MF2:MOT_1",
pvname_y="SARES20-MF2:MOT_2",
pvname_z="SARES20-MF2:MOT_3",
name="target_stages",
is_status="recursive",
)
self.target = self.target_stages.presets
# self._append(MotorRecord, "SARES20-MF2:MOT_1", name="x_target", is_setting=True)
# self._append(MotorRecord, "SARES20-MF2:MOT_2", name="y_target", is_setting=True)
# self._append(MotorRecord, "SARES20-MF2:MOT_3", name="z_target", is_setting=True)
self._append(MotorRecord,
"SARES20-MF2:MOT_4",
name="zoom_microscope",
is_setting=True)
self._append(
SmaractStreamdevice,
pvname_mirror,
name="x_mirror_microscope",
is_setting=True,
is_status=False,
)
self._append(
AdjustableVirtual,
[self.x_mirror_microscope],
lambda v: abs(v - self.mirror_in_position) < 0.003,
lambda v: self.mirror_in_position
if v else self.mirror_out_position,
name="mirror_in",
is_setting=True,
is_status=True,
)
self._append(
CameraBasler,
"SARES20-PROF141-M1",
name="camera_microscope",
is_setting=True,
is_status=False,
)
self._append(MotorRecord,
pvname_zoom,
name="zoom",
is_setting=True,
is_status=True)
self._append(
CameraPCO,
"SARES20-CAMS142-M5",
name="camera_spectrometer",
is_setting=True,
is_status=False,
)
self._append(
PvRecord,
pvsetname="SLAAR21-LMNP-ESBIR11:DRIVE",
pvreadbackname="SLAAR21-LMNP-ESBIR11:MOTRBV",
name="las_in_ry",
accuracy=10,
is_setting=True,
)
self._append(
PvRecord,
pvsetname="SLAAR21-LMNP-ESBIR12:DRIVE",
pvreadbackname="SLAAR21-LMNP-ESBIR12:MOTRBV",
name="las_in_rx",
accuracy=10,
is_setting=True,
)
self._append(
PvRecord,
pvsetname="SLAAR21-LMNP-ESBIR13:DRIVE",
pvreadbackname="SLAAR21-LMNP-ESBIR13:MOTRBV",
name="las_out_rx",
accuracy=10,
is_setting=True,
)
self._append(
PvRecord,
pvsetname="SLAAR21-LMNP-ESBIR14:DRIVE",
pvreadbackname="SLAAR21-LMNP-ESBIR14:MOTRBV",
name="las_out_ry",
accuracy=10,
is_setting=True,
)
def get_online_data(self):
self.online_monitor = TtProcessor()
def start_online_monitor(self):
print(f"Starting online data acquisition ...")
self.get_online_data()
print(f"... done, waiting for data coming in ...")
sleep(5)
print(f"... done, starting online plot.")
self.online_monitor.plot_animation()
def get_proc_config(self):
return self.proc_client.get_pipeline_config(self.proc_pipeline)
def update_proc_config(self, cfg_dict):
cfg = self.get_proc_config()
cfg.update(cfg_dict)
self.proc_client.set_instance_config(self.proc_instance, cfg)
def acquire_and_plot_spectrometer_image(self, N_pulses=50):
with source(channels=[self.spectrometer_camera_channel]) as s:
im = []
while True:
m = s.receive()
tim = m.data.data[self.spectrometer_camera_channel]
if not tim:
continue
if len(im) > N_pulses:
break
im.append(tim.value)
im = np.asarray(im).mean(axis=0)
fig = plt.figure("bsen spectrometer pattern")
fig.clf()
ax = fig.add_subplot(111)
ax.imshow(im)
from cam_server import PipelineClient
client = PipelineClient()
instance_id = "SARES11-SPEC125-M2_psen_db1"
# New parameters to set.
# [offset_x, size_x, offset_y, size_y] - the offsets are calculated from the top left corner of the image.
parameters = {
"roi_signal": [300, 2048, 1150, 400],
"roi_background": [300, 2048, 600, 400],
}
client.set_instance_config(instance_id, parameters)
import pickle
import base64
from cam_server import PipelineClient
pipeline_client = PipelineClient("http://sf-daqsync-01:8889/")
bg = pipeline_client.get_latest_background("SARBD02-DSCR050")
image = pipeline_client.get_background_image_bytes(bg)
dtype = image["dtype"]
shape = image["shape"]
bytes = base64.b64decode(image["bytes"].encode())
with open('./bytes.pkl', 'wb') as f:
pickle.dump(bytes, f)
class CameraClientProxyTest(unittest.TestCase):
def setUp(self):
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.temp_folder = os.path.join(test_base_dir, "temp/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
require_folder(self.temp_folder)
self.host = "0.0.0.0"
self.cam_server_ports = [8880, 8881]
self.cam_manager_port = 8888
self.pipeline_server_ports = [8890, 8891]
self.pipeline_manager_port = 8889
self.cam_server_address = []
self.process_camserver = []
port_range = config.CAMERA_STREAM_PORT_RANGE
for p in self.cam_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.cam_server_address.append("http://%s:%s" % (self.host, p))
process = Process(target=start_camera_worker,
args=(self.host, p, self.user_scripts_folder,
None, port_range))
self.process_camserver.append(process)
process.start()
self.cam_proxy_host = "0.0.0.0"
self.process_camproxy = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port,
",".join(self.cam_server_address), self.cam_config_folder,
self.user_scripts_folder))
self.process_camproxy.start()
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_manager_port)
self.pipeline_server_address = []
self.process_pipelineserver = []
port_range = config.PIPELINE_STREAM_PORT_RANGE
for p in self.pipeline_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.pipeline_server_address.append("http://%s:%s" %
(self.host, p))
process = Process(target=start_pipeline_worker,
args=(self.host, p, self.temp_folder,
self.temp_folder, cam_server_proxy_address,
None, port_range))
self.process_pipelineserver.append(process)
process.start()
cfg = ",".join(self.pipeline_server_address)
self.pipeline_proxy_process = Process(
target=start_pipeline_manager,
args=(self.host, self.pipeline_manager_port, cfg,
self.pipeline_config_folder, self.background_config_folder,
config.DEFAULT_BACKGROUND_FILES_DAYS_TO_LIVE,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
sleep(1.0) # Give it some time to start.
cam_server_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
self.cam_client = CamClient(cam_server_address)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_manager_port)
self.pipeline_client = PipelineClient(pipeline_server_address)
self.cam_proxy_client = ProxyClient(cam_server_address)
self.pipeline_proxy_client = ProxyClient(pipeline_server_address)
def tearDown(self):
test_cleanup([self.cam_client, self.pipeline_client], [
self.pipeline_proxy_process,
] + self.process_pipelineserver + [
self.process_camproxy,
] + self.process_camserver, [
self.background_config_folder, self.user_scripts_folder,
self.temp_folder
])
def test_manager(self):
#Creating instances from name
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name(
"simulation_sp")
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_name(
"simulation2_sp")
print(instance_id_1, instance_stream_1)
print(instance_id_2, instance_stream_2)
# Check if streams are alive
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_1)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
#check client
server_info = self.cam_proxy_client.get_servers_info()
status_info = self.cam_proxy_client.get_status_info()
instance_info = self.cam_proxy_client.get_instances_info()
#Check if camera streams are equally distributed
self.assertEqual(server_info[self.cam_server_address[0]]["load"], 1)
self.assertEqual(server_info[self.cam_server_address[1]]["load"], 1)
server_info = self.pipeline_proxy_client.get_servers_info()
status_info = self.pipeline_proxy_client.get_status_info()
instance_info = self.pipeline_proxy_client.get_instances_info()
print(server_info)
print(instance_info)
#Check if pipeline are equally distributed
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
1)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
1)
# Check if instance information is available for each server instance
for instance in server_info[
self.pipeline_server_address[0]]["instances"]:
self.assertIn(instance, instance_info)
for instance in server_info[
self.pipeline_server_address[1]]["instances"]:
self.assertIn(instance, instance_info)
#Test stopping instances
self.pipeline_client.stop_instance(instance_id_1)
self.pipeline_client.stop_instance(instance_id_2)
server_info = self.pipeline_proxy_client.get_servers_info()
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
0)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
0)
#Creating instances from config
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation2"})
print(instance_id_1, instance_stream_1)
print(instance_id_2, instance_stream_2)
# Check if streams are alive
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_1)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
server_info = self.pipeline_proxy_client.get_servers_info()
status_info = self.pipeline_proxy_client.get_status_info()
instance_info = self.pipeline_proxy_client.get_instances_info()
print(server_info)
print(instance_info)
#Check if pipeline are equally distributed
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
1)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
1)
# Check if instance information is available for each server instance
for instance in server_info[
self.pipeline_server_address[0]]["instances"]:
self.assertIn(instance, instance_info)
for instance in server_info[
self.pipeline_server_address[1]]["instances"]:
self.assertIn(instance, instance_info)
#Test stopping instances
self.pipeline_client.stop_instance(instance_id_1)
self.pipeline_client.stop_instance(instance_id_2)
server_info = self.pipeline_proxy_client.get_servers_info()
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
0)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
0)
#Server Config
self.cam_client.stop_all_instances()
self.assertEqual(
self.pipeline_proxy_client.get_config(), {
'http://0.0.0.0:8890': {
'expanding': True
},
'http://0.0.0.0:8891': {
'expanding': True
}
})
self.pipeline_proxy_client.set_config({
'http://0.0.0.0:8890': {
'expanding': True
},
'http://0.0.0.0:8891': {
"instances": ["DUMMY"],
'expanding': False
}
})
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation2"})
#Check if streams are alive
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_1)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
server_info = self.pipeline_proxy_client.get_servers_info()
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
2)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
0)
#Version
self.assertEqual(self.cam_client.get_version(), __VERSION__)
self.assertEqual(self.pipeline_client.get_version(), __VERSION__)
self.assertEqual(self.pipeline_proxy_client.get_version(), __VERSION__)
self.assertEqual(self.pipeline_proxy_client.get_version(), __VERSION__)
"""
def test_persisted_config(self):
cfg = self.pipeline_proxy_client.get_config()
#Server Config
self.assertEqual(cfg, {
"http://localhost:8889": {
"instances": [
"simulation_sp"
],
"cameras": [
"simulation"
],
"expanding": True
}
})
cfg["http://localhost:8889"]["expanding"] = False
self.pipeline_proxy_client.set_config(cfg)
self.assertEqual(self.pipeline_proxy_client.get_config(), {
"http://localhost:8889": {
"instances": [
"simulation_sp"
],
"cameras": [
"simulation"
],
"expanding": False
}
})
"""
def test_permanent_pipelines(self):
pp = self.pipeline_proxy_client.get_permanent_instances()
pp["cxx"] = "yyy"
pp["simulation_sp"] = "pp"
self.pipeline_proxy_client.set_permanent_instances(pp)
pp = self.pipeline_proxy_client.get_permanent_instances()
self.assertIn(("simulation_sp", "pp"), pp.items())
self.pipeline_proxy_client.set_permanent_instances({})
def test_file_dump_pipeline(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_dump", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5"})
print(instance_id_1, instance_stream_1)
time.sleep(1.0)
self.pipeline_client.stop_instance(instance_id_1)
def test_file_dump_pipeline2(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_dump", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5",
"layout":"FLAT", "localtime":False})
print(instance_id_1, instance_stream_1)
time.sleep(1.0)
self.pipeline_client.stop_instance(instance_id_1)
def test_pipeline_pid_range(self):
pids = list(range(5, 11))
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "pid_range":[pids[0], pids[-1]]})
print(instance_id_1, instance_stream_1)
self.pipeline_client.wait_instance_completed(instance_id_1)
with h5py.File(self.temp_folder + "data.h5", 'r') as file:
self.assertEqual(list(file['header1/pulse_id']), pids)
def test_pipeline_pid_range2(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "pid_range":[0, 0]})
print(instance_id_1, instance_stream_1)
time.sleep(0.5)
self.pipeline_client.set_instance_config(instance_id_1,
{"pid_range": [5, 0]})
time.sleep(0.5)
self.pipeline_client.set_instance_config(instance_id_1,
{"pid_range": [5, 6]})
self.pipeline_client.wait_instance_completed(instance_id_1)
def test_pause(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5"})
time.sleep(0.5)
self.pipeline_client.set_instance_config(instance_id_1,
{"pause": True})
time.sleep(0.5)
self.pipeline_client.set_instance_config(instance_id_1,
{"pause": False})
time.sleep(0.5)
def test_pipeline_records(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_records", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "records":12})
print(instance_id_1, instance_stream_1)
self.pipeline_client.wait_instance_completed(instance_id_1)
with h5py.File(self.temp_folder + "data.h5", 'r') as file:
self.assertEqual(len(file['header1/pulse_id']), 12)
def test_exit_code(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"no_client_timeout": 1000
})
self.pipeline_client.stop_instance(instance_id_1)
#When instance is stopped, it is deleted immediately, and there is no reference to the process
self.pipeline_client.get_instance_exit_code(instance_id_1)
exit_code = self.pipeline_client.get_instance_exit_code(instance_id_1)
self.assertEqual(exit_code, None)
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "pid_range":[5, 10]})
with self.assertRaisesRegex(
ValueError, "Instance 'simulation_file_range' still running."):
self.pipeline_client.get_instance_exit_code(instance_id_1)
self.pipeline_client.wait_instance_completed(instance_id_1)
exit_code = self.pipeline_client.get_instance_exit_code(instance_id_1)
self.assertEqual(exit_code, 0)
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "rotation":{"angle":True, "order":6}})
self.pipeline_client.wait_instance_completed(instance_id_1)
exit_code = self.pipeline_client.get_instance_exit_code(instance_id_1)
self.assertEqual(exit_code, 2)
def get_pipelineclient():
global PIPELINE_CLIENT
if not PIPELINE_CLIENT:
PIPELINE_CLIENT = PipelineClient()
return PIPELINE_CLIENT
class SpatialEncoder:
def __init__(
self,
name=None,
reduction_client_address="http://sf-daqsync-02:12003/",
delay_stages={"spatial_tt": "SLAAR21-LMOT-M522:MOTOR_1"},
pipeline_id="SARES20-CAMS142-M4_psen_db1",
):
self.name = name
self.alias = Alias(name)
# append_object_to_object(self,MotorRecord,pvname+":MOTOR_X1",name='x_target')
# append_object_to_object(self,MotorRecord,pvname+":MOTOR_Y1",name='y_target')
if delay_stages:
for key, pv in delay_stages.items():
tname = "delay_" + key + "_stg"
append_object_to_object(self, MotorRecord, pv, name=tname)
append_object_to_object(self,
DelayTime,
self.__dict__[tname],
name="delay_" + key)
# self.delay = MotorRecord(self.Id + "-M424:MOT")
# self.delayTime = DelayStage(self.delay)
# self.data_reduction_client = PsenProcessingClient(address=reduction_client_address)
self._camera_server_client = PipelineClient()
self._camera_server_pipeline_id = pipeline_id
# @property
# def roi(self):
# return self.data_reduction_client.get_roi_signal()
# @roi.setter
# def roi(self,values):
# self.data_reduction_client.set_roi_signal(values)
# @property
# def roi_background(self):
# return self.data_reduction_client.get_roi_background()
# @roi_background.setter
# def roi_background(self,values):
# self.data_reduction_client.set_roi_background(values)
@property
def roi(self):
return self._camera_server_client.get_instance_config(
self._camera_server_pipeline_id)["roi_signal"]
@roi.setter
def roi(self, values):
self._camera_server_client.set_instance_config(
self._camera_server_pipeline_id, {"roi_signal": values})
@property
def roi_background(self):
return self._camera_server_client.get_instance_config(
self._camera_server_pipeline_id)["roi_background"]
@roi_background.setter
def roi_background(self, values):
self._camera_server_client.set_instance_config(
self._camera_server_pipeline_id, {"roi_background": values})
def __repr__(self):
s = [f"Status {self.name}"]
# s.append(str(self.x_target))
# s.append(str(self.y_target))
s.append(f"Data reduction is on")
s.append(f" roi {self.roi}")
s.append(f" roi_background {self.roi_background}")
return "\n".join(s)
def get_image(Device_params, numImg, angle=None):
if angle == None:
angle = 0
pipeline_client = PipelineClient()
# pipeline_config = {"camera_name": camName, "image_region_of_interest": ROI}
pipeline_config = {
"camera_name": Device_params['FullName'],
"rotation": angle
}
instance_id, pipeline_stream_address = pipeline_client.create_instance_from_config(
pipeline_config)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
pipeline_stream_address)
img = []
x_profile = []
y_profile = []
x_fit_gauss_function = []
y_fit_gauss_function = []
x_fit_mean = []
y_fit_mean = []
x_fit_standard_deviation = []
y_fit_standard_deviation = []
x_fwhm = []
y_fwhm = []
x_center_of_mass = []
y_center_of_mass = []
intensity = []
max_value = []
width = []
height = []
with source(host=pipeline_host, port=pipeline_port, mode=SUB) as stream:
for i in range(0, numImg):
data = stream.receive()
img.append(data.data.data["image"].value)
x_profile.append(data.data.data["x_profile"].value)
y_profile.append(data.data.data["y_profile"].value)
x_fit_gauss_function.append(
data.data.data["x_fit_gauss_function"].value)
y_fit_gauss_function.append(
data.data.data["y_fit_gauss_function"].value)
x_fwhm.append(data.data.data["x_fwhm"].value)
y_fwhm.append(data.data.data["y_fwhm"].value)
x_center_of_mass.append(data.data.data["x_center_of_mass"].value)
y_center_of_mass.append(data.data.data["y_center_of_mass"].value)
x_fit_mean.append(data.data.data["x_fit_mean"].value)
y_fit_mean.append(data.data.data["y_fit_mean"].value)
x_fit_standard_deviation.append(
data.data.data["x_fit_standard_deviation"].value)
y_fit_standard_deviation.append(
data.data.data["y_fit_standard_deviation"].value)
intensity.append(data.data.data["intensity"].value)
width.append(data.data.data["width"].value)
height.append(data.data.data["height"].value)
max_value.append(data.data.data["max_value"].value)
print(list(data.data.data.keys()))
img = np.asarray(img)
# Take metedata
PhotonEnergy = ep.caget('SARUN08-UIND030:FELPHOTENE.VAL')
PulseEnergy = ep.caget('SARFE10-PBPG050:PHOTON-ENERGY-PER-PULSE-US')
ATT053 = ep.caget('SARFE10-OATT053:TRANS_RB')
APU044_x_pos = ep.caget('SARFE10-OAPU044:MOTOR_X.RBV')
APU044_y_pos = ep.caget('SARFE10-OAPU044:MOTOR_Y.RBV')
APU044_w_pos = ep.caget('SARFE10-OAPU044:MOTOR_W.RBV')
APU044_h_pos = ep.caget('SARFE10-OAPU044:MOTOR_H.RBV')
dataout = {
"mean": img.mean(axis=0),
"image": img,
"x_profile": x_profile,
"y_profile": y_profile,
"x_fwhm": x_fwhm,
"y_fwhm": y_fwhm,
"x_center_of_mass": x_center_of_mass,
"y_center_of_mass": y_center_of_mass,
"intensity": intensity,
"max_value": max_value,
"width": width,
"height": height,
"camera_name": Device_params['FullName'],
"Bit_depth": Device_params['BitDepth'],
"PhotonEnergy": PhotonEnergy,
"PulseEnergy": PulseEnergy,
"x_fit_gauss_function": x_fit_gauss_function,
"y_fit_gauss_function": y_fit_gauss_function,
"x_fit_mean": x_fit_mean,
"y_fit_mean": y_fit_mean,
"x_fit_standard_deviation": x_fit_standard_deviation,
"y_fit_standard_deviation": y_fit_standard_deviation,
"x_axis": data.data.data["x_axis"].value,
"y_axis": data.data.data["y_axis"].value,
"APU044_x_pos": APU044_x_pos,
"APU044_y_pos": APU044_y_pos,
"APU044_w_pos": APU044_w_pos,
"APU044_h_pos": APU044_h_pos,
"ATT053": ATT053
}
return dataout
class PipelineClientTest(unittest.TestCase):
def setUp(self):
self.host = "0.0.0.0"
self.cam_port = 8880
self.pipeline_port = 8881
self.cam_manager_port = 8888
self.pipeline_manager_port = 8889
for port in self.cam_port, self.pipeline_port, self.cam_manager_port, self.pipeline_manager_port:
print("Port ", port, "avilable: ", is_port_available(port))
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.temp_folder = os.path.join(test_base_dir, "temp/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
require_folder(self.temp_folder)
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_manager_port)
self.cam_process = Process(target=start_camera_worker,
args=(self.host, self.cam_port,
self.user_scripts_folder))
self.cam_process.start()
self.pipeline_process = Process(
target=start_pipeline_worker,
args=(self.host, self.pipeline_port, self.temp_folder,
self.temp_folder, cam_server_proxy_address))
self.pipeline_process.start()
self.cam_proxy_process = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port, cam_server_address,
self.cam_config_folder, self.user_scripts_folder))
self.cam_proxy_process.start()
self.pipeline_proxy_process = Process(
target=start_pipeline_manager,
args=(self.host, self.pipeline_manager_port,
pipeline_server_address, self.pipeline_config_folder,
self.background_config_folder,
config.DEFAULT_BACKGROUND_FILES_DAYS_TO_LIVE,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
self.cam_client = CamClient(cam_server_proxy_address)
self.pipeline_client = PipelineClient(pipeline_server_proxy_address)
# Give it some time to start.
sleep(1.0) # Give it some time to start.
def tearDown(self):
test_cleanup([self.pipeline_client, self.cam_client], [
self.pipeline_proxy_process, self.cam_proxy_process,
self.pipeline_process, self.cam_process
], [
os.path.join(self.pipeline_config_folder, "testing_config.json"),
os.path.join(self.temp_folder, "Test.py"),
os.path.join(self.temp_folder, "Test2.py"),
os.path.join(self.user_scripts_folder, "Test.py"),
os.path.join(self.user_scripts_folder, "Test2.py"),
self.background_config_folder,
self.user_scripts_folder,
self.temp_folder,
])
def test_client(self):
expected_pipelines = [
"pipeline_example_1", "pipeline_example_2", "pipeline_example_3",
"pipeline_example_4"
]
for pipeline in expected_pipelines:
self.assertIn(pipeline, self.pipeline_client.get_pipelines(),
"Test config pipelines have changed?")
expected_pipelines = set(self.pipeline_client.get_pipelines())
camera_config = self.pipeline_client.get_pipeline_config(
"pipeline_example_4")
self.pipeline_client.save_pipeline_config("testing_config",
camera_config)
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.save_pipeline_config("testing_config", {})
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.save_pipeline_config("testing_config",
{"invalid": "config"})
with self.assertRaisesRegex(
ValueError,
"pipeline_type 'invalid' not present in mapping. Available"):
self.pipeline_client.save_pipeline_config(
"testing_config", {
"camera_name": "simulation",
"pipeline_type": "invalid"
})
expected_pipelines.add("testing_config")
self.assertSetEqual(set(self.pipeline_client.get_pipelines()),
expected_pipelines,
"Testing config was not added.")
stream_address_1 = self.pipeline_client.get_instance_stream(
"testing_config")
stream_address_2 = self.pipeline_client.get_instance_stream(
"testing_config")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 1,
"Instance not started or too many instances started.")
with self.assertRaisesRegex(ValueError,
"Config updates cannot be empty."):
self.pipeline_client.set_instance_config("testing_config", {})
with self.assertRaisesRegex(
ValueError, "Cannot set config on a read only instance."):
self.pipeline_client.set_instance_config(
"testing_config", {"camera_name": "simulation"})
self.assertEqual(stream_address_1, stream_address_2,
"Stream addresses should be equal.")
self.pipeline_client.stop_instance("testing_config")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 0,
"The instance should be stopped.")
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name(
"testing_config")
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_name(
"testing_config")
instance_info = self.pipeline_client.get_instance_info(instance_id_1)
self.assertTrue("last_start_time" in instance_info)
self.assertTrue("statistics" in instance_info)
with self.assertRaisesRegex(
ValueError,
"Cannot change the camera name on a running instance. "
"Stop the instance first."):
self.pipeline_client.set_instance_config(
instance_id_1, {"camera_name": "different_camera"})
self.pipeline_client.set_instance_config(
instance_id_2, {
"image_threshold": 99999,
"image_region_of_interest": [0, 200, 0, 200]
})
# Wait for the config update.
sleep(0.5)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data,
"This should really not happen anymore.")
# shape 200, 200 -> Account for the region of interest change.
self.assertTrue(
numpy.array_equal(data.data.data["image"].value,
numpy.zeros(shape=(200, 200))),
"Array should be all zeros, because of the threshold config.")
# Adjust width and height with the region of interest.
self.assertEqual(data.data.data["width"].value, 200,
"Region of interest not takes into account.")
self.assertEqual(data.data.data["height"].value, 200,
"Region of interest not takes into account.")
self.assertNotEqual(instance_id_1, instance_id_2,
"Instances should be different.")
self.assertNotEqual(instance_stream_1, instance_stream_2,
"Stream addresses should be different.")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 2,
"Two instances should be running.")
instance_stream_3 = self.pipeline_client.get_instance_stream(
instance_id_2)
self.assertEqual(instance_stream_2, instance_stream_3,
"Stream addresses should be equal.")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 2,
"Two instances should be running, get does not increase the number of instances."
)
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 4,
"Two new instances should be created.")
with self.assertRaisesRegex(
ValueError, "You must specify either the pipeline name or the "
"configuration for the pipeline."):
self.pipeline_client.create_instance_from_config({})
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.create_instance_from_config(
{"invalid": "config"})
background_id = self.pipeline_client.collect_background("simulation")
expected_background_file = os.path.join(self.background_config_folder,
background_id + ".npy")
# Unfortunately there is not way of knowing (from client side) how many images were collected.
background_id_2 = self.pipeline_client.collect_background(
"simulation", 5)
self.assertNotEqual(background_id, background_id_2,
"Background should be different.")
with self.assertRaisesRegex(Exception, "n_images must be a number."):
self.pipeline_client.collect_background("simulation",
"invalid_number")
pipeline_config = {
"camera_name": "simulation",
"background_image": background_id
}
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
pipeline_config)
instance_config = self.pipeline_client.get_instance_config(instance_id)
# We need to account for the expended config.
expected_config = {}
expected_config.update(PipelineConfig.DEFAULT_CONFIGURATION)
expected_config.update(pipeline_config)
self.assertDictEqual(expected_config, instance_config,
"Set and retrieved instances are not equal.")
instance_info = self.pipeline_client.get_instance_info(instance_id)
self.assertEqual(instance_info["instance_id"], instance_id,
"Requested and retireved instances are different.")
self.assertEqual(instance_info["stream_address"], stream_address,
"Different stream address.")
self.assertTrue(instance_info["is_stream_active"],
"Stream should be active.")
self.assertFalse(instance_info["read_only"],
"It should not be read only.")
self.assertEqual(instance_info["camera_name"], "simulation",
"Wrong camera name.")
self.assertDictEqual(instance_info["config"], expected_config,
"Config is not equal")
self.pipeline_client.stop_instance(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance '%s' does not exist." % instance_id):
self.pipeline_client.get_instance_info(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance 'invalid instance' does not exist."):
self.pipeline_client.get_instance_info("invalid instance")
self.assertTrue(os.path.exists(expected_background_file))
os.remove(expected_background_file)
self.pipeline_client.stop_all_instances()
self.assertTrue(
"testing_config" in self.pipeline_client.get_pipelines(),
"Pre requirement for next test.")
self.pipeline_client.delete_pipeline_config("testing_config")
self.assertFalse(
"testing_config" in self.pipeline_client.get_pipelines(),
"Pipeline should not exist anymore.")
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"image_threshold": 10
}, "custom_instance")
self.assertEqual(instance_id, "custom_instance",
"Custom instance not set properly.")
self.assertTrue(
self.pipeline_client.is_instance_running("custom_instance"),
"Instance with custom instance id not present.")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 10, "Config not set on custom instance id.")
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance if you want "
"to reuse the same instance_id."):
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"}, "custom_instance")
self.pipeline_client.stop_instance("custom_instance")
# The instance is now stopped, it should overwrite it.
self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"image_threshold": 20
}, "custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config(
"custom_instance")["image_threshold"], 20,
"Instance with custom id not overwritten.")
self.pipeline_client.save_pipeline_config("testing_config", {
"camera_name": "simulation",
"image_threshold": 30
})
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance if you want "
"to reuse the same instance_id."):
self.pipeline_client.create_instance_from_name(
"testing_config", "custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 20, "Instance should not have changed.")
data = self.pipeline_client.get_instance_message("custom_instance")
self.assertIsNotNone(data)
self.assertTrue("image" in data.data.data)
self.pipeline_client.stop_instance("custom_instance")
self.pipeline_client.create_instance_from_name("testing_config",
"custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 30, "Instance should have changed.")
background_1 = self.pipeline_client.collect_background("simulation")
background_2 = self.pipeline_client.collect_background("simulation")
self.assertNotEqual(background_1, background_2)
latest_background = self.pipeline_client.get_latest_background(
"simulation")
self.assertEqual(latest_background, background_2,
"Wrong background set as latest.")
with self.assertRaisesRegex(
ValueError,
"No background matches for the specified prefix 'does not exist'."
):
self.pipeline_client.get_latest_background("does not exist")
expected_cameras = [
'camera_example_1', 'camera_example_3', 'camera_example_2',
'camera_example_4', 'simulation', 'simulation2'
]
#self.assertEqual(set(self.pipeline_client.get_cameras()), set(expected_cameras),
# "Expected cameras not present.")
for camera in expected_cameras:
self.assertIn(camera, set(self.pipeline_client.get_cameras()),
"Not getting expected camera: " + camera)
configuration = {"camera_name": "simulation", "threshold": 50}
stream_address_1 = self.pipeline_client.get_instance_stream_from_config(
configuration)
stream_address_2 = self.pipeline_client.get_instance_stream_from_config(
configuration)
self.assertEqual(
stream_address_1, stream_address_2,
"Requesting the same config should give you the same instance.")
self.pipeline_client.stop_all_instances()
self.cam_client.stop_all_instances()
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"pipeline_type": "store",
'stream_port': 10000
})
with self.assertRaisesRegex(
ValueError, "Cannot get message from 'store' pipeline type."):
self.pipeline_client.get_instance_message(instance_id)
host, port = get_host_port_from_stream_address(stream_address)
with source(host=host, port=port, mode=PULL) as stream:
data = stream.receive()
self.assertIsNotNone(data)
self.assertEqual(
len(data.data.data), 1,
"Only the image should be present in the received data.")
self.assertTrue(
"simulation" + config.EPICS_PV_SUFFIX_IMAGE in data.data.data,
"Camera name should be used instead of 'image'.")
self.pipeline_client.stop_all_instances()
#Transparent pipeline
instance_id, instance_stream = self.pipeline_client.create_instance_from_name(
"simulation")
cfg = self.pipeline_client.get_instance_config(instance_id)
cfg["function"] = "transparent"
self.pipeline_client.set_instance_config(instance_id, cfg)
time.sleep(.5)
data = self.pipeline_client.get_instance_message(instance_id)
self.assertIsNotNone(data)
# Cam_server fields + processing_parameters
required_fields = set([
"image", "timestamp", "width", "height", "x_axis", "y_axis",
"processing_parameters"
])
self.assertSetEqual(required_fields, set(data.data.data.keys()),
"Bad transparent pipeline fields.")
self.pipeline_client.stop_all_instances()
time.sleep(1.0)
def test_background_file(self):
bg = self.pipeline_client.get_latest_background("simulation")
image = self.pipeline_client.get_background_image(bg)
self.assertGreater(len(image.content), 0)
image = self.pipeline_client.get_background_image_bytes(bg)
dtype = image["dtype"]
shape = image["shape"]
bytes = base64.b64decode(image["bytes"].encode())
x_size, y_size = get_simulated_camera().get_geometry()
self.assertEqual(shape, [y_size, x_size])
image_array = numpy.frombuffer(bytes, dtype=dtype).reshape(shape)
self.assertIsNotNone(image_array)
def test_user_scripts(self):
script_name = "Test.py"
script_content = "print('Hello world')"
self.pipeline_client.set_user_script(script_name, script_content)
scripts = self.pipeline_client.get_user_scripts()
self.assertIn(script_name, scripts)
ret = self.pipeline_client.get_user_script(script_name)
self.assertEqual(ret, script_content)
filename = "temp/Test2.py"
with open(filename, "w") as data_file:
data_file.write(script_content)
self.pipeline_client.upload_user_script(filename)
os.remove(filename)
self.pipeline_client.download_user_script(filename)
with open(filename, "r") as data_file:
ret = data_file.read()
self.assertEqual(ret, script_content)
script_content = """
from cam_server.pipeline.data_processing import functions, processor
def process_image(image, pulse_id, timestamp, x_axis, y_axis, parameters, bsdata):
ret = processor.process_image(image, pulse_id, timestamp, x_axis, y_axis, parameters, bsdata)
ret["average_value"] = float(ret["intensity"]) / len(ret["x_axis"]) / len(ret["y_axis"])
return ret
"""
filename = "temp/Test.py"
with open(filename, "w") as data_file:
data_file.write(script_content)
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
host, port = get_host_port_from_stream_address(stream_address)
with source(host=host, port=port, mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data,
"This should really not happen anymore.")
self.assertIsNotNone(data.data.data["width"].value)
self.assertIsNotNone(data.data.data["height"].value)
self.pipeline_client.set_function_script(instance_id, filename)
time.sleep(1.0)
with source(host=host, port=port, mode=SUB) as stream:
data = stream.receive()
print(data.data.data.keys())
self.assertIsNotNone(data.data.data["average_value"].value)
def __init__(
self,
name=None,
processing_pipeline="SARES20-CAMS142-M5_psen_db",
processing_instance="SARES20-CAMS142-M5_psen_db1",
spectrometer_camera_channel="SARES20-CAMS142-M5:FPICTURE",
delaystage_PV="SLAAR21-LMOT-M524:MOTOR_1",
pvname_mirror="SARES23-LIC9",
pvname_zoom="SARES20-MF1:MOT_8",
mirror_in=15,
mirror_out=-5,
):
super().__init__(name=name)
self.mirror_in_position = mirror_in
self.mirror_out_position = mirror_out
# Table 1, Benrina hutch
self._append(MotorRecord,
delaystage_PV,
name="delaystage_tt_usd",
is_setting=True)
self._append(DelayTime,
self.delaystage_tt_usd,
name="delay",
is_setting=True)
self.proc_client = PipelineClient()
self.proc_pipeline = processing_pipeline
self.proc_instance = processing_instance
self.spectrometer_camera_channel = spectrometer_camera_channel
self._append(
Target_xyz,
pvname_x="SARES20-MF2:MOT_1",
pvname_y="SARES20-MF2:MOT_2",
pvname_z="SARES20-MF2:MOT_3",
name="target_stages",
is_status="recursive",
)
self.target = self.target_stages.presets
# self._append(MotorRecord, "SARES20-MF2:MOT_1", name="x_target", is_setting=True)
# self._append(MotorRecord, "SARES20-MF2:MOT_2", name="y_target", is_setting=True)
# self._append(MotorRecord, "SARES20-MF2:MOT_3", name="z_target", is_setting=True)
self._append(MotorRecord,
"SARES20-MF2:MOT_4",
name="zoom_microscope",
is_setting=True)
self._append(
SmaractStreamdevice,
pvname_mirror,
name="x_mirror_microscope",
is_setting=True,
is_status=False,
)
self._append(
AdjustableVirtual,
[self.x_mirror_microscope],
lambda v: abs(v - self.mirror_in_position) < 0.003,
lambda v: self.mirror_in_position
if v else self.mirror_out_position,
name="mirror_in",
is_setting=True,
is_status=True,
)
self._append(
CameraBasler,
"SARES20-PROF141-M1",
name="camera_microscope",
is_setting=True,
is_status=False,
)
self._append(MotorRecord,
pvname_zoom,
name="zoom",
is_setting=True,
is_status=True)
self._append(
CameraPCO,
"SARES20-CAMS142-M5",
name="camera_spectrometer",
is_setting=True,
is_status=False,
)
self._append(
PvRecord,
pvsetname="SLAAR21-LMNP-ESBIR11:DRIVE",
pvreadbackname="SLAAR21-LMNP-ESBIR11:MOTRBV",
name="las_in_ry",
accuracy=10,
is_setting=True,
)
self._append(
PvRecord,
pvsetname="SLAAR21-LMNP-ESBIR12:DRIVE",
pvreadbackname="SLAAR21-LMNP-ESBIR12:MOTRBV",
name="las_in_rx",
accuracy=10,
is_setting=True,
)
self._append(
PvRecord,
pvsetname="SLAAR21-LMNP-ESBIR13:DRIVE",
pvreadbackname="SLAAR21-LMNP-ESBIR13:MOTRBV",
name="las_out_rx",
accuracy=10,
is_setting=True,
)
self._append(
PvRecord,
pvsetname="SLAAR21-LMNP-ESBIR14:DRIVE",
pvreadbackname="SLAAR21-LMNP-ESBIR14:MOTRBV",
name="las_out_ry",
accuracy=10,
is_setting=True,
)
class PipelineClientTest(unittest.TestCase):
def setUp(self):
self.host = "localhost"
self.cam_port = 8888
self.pipeline_port = 8889
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
self.cam_process = Process(target=start_camera_server,
args=(self.host, self.cam_port,
self.cam_config_folder, None))
self.cam_process.start()
self.pipeline_process = Process(
target=start_pipeline_server,
args=(self.host, self.pipeline_port, self.pipeline_config_folder,
self.background_config_folder, self.user_scripts_folder,
cam_server_address))
self.pipeline_process.start()
# Give it some time to start.
sleep(1)
self.cam_client = CamClient(cam_server_address)
self.pipeline_client = PipelineClient(pipeline_server_address)
def tearDown(self):
test_cleanup([self.pipeline_client, self.cam_client], [
self.pipeline_process, self.cam_process
], [
os.path.join(self.pipeline_config_folder, "testing_config.json"),
os.path.join(self.pipeline_config_folder, "test_pipeline.json"),
os.path.join(self.cam_config_folder, "test_camera.json"),
"testing_dump.h5",
self.background_config_folder,
self.user_scripts_folder,
])
def test_a_quick_start(self):
from cam_server import PipelineClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Create a pipeline client.
client = PipelineClient()
# ADDITIONAL, FOR THE TEST
camera_name = "simulation"
client = self.pipeline_client
client.create_instance_from_config(
configuration={"camera_name": camera_name},
instance_id=camera_name + "_sp1")
# Define the camera name you want to read. This should be the same camera you are streaming in screen panel.
camera_name = "simulation"
# Format of the instance id when screen_panel creates a pipeline.
pipeline_instance_id = camera_name + "_sp1"
# Get the stream for the pipelie instance.
stream_address = client.get_instance_stream(pipeline_instance_id)
# Extract the stream host and port from the stream_address.
stream_host, stream_port = get_host_port_from_stream_address(
stream_address)
# Open connection to the stream. When exiting the 'with' section, the source disconnects by itself.
with source(host=stream_host, port=stream_port,
mode=SUB) as input_stream:
input_stream.connect()
# Read one message.
message = input_stream.receive()
# Print out the received stream data - dictionary.
print("Dictionary with data:\n", message.data.data)
# Print out the X center of mass.
print("X center of mass: ",
message.data.data["x_center_of_mass"].value)
def test_get_simulated_camera(self):
from cam_server import CamClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Change to match your camera server
server_address = "http://0.0.0.0:8888"
# Initialize the client.
camera_client = CamClient(server_address)
# Get stream address of simulation camera. Stream address in format tcp://hostname:port.
camera_stream_address = camera_client.get_instance_stream("simulation")
# Extract the stream hostname and port from the stream address.
camera_host, camera_port = get_host_port_from_stream_address(
camera_stream_address)
# Subscribe to the stream.
with source(host=camera_host, port=camera_port, mode=SUB) as stream:
# Receive next message.
data = stream.receive()
image_width = data.data.data["width"].value
image_height = data.data.data["height"].value
image_bytes = data.data.data["image"].value
print("Image size: %d x %d" % (image_width, image_height))
print("Image data: %s" % image_bytes)
x_size, y_size = get_simulated_camera().get_geometry()
self.assertIsNotNone(data)
self.assertEqual(image_width, x_size)
self.assertIsNotNone(image_height, y_size)
def test_get_basic_pipeline_with_simulated_camera(self):
from cam_server import PipelineClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Change to match your pipeline server
server_address = "http://0.0.0.0:8889"
# Initialize the client.
pipeline_client = PipelineClient(server_address)
# Setup the pipeline config. Use the simulation camera as the pipeline source.
pipeline_config = {"camera_name": "simulation"}
# Create a new pipeline with the provided configuration. Stream address in format tcp://hostname:port.
instance_id, pipeline_stream_address = pipeline_client.create_instance_from_config(
pipeline_config)
# Extract the stream hostname and port from the stream address.
pipeline_host, pipeline_port = get_host_port_from_stream_address(
pipeline_stream_address)
# Subscribe to the stream.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
# Receive next message.
data = stream.receive()
image_width = data.data.data["width"].value
image_height = data.data.data["height"].value
image_bytes = data.data.data["image"].value
print("Image size: %d x %d" % (image_width, image_height))
print("Image data: %s" % image_bytes)
x_size, y_size = get_simulated_camera().get_geometry()
self.assertIsNotNone(data)
self.assertEqual(image_width, x_size)
self.assertIsNotNone(image_height, y_size)
def test_create_pipeline_with_background(self):
from cam_server import PipelineClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Change to match your pipeline server
server_address = "http://0.0.0.0:8889"
camera_name = "simulation"
# Initialize the client.
pipeline_client = PipelineClient(server_address)
# Collect the background for the given camera.
background_id = pipeline_client.collect_background(camera_name)
# Setup the pipeline config. Use the simulation camera as the pipeline source, and the collected background.
pipeline_config = {
"camera_name": camera_name,
"background_id": background_id
}
# Create a new pipeline with the provided configuration. Stream address in format tcp://hostname:port.
instance_id, pipeline_stream_address = pipeline_client.create_instance_from_config(
pipeline_config)
# Extract the stream hostname and port from the stream address.
pipeline_host, pipeline_port = get_host_port_from_stream_address(
pipeline_stream_address)
# Subscribe to the stream.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
# Receive next message.
data = stream.receive()
image_width = data.data.data["width"].value
image_height = data.data.data["height"].value
image_bytes = data.data.data["image"].value
print("Image size: %d x %d" % (image_width, image_height))
print("Image data: %s" % image_bytes)
x_size, y_size = get_simulated_camera().get_geometry()
self.assertIsNotNone(data)
self.assertEqual(image_width, x_size)
self.assertIsNotNone(image_height, y_size)
def test_modify_camera_config(self):
self.cam_client.set_camera_config(
"test_camera",
self.cam_client.get_camera_config("camera_example_2"))
from cam_server import CamClient
# Initialize the camera client.
cam_client = CamClient()
cam_client = self.cam_client
# Print the list of available cameras.
print(cam_client.get_cameras())
# Put the name of the camera you want to modify.
camera_to_modify = "test_camera"
# Retrieve the camera config.
camera_config = cam_client.get_camera_config(camera_to_modify)
# Change the mirror_x setting.
camera_config["mirror_x"] = False
# Change the camera_calibration setting.
camera_config["camera_calibration"] = {
"reference_marker": [0, 0, 100, 100],
"reference_marker_width": 100.0,
"reference_marker_height": 100.0,
"angle_horizontal": 0.0,
"angle_vertical": 0.0
}
# Save the camera configuration.
cam_client.set_camera_config(camera_to_modify, camera_config)
# You can also save the same (or another) config under a different camera name.
cam_client.set_camera_config("camera_to_delete", camera_config)
# And also delete camera configs.
cam_client.delete_camera_config("camera_to_delete")
def test_modify_pipeline_config(self):
self.pipeline_client.save_pipeline_config(
"test_pipeline",
self.pipeline_client.get_pipeline_config("pipeline_example_1"))
from cam_server import PipelineClient
# Initialize the pipeline client.
pipeline_client = PipelineClient()
pipeline_client = self.pipeline_client
# Print the list of available pipelines.
print(pipeline_client.get_pipelines())
# Put the name of the pipeline you want to modify.
pipeline_to_modify = "test_pipeline"
# Retrieve the camera config.
pipeline_config = pipeline_client.get_pipeline_config(
pipeline_to_modify)
# Change the image threshold.
pipeline_config["image_threshold"] = 0.5
# Change the image region of interest.
pipeline_config["image_region_of_interest"] = [0, 100, 0, 100]
# Save the camera configuration.
pipeline_client.save_pipeline_config(pipeline_to_modify,
pipeline_config)
# You can also save the same (or another) config under a different camera name.
pipeline_client.save_pipeline_config("pipeline_to_delete",
pipeline_config)
# And also delete camera configs.
pipeline_client.delete_pipeline_config("pipeline_to_delete")
def test_create_camera(self):
# Specify the desired camera config.
camera_config = {
"name": "camera_example_3",
"source": "EPICS:CAM1:EXAMPLE",
"source_type": "epics",
"mirror_x": False,
"mirror_y": False,
"rotate": 0,
"camera_calibration": {
"reference_marker": [0, 0, 100, 100],
"reference_marker_width": 100.0,
"reference_marker_height": 100.0,
"angle_horizontal": 0.0,
"angle_vertical": 0.0
}
}
# Specify the new camera name.
new_camera_name = "new_camera_name"
# Save the camera configuration.
self.cam_client.set_camera_config(new_camera_name, camera_config)
self.assertTrue(new_camera_name in self.cam_client.get_cameras())
# Delete the camera config you just added.
self.cam_client.delete_camera_config(new_camera_name)
self.assertTrue(new_camera_name not in self.cam_client.get_cameras())
def get_single_message(self):
self.pipeline_client.create_instance_from_name("pipeline_example_1",
"simulation_sp1")
# Name of the camera we want to get a message from.
camera_name = "simulation"
# Screen panel defines the instance name as: [CAMERA_NAME]_sp1
instance_name = camera_name + "_sp1"
# Get the data.
data = self.pipeline_client.get_instance_message(instance_name)
self.assertIsNotNone(data)
self.assertTrue("image" in data.data.data)
def test_save_camera_stream_to_h5(self):
from bsread import h5, SUB
camera_name = "simulation"
file_name = "testing_dump.h5"
n_messages = 10
# Change to match your camera server
server_address = "http://0.0.0.0:8889"
client = PipelineClient(server_address)
instance_id, stream_address = client.create_instance_from_config(
{"camera_name": camera_name})
# The output file 'output.h5' has 10 images from the simulation camera stream.
h5.receive(source=stream_address,
file_name=file_name,
mode=SUB,
n_messages=n_messages)
self.assertTrue(os.path.exists(file_name))
def setUp(self):
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.temp_folder = os.path.join(test_base_dir, "temp/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
require_folder(self.temp_folder)
self.host = "localhost"
self.cam_server_ports = [8880, 8881]
self.cam_manager_port = 8888
self.pipeline_server_ports = [8890, 8891]
self.pipeline_manager_port = 8889
self.cam_server_address = []
self.process_camserver = []
port_range = config.CAMERA_STREAM_PORT_RANGE
for p in self.cam_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.cam_server_address.append("http://%s:%s" % (self.host, p))
process = Process(target=start_camera_worker,
args=(self.host, p, self.user_scripts_folder,
None, port_range))
self.process_camserver.append(process)
process.start()
self.cam_proxy_host = "0.0.0.0"
self.process_camproxy = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port,
",".join(self.cam_server_address), self.cam_config_folder,
self.user_scripts_folder))
self.process_camproxy.start()
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_manager_port)
self.pipeline_server_address = []
self.process_pipelineserver = []
port_range = config.PIPELINE_STREAM_PORT_RANGE
for p in self.pipeline_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.pipeline_server_address.append("http://%s:%s" %
(self.host, p))
process = Process(target=start_pipeline_worker,
args=(self.host, p, self.temp_folder,
self.temp_folder, cam_server_proxy_address,
None, port_range))
self.process_pipelineserver.append(process)
process.start()
cfg = """{
"http://localhost:8890": {
"expanding": true
}, "http://localhost:8891": {
"cameras": [
"simulation3"
],
"expanding": false,
"instances": [
"simulation_sp:10123"
]
}
}"""
self.pipeline_proxy_process = Process(
target=start_pipeline_manager,
args=(self.host, self.pipeline_manager_port, cfg,
self.pipeline_config_folder, self.background_config_folder,
config.DEFAULT_BACKGROUND_FILES_DAYS_TO_LIVE,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
sleep(1.0) # Give it some time to start.
cam_server_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
self.cam_client = CamClient(cam_server_address)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_manager_port)
self.pipeline_client = PipelineClient(pipeline_server_address)
self.cam_proxy_client = ProxyClient(cam_server_address)
self.pipeline_proxy_client = ProxyClient(pipeline_server_address)
class PipelineClientTest(unittest.TestCase):
def setUp(self):
self.host = "0.0.0.0"
self.cam_port = 8888
self.pipeline_port = 8889
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
self.cam_process = Process(target=start_camera_server,
args=(self.host, self.cam_port,
self.cam_config_folder))
self.cam_process.start()
self.pipeline_process = Process(
target=start_pipeline_server,
args=(self.host, self.pipeline_port, self.pipeline_config_folder,
self.background_config_folder, cam_server_address))
self.pipeline_process.start()
self.cam_client = CamClient(cam_server_address)
self.pipeline_client = PipelineClient(pipeline_server_address)
# Give it some time to start.
sleep(1)
def tearDown(self):
os.kill(self.cam_process.pid, signal.SIGINT)
os.kill(self.pipeline_process.pid, signal.SIGINT)
self.cam_process.join()
self.pipeline_process.join()
try:
os.remove(
os.path.join(self.pipeline_config_folder,
"testing_config.json"))
except:
pass
# Wait for the server to die.
sleep(1)
def test_client(self):
expected_pipelines = set([
"pipeline_example_1", "pipeline_example_2", "pipeline_example_3",
"pipeline_example_4"
])
self.assertSetEqual(set(self.pipeline_client.get_pipelines()),
expected_pipelines,
"Test config pipelines have changed?")
camera_config = self.pipeline_client.get_pipeline_config(
"pipeline_example_4")
self.pipeline_client.save_pipeline_config("testing_config",
camera_config)
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.save_pipeline_config("testing_config", {})
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.save_pipeline_config("testing_config",
{"invalid": "config"})
with self.assertRaisesRegex(
ValueError,
"pipeline_type 'invalid' not present in mapping. Available"):
self.pipeline_client.save_pipeline_config(
"testing_config", {
"camera_name": "simulation",
"pipeline_type": "invalid"
})
expected_pipelines.add("testing_config")
self.assertSetEqual(set(self.pipeline_client.get_pipelines()),
expected_pipelines,
"Testing config was not added.")
stream_address_1 = self.pipeline_client.get_instance_stream(
"testing_config")
stream_address_2 = self.pipeline_client.get_instance_stream(
"testing_config")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 1,
"Instance not started or too many instances started.")
with self.assertRaisesRegex(ValueError,
"Config updates cannot be empty."):
self.pipeline_client.set_instance_config("testing_config", {})
with self.assertRaisesRegex(
ValueError, "Cannot set config on a read only instance."):
self.pipeline_client.set_instance_config(
"testing_config", {"camera_name": "simulation"})
self.assertEqual(stream_address_1, stream_address_2,
"Stream addresses should be equal.")
self.pipeline_client.stop_instance("testing_config")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 0,
"The instance should be stopped.")
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name(
"testing_config")
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_name(
"testing_config")
instance_info = self.pipeline_client.get_instance_info(instance_id_1)
self.assertTrue("last_start_time" in instance_info)
self.assertTrue("statistics" in instance_info)
with self.assertRaisesRegex(
ValueError,
"Cannot change the camera name on a running instance. "
"Stop the instance first."):
self.pipeline_client.set_instance_config(
instance_id_1, {"camera_name": "different_camera"})
self.pipeline_client.set_instance_config(
instance_id_2, {
"image_threshold": 99999,
"image_region_of_interest": [0, 200, 0, 200]
})
# Wait for the config update.
sleep(0.5)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data,
"This should really not happen anymore.")
# shape 200, 200 -> Account for the region of interest change.
self.assertTrue(
numpy.array_equal(data.data.data["image"].value,
numpy.zeros(shape=(200, 200))),
"Array should be all zeros, because of the threshold config.")
# Adjust width and height with the region of interest.
self.assertEqual(data.data.data["width"].value, 200,
"Region of interest not takes into account.")
self.assertEqual(data.data.data["height"].value, 200,
"Region of interest not takes into account.")
self.assertNotEqual(instance_id_1, instance_id_2,
"Instances should be different.")
self.assertNotEqual(instance_stream_1, instance_stream_2,
"Stream addresses should be different.")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 2,
"Two instances should be running.")
instance_stream_3 = self.pipeline_client.get_instance_stream(
instance_id_2)
self.assertEqual(instance_stream_2, instance_stream_3,
"Stream addresses should be equal.")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 2,
"Two instances should be running, get does not increase the number of instances."
)
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 4,
"Two new instances should be created.")
with self.assertRaisesRegex(
ValueError, "You must specify either the pipeline name or the "
"configuration for the pipeline."):
self.pipeline_client.create_instance_from_config({})
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.create_instance_from_config(
{"invalid": "config"})
background_id = self.pipeline_client.collect_background("simulation")
expected_background_file = os.path.join(self.background_config_folder,
background_id + ".npy")
# Unfortunately there is not way of knowing (from client side) how many images were collected.
background_id_2 = self.pipeline_client.collect_background(
"simulation", 5)
self.assertNotEqual(background_id, background_id_2,
"Background should be different.")
with self.assertRaisesRegex(Exception, "n_images must be a number."):
self.pipeline_client.collect_background("simulation",
"invalid_number")
pipeline_config = {
"camera_name": "simulation",
"background_image": background_id
}
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
pipeline_config)
instance_config = self.pipeline_client.get_instance_config(instance_id)
# We need to account for the expended config.
expected_config = {}
expected_config.update(PipelineConfig.DEFAULT_CONFIGURATION)
expected_config.update(pipeline_config)
self.assertDictEqual(expected_config, instance_config,
"Set and retrieved instances are not equal.")
instance_info = self.pipeline_client.get_instance_info(instance_id)
self.assertEqual(instance_info["instance_id"], instance_id,
"Requested and retireved instances are different.")
self.assertEqual(instance_info["stream_address"], stream_address,
"Different stream address.")
self.assertTrue(instance_info["is_stream_active"],
"Stream should be active.")
self.assertFalse(instance_info["read_only"],
"It should not be read only.")
self.assertEqual(instance_info["camera_name"], "simulation",
"Wrong camera name.")
self.assertDictEqual(instance_info["config"], expected_config,
"Config is not equal")
self.pipeline_client.stop_instance(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance '%s' does not exist." % instance_id):
self.pipeline_client.get_instance_info(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance 'invalid instance' does not exist."):
self.pipeline_client.get_instance_info("invalid instance")
self.assertTrue(os.path.exists(expected_background_file))
os.remove(expected_background_file)
self.pipeline_client.stop_all_instances()
self.assertTrue(
"testing_config" in self.pipeline_client.get_pipelines(),
"Pre requirement for next test.")
self.pipeline_client.delete_pipeline_config("testing_config")
self.assertFalse(
"testing_config" in self.pipeline_client.get_pipelines(),
"Pipeline should not exist anymore.")
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"image_threshold": 10
}, "custom_instance")
self.assertEqual(instance_id, "custom_instance",
"Custom instance not set properly.")
self.assertTrue(
"custom_instance"
in self.pipeline_client.get_server_info()["active_instances"],
"Instance with custom instance id not present.")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 10, "Config not set on custom instance id.")
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance if you want "
"to reuse the same instance_id."):
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"}, "custom_instance")
self.pipeline_client.stop_instance("custom_instance")
# The instance is now stopped, it should overwrite it.
self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"image_threshold": 20
}, "custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config(
"custom_instance")["image_threshold"], 20,
"Instance with custom id not overwritten.")
self.pipeline_client.save_pipeline_config("testing_config", {
"camera_name": "simulation",
"image_threshold": 30
})
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance if you want "
"to reuse the same instance_id."):
self.pipeline_client.create_instance_from_name(
"testing_config", "custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 20, "Instance should not have changed.")
data = self.pipeline_client.get_instance_message("custom_instance")
self.assertIsNotNone(data)
self.assertTrue("image" in data.data.data)
self.pipeline_client.stop_instance("custom_instance")
self.pipeline_client.create_instance_from_name("testing_config",
"custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 30, "Instance should have changed.")
background_1 = self.pipeline_client.collect_background("simulation")
background_2 = self.pipeline_client.collect_background("simulation")
self.assertNotEqual(background_1, background_2)
latest_background = self.pipeline_client.get_latest_background(
"simulation")
self.assertEqual(latest_background, background_2,
"Wrong background set as latest.")
with self.assertRaisesRegex(
ValueError,
"No background matches for the specified prefix 'does not exist'."
):
self.pipeline_client.get_latest_background("does not exist")
expected_cameras = [
'camera_example_1', 'camera_example_3', 'camera_example_2',
'camera_example_4', 'simulation'
]
self.assertEqual(set(self.pipeline_client.get_cameras()),
set(expected_cameras),
"Expected cameras not present.")
configuration = {"camera_name": "simulation", "threshold": 50}
stream_address_1 = self.pipeline_client.get_instance_stream_from_config(
configuration)
stream_address_2 = self.pipeline_client.get_instance_stream_from_config(
configuration)
self.assertEqual(
stream_address_1, stream_address_2,
"Requesting the same config should give you the same instance.")
self.pipeline_client.stop_all_instances()
self.cam_client.stop_all_cameras()
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"pipeline_type": "store"
})
with self.assertRaisesRegex(
ValueError, "Cannot get message from 'store' pipeline type."):
self.pipeline_client.get_instance_message(instance_id)
host, port = get_host_port_from_stream_address(stream_address)
with source(host=host, port=port, mode=PULL) as stream:
data = stream.receive()
self.assertIsNotNone(data)
self.assertEqual(
len(data.data.data), 1,
"Only the image should be present in the received data.")
self.assertTrue("simulation" in data.data.data,
"Camera name should be used instead of 'image'.")
self.pipeline_client.stop_all_instances()
