def setUp(self):
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.host = "0.0.0.0"
self.cam_server_port = 8888
self.cam_proxy_port = 8898
cam_server_address = "http://%s:%s" % (self.host, self.cam_server_port)
self.process_camserver = Process(target=start_camera_server,
args=(self.host, self.cam_server_port,
self.config_folder,
self.user_scripts_folder))
self.process_camserver.start()
self.cam_proxy_host = "0.0.0.0"
self.process_camproxy = Process(target=start_camera_proxy,
args=(self.host, self.cam_proxy_port,
cam_server_address,
self.config_folder))
self.process_camproxy.start()
sleep(1.0) # Give it some time to start.
server_address = "http://%s:%s" % (self.host, self.cam_proxy_port)
self.client = CamClient(server_address)
def test_get_simulated_camera(self):
from cam_server import CamClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Change to match your camera server
server_address = "http://0.0.0.0:8888"
# Initialize the client.
camera_client = CamClient(server_address)
# Get stream address of simulation camera. Stream address in format tcp://hostname:port.
camera_stream_address = camera_client.get_instance_stream("simulation")
# Extract the stream hostname and port from the stream address.
camera_host, camera_port = get_host_port_from_stream_address(
camera_stream_address)
# Subscribe to the stream.
with source(host=camera_host, port=camera_port, mode=SUB) as stream:
# Receive next message.
data = stream.receive()
image_width = data.data.data["width"].value
image_height = data.data.data["height"].value
image_bytes = data.data.data["image"].value
print("Image size: %d x %d" % (image_width, image_height))
print("Image data: %s" % image_bytes)
x_size, y_size = get_simulated_camera().get_geometry()
self.assertIsNotNone(data)
self.assertEqual(image_width, x_size)
self.assertIsNotNone(image_height, y_size)
def setUp(self):
self.host = "0.0.0.0"
self.cam_port = 8888
self.pipeline_port = 8889
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
self.cam_process = Process(target=start_camera_server,
args=(self.host, self.cam_port,
self.cam_config_folder))
self.cam_process.start()
self.pipeline_process = Process(
target=start_pipeline_server,
args=(self.host, self.pipeline_port, self.pipeline_config_folder,
self.background_config_folder, cam_server_address))
self.pipeline_process.start()
# Give it some time to start.
sleep(1)
self.cam_client = CamClient(cam_server_address)
self.pipeline_client = PipelineClient(pipeline_server_address)
def setUp(self):
self.host = "0.0.0.0"
self.cam_port = 8888
self.pipeline_port = 8889
self.cam_proxy_port = 8898
self.pipeline_proxy_port = 8899
for port in self.cam_port, self.pipeline_port, self.cam_proxy_port, self.pipeline_proxy_port:
print("Port ", port, "avilable: ", is_port_available(port))
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_proxy_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_proxy_port)
self.cam_process = Process(target=start_camera_server,
args=(self.host, self.cam_port,
self.cam_config_folder,
self.user_scripts_folder))
self.cam_process.start()
self.cam_proxy_process = Process(target=start_camera_proxy,
args=(self.host, self.cam_proxy_port,
cam_server_address,
self.cam_config_folder))
self.cam_proxy_process.start()
self.pipeline_process = Process(
target=start_pipeline_server,
args=(self.host, self.pipeline_port, self.pipeline_config_folder,
self.background_config_folder, self.user_scripts_folder,
cam_server_proxy_address))
self.pipeline_process.start()
self.pipeline_proxy_process = Process(
target=start_pipeline_proxy,
args=(self.host, self.pipeline_proxy_port, pipeline_server_address,
self.pipeline_config_folder, self.background_config_folder,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
self.cam_client = CamClient(cam_server_proxy_address)
self.pipeline_client = PipelineClient(pipeline_server_proxy_address)
# Give it some time to start.
sleep(1.0) # Give it some time to start.
def get_test_pipeline_manager_with_real_cam():
config_manager = PipelineConfigManager(config_provider=MockConfigStorage())
pipeline_instance_manager = PipelineInstanceManager(
config_manager, MockBackgroundManager(), None,
CamClient("http://0.0.0.0:8888"))
return pipeline_instance_manager
def setUp(self):
self.host = "0.0.0.0"
self.port = 8888
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.process = Process(target=start_camera_server,
args=(self.host, self.port, self.config_folder))
self.process.start()
# Give it some time to start.
sleep(0.5)
server_address = "http://%s:%s" % (self.host, self.port)
self.client = CamClient(server_address)
def start_record_server(host,
port,
config_directory,
cam_server_api_address,
hostname=None):
if not os.path.isdir(config_directory):
_logger.error("Configuration directory '%s' does not exist." %
config_directory)
exit(-1)
if hostname:
_logger.warning("Using custom hostname '%s'." % hostname)
# TODO: Do something with the hostname.
app = bottle.Bottle()
cam_client = CamClient(cam_server_api_address)
config_manager = CamRecordConfigManager(
config_provider=ConfigFileStorage(config_directory))
instance_manager = CamRecordInstanceManager(cam_client=cam_client,
config_manager=config_manager)
instance_manager.start_all_cameras()
register_rest_interface(app, instance_manager)
try:
bottle.run(app=app, host=host, port=port)
finally:
instance_manager.stop_all_cameras()
def setUp(self):
self.host = "127.0.0.1"
self.port = 8888
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.process = Process(target=start_camera_server,
args=(self.host, self.port, self.config_folder,
self.user_scripts_folder))
self.process.start()
# Give it some time to start.
sleep(1.0)
self.rest_api_endpoint = "http://%s:%s" % (self.host, self.port)
self.client = CamClient(self.rest_api_endpoint)
def start_pipeline_server(host,
port,
config_base,
background_base,
scripts_base,
cam_server_api_address,
hostname=None,
port_range=None,
web_server=config.DEFAULT_WEB_SERVER,
web_server_args={}):
# Check if config directory exists
if not os.path.isdir(config_base):
_logger.error("Configuration directory '%s' does not exist." %
config_base)
exit(-1)
if not os.path.isdir(background_base):
_logger.error("Background image directory '%s' does not exist." %
background_base)
exit(-1)
if not os.path.isdir(scripts_base):
_logger.error("Scripts directory '%s' does not exist." % scripts_base)
exit(-1)
if hostname:
_logger.warning("Using custom hostname '%s'." % hostname)
cam_server_client = CamClient(cam_server_api_address)
config_manager = PipelineConfigManager(
config_provider=ConfigFileStorage(config_base))
background_manager = BackgroundImageManager(background_base)
user_scripts_manager = UserScriptsManager(scripts_base)
pipeline_instance_manager = PipelineInstanceManager(config_manager,
background_manager,
user_scripts_manager,
cam_server_client,
hostname=hostname,
port_range=port_range)
app = bottle.Bottle()
register_pipeline_rest_interface(
app=app, instance_manager=pipeline_instance_manager)
try:
bottle.run(app=app,
server=validate_web_server(web_server),
host=host,
port=port,
**web_server_args)
finally:
# Close the external processor when terminating the web server.
pipeline_instance_manager.stop_all_instances()
def test_modify_camera_config(self):
self.cam_client.set_camera_config(
"test_camera",
self.cam_client.get_camera_config("camera_example_2"))
from cam_server import CamClient
# Initialize the camera client.
cam_client = CamClient()
cam_client = self.cam_client
# Print the list of available cameras.
print(cam_client.get_cameras())
# Put the name of the camera you want to modify.
camera_to_modify = "test_camera"
# Retrieve the camera config.
camera_config = cam_client.get_camera_config(camera_to_modify)
# Change the mirror_x setting.
camera_config["mirror_x"] = False
# Change the camera_calibration setting.
camera_config["camera_calibration"] = {
"reference_marker": [0, 0, 100, 100],
"reference_marker_width": 100.0,
"reference_marker_height": 100.0,
"angle_horizontal": 0.0,
"angle_vertical": 0.0
}
# Save the camera configuration.
cam_client.set_camera_config(camera_to_modify, camera_config)
# You can also save the same (or another) config under a different camera name.
cam_client.set_camera_config("camera_to_delete", camera_config)
# And also delete camera configs.
cam_client.delete_camera_config("camera_to_delete")
def get_axis(screen):
camera_client = CamClient()
camera_stream_adress = camera_client.get_instance_stream(screen)
host, port = get_host_port_from_stream_address(camera_stream_adress)
max_tries = 3
for _try in range(max_tries):
try:
with source(host=host, port=port, mode=SUB) as stream:
data = stream.receive()
print('Received image')
x_axis = np.array(data.data.data['x_axis'].value) * 1e-6
y_axis = np.array(data.data.data['y_axis'].value) * 1e-6
except Exception as e:
print('Try %i' % _try, e)
pass
else:
break
else:
raise
return x_axis, y_axis
def setUp(self):
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.host = "0.0.0.0"
self.cam_server_ports = [8880, 8881]
self.cam_manager_port = 8888
self.cam_server_address = []
self.process_camserver = []
port_range = config.CAMERA_STREAM_PORT_RANGE
for port in self.cam_server_ports:
print(port, is_port_available(port))
for p in self.cam_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.cam_server_address.append("http://%s:%s" % (self.host, p))
process = Process(target=start_camera_worker,
args=(self.host, p, self.user_scripts_folder,
None, port_range))
self.process_camserver.append(process)
process.start()
self.cam_proxy_host = "0.0.0.0"
self.process_camproxy = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port,
",".join(self.cam_server_address), self.config_folder,
self.user_scripts_folder))
self.process_camproxy.start()
sleep(1.0) # Give it some time to start.
server_address = "http://%s:%s" % (self.host, self.cam_manager_port)
self.client = CamClient(server_address)
self.proxy_client = ProxyClient(server_address)
def test_custom_hostname(self):
config_manager = PipelineConfigManager(
config_provider=MockConfigStorage())
pipeline_instance_manager = PipelineInstanceManager(
config_manager,
MockBackgroundManager(),
CamClient("http://0.0.0.0:8888"),
hostname="custom_cam_hostname")
_, stream_address = pipeline_instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
self.assertTrue(stream_address.startswith("tcp://custom_cam_hostname"))
pipeline_instance_manager.stop_all_instances()
def start_pipeline_proxy(host,
port,
server_config,
config_base,
background_base,
scripts_base,
cam_server_api_address,
web_server=config.DEFAULT_WEB_SERVER,
web_server_args={}):
# Check if config directory exists
if not os.path.isdir(config_base):
_logger.error("Configuration directory '%s' does not exist." %
config_base)
exit(-1)
if not os.path.isdir(background_base):
_logger.error("Background image directory '%s' does not exist." %
background_base)
exit(-1)
cam_server_client = CamClient(cam_server_api_address)
config_manager = PipelineConfigManager(
config_provider=ConfigFileStorage(config_base))
background_manager = BackgroundImageManager(background_base)
app = bottle.Bottle()
proxy = PipelineProxy(config_manager, background_manager,
cam_server_client, server_config)
register_pipeline_rest_interface(app=app, instance_manager=proxy)
proxy.register_rest_interface(app)
proxy.register_management_page(app)
try:
bottle.run(app=app,
server=validate_web_server(web_server),
host=host,
port=port,
**web_server_args)
finally:
#clenup
pass
def start_pipeline_manager(host, port, server_config, config_base, background_base, background_files_days_to_live,
scripts_base, cam_server_api_address, client_timeout=None, info_update_timeout=None,
web_server=config.DEFAULT_WEB_SERVER, web_server_args={}):
# Check if config directory exists
if not os.path.isdir(config_base):
_logger.error("Configuration directory '%s' does not exist." % config_base)
exit(-1)
if not os.path.isdir(background_base):
_logger.error("Background image directory '%s' does not exist." % background_base)
exit(-1)
if not os.path.isdir(scripts_base):
_logger.error("Scripts directory '%s' does not exist." % scripts_base)
exit(-1)
cam_server_client = CamClient(cam_server_api_address)
config_manager = PipelineConfigManager(config_provider=ConfigFileStorage(config_base))
background_manager = BackgroundImageManager(background_base)
user_scripts_manager = UserScriptsManager(scripts_base)
app = bottle.Bottle()
proxy = PipelineManager(config_manager, background_manager,user_scripts_manager,
cam_server_client, server_config, int(background_files_days_to_live),
float(client_timeout) if client_timeout else None,
float(info_update_timeout) if info_update_timeout else None)
register_pipeline_rest_interface(app=app, instance_manager=proxy)
proxy.register_rest_interface(app)
proxy.register_management_page(app)
try:
bottle.run(app=app, server=validate_web_server(web_server), host=host, port=port, **web_server_args)
finally:
#cleanup
pass
class PipelineClientTest(unittest.TestCase):
def setUp(self):
self.host = "0.0.0.0"
self.cam_port = 8888
self.pipeline_port = 8889
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
self.cam_process = Process(target=start_camera_server,
args=(self.host, self.cam_port,
self.cam_config_folder))
self.cam_process.start()
self.pipeline_process = Process(
target=start_pipeline_server,
args=(self.host, self.pipeline_port, self.pipeline_config_folder,
self.background_config_folder, cam_server_address))
self.pipeline_process.start()
self.cam_client = CamClient(cam_server_address)
self.pipeline_client = PipelineClient(pipeline_server_address)
# Give it some time to start.
sleep(1)
def tearDown(self):
os.kill(self.cam_process.pid, signal.SIGINT)
os.kill(self.pipeline_process.pid, signal.SIGINT)
self.cam_process.join()
self.pipeline_process.join()
try:
os.remove(
os.path.join(self.pipeline_config_folder,
"testing_config.json"))
except:
pass
# Wait for the server to die.
sleep(1)
def test_client(self):
expected_pipelines = set([
"pipeline_example_1", "pipeline_example_2", "pipeline_example_3",
"pipeline_example_4"
])
self.assertSetEqual(set(self.pipeline_client.get_pipelines()),
expected_pipelines,
"Test config pipelines have changed?")
camera_config = self.pipeline_client.get_pipeline_config(
"pipeline_example_4")
self.pipeline_client.save_pipeline_config("testing_config",
camera_config)
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.save_pipeline_config("testing_config", {})
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.save_pipeline_config("testing_config",
{"invalid": "config"})
with self.assertRaisesRegex(
ValueError,
"pipeline_type 'invalid' not present in mapping. Available"):
self.pipeline_client.save_pipeline_config(
"testing_config", {
"camera_name": "simulation",
"pipeline_type": "invalid"
})
expected_pipelines.add("testing_config")
self.assertSetEqual(set(self.pipeline_client.get_pipelines()),
expected_pipelines,
"Testing config was not added.")
stream_address_1 = self.pipeline_client.get_instance_stream(
"testing_config")
stream_address_2 = self.pipeline_client.get_instance_stream(
"testing_config")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 1,
"Instance not started or too many instances started.")
with self.assertRaisesRegex(ValueError,
"Config updates cannot be empty."):
self.pipeline_client.set_instance_config("testing_config", {})
with self.assertRaisesRegex(
ValueError, "Cannot set config on a read only instance."):
self.pipeline_client.set_instance_config(
"testing_config", {"camera_name": "simulation"})
self.assertEqual(stream_address_1, stream_address_2,
"Stream addresses should be equal.")
self.pipeline_client.stop_instance("testing_config")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 0,
"The instance should be stopped.")
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name(
"testing_config")
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_name(
"testing_config")
instance_info = self.pipeline_client.get_instance_info(instance_id_1)
self.assertTrue("last_start_time" in instance_info)
self.assertTrue("statistics" in instance_info)
with self.assertRaisesRegex(
ValueError,
"Cannot change the camera name on a running instance. "
"Stop the instance first."):
self.pipeline_client.set_instance_config(
instance_id_1, {"camera_name": "different_camera"})
self.pipeline_client.set_instance_config(
instance_id_2, {
"image_threshold": 99999,
"image_region_of_interest": [0, 200, 0, 200]
})
# Wait for the config update.
sleep(0.5)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data,
"This should really not happen anymore.")
# shape 200, 200 -> Account for the region of interest change.
self.assertTrue(
numpy.array_equal(data.data.data["image"].value,
numpy.zeros(shape=(200, 200))),
"Array should be all zeros, because of the threshold config.")
# Adjust width and height with the region of interest.
self.assertEqual(data.data.data["width"].value, 200,
"Region of interest not takes into account.")
self.assertEqual(data.data.data["height"].value, 200,
"Region of interest not takes into account.")
self.assertNotEqual(instance_id_1, instance_id_2,
"Instances should be different.")
self.assertNotEqual(instance_stream_1, instance_stream_2,
"Stream addresses should be different.")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 2,
"Two instances should be running.")
instance_stream_3 = self.pipeline_client.get_instance_stream(
instance_id_2)
self.assertEqual(instance_stream_2, instance_stream_3,
"Stream addresses should be equal.")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 2,
"Two instances should be running, get does not increase the number of instances."
)
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 4,
"Two new instances should be created.")
with self.assertRaisesRegex(
ValueError, "You must specify either the pipeline name or the "
"configuration for the pipeline."):
self.pipeline_client.create_instance_from_config({})
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.create_instance_from_config(
{"invalid": "config"})
background_id = self.pipeline_client.collect_background("simulation")
expected_background_file = os.path.join(self.background_config_folder,
background_id + ".npy")
# Unfortunately there is not way of knowing (from client side) how many images were collected.
background_id_2 = self.pipeline_client.collect_background(
"simulation", 5)
self.assertNotEqual(background_id, background_id_2,
"Background should be different.")
with self.assertRaisesRegex(Exception, "n_images must be a number."):
self.pipeline_client.collect_background("simulation",
"invalid_number")
pipeline_config = {
"camera_name": "simulation",
"background_image": background_id
}
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
pipeline_config)
instance_config = self.pipeline_client.get_instance_config(instance_id)
# We need to account for the expended config.
expected_config = {}
expected_config.update(PipelineConfig.DEFAULT_CONFIGURATION)
expected_config.update(pipeline_config)
self.assertDictEqual(expected_config, instance_config,
"Set and retrieved instances are not equal.")
instance_info = self.pipeline_client.get_instance_info(instance_id)
self.assertEqual(instance_info["instance_id"], instance_id,
"Requested and retireved instances are different.")
self.assertEqual(instance_info["stream_address"], stream_address,
"Different stream address.")
self.assertTrue(instance_info["is_stream_active"],
"Stream should be active.")
self.assertFalse(instance_info["read_only"],
"It should not be read only.")
self.assertEqual(instance_info["camera_name"], "simulation",
"Wrong camera name.")
self.assertDictEqual(instance_info["config"], expected_config,
"Config is not equal")
self.pipeline_client.stop_instance(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance '%s' does not exist." % instance_id):
self.pipeline_client.get_instance_info(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance 'invalid instance' does not exist."):
self.pipeline_client.get_instance_info("invalid instance")
self.assertTrue(os.path.exists(expected_background_file))
os.remove(expected_background_file)
self.pipeline_client.stop_all_instances()
self.assertTrue(
"testing_config" in self.pipeline_client.get_pipelines(),
"Pre requirement for next test.")
self.pipeline_client.delete_pipeline_config("testing_config")
self.assertFalse(
"testing_config" in self.pipeline_client.get_pipelines(),
"Pipeline should not exist anymore.")
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"image_threshold": 10
}, "custom_instance")
self.assertEqual(instance_id, "custom_instance",
"Custom instance not set properly.")
self.assertTrue(
"custom_instance"
in self.pipeline_client.get_server_info()["active_instances"],
"Instance with custom instance id not present.")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 10, "Config not set on custom instance id.")
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance if you want "
"to reuse the same instance_id."):
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"}, "custom_instance")
self.pipeline_client.stop_instance("custom_instance")
# The instance is now stopped, it should overwrite it.
self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"image_threshold": 20
}, "custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config(
"custom_instance")["image_threshold"], 20,
"Instance with custom id not overwritten.")
self.pipeline_client.save_pipeline_config("testing_config", {
"camera_name": "simulation",
"image_threshold": 30
})
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance if you want "
"to reuse the same instance_id."):
self.pipeline_client.create_instance_from_name(
"testing_config", "custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 20, "Instance should not have changed.")
data = self.pipeline_client.get_instance_message("custom_instance")
self.assertIsNotNone(data)
self.assertTrue("image" in data.data.data)
self.pipeline_client.stop_instance("custom_instance")
self.pipeline_client.create_instance_from_name("testing_config",
"custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 30, "Instance should have changed.")
background_1 = self.pipeline_client.collect_background("simulation")
background_2 = self.pipeline_client.collect_background("simulation")
self.assertNotEqual(background_1, background_2)
latest_background = self.pipeline_client.get_latest_background(
"simulation")
self.assertEqual(latest_background, background_2,
"Wrong background set as latest.")
with self.assertRaisesRegex(
ValueError,
"No background matches for the specified prefix 'does not exist'."
):
self.pipeline_client.get_latest_background("does not exist")
expected_cameras = [
'camera_example_1', 'camera_example_3', 'camera_example_2',
'camera_example_4', 'simulation'
]
self.assertEqual(set(self.pipeline_client.get_cameras()),
set(expected_cameras),
"Expected cameras not present.")
configuration = {"camera_name": "simulation", "threshold": 50}
stream_address_1 = self.pipeline_client.get_instance_stream_from_config(
configuration)
stream_address_2 = self.pipeline_client.get_instance_stream_from_config(
configuration)
self.assertEqual(
stream_address_1, stream_address_2,
"Requesting the same config should give you the same instance.")
self.pipeline_client.stop_all_instances()
self.cam_client.stop_all_cameras()
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"pipeline_type": "store"
})
with self.assertRaisesRegex(
ValueError, "Cannot get message from 'store' pipeline type."):
self.pipeline_client.get_instance_message(instance_id)
host, port = get_host_port_from_stream_address(stream_address)
with source(host=host, port=port, mode=PULL) as stream:
data = stream.receive()
self.assertIsNotNone(data)
self.assertEqual(
len(data.data.data), 1,
"Only the image should be present in the received data.")
self.assertTrue("simulation" in data.data.data,
"Camera name should be used instead of 'image'.")
self.pipeline_client.stop_all_instances()
class CameraClientProxyTest(unittest.TestCase):
def setUp(self):
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.temp_folder = os.path.join(test_base_dir, "temp/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
require_folder(self.temp_folder)
self.host = "0.0.0.0"
self.cam_server_ports = [8880, 8881]
self.cam_manager_port = 8888
self.pipeline_server_ports = [8890, 8891]
self.pipeline_manager_port = 8889
self.cam_server_address = []
self.process_camserver = []
port_range = config.CAMERA_STREAM_PORT_RANGE
for p in self.cam_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.cam_server_address.append("http://%s:%s" % (self.host, p))
process = Process(target=start_camera_worker,
args=(self.host, p, self.user_scripts_folder,
None, port_range))
self.process_camserver.append(process)
process.start()
self.cam_proxy_host = "0.0.0.0"
self.process_camproxy = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port,
",".join(self.cam_server_address), self.cam_config_folder,
self.user_scripts_folder))
self.process_camproxy.start()
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_manager_port)
self.pipeline_server_address = []
self.process_pipelineserver = []
port_range = config.PIPELINE_STREAM_PORT_RANGE
for p in self.pipeline_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.pipeline_server_address.append("http://%s:%s" %
(self.host, p))
process = Process(target=start_pipeline_worker,
args=(self.host, p, self.temp_folder,
self.temp_folder, cam_server_proxy_address,
None, port_range))
self.process_pipelineserver.append(process)
process.start()
cfg = ",".join(self.pipeline_server_address)
self.pipeline_proxy_process = Process(
target=start_pipeline_manager,
args=(self.host, self.pipeline_manager_port, cfg,
self.pipeline_config_folder, self.background_config_folder,
config.DEFAULT_BACKGROUND_FILES_DAYS_TO_LIVE,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
sleep(1.0) # Give it some time to start.
cam_server_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
self.cam_client = CamClient(cam_server_address)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_manager_port)
self.pipeline_client = PipelineClient(pipeline_server_address)
self.cam_proxy_client = ProxyClient(cam_server_address)
self.pipeline_proxy_client = ProxyClient(pipeline_server_address)
def tearDown(self):
test_cleanup([self.cam_client, self.pipeline_client], [
self.pipeline_proxy_process,
] + self.process_pipelineserver + [
self.process_camproxy,
] + self.process_camserver, [
self.background_config_folder, self.user_scripts_folder,
self.temp_folder
])
def test_manager(self):
#Creating instances from name
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name(
"simulation_sp")
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_name(
"simulation2_sp")
print(instance_id_1, instance_stream_1)
print(instance_id_2, instance_stream_2)
# Check if streams are alive
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_1)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
#check client
server_info = self.cam_proxy_client.get_servers_info()
status_info = self.cam_proxy_client.get_status_info()
instance_info = self.cam_proxy_client.get_instances_info()
#Check if camera streams are equally distributed
self.assertEqual(server_info[self.cam_server_address[0]]["load"], 1)
self.assertEqual(server_info[self.cam_server_address[1]]["load"], 1)
server_info = self.pipeline_proxy_client.get_servers_info()
status_info = self.pipeline_proxy_client.get_status_info()
instance_info = self.pipeline_proxy_client.get_instances_info()
print(server_info)
print(instance_info)
#Check if pipeline are equally distributed
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
1)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
1)
# Check if instance information is available for each server instance
for instance in server_info[
self.pipeline_server_address[0]]["instances"]:
self.assertIn(instance, instance_info)
for instance in server_info[
self.pipeline_server_address[1]]["instances"]:
self.assertIn(instance, instance_info)
#Test stopping instances
self.pipeline_client.stop_instance(instance_id_1)
self.pipeline_client.stop_instance(instance_id_2)
server_info = self.pipeline_proxy_client.get_servers_info()
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
0)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
0)
#Creating instances from config
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation2"})
print(instance_id_1, instance_stream_1)
print(instance_id_2, instance_stream_2)
# Check if streams are alive
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_1)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
server_info = self.pipeline_proxy_client.get_servers_info()
status_info = self.pipeline_proxy_client.get_status_info()
instance_info = self.pipeline_proxy_client.get_instances_info()
print(server_info)
print(instance_info)
#Check if pipeline are equally distributed
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
1)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
1)
# Check if instance information is available for each server instance
for instance in server_info[
self.pipeline_server_address[0]]["instances"]:
self.assertIn(instance, instance_info)
for instance in server_info[
self.pipeline_server_address[1]]["instances"]:
self.assertIn(instance, instance_info)
#Test stopping instances
self.pipeline_client.stop_instance(instance_id_1)
self.pipeline_client.stop_instance(instance_id_2)
server_info = self.pipeline_proxy_client.get_servers_info()
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
0)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
0)
#Server Config
self.cam_client.stop_all_instances()
self.assertEqual(
self.pipeline_proxy_client.get_config(), {
'http://0.0.0.0:8890': {
'expanding': True
},
'http://0.0.0.0:8891': {
'expanding': True
}
})
self.pipeline_proxy_client.set_config({
'http://0.0.0.0:8890': {
'expanding': True
},
'http://0.0.0.0:8891': {
"instances": ["DUMMY"],
'expanding': False
}
})
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation2"})
#Check if streams are alive
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_1)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
server_info = self.pipeline_proxy_client.get_servers_info()
self.assertEqual(server_info[self.pipeline_server_address[0]]["load"],
2)
self.assertEqual(server_info[self.pipeline_server_address[1]]["load"],
0)
#Version
self.assertEqual(self.cam_client.get_version(), __VERSION__)
self.assertEqual(self.pipeline_client.get_version(), __VERSION__)
self.assertEqual(self.pipeline_proxy_client.get_version(), __VERSION__)
self.assertEqual(self.pipeline_proxy_client.get_version(), __VERSION__)
"""
def test_persisted_config(self):
cfg = self.pipeline_proxy_client.get_config()
#Server Config
self.assertEqual(cfg, {
"http://localhost:8889": {
"instances": [
"simulation_sp"
],
"cameras": [
"simulation"
],
"expanding": True
}
})
cfg["http://localhost:8889"]["expanding"] = False
self.pipeline_proxy_client.set_config(cfg)
self.assertEqual(self.pipeline_proxy_client.get_config(), {
"http://localhost:8889": {
"instances": [
"simulation_sp"
],
"cameras": [
"simulation"
],
"expanding": False
}
})
"""
def test_permanent_pipelines(self):
pp = self.pipeline_proxy_client.get_permanent_instances()
pp["cxx"] = "yyy"
pp["simulation_sp"] = "pp"
self.pipeline_proxy_client.set_permanent_instances(pp)
pp = self.pipeline_proxy_client.get_permanent_instances()
self.assertIn(("simulation_sp", "pp"), pp.items())
self.pipeline_proxy_client.set_permanent_instances({})
def test_file_dump_pipeline(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_dump", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5"})
print(instance_id_1, instance_stream_1)
time.sleep(1.0)
self.pipeline_client.stop_instance(instance_id_1)
def test_file_dump_pipeline2(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_dump", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5",
"layout":"FLAT", "localtime":False})
print(instance_id_1, instance_stream_1)
time.sleep(1.0)
self.pipeline_client.stop_instance(instance_id_1)
def test_pipeline_pid_range(self):
pids = list(range(5, 11))
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "pid_range":[pids[0], pids[-1]]})
print(instance_id_1, instance_stream_1)
self.pipeline_client.wait_instance_completed(instance_id_1)
with h5py.File(self.temp_folder + "data.h5", 'r') as file:
self.assertEqual(list(file['header1/pulse_id']), pids)
def test_pipeline_pid_range2(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "pid_range":[0, 0]})
print(instance_id_1, instance_stream_1)
time.sleep(0.5)
self.pipeline_client.set_instance_config(instance_id_1,
{"pid_range": [5, 0]})
time.sleep(0.5)
self.pipeline_client.set_instance_config(instance_id_1,
{"pid_range": [5, 6]})
self.pipeline_client.wait_instance_completed(instance_id_1)
def test_pause(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5"})
time.sleep(0.5)
self.pipeline_client.set_instance_config(instance_id_1,
{"pause": True})
time.sleep(0.5)
self.pipeline_client.set_instance_config(instance_id_1,
{"pause": False})
time.sleep(0.5)
def test_pipeline_records(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_records", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "records":12})
print(instance_id_1, instance_stream_1)
self.pipeline_client.wait_instance_completed(instance_id_1)
with h5py.File(self.temp_folder + "data.h5", 'r') as file:
self.assertEqual(len(file['header1/pulse_id']), 12)
def test_exit_code(self):
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"no_client_timeout": 1000
})
self.pipeline_client.stop_instance(instance_id_1)
#When instance is stopped, it is deleted immediately, and there is no reference to the process
self.pipeline_client.get_instance_exit_code(instance_id_1)
exit_code = self.pipeline_client.get_instance_exit_code(instance_id_1)
self.assertEqual(exit_code, None)
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "pid_range":[5, 10]})
with self.assertRaisesRegex(
ValueError, "Instance 'simulation_file_range' still running."):
self.pipeline_client.get_instance_exit_code(instance_id_1)
self.pipeline_client.wait_instance_completed(instance_id_1)
exit_code = self.pipeline_client.get_instance_exit_code(instance_id_1)
self.assertEqual(exit_code, 0)
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name("simulation_sp", \
instance_id = "simulation_file_range", \
additional_config = {"mode":"FILE", "file":self.temp_folder+"data.h5", "rotation":{"angle":True, "order":6}})
self.pipeline_client.wait_instance_completed(instance_id_1)
exit_code = self.pipeline_client.get_instance_exit_code(instance_id_1)
self.assertEqual(exit_code, 2)
def setUp(self):
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.temp_folder = os.path.join(test_base_dir, "temp/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
require_folder(self.temp_folder)
self.host = "localhost"
self.cam_server_ports = [8880, 8881]
self.cam_manager_port = 8888
self.pipeline_server_ports = [8890, 8891]
self.pipeline_manager_port = 8889
self.cam_server_address = []
self.process_camserver = []
port_range = config.CAMERA_STREAM_PORT_RANGE
for p in self.cam_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.cam_server_address.append("http://%s:%s" % (self.host, p))
process = Process(target=start_camera_worker,
args=(self.host, p, self.user_scripts_folder,
None, port_range))
self.process_camserver.append(process)
process.start()
self.cam_proxy_host = "0.0.0.0"
self.process_camproxy = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port,
",".join(self.cam_server_address), self.cam_config_folder,
self.user_scripts_folder))
self.process_camproxy.start()
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_manager_port)
self.pipeline_server_address = []
self.process_pipelineserver = []
port_range = config.PIPELINE_STREAM_PORT_RANGE
for p in self.pipeline_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.pipeline_server_address.append("http://%s:%s" %
(self.host, p))
process = Process(target=start_pipeline_worker,
args=(self.host, p, self.temp_folder,
self.temp_folder, cam_server_proxy_address,
None, port_range))
self.process_pipelineserver.append(process)
process.start()
cfg = """{
"http://localhost:8890": {
"expanding": true
}, "http://localhost:8891": {
"cameras": [
"simulation3"
],
"expanding": false,
"instances": [
"simulation_sp:10123"
]
}
}"""
self.pipeline_proxy_process = Process(
target=start_pipeline_manager,
args=(self.host, self.pipeline_manager_port, cfg,
self.pipeline_config_folder, self.background_config_folder,
config.DEFAULT_BACKGROUND_FILES_DAYS_TO_LIVE,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
sleep(1.0) # Give it some time to start.
cam_server_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
self.cam_client = CamClient(cam_server_address)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_manager_port)
self.pipeline_client = PipelineClient(pipeline_server_address)
self.cam_proxy_client = ProxyClient(cam_server_address)
self.pipeline_proxy_client = ProxyClient(pipeline_server_address)
## Initialize the camera client.
from cam_server import CamClient
cam_client = CamClient()
# Print the list of available cameras.
print(cam_client.get_cameras())
# If modifying: Put the name of the camera you want to modify.
#camera_to_modify = "SATOP31-PSCR136"
# Retrieve the camera config.
#camera_config = cam_client.get_camera_config(camera_to_modify)
#print(camera_config)
# Specify the desired camera config.
#camera_config={
# 'name': 'SATOP31-PSCR136',
# 'source': 'SATOP31-PSCR136',
# 'source_type': 'epics',
# 'mirror_x': False,
# 'mirror_y': False,
# 'rotate': 0,
#
# 'camera_calibration': {
# 'reference_marker': [0, 0, 100, 100],
# 'reference_marker_width': 100.0,
# 'reference_marker_height': 100.0,
# 'angle_horizontal': 0.0,
# 'angle_vertical': 0.0
# }
#}
class CameraClientTest(unittest.TestCase):
def setUp(self):
self.host = "0.0.0.0"
self.port = 8888
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.process = Process(target=start_camera_server,
args=(self.host, self.port, self.config_folder))
self.process.start()
# Give it some time to start.
sleep(0.5)
server_address = "http://%s:%s" % (self.host, self.port)
self.client = CamClient(server_address)
def tearDown(self):
self.client.stop_all_cameras()
os.kill(self.process.pid, signal.SIGINT)
try:
os.remove(os.path.join(self.config_folder, "testing_camera.json"))
except:
pass
# Wait for the server to die.
sleep(1)
def test_client(self):
server_info = self.client.get_server_info()
self.assertIsNot(server_info["active_instances"],
"There should be no running instances.")
expected_cameras = set([
"camera_example_1", "camera_example_2", "camera_example_3",
"camera_example_4", "simulation"
])
self.assertSetEqual(set(self.client.get_cameras()), expected_cameras,
"Not getting all expected cameras")
camera_stream_address = self.client.get_camera_stream("simulation")
self.assertTrue(bool(camera_stream_address),
"Camera stream address cannot be empty.")
self.assertTrue(
"simulation" in self.client.get_server_info()["active_instances"],
"Simulation camera not present in server info.")
# Check if we can connect to the stream and receive data (in less than 2 seconds).
host, port = get_host_port_from_stream_address(camera_stream_address)
with source(host=host, port=port, receive_timeout=2000,
mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Received data was none.")
required_fields = set(
["image", "timestamp", "width", "height", "x_axis", "y_axis"])
self.assertSetEqual(required_fields, set(data.data.data.keys()),
"Required fields missing.")
image = data.data.data["image"].value
x_size, y_size = CameraSimulation(
CameraConfig("simulation")).get_geometry()
self.assertListEqual(
list(image.shape), [y_size, x_size],
"Original and received image are not the same.")
self.assertEqual(data.data.data["width"].value, x_size,
"Width not correct.")
self.assertEqual(data.data.data["height"].value, y_size,
"Height not correct.")
# Stop the simulation instance.
self.client.stop_camera("simulation")
self.assertTrue(
"simulation"
not in self.client.get_server_info()["active_instances"],
"Camera simulation did not stop.")
self.client.get_camera_stream("simulation")
self.assertTrue(
"simulation" in self.client.get_server_info()["active_instances"],
"Camera simulation did not start.")
self.client.stop_all_cameras()
self.assertTrue(
"simulation"
not in self.client.get_server_info()["active_instances"],
"Camera simulation did not stop.")
example_1_config = self.client.get_camera_config("camera_example_1")
self.assertTrue(bool(example_1_config), "Cannot retrieve config.")
# Change the name to reflect tha camera.
example_1_config["name"] = "testing_camera"
self.client.set_camera_config("testing_camera", example_1_config)
testing_camera_config = self.client.get_camera_config("testing_camera")
self.assertDictEqual(example_1_config, testing_camera_config,
"Saved and loaded configs are not the same.")
geometry = self.client.get_camera_geometry("simulation")
simulated_camera = CameraSimulation(CameraConfig("simulation"))
size_x, size_y = simulated_camera.get_geometry()
self.assertListEqual(
geometry, [size_x, size_y],
'The geometry of the simulated camera is not correct.')
self.assertTrue("testing_camera" in self.client.get_cameras(),
"Testing camera should be present.")
self.client.delete_camera_config("testing_camera")
self.assertTrue("testing_camera" not in self.client.get_cameras(),
"Testing camera should not be present.")
# Test if it fails quickly enough.
with self.assertRaisesRegex(
ValueError,
"Camera with prefix EPICS_example_1 not online - Status None"):
self.client.get_camera_stream("camera_example_1")
self.assertTrue(self.client.is_camera_online("simulation"),
"Simulation should be always online")
self.assertFalse(self.client.is_camera_online("camera_example_1"),
"Epics not working in this tests.")
self.client.set_camera_config(
"simulation_temp", self.client.get_camera_config("simulation"))
stream_address = self.client.get_camera_stream("simulation_temp")
camera_host, camera_port = get_host_port_from_stream_address(
stream_address)
sim_x, sim_y = CameraSimulation(
CameraConfig("simulation")).get_geometry()
instance_info = self.client.get_server_info(
)["active_instances"]["simulation_temp"]
self.assertTrue("last_start_time" in instance_info)
self.assertTrue("statistics" in instance_info)
# Collect from the pipeline.
with source(host=camera_host, port=camera_port, mode=SUB) as stream:
data = stream.receive()
x_size = data.data.data["width"].value
y_size = data.data.data["height"].value
self.assertEqual(x_size, sim_x)
self.assertEqual(y_size, sim_y)
x_axis_1 = data.data.data["x_axis"].value
y_axis_1 = data.data.data["y_axis"].value
self.assertEqual(x_axis_1.shape[0], sim_x)
self.assertEqual(y_axis_1.shape[0], sim_y)
camera_config = self.client.get_camera_config("simulation_temp")
camera_config["rotate"] = 1
self.client.set_camera_config("simulation_temp", camera_config)
sleep(0.5)
# Collect from the pipeline.
with source(host=camera_host, port=camera_port, mode=SUB) as stream:
data = stream.receive()
x_size = data.data.data["width"].value
y_size = data.data.data["height"].value
# We rotate the image for 90 degrees - X and Y size should be inverted.
self.assertEqual(x_size, sim_y)
self.assertEqual(y_size, sim_x)
x_axis_2 = data.data.data["x_axis"].value
y_axis_2 = data.data.data["y_axis"].value
# We rotate the image for 90 degrees - X and Y size should be inverted.
self.assertEqual(x_axis_2.shape[0], sim_y)
self.assertEqual(y_axis_2.shape[0], sim_x)
self.client.delete_camera_config("simulation_temp")
image = self.client.get_camera_image("simulation")
self.assertGreater(len(image.content), 0)
image = self.client.get_camera_image_bytes("simulation")
dtype = image["dtype"]
shape = image["shape"]
bytes = base64.b64decode(image["bytes"].encode())
x_size, y_size = CameraSimulation(
CameraConfig("simulation")).get_geometry()
self.assertEqual(shape, [y_size, x_size])
image_array = numpy.frombuffer(bytes, dtype=dtype).reshape(shape)
self.assertIsNotNone(image_array)
self.client.stop_all_cameras()
class CameraClientProxyTest(unittest.TestCase):
def setUp(self):
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.host = "0.0.0.0"
self.cam_server_ports = [8880, 8881]
self.cam_manager_port = 8888
self.cam_server_address = []
self.process_camserver = []
port_range = config.CAMERA_STREAM_PORT_RANGE
for port in self.cam_server_ports:
print(port, is_port_available(port))
for p in self.cam_server_ports:
port_range = [port_range[0] + 10000, port_range[1] + 10000]
self.cam_server_address.append("http://%s:%s" % (self.host, p))
process = Process(target=start_camera_worker,
args=(self.host, p, self.user_scripts_folder,
None, port_range))
self.process_camserver.append(process)
process.start()
self.cam_proxy_host = "0.0.0.0"
self.process_camproxy = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port,
",".join(self.cam_server_address), self.config_folder,
self.user_scripts_folder))
self.process_camproxy.start()
sleep(1.0) # Give it some time to start.
server_address = "http://%s:%s" % (self.host, self.cam_manager_port)
self.client = CamClient(server_address)
self.proxy_client = ProxyClient(server_address)
def tearDown(self):
test_cleanup([self.client], [
self.process_camproxy,
] + self.process_camserver, [])
def test_manager(self):
stream_address_1 = self.client.get_instance_stream("simulation")
stream_address_2 = self.client.get_instance_stream("simulation2")
#Check if streams are alive
camera_host, camera_port = get_host_port_from_stream_address(
stream_address_1)
with source(host=camera_host, port=camera_port, mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
camera_host, camera_port = get_host_port_from_stream_address(
stream_address_2)
with source(host=camera_host, port=camera_port, mode=SUB) as stream:
data = stream.receive()
for key in ["image", "width", "height"]:
self.assertIn(key, data.data.data.keys())
server_info = self.proxy_client.get_servers_info()
status_info = self.proxy_client.get_status_info()
instance_info = self.proxy_client.get_instances_info()
#Check if streams are equally distributed
self.assertEqual(server_info[self.cam_server_address[0]]["load"], 1)
self.assertEqual(server_info[self.cam_server_address[1]]["load"], 1)
# Check if instance information is available for each server instance
for instance in server_info[self.cam_server_address[0]]["instances"]:
self.assertIn(instance, instance_info)
for instance in server_info[self.cam_server_address[1]]["instances"]:
self.assertIn(instance, instance_info)
self.client.stop_all_instances()
#Server Config
self.assertEqual(
self.proxy_client.get_config(), {
'http://0.0.0.0:8880': {
'expanding': True
},
'http://0.0.0.0:8881': {
'expanding': True
}
})
self.proxy_client.set_config({
'http://0.0.0.0:8880': {
'expanding': True
},
'http://0.0.0.0:8881': {
"instances": ["DUMMY"],
'expanding': False
}
})
stream_address_1 = self.client.get_instance_stream("simulation")
stream_address_2 = self.client.get_instance_stream("simulation2")
#Check if streams are alive
camera_host, camera_port = get_host_port_from_stream_address(
stream_address_1)
camera_host, camera_port = get_host_port_from_stream_address(
stream_address_2)
server_info = self.proxy_client.get_servers_info()
status_info = self.proxy_client.get_status_info()
instance_info = self.proxy_client.get_instances_info()
self.assertEqual(server_info[self.cam_server_address[0]]["load"], 2)
self.assertEqual(server_info[self.cam_server_address[1]]["load"], 0)
"""
class PipelineTransceiverTest(unittest.TestCase):
def setUp(self):
self.host = "0.0.0.0"
self.port = 8888
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.process = Process(target=start_camera_server,
args=(self.host, self.port, self.config_folder))
self.process.start()
# Give it some time to start.
sleep(0.5)
self.rest_api_endpoint = "http://%s:%s" % (self.host, self.port)
self.client = CamClient(self.rest_api_endpoint)
def tearDown(self):
self.client.stop_all_cameras()
sleep(1)
try:
os.kill(self.process.pid, signal.SIGINT)
except:
pass
try:
os.remove(os.path.join(self.config_folder, "testing_camera.json"))
except:
pass
# Wait for the server to die.
sleep(2)
def test_pipeline_with_simulation_camera(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
pipeline_config = PipelineConfig("test_pipeline")
def send():
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12000,
MockBackgroundManager())
thread = Thread(target=send)
thread.start()
with source(host="127.0.0.1", port=12000, mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
required_keys = set([
'x_fit_standard_deviation', 'x_axis',
'y_fit_standard_deviation', 'x_center_of_mass',
'x_fit_amplitude', 'y_fit_mean', 'processing_parameters',
'timestamp', 'y_fit_gauss_function', 'y_profile',
'y_center_of_mass', 'x_fit_gauss_function', 'x_rms', 'y_rms',
'y_fit_amplitude', 'image', 'y_axis', 'min_value',
'x_fit_mean', 'max_value', 'x_fit_offset', 'x_profile',
'y_fit_offset', 'width', 'height', 'intensity'
])
self.assertSetEqual(
required_keys, set(data.data.data.keys()),
"Missing required keys in pipeline output bsread message.")
stop_event.set()
thread.join()
def test_pipeline_background_manager(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
pipeline_config = PipelineConfig("test_pipeline",
parameters={
"camera_name": "simulation",
"image_background":
"full_background",
"image_background_enable": True,
"image_threshold": 0
})
background_manager = MockBackgroundManager()
with self.assertRaises(Exception):
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12000,
background_manager)
simulated_camera_shape = (960, 1280)
background_array = numpy.zeros(shape=simulated_camera_shape)
background_array.fill(99999)
background_manager.save_background("full_background",
background_array,
append_timestamp=False)
def send():
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12000,
background_manager)
thread = Thread(target=send)
thread.start()
with source(host="127.0.0.1", port=12000, mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
self.assertTrue(
numpy.array_equal(data.data.data["image"].value,
numpy.zeros(shape=simulated_camera_shape)))
stop_event.set()
thread.join()
def test_rotate_camera(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
self.client.set_camera_config(
"simulation_temp", self.client.get_camera_config("simulation"))
pipeline_config = PipelineConfig(
"test_pipeline", parameters={"camera_name": "simulation_temp"})
background_manager = MockBackgroundManager()
simulated_camera_shape = (960, 1280)
def send():
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12000,
background_manager)
thread = Thread(target=send)
thread.start()
with source(host="127.0.0.1", port=12000, mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
x_size = data.data.data["width"].value
y_size = data.data.data["height"].value
x_axis = data.data.data["x_axis"].value
y_axis = data.data.data["y_axis"].value
image_shape = data.data.data["image"].value.shape
self.assertEqual(x_size, simulated_camera_shape[1])
self.assertEqual(y_size, simulated_camera_shape[0])
# Sanity checks.
self.assertEqual(x_size, len(x_axis))
self.assertEqual(y_size, len(y_axis))
self.assertEqual(image_shape, (y_size, x_size))
# Rotate the image by 90 degree.
camera_config = self.client.get_camera_config("simulation_temp")
camera_config["rotate"] = 1
self.client.set_camera_config("simulation_temp", camera_config)
# Make a few frames pass.
for _ in range(5):
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
x_size = data.data.data["width"].value
y_size = data.data.data["height"].value
x_axis = data.data.data["x_axis"].value
y_axis = data.data.data["y_axis"].value
image_shape = data.data.data["image"].value.shape
# X and Y size should be inverted.
self.assertEqual(x_size, simulated_camera_shape[0])
self.assertEqual(y_size, simulated_camera_shape[1])
# Sanity checks.
self.assertEqual(x_size, len(x_axis))
self.assertEqual(y_size, len(y_axis))
self.assertEqual(image_shape, (y_size, x_size))
self.client.delete_camera_config("simulation_temp")
stop_event.set()
thread.join()
def test_store_pipeline_with_simulated_camera(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
self.client.stop_all_cameras()
pipeline_config = PipelineConfig("test_pipeline")
def send():
store_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12000,
MockBackgroundManager())
thread = Thread(target=send)
thread.start()
sleep(0.5)
with source(host="127.0.0.1", port=12000, mode=PULL) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
required_keys = set(["simulation"])
self.assertSetEqual(
required_keys, set(data.data.data.keys()),
"Missing required keys in pipeline output bsread message.")
stop_event.set()
thread.join()
def get_camclient():
global CAM_CLIENT
if not CAM_CLIENT:
CAM_CLIENT = CamClient()
return CAM_CLIENT
class PipelineClientTest(unittest.TestCase):
def setUp(self):
self.host = "0.0.0.0"
self.cam_port = 8880
self.pipeline_port = 8881
self.cam_manager_port = 8888
self.pipeline_manager_port = 8889
for port in self.cam_port, self.pipeline_port, self.cam_manager_port, self.pipeline_manager_port:
print("Port ", port, "avilable: ", is_port_available(port))
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.temp_folder = os.path.join(test_base_dir, "temp/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
require_folder(self.temp_folder)
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
cam_server_proxy_address = "http://%s:%s" % (self.host,
self.cam_manager_port)
pipeline_server_proxy_address = "http://%s:%s" % (
self.host, self.pipeline_manager_port)
self.cam_process = Process(target=start_camera_worker,
args=(self.host, self.cam_port,
self.user_scripts_folder))
self.cam_process.start()
self.pipeline_process = Process(
target=start_pipeline_worker,
args=(self.host, self.pipeline_port, self.temp_folder,
self.temp_folder, cam_server_proxy_address))
self.pipeline_process.start()
self.cam_proxy_process = Process(
target=start_camera_manager,
args=(self.host, self.cam_manager_port, cam_server_address,
self.cam_config_folder, self.user_scripts_folder))
self.cam_proxy_process.start()
self.pipeline_proxy_process = Process(
target=start_pipeline_manager,
args=(self.host, self.pipeline_manager_port,
pipeline_server_address, self.pipeline_config_folder,
self.background_config_folder,
config.DEFAULT_BACKGROUND_FILES_DAYS_TO_LIVE,
self.user_scripts_folder, cam_server_proxy_address))
self.pipeline_proxy_process.start()
self.cam_client = CamClient(cam_server_proxy_address)
self.pipeline_client = PipelineClient(pipeline_server_proxy_address)
# Give it some time to start.
sleep(1.0) # Give it some time to start.
def tearDown(self):
test_cleanup([self.pipeline_client, self.cam_client], [
self.pipeline_proxy_process, self.cam_proxy_process,
self.pipeline_process, self.cam_process
], [
os.path.join(self.pipeline_config_folder, "testing_config.json"),
os.path.join(self.temp_folder, "Test.py"),
os.path.join(self.temp_folder, "Test2.py"),
os.path.join(self.user_scripts_folder, "Test.py"),
os.path.join(self.user_scripts_folder, "Test2.py"),
self.background_config_folder,
self.user_scripts_folder,
self.temp_folder,
])
def test_client(self):
expected_pipelines = [
"pipeline_example_1", "pipeline_example_2", "pipeline_example_3",
"pipeline_example_4"
]
for pipeline in expected_pipelines:
self.assertIn(pipeline, self.pipeline_client.get_pipelines(),
"Test config pipelines have changed?")
expected_pipelines = set(self.pipeline_client.get_pipelines())
camera_config = self.pipeline_client.get_pipeline_config(
"pipeline_example_4")
self.pipeline_client.save_pipeline_config("testing_config",
camera_config)
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.save_pipeline_config("testing_config", {})
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.save_pipeline_config("testing_config",
{"invalid": "config"})
with self.assertRaisesRegex(
ValueError,
"pipeline_type 'invalid' not present in mapping. Available"):
self.pipeline_client.save_pipeline_config(
"testing_config", {
"camera_name": "simulation",
"pipeline_type": "invalid"
})
expected_pipelines.add("testing_config")
self.assertSetEqual(set(self.pipeline_client.get_pipelines()),
expected_pipelines,
"Testing config was not added.")
stream_address_1 = self.pipeline_client.get_instance_stream(
"testing_config")
stream_address_2 = self.pipeline_client.get_instance_stream(
"testing_config")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 1,
"Instance not started or too many instances started.")
with self.assertRaisesRegex(ValueError,
"Config updates cannot be empty."):
self.pipeline_client.set_instance_config("testing_config", {})
with self.assertRaisesRegex(
ValueError, "Cannot set config on a read only instance."):
self.pipeline_client.set_instance_config(
"testing_config", {"camera_name": "simulation"})
self.assertEqual(stream_address_1, stream_address_2,
"Stream addresses should be equal.")
self.pipeline_client.stop_instance("testing_config")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 0,
"The instance should be stopped.")
instance_id_1, instance_stream_1 = self.pipeline_client.create_instance_from_name(
"testing_config")
instance_id_2, instance_stream_2 = self.pipeline_client.create_instance_from_name(
"testing_config")
instance_info = self.pipeline_client.get_instance_info(instance_id_1)
self.assertTrue("last_start_time" in instance_info)
self.assertTrue("statistics" in instance_info)
with self.assertRaisesRegex(
ValueError,
"Cannot change the camera name on a running instance. "
"Stop the instance first."):
self.pipeline_client.set_instance_config(
instance_id_1, {"camera_name": "different_camera"})
self.pipeline_client.set_instance_config(
instance_id_2, {
"image_threshold": 99999,
"image_region_of_interest": [0, 200, 0, 200]
})
# Wait for the config update.
sleep(0.5)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
instance_stream_2)
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data,
"This should really not happen anymore.")
# shape 200, 200 -> Account for the region of interest change.
self.assertTrue(
numpy.array_equal(data.data.data["image"].value,
numpy.zeros(shape=(200, 200))),
"Array should be all zeros, because of the threshold config.")
# Adjust width and height with the region of interest.
self.assertEqual(data.data.data["width"].value, 200,
"Region of interest not takes into account.")
self.assertEqual(data.data.data["height"].value, 200,
"Region of interest not takes into account.")
self.assertNotEqual(instance_id_1, instance_id_2,
"Instances should be different.")
self.assertNotEqual(instance_stream_1, instance_stream_2,
"Stream addresses should be different.")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 2,
"Two instances should be running.")
instance_stream_3 = self.pipeline_client.get_instance_stream(
instance_id_2)
self.assertEqual(instance_stream_2, instance_stream_3,
"Stream addresses should be equal.")
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 2,
"Two instances should be running, get does not increase the number of instances."
)
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
self.assertEqual(
len(self.pipeline_client.get_server_info()["active_instances"]), 4,
"Two new instances should be created.")
with self.assertRaisesRegex(
ValueError, "You must specify either the pipeline name or the "
"configuration for the pipeline."):
self.pipeline_client.create_instance_from_config({})
with self.assertRaisesRegex(
ValueError, "Camera name not specified in configuration."):
self.pipeline_client.create_instance_from_config(
{"invalid": "config"})
background_id = self.pipeline_client.collect_background("simulation")
expected_background_file = os.path.join(self.background_config_folder,
background_id + ".npy")
# Unfortunately there is not way of knowing (from client side) how many images were collected.
background_id_2 = self.pipeline_client.collect_background(
"simulation", 5)
self.assertNotEqual(background_id, background_id_2,
"Background should be different.")
with self.assertRaisesRegex(Exception, "n_images must be a number."):
self.pipeline_client.collect_background("simulation",
"invalid_number")
pipeline_config = {
"camera_name": "simulation",
"background_image": background_id
}
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
pipeline_config)
instance_config = self.pipeline_client.get_instance_config(instance_id)
# We need to account for the expended config.
expected_config = {}
expected_config.update(PipelineConfig.DEFAULT_CONFIGURATION)
expected_config.update(pipeline_config)
self.assertDictEqual(expected_config, instance_config,
"Set and retrieved instances are not equal.")
instance_info = self.pipeline_client.get_instance_info(instance_id)
self.assertEqual(instance_info["instance_id"], instance_id,
"Requested and retireved instances are different.")
self.assertEqual(instance_info["stream_address"], stream_address,
"Different stream address.")
self.assertTrue(instance_info["is_stream_active"],
"Stream should be active.")
self.assertFalse(instance_info["read_only"],
"It should not be read only.")
self.assertEqual(instance_info["camera_name"], "simulation",
"Wrong camera name.")
self.assertDictEqual(instance_info["config"], expected_config,
"Config is not equal")
self.pipeline_client.stop_instance(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance '%s' does not exist." % instance_id):
self.pipeline_client.get_instance_info(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance 'invalid instance' does not exist."):
self.pipeline_client.get_instance_info("invalid instance")
self.assertTrue(os.path.exists(expected_background_file))
os.remove(expected_background_file)
self.pipeline_client.stop_all_instances()
self.assertTrue(
"testing_config" in self.pipeline_client.get_pipelines(),
"Pre requirement for next test.")
self.pipeline_client.delete_pipeline_config("testing_config")
self.assertFalse(
"testing_config" in self.pipeline_client.get_pipelines(),
"Pipeline should not exist anymore.")
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"image_threshold": 10
}, "custom_instance")
self.assertEqual(instance_id, "custom_instance",
"Custom instance not set properly.")
self.assertTrue(
self.pipeline_client.is_instance_running("custom_instance"),
"Instance with custom instance id not present.")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 10, "Config not set on custom instance id.")
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance if you want "
"to reuse the same instance_id."):
self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"}, "custom_instance")
self.pipeline_client.stop_instance("custom_instance")
# The instance is now stopped, it should overwrite it.
self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"image_threshold": 20
}, "custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config(
"custom_instance")["image_threshold"], 20,
"Instance with custom id not overwritten.")
self.pipeline_client.save_pipeline_config("testing_config", {
"camera_name": "simulation",
"image_threshold": 30
})
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance if you want "
"to reuse the same instance_id."):
self.pipeline_client.create_instance_from_name(
"testing_config", "custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 20, "Instance should not have changed.")
data = self.pipeline_client.get_instance_message("custom_instance")
self.assertIsNotNone(data)
self.assertTrue("image" in data.data.data)
self.pipeline_client.stop_instance("custom_instance")
self.pipeline_client.create_instance_from_name("testing_config",
"custom_instance")
self.assertEqual(
self.pipeline_client.get_instance_config("custom_instance")
["image_threshold"], 30, "Instance should have changed.")
background_1 = self.pipeline_client.collect_background("simulation")
background_2 = self.pipeline_client.collect_background("simulation")
self.assertNotEqual(background_1, background_2)
latest_background = self.pipeline_client.get_latest_background(
"simulation")
self.assertEqual(latest_background, background_2,
"Wrong background set as latest.")
with self.assertRaisesRegex(
ValueError,
"No background matches for the specified prefix 'does not exist'."
):
self.pipeline_client.get_latest_background("does not exist")
expected_cameras = [
'camera_example_1', 'camera_example_3', 'camera_example_2',
'camera_example_4', 'simulation', 'simulation2'
]
#self.assertEqual(set(self.pipeline_client.get_cameras()), set(expected_cameras),
# "Expected cameras not present.")
for camera in expected_cameras:
self.assertIn(camera, set(self.pipeline_client.get_cameras()),
"Not getting expected camera: " + camera)
configuration = {"camera_name": "simulation", "threshold": 50}
stream_address_1 = self.pipeline_client.get_instance_stream_from_config(
configuration)
stream_address_2 = self.pipeline_client.get_instance_stream_from_config(
configuration)
self.assertEqual(
stream_address_1, stream_address_2,
"Requesting the same config should give you the same instance.")
self.pipeline_client.stop_all_instances()
self.cam_client.stop_all_instances()
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{
"camera_name": "simulation",
"pipeline_type": "store",
'stream_port': 10000
})
with self.assertRaisesRegex(
ValueError, "Cannot get message from 'store' pipeline type."):
self.pipeline_client.get_instance_message(instance_id)
host, port = get_host_port_from_stream_address(stream_address)
with source(host=host, port=port, mode=PULL) as stream:
data = stream.receive()
self.assertIsNotNone(data)
self.assertEqual(
len(data.data.data), 1,
"Only the image should be present in the received data.")
self.assertTrue(
"simulation" + config.EPICS_PV_SUFFIX_IMAGE in data.data.data,
"Camera name should be used instead of 'image'.")
self.pipeline_client.stop_all_instances()
#Transparent pipeline
instance_id, instance_stream = self.pipeline_client.create_instance_from_name(
"simulation")
cfg = self.pipeline_client.get_instance_config(instance_id)
cfg["function"] = "transparent"
self.pipeline_client.set_instance_config(instance_id, cfg)
time.sleep(.5)
data = self.pipeline_client.get_instance_message(instance_id)
self.assertIsNotNone(data)
# Cam_server fields + processing_parameters
required_fields = set([
"image", "timestamp", "width", "height", "x_axis", "y_axis",
"processing_parameters"
])
self.assertSetEqual(required_fields, set(data.data.data.keys()),
"Bad transparent pipeline fields.")
self.pipeline_client.stop_all_instances()
time.sleep(1.0)
def test_background_file(self):
bg = self.pipeline_client.get_latest_background("simulation")
image = self.pipeline_client.get_background_image(bg)
self.assertGreater(len(image.content), 0)
image = self.pipeline_client.get_background_image_bytes(bg)
dtype = image["dtype"]
shape = image["shape"]
bytes = base64.b64decode(image["bytes"].encode())
x_size, y_size = get_simulated_camera().get_geometry()
self.assertEqual(shape, [y_size, x_size])
image_array = numpy.frombuffer(bytes, dtype=dtype).reshape(shape)
self.assertIsNotNone(image_array)
def test_user_scripts(self):
script_name = "Test.py"
script_content = "print('Hello world')"
self.pipeline_client.set_user_script(script_name, script_content)
scripts = self.pipeline_client.get_user_scripts()
self.assertIn(script_name, scripts)
ret = self.pipeline_client.get_user_script(script_name)
self.assertEqual(ret, script_content)
filename = "temp/Test2.py"
with open(filename, "w") as data_file:
data_file.write(script_content)
self.pipeline_client.upload_user_script(filename)
os.remove(filename)
self.pipeline_client.download_user_script(filename)
with open(filename, "r") as data_file:
ret = data_file.read()
self.assertEqual(ret, script_content)
script_content = """
from cam_server.pipeline.data_processing import functions, processor
def process_image(image, pulse_id, timestamp, x_axis, y_axis, parameters, bsdata):
ret = processor.process_image(image, pulse_id, timestamp, x_axis, y_axis, parameters, bsdata)
ret["average_value"] = float(ret["intensity"]) / len(ret["x_axis"]) / len(ret["y_axis"])
return ret
"""
filename = "temp/Test.py"
with open(filename, "w") as data_file:
data_file.write(script_content)
instance_id, stream_address = self.pipeline_client.create_instance_from_config(
{"camera_name": "simulation"})
host, port = get_host_port_from_stream_address(stream_address)
with source(host=host, port=port, mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data,
"This should really not happen anymore.")
self.assertIsNotNone(data.data.data["width"].value)
self.assertIsNotNone(data.data.data["height"].value)
self.pipeline_client.set_function_script(instance_id, filename)
time.sleep(1.0)
with source(host=host, port=port, mode=SUB) as stream:
data = stream.receive()
print(data.data.data.keys())
self.assertIsNotNone(data.data.data["average_value"].value)
class PipelineClientTest(unittest.TestCase):
def setUp(self):
self.host = "localhost"
self.cam_port = 8888
self.pipeline_port = 8889
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.cam_config_folder = os.path.join(test_base_dir, "camera_config/")
self.pipeline_config_folder = os.path.join(test_base_dir,
"pipeline_config/")
self.background_config_folder = os.path.join(test_base_dir,
"background_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
require_folder(self.background_config_folder)
require_folder(self.user_scripts_folder)
cam_server_address = "http://%s:%s" % (self.host, self.cam_port)
pipeline_server_address = "http://%s:%s" % (self.host,
self.pipeline_port)
self.cam_process = Process(target=start_camera_server,
args=(self.host, self.cam_port,
self.cam_config_folder, None))
self.cam_process.start()
self.pipeline_process = Process(
target=start_pipeline_server,
args=(self.host, self.pipeline_port, self.pipeline_config_folder,
self.background_config_folder, self.user_scripts_folder,
cam_server_address))
self.pipeline_process.start()
# Give it some time to start.
sleep(1)
self.cam_client = CamClient(cam_server_address)
self.pipeline_client = PipelineClient(pipeline_server_address)
def tearDown(self):
test_cleanup([self.pipeline_client, self.cam_client], [
self.pipeline_process, self.cam_process
], [
os.path.join(self.pipeline_config_folder, "testing_config.json"),
os.path.join(self.pipeline_config_folder, "test_pipeline.json"),
os.path.join(self.cam_config_folder, "test_camera.json"),
"testing_dump.h5",
self.background_config_folder,
self.user_scripts_folder,
])
def test_a_quick_start(self):
from cam_server import PipelineClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Create a pipeline client.
client = PipelineClient()
# ADDITIONAL, FOR THE TEST
camera_name = "simulation"
client = self.pipeline_client
client.create_instance_from_config(
configuration={"camera_name": camera_name},
instance_id=camera_name + "_sp1")
# Define the camera name you want to read. This should be the same camera you are streaming in screen panel.
camera_name = "simulation"
# Format of the instance id when screen_panel creates a pipeline.
pipeline_instance_id = camera_name + "_sp1"
# Get the stream for the pipelie instance.
stream_address = client.get_instance_stream(pipeline_instance_id)
# Extract the stream host and port from the stream_address.
stream_host, stream_port = get_host_port_from_stream_address(
stream_address)
# Open connection to the stream. When exiting the 'with' section, the source disconnects by itself.
with source(host=stream_host, port=stream_port,
mode=SUB) as input_stream:
input_stream.connect()
# Read one message.
message = input_stream.receive()
# Print out the received stream data - dictionary.
print("Dictionary with data:\n", message.data.data)
# Print out the X center of mass.
print("X center of mass: ",
message.data.data["x_center_of_mass"].value)
def test_get_simulated_camera(self):
from cam_server import CamClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Change to match your camera server
server_address = "http://0.0.0.0:8888"
# Initialize the client.
camera_client = CamClient(server_address)
# Get stream address of simulation camera. Stream address in format tcp://hostname:port.
camera_stream_address = camera_client.get_instance_stream("simulation")
# Extract the stream hostname and port from the stream address.
camera_host, camera_port = get_host_port_from_stream_address(
camera_stream_address)
# Subscribe to the stream.
with source(host=camera_host, port=camera_port, mode=SUB) as stream:
# Receive next message.
data = stream.receive()
image_width = data.data.data["width"].value
image_height = data.data.data["height"].value
image_bytes = data.data.data["image"].value
print("Image size: %d x %d" % (image_width, image_height))
print("Image data: %s" % image_bytes)
x_size, y_size = get_simulated_camera().get_geometry()
self.assertIsNotNone(data)
self.assertEqual(image_width, x_size)
self.assertIsNotNone(image_height, y_size)
def test_get_basic_pipeline_with_simulated_camera(self):
from cam_server import PipelineClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Change to match your pipeline server
server_address = "http://0.0.0.0:8889"
# Initialize the client.
pipeline_client = PipelineClient(server_address)
# Setup the pipeline config. Use the simulation camera as the pipeline source.
pipeline_config = {"camera_name": "simulation"}
# Create a new pipeline with the provided configuration. Stream address in format tcp://hostname:port.
instance_id, pipeline_stream_address = pipeline_client.create_instance_from_config(
pipeline_config)
# Extract the stream hostname and port from the stream address.
pipeline_host, pipeline_port = get_host_port_from_stream_address(
pipeline_stream_address)
# Subscribe to the stream.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
# Receive next message.
data = stream.receive()
image_width = data.data.data["width"].value
image_height = data.data.data["height"].value
image_bytes = data.data.data["image"].value
print("Image size: %d x %d" % (image_width, image_height))
print("Image data: %s" % image_bytes)
x_size, y_size = get_simulated_camera().get_geometry()
self.assertIsNotNone(data)
self.assertEqual(image_width, x_size)
self.assertIsNotNone(image_height, y_size)
def test_create_pipeline_with_background(self):
from cam_server import PipelineClient
from cam_server.utils import get_host_port_from_stream_address
from bsread import source, SUB
# Change to match your pipeline server
server_address = "http://0.0.0.0:8889"
camera_name = "simulation"
# Initialize the client.
pipeline_client = PipelineClient(server_address)
# Collect the background for the given camera.
background_id = pipeline_client.collect_background(camera_name)
# Setup the pipeline config. Use the simulation camera as the pipeline source, and the collected background.
pipeline_config = {
"camera_name": camera_name,
"background_id": background_id
}
# Create a new pipeline with the provided configuration. Stream address in format tcp://hostname:port.
instance_id, pipeline_stream_address = pipeline_client.create_instance_from_config(
pipeline_config)
# Extract the stream hostname and port from the stream address.
pipeline_host, pipeline_port = get_host_port_from_stream_address(
pipeline_stream_address)
# Subscribe to the stream.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
# Receive next message.
data = stream.receive()
image_width = data.data.data["width"].value
image_height = data.data.data["height"].value
image_bytes = data.data.data["image"].value
print("Image size: %d x %d" % (image_width, image_height))
print("Image data: %s" % image_bytes)
x_size, y_size = get_simulated_camera().get_geometry()
self.assertIsNotNone(data)
self.assertEqual(image_width, x_size)
self.assertIsNotNone(image_height, y_size)
def test_modify_camera_config(self):
self.cam_client.set_camera_config(
"test_camera",
self.cam_client.get_camera_config("camera_example_2"))
from cam_server import CamClient
# Initialize the camera client.
cam_client = CamClient()
cam_client = self.cam_client
# Print the list of available cameras.
print(cam_client.get_cameras())
# Put the name of the camera you want to modify.
camera_to_modify = "test_camera"
# Retrieve the camera config.
camera_config = cam_client.get_camera_config(camera_to_modify)
# Change the mirror_x setting.
camera_config["mirror_x"] = False
# Change the camera_calibration setting.
camera_config["camera_calibration"] = {
"reference_marker": [0, 0, 100, 100],
"reference_marker_width": 100.0,
"reference_marker_height": 100.0,
"angle_horizontal": 0.0,
"angle_vertical": 0.0
}
# Save the camera configuration.
cam_client.set_camera_config(camera_to_modify, camera_config)
# You can also save the same (or another) config under a different camera name.
cam_client.set_camera_config("camera_to_delete", camera_config)
# And also delete camera configs.
cam_client.delete_camera_config("camera_to_delete")
def test_modify_pipeline_config(self):
self.pipeline_client.save_pipeline_config(
"test_pipeline",
self.pipeline_client.get_pipeline_config("pipeline_example_1"))
from cam_server import PipelineClient
# Initialize the pipeline client.
pipeline_client = PipelineClient()
pipeline_client = self.pipeline_client
# Print the list of available pipelines.
print(pipeline_client.get_pipelines())
# Put the name of the pipeline you want to modify.
pipeline_to_modify = "test_pipeline"
# Retrieve the camera config.
pipeline_config = pipeline_client.get_pipeline_config(
pipeline_to_modify)
# Change the image threshold.
pipeline_config["image_threshold"] = 0.5
# Change the image region of interest.
pipeline_config["image_region_of_interest"] = [0, 100, 0, 100]
# Save the camera configuration.
pipeline_client.save_pipeline_config(pipeline_to_modify,
pipeline_config)
# You can also save the same (or another) config under a different camera name.
pipeline_client.save_pipeline_config("pipeline_to_delete",
pipeline_config)
# And also delete camera configs.
pipeline_client.delete_pipeline_config("pipeline_to_delete")
def test_create_camera(self):
# Specify the desired camera config.
camera_config = {
"name": "camera_example_3",
"source": "EPICS:CAM1:EXAMPLE",
"source_type": "epics",
"mirror_x": False,
"mirror_y": False,
"rotate": 0,
"camera_calibration": {
"reference_marker": [0, 0, 100, 100],
"reference_marker_width": 100.0,
"reference_marker_height": 100.0,
"angle_horizontal": 0.0,
"angle_vertical": 0.0
}
}
# Specify the new camera name.
new_camera_name = "new_camera_name"
# Save the camera configuration.
self.cam_client.set_camera_config(new_camera_name, camera_config)
self.assertTrue(new_camera_name in self.cam_client.get_cameras())
# Delete the camera config you just added.
self.cam_client.delete_camera_config(new_camera_name)
self.assertTrue(new_camera_name not in self.cam_client.get_cameras())
def get_single_message(self):
self.pipeline_client.create_instance_from_name("pipeline_example_1",
"simulation_sp1")
# Name of the camera we want to get a message from.
camera_name = "simulation"
# Screen panel defines the instance name as: [CAMERA_NAME]_sp1
instance_name = camera_name + "_sp1"
# Get the data.
data = self.pipeline_client.get_instance_message(instance_name)
self.assertIsNotNone(data)
self.assertTrue("image" in data.data.data)
def test_save_camera_stream_to_h5(self):
from bsread import h5, SUB
camera_name = "simulation"
file_name = "testing_dump.h5"
n_messages = 10
# Change to match your camera server
server_address = "http://0.0.0.0:8889"
client = PipelineClient(server_address)
instance_id, stream_address = client.create_instance_from_config(
{"camera_name": camera_name})
# The output file 'output.h5' has 10 images from the simulation camera stream.
h5.receive(source=stream_address,
file_name=file_name,
mode=SUB,
n_messages=n_messages)
self.assertTrue(os.path.exists(file_name))
class PipelineTransceiverTest(unittest.TestCase):
def setUp(self):
self.host = "127.0.0.1"
self.port = 8888
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.user_scripts_folder = os.path.join(test_base_dir, "user_scripts/")
self.process = Process(target=start_camera_server,
args=(self.host, self.port, self.config_folder,
self.user_scripts_folder))
self.process.start()
# Give it some time to start.
sleep(1.0)
self.rest_api_endpoint = "http://%s:%s" % (self.host, self.port)
self.client = CamClient(self.rest_api_endpoint)
def tearDown(self):
test_cleanup(
[self.client], [self.process],
[os.path.join(self.config_folder, "simulation_temp.json")])
def test_pipeline_with_simulation_camera(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
pipeline_config = PipelineConfig("test_pipeline")
def send():
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12000,
MockBackgroundManager())
thread = Thread(target=send)
thread.start()
with source(host="127.0.0.1",
port=12000,
mode=SUB,
receive_timeout=3000) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
required_keys = set([
'x_fit_standard_deviation', 'x_axis',
'y_fit_standard_deviation', 'x_center_of_mass',
'x_fit_amplitude', 'y_fit_mean', 'processing_parameters',
'timestamp', 'y_fit_gauss_function', 'y_profile',
'y_center_of_mass', 'x_fit_gauss_function', 'x_rms', 'y_rms',
'y_fit_amplitude', 'image', 'y_axis', 'min_value',
'x_fit_mean', 'max_value', 'x_fit_offset', 'x_profile',
'y_fit_offset', 'width', 'height', 'intensity', "x_fwhm",
"y_fwhm"
])
self.assertSetEqual(
required_keys, set(data.data.data.keys()),
"Missing required keys in pipeline output bsread message.")
stop_event.set()
thread.join()
def test_system_exit(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
pipeline_config = PipelineConfig("test_pipeline",
parameters={
"camera_name": "simulation",
"image_background":
"full_background",
"image_background_enable": True,
"image_threshold": 0
})
background_manager = MockBackgroundManager()
with self.assertRaises(SystemExit):
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12001,
background_manager)
def test_pipeline_background_manager(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
pipeline_config = PipelineConfig("test_pipeline",
parameters={
"camera_name": "simulation",
"image_background":
"full_background",
"image_background_enable": True,
"image_threshold": 0
})
background_manager = MockBackgroundManager()
x_size, y_size = get_simulated_camera().get_geometry()
simulated_camera_shape = (y_size, x_size)
background_array = numpy.zeros(shape=simulated_camera_shape)
background_array.fill(65535)
background_manager.save_background("full_background",
background_array,
append_timestamp=False)
def send():
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12001,
background_manager)
thread = Thread(target=send)
thread.start()
with source(host="127.0.0.1",
port=12001,
mode=SUB,
receive_timeout=3000) as stream:
import time
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
self.assertTrue(
numpy.array_equal(data.data.data["image"].value,
numpy.zeros(shape=simulated_camera_shape)))
stop_event.set()
thread.join()
def test_rotate_camera(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
self.client.set_camera_config(
"simulation_temp", self.client.get_camera_config("simulation"))
pipeline_config = PipelineConfig(
"test_pipeline", parameters={"camera_name": "simulation_temp"})
background_manager = MockBackgroundManager()
x_size, y_size = get_simulated_camera().get_geometry()
simulated_camera_shape = (y_size, x_size)
def send():
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12002,
background_manager)
thread = Thread(target=send)
thread.start()
with source(host="127.0.0.1",
port=12002,
mode=SUB,
receive_timeout=3000) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
x_size = data.data.data["width"].value
y_size = data.data.data["height"].value
x_axis = data.data.data["x_axis"].value
y_axis = data.data.data["y_axis"].value
image_shape = data.data.data["image"].value.shape
self.assertEqual(x_size, simulated_camera_shape[1])
self.assertEqual(y_size, simulated_camera_shape[0])
# Sanity checks.
self.assertEqual(x_size, len(x_axis))
self.assertEqual(y_size, len(y_axis))
self.assertEqual(image_shape, (y_size, x_size))
# Rotate the image by 90 degree.
camera_config = self.client.get_camera_config("simulation_temp")
camera_config["rotate"] = 1
self.client.set_camera_config("simulation_temp", camera_config)
# Make a few frames pass.
for _ in range(5):
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
x_size = data.data.data["width"].value
y_size = data.data.data["height"].value
x_axis = data.data.data["x_axis"].value
y_axis = data.data.data["y_axis"].value
image_shape = data.data.data["image"].value.shape
# X and Y size should be inverted.
self.assertEqual(x_size, simulated_camera_shape[0])
self.assertEqual(y_size, simulated_camera_shape[1])
# Sanity checks.
self.assertEqual(x_size, len(x_axis))
self.assertEqual(y_size, len(y_axis))
self.assertEqual(image_shape, (y_size, x_size))
self.client.delete_camera_config("simulation_temp")
stop_event.set()
thread.join()
def test_store_pipeline_with_simulated_camera(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
self.client.stop_all_instances()
pipeline_config = PipelineConfig("test_pipeline")
def send():
store_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12003,
MockBackgroundManager())
thread = Thread(target=send)
thread.start()
sleep(0.5)
with source(host="127.0.0.1",
port=12003,
mode=PULL,
receive_timeout=3000) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Receiving timeout.")
required_keys = set(["simulation" + config.EPICS_PV_SUFFIX_IMAGE])
self.assertSetEqual(
required_keys, set(data.data.data.keys()),
"Missing required keys in pipeline output bsread message.")
stop_event.set()
thread.join()
def test_transparent_pipeline(self):
manager = multiprocessing.Manager()
stop_event = multiprocessing.Event()
statistics = manager.Namespace()
parameter_queue = multiprocessing.Queue()
pipeline_config = PipelineConfig("test_pipeline",
parameters={
"camera_name": "simulation",
"function": "transparent"
})
def send():
processing_pipeline(stop_event, statistics, parameter_queue,
self.client, pipeline_config, 12004,
MockBackgroundManager())
thread = Thread(target=send)
thread.start()
with source(host="127.0.0.1",
port=12004,
mode=SUB,
receive_timeout=3000) as stream:
data = stream.receive()
self.assertIsNotNone(data, "Received None message.")
required_keys = set([
"image", "timestamp", "width", "height", "x_axis", "y_axis",
"processing_parameters"
])
self.assertSetEqual(
required_keys, set(data.data.data.keys()),
"Missing required keys in pipeline output bsread message.")
stop_event.set()
thread.join(5.0)
class PipelineManagerTest(unittest.TestCase):
def setUp(self):
self.host = "0.0.0.0"
self.port = 8888
test_base_dir = os.path.split(os.path.abspath(__file__))[0]
self.config_folder = os.path.join(test_base_dir, "camera_config/")
self.background_folder = os.path.join(test_base_dir,
"background_config/")
self.process = Process(target=start_camera_server,
args=(self.host, self.port, self.config_folder))
self.process.start()
# Give it some time to start.
sleep(0.5)
server_address = "http://%s:%s" % (self.host, self.port)
self.client = CamClient(server_address)
def tearDown(self):
self.client.stop_all_cameras()
try:
os.kill(self.process.pid, signal.SIGINT)
except:
pass
try:
os.remove(os.path.join(self.config_folder, "testing_camera.json"))
except:
pass
try:
os.remove(
os.path.join(self.background_folder, "white_background.npy"))
except:
pass
# Wait for the server to die.
sleep(1)
def test_get_pipeline_list(self):
pipeline_manager = get_test_pipeline_manager()
self.assertEqual(len(pipeline_manager.get_pipeline_list()), 0,
"Pipeline manager should be empty by default.")
initial_config = {"test_pipeline1": {}, "test_pipeline2": {}}
pipeline_manager.config_manager.config_provider.configs = initial_config
self.assertListEqual(sorted(list(initial_config.keys())),
sorted(pipeline_manager.get_pipeline_list()),
"Set and received lists are not the same.")
def test_create_pipeline_instance(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
pipeline_config = {"camera_name": "simulation"}
instance_manager.config_manager.save_pipeline_config(
"test_pipeline", pipeline_config)
pipeline_id_1, stream_address_1 = instance_manager.create_pipeline(
"test_pipeline")
pipeline_id_2, stream_address_2 = instance_manager.create_pipeline(
"test_pipeline")
self.assertNotEqual(pipeline_id_1, pipeline_id_2,
"The pipeline ids should be different.")
self.assertNotEqual(stream_address_1, stream_address_2,
"The pipeline streams should be different.")
self.assertFalse(
instance_manager.get_info()["active_instances"][pipeline_id_1]
["read_only"], "Instance should not be read only.")
self.assertFalse(
instance_manager.get_info()["active_instances"][pipeline_id_2]
["read_only"], "Instance should not be read only.")
pipeline_id_3, stream_address_3 = instance_manager.create_pipeline(
configuration=pipeline_config)
self.assertNotEqual(pipeline_id_2, pipeline_id_3,
"The pipeline ids should be different.")
self.assertNotEqual(stream_address_2, stream_address_3,
"The pipeline streams should be different.")
self.assertFalse(
instance_manager.get_info()["active_instances"][pipeline_id_3]
["read_only"], "Instance should not be read only.")
with self.assertRaisesRegex(
ValueError, "You must specify either the pipeline name or the "
"configuration for the pipeline."):
instance_manager.create_pipeline(pipeline_name="test_pipeline",
configuration=pipeline_config)
instance_manager.stop_all_instances()
def test_get_instance_stream(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
instance_manager.stop_all_instances()
self.assertTrue(
len(instance_manager.get_info()["active_instances"]) == 0,
"There should be no running instances.")
pipeline_id = "test_pipeline"
pipeline_config = PipelineConfig(
pipeline_id, parameters={"camera_name": "simulation"})
instance_manager.config_manager.save_pipeline_config(
pipeline_id, pipeline_config.get_configuration())
instance_stream_1 = instance_manager.get_instance_stream(pipeline_id)
self.assertTrue(
instance_manager.get_info()["active_instances"][pipeline_id]
["read_only"], "Instance should be read only.")
with self.assertRaisesRegex(
ValueError, "Cannot set config on a read only instance."):
instance_manager.get_instance(pipeline_id).set_parameter({})
instance_stream_2 = instance_manager.get_instance_stream(pipeline_id)
self.assertEqual(instance_stream_1, instance_stream_2,
"Only one instance should be present.")
instance_manager.stop_all_instances()
def test_collect_background(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
pipeline_id = "test_pipeline"
number_of_images = 10
pipeline_config = PipelineConfig(
pipeline_id, parameters={"camera_name": "simulation"})
instance_manager.config_manager.save_pipeline_config(
pipeline_id, pipeline_config.get_configuration())
pipeline_stream_address = instance_manager.get_instance_stream(
pipeline_id)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
pipeline_stream_address)
# Collect from the pipeline.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data,
"This should really not happen anymore.")
camera_name = instance_manager.get_instance(
pipeline_id).get_info()["camera_name"]
background_id = collect_background(camera_name,
pipeline_stream_address,
number_of_images,
instance_manager.background_manager)
self.assertTrue(background_id.startswith("simulation"),
"Background id not as expected.")
host, port = get_host_port_from_stream_address(
instance_manager.cam_server_client.get_camera_stream("simulation"))
# Collect from the camera.
with source(host=host, port=port, mode=SUB) as stream:
data = stream.receive()
self.assertIsNotNone(data,
"This should really not happen anymore.")
self.assertEqual(
instance_manager.background_manager.get_background(
background_id).shape, data.data.data["image"].value.shape,
"Background and image have to be of the same shape.")
self.assertEqual(
instance_manager.background_manager.get_background(
background_id).dtype, data.data.data["image"].value.dtype,
"Background and image have to be of the same dtype.")
instance_manager.stop_all_instances()
def test_custom_hostname(self):
config_manager = PipelineConfigManager(
config_provider=MockConfigStorage())
pipeline_instance_manager = PipelineInstanceManager(
config_manager,
MockBackgroundManager(),
CamClient("http://0.0.0.0:8888"),
hostname="custom_cam_hostname")
_, stream_address = pipeline_instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
self.assertTrue(stream_address.startswith("tcp://custom_cam_hostname"))
pipeline_instance_manager.stop_all_instances()
def test_update_instance_config_without_running(self):
pipeline_manager = get_test_pipeline_manager()
instance_id, _ = pipeline_manager.create_pipeline(
configuration={"camera_name": "simulation"})
config_updates = {
"camera_name": "different_name",
}
with self.assertRaisesRegex(
ValueError,
"Cannot change the camera name on a running instance. "
"Stop the instance first."):
pipeline_manager.update_instance_config(instance_id,
config_updates)
config_updates = {
"camera_name": "simulation",
"camera_calibration": {
"reference_marker": [1, 2, 3, 4],
"reference_marker_width": 5.0,
"reference_marker_height": 6.0,
"angle_horizontal": 7.0,
"angle_vertical": 8.0
},
"image_background": None,
"image_background_enable": False,
"image_threshold": 2,
"image_region_of_interest": [3, 4, 5, 6],
"image_good_region": {
"threshold": 0.9,
"gfscale": 3.6
},
"image_slices": {
"number_of_slices": 6,
"scale": 7,
"orientation": "vertical"
},
"pipeline_type": "processing"
}
pipeline_manager.update_instance_config(instance_id, config_updates)
self.assertDictEqual(
pipeline_manager.get_instance(instance_id).get_configuration(),
config_updates, "Update was not successful.")
self.assertDictEqual(
pipeline_manager.get_instance(
instance_id).get_configuration()["camera_calibration"],
config_updates["camera_calibration"], "Update was not successful.")
self.assertDictEqual(
pipeline_manager.get_instance(
instance_id).get_configuration()["image_good_region"],
config_updates["image_good_region"], "Update was not successful.")
self.assertDictEqual(
pipeline_manager.get_instance(
instance_id).get_configuration()["image_slices"],
config_updates["image_slices"], "Update was not successful.")
with self.assertRaisesRegex(
ValueError,
"Requested background 'non_existing' does not exist."):
pipeline_manager.update_instance_config(
instance_id, {"image_background": "non_existing"})
pipeline_manager.background_manager.save_background(
"non_existing", None, append_timestamp=False)
pipeline_manager.update_instance_config(
instance_id, {"image_background": "non_existing"})
self.assertEqual(
pipeline_manager.get_instance(instance_id).get_configuration()
["image_background"], "non_existing", "Background not updated.")
pipeline_manager.update_instance_config(instance_id,
{"image_background": None})
self.assertIsNone(
pipeline_manager.get_instance(instance_id).get_configuration()
["image_background"], "Background should be None.")
pipeline_manager.update_instance_config(instance_id,
{"image_slices": None})
self.assertIsNone(
pipeline_manager.get_instance(instance_id).get_configuration()
["image_slices"], "Sub dictionary not set to None.")
pipeline_manager.update_instance_config(
instance_id, {"image_good_region": {
"gfscale": None
}})
self.assertEqual(
pipeline_manager.get_instance(instance_id).get_configuration()
["image_good_region"]["gfscale"],
PipelineConfig.DEFAULT_IMAGE_GOOD_REGION["gfscale"],
"Default value not set correctly.")
pipeline_manager.update_instance_config(instance_id,
{"pipeline_type": None})
self.assertEqual(
pipeline_manager.get_instance(
instance_id).get_configuration()["pipeline_type"],
PipelineConfig.DEFAULT_CONFIGURATION["pipeline_type"],
"Default value not set correctly.")
pipeline_manager.stop_all_instances()
def test_update_instance_config_with_running(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
instance_manager.background_manager = BackgroundImageManager(
self.background_folder)
pipeline_config = {"camera_name": "simulation"}
black_image = numpy.zeros(shape=(960, 1280))
pipeline_id, pipeline_stream_address = instance_manager.create_pipeline(
configuration=pipeline_config)
pipeline_host, pipeline_port = get_host_port_from_stream_address(
pipeline_stream_address)
# Collect from the pipeline.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
normal_image_data = stream.receive()
self.assertIsNotNone(normal_image_data,
"This should really not happen anymore.")
self.assertFalse(
numpy.array_equal(normal_image_data.data.data['image'].value,
black_image),
"There should be a non black image.")
self.assertEqual(normal_image_data.data.data["width"].value, 1280,
"Incorrect width.")
self.assertEqual(normal_image_data.data.data["height"].value, 960,
"Incorrect height.")
white_image = numpy.zeros(shape=(960, 1280))
white_image.fill(99999)
instance_manager.background_manager.save_background(
"white_background", white_image, append_timestamp=False)
instance_manager.update_instance_config(
pipeline_id, {"image_background": "white_background"})
# Give it some time to load the background.
sleep(0.5)
# Collect from the camera.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
image_data = stream.receive()
self.assertIsNotNone(image_data,
"This should really not happen anymore.")
# Because we didn't set the "image_background_enable" yet.
self.assertFalse(
numpy.array_equal(image_data.data.data["image"].value,
black_image),
"Background should not be yet applied.")
instance_manager.update_instance_config(pipeline_id, {
"image_background_enable": True,
"image_threshold": 0
})
# Give it some time to load the background.
sleep(0.5)
# Collect from the camera.
with source(host=pipeline_host, port=pipeline_port,
mode=SUB) as stream:
black_image_data = stream.receive()
self.assertIsNotNone(black_image_data,
"This should really not happen anymore.")
self.assertTrue(
numpy.array_equal(black_image_data.data.data["image"].value,
black_image), "Now background should work.")
instance_manager.stop_all_instances()
def test_delete_pipeline(self):
instance_manager = get_test_pipeline_manager()
self.assertEqual(len(instance_manager.get_pipeline_list()), 0,
"Pipeline should be empty")
instance_manager.config_manager.save_pipeline_config(
"test", {"camera_name": "simulation"})
self.assertEqual(len(instance_manager.get_pipeline_list()), 1,
"Pipeline should not be empty")
instance_manager.config_manager.delete_pipeline_config("test")
self.assertEqual(len(instance_manager.get_pipeline_list()), 0,
"Pipeline should be empty")
def test_custom_instance_id(self):
instance_manager = get_test_pipeline_manager()
instance_manager.config_manager.save_pipeline_config(
"test_pipeline", {"camera_name": "simulation"})
instance_id, stream_address = instance_manager.create_pipeline(
"test_pipeline", instance_id="custom_instance")
self.assertEqual(instance_id, "custom_instance",
"Custom instance name was not set.")
with self.assertRaisesRegex(
ValueError,
"Instance with id 'custom_instance' is already present and running. "
"Use another instance_id or stop the current instance "
"if you want to reuse the same instance_id."):
instance_manager.create_pipeline("test_pipeline",
instance_id="custom_instance")
instance_manager.stop_instance("custom_instance")
instance_id, stream_address = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"},
instance_id="custom_instance")
self.assertEqual(instance_id, "custom_instance",
"Custom instance name was not set.")
instance_manager.stop_all_instances()
def test_reload_after_stop(self):
instance_manager = get_test_pipeline_manager()
instance_id = "trying_to_test"
initial_config = {"camera_name": "simulation"}
instance_manager.config_manager.save_pipeline_config(
instance_id, initial_config)
instance_manager.get_instance_stream(instance_id)
self.assertEqual(
instance_manager.get_instance(instance_id).get_configuration()
["image_threshold"], None)
updated_config = {"camera_name": "simulation", "image_threshold": 9999}
instance_manager.config_manager.save_pipeline_config(
instance_id, updated_config)
self.assertEqual(
instance_manager.get_instance(instance_id).get_configuration()
["image_threshold"], None, "It should not change at this point.")
instance_manager.stop_instance(instance_id)
instance_manager.get_instance_stream(instance_id)
self.assertEqual(
instance_manager.get_instance(
instance_id).get_configuration()["image_threshold"], 9999,
"It should have changed now - reload should happen.")
instance_manager.stop_all_instances()
def test_update_stopped_instance(self):
instance_manager = get_test_pipeline_manager()
instance_id, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_manager.update_instance_config(instance_id, {
"camera_name": "simulation",
"image_threshold": 9999
})
instance_manager.stop_instance(instance_id)
with self.assertRaisesRegex(
ValueError, "Instance '%s' does not exist." % instance_id):
instance_manager.update_instance_config(instance_id, {
"camera_name": "simulation",
"image_threshold": 9999
})
instance_manager.stop_all_instances()
def test_multiple_get_stream(self):
instance_manager = get_test_pipeline_manager()
instance_manager.config_manager.save_pipeline_config(
"simulation", {"camera_name": "simulation"})
stream_address_1 = instance_manager.get_instance_stream("simulation")
stream_address_2 = instance_manager.get_instance_stream("simulation")
self.assertEqual(stream_address_1, stream_address_2)
self.assertTrue(instance_manager.is_instance_present("simulation"))
instance_port = instance_manager.get_instance(
"simulation").get_stream_port()
self.assertEqual(instance_manager._used_ports[instance_port],
"simulation")
instance_manager.stop_instance("simulation")
self.assertEqual(len(instance_manager._used_ports), 0)
self.assertFalse(instance_manager.is_instance_present("simulation"),
"Instance should have been deleted.")
stream_address_3 = instance_manager.get_instance_stream("simulation")
self.assertNotEqual(
stream_address_1, stream_address_3,
"The instance was stopped, the stream should have changed.")
instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
self.assertEqual(len(instance_manager._used_ports), 4,
"There should be 4 used ports.")
instance_manager.stop_all_instances()
self.assertEqual(len(instance_manager._used_ports), 0,
"All ports should be free now.")
def test_out_of_ports(self):
config_manager = PipelineConfigManager(
config_provider=MockConfigStorage())
instance_manager = PipelineInstanceManager(config_manager,
MockBackgroundManager(),
MockCamServerClient(),
port_range=(12000, 12003))
instance_id_0, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_id_1, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_id_2, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
self.assertEqual(
instance_manager.get_instance(instance_id_0).get_stream_port(),
12000)
self.assertEqual(
instance_manager.get_instance(instance_id_1).get_stream_port(),
12001)
self.assertEqual(
instance_manager.get_instance(instance_id_2).get_stream_port(),
12002)
with self.assertRaisesRegex(
Exception,
"All ports are used. Stop some instances before opening a new stream."
):
instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_manager.stop_instance(instance_id_1)
instance_id_1, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
self.assertEqual(
instance_manager.get_instance(instance_id_1).get_stream_port(),
12001,
"Instance_id_1 should have freeded the port 10001, but some other port was assigned."
)
instance_manager.stop_all_instances()
# Check if the port rotation works as expected.
instance_id_2, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_id_0, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_id_1, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
self.assertEqual(
instance_manager.get_instance(instance_id_2).get_stream_port(),
12002)
self.assertEqual(
instance_manager.get_instance(instance_id_0).get_stream_port(),
12000)
self.assertEqual(
instance_manager.get_instance(instance_id_1).get_stream_port(),
12001)
instance_manager.stop_all_instances()
old_timeout = config.MFLOW_NO_CLIENTS_TIMEOUT
config.MFLOW_NO_CLIENTS_TIMEOUT = 1
# Test the cleanup procedure.
instance_id_2, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_id_0, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_id_1, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
self.assertEqual(len(instance_manager.get_info()["active_instances"]),
3, "All 3 instances should be running")
with self.assertRaisesRegex(
Exception,
"All ports are used. Stop some instances before opening a new stream."
):
instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
# Wait for the instances to die.
sleep(5)
self.assertEqual(len(instance_manager.get_info()["active_instances"]),
0, "All instances should be dead by now.")
instance_id_2, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_id_0, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
instance_id_1, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
# Restore previous state.
config.MFLOW_NO_CLIENTS_TIMEOUT = old_timeout
instance_manager.stop_all_instances()
def test_camera_offline(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
with self.assertRaisesRegex(
ValueError,
"Camera camera_example_1 is not online. Cannot start pipeline."
):
instance_manager.create_pipeline(
configuration={"camera_name": "camera_example_1"})
def test_last_start_time(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
instance_id_0, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
pipeline_info = instance_manager.get_instance(instance_id_0).get_info()
self.assertTrue("last_start_time" in pipeline_info)
last_time = pipeline_info["last_start_time"]
new_last_time = instance_manager.get_instance(
instance_id_0).get_info()["last_start_time"]
self.assertEqual(
last_time, new_last_time,
"The instance was still running, the times should be the same.")
instance_manager.stop_all_instances()
def test_statistics(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
instance_id_0, _ = instance_manager.create_pipeline(
configuration={"camera_name": "simulation"})
latest_statistics = instance_manager.get_instance(
instance_id_0).get_statistics()
self.assertTrue("n_clients" in latest_statistics)
self.assertTrue("input_throughput" in latest_statistics)
instance_manager.stop_all_instances()
def test_get_instance_from_config(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
configuration_1 = {
"camera_name": "simulation",
"pipeline_type": "store"
}
instance_id_1, instance_stream_1 = instance_manager.get_instance_stream_from_config(
configuration_1)
instance_id_2, instance_stream_2 = instance_manager.get_instance_stream_from_config(
configuration_1)
self.assertEqual(instance_id_1, instance_id_2)
self.assertEqual(instance_stream_1, instance_stream_2)
self.assertTrue(
instance_manager.get_instance(instance_id_1).is_read_only_config())
configuration_2 = {
"camera_name": "simulation",
"pipeline_type": "processing"
}
instance_id_3, instance_stream_3 = instance_manager.get_instance_stream_from_config(
configuration_2)
self.assertTrue(
instance_manager.get_instance(instance_id_3).is_read_only_config())
self.assertNotEqual(instance_id_1, instance_id_3)
self.assertNotEqual(instance_stream_1, instance_stream_3)
instance_id_4, instance_stream_4 = instance_manager.create_pipeline(
configuration=configuration_1)
instance_id_5, instance_stream_5 = instance_manager.get_instance_stream_from_config(
configuration_1)
self.assertFalse(
instance_manager.get_instance(instance_id_4).is_read_only_config())
self.assertTrue(
instance_manager.get_instance(instance_id_5).is_read_only_config())
self.assertNotEqual(
instance_id_4, instance_id_5,
"Only read only instances can be returned by get_instance_stream_from_config."
)
self.assertNotEqual(
instance_stream_4, instance_stream_5,
"Only read only instances can be returned by get_instance_stream_from_config."
)
instance_manager.stop_all_instances()
old_timeout = config.MFLOW_NO_CLIENTS_TIMEOUT
config.MFLOW_NO_CLIENTS_TIMEOUT = 1
instance_id_6, instance_stream_6 = instance_manager.get_instance_stream_from_config(
configuration_1)
# Lets be sure that the instance goes down.
sleep(2)
self.assertFalse(
instance_id_6 in instance_manager.get_info()["active_instances"],
"Instance not stopped yet.")
instance_id_7, instance_stream_7 = instance_manager.get_instance_stream_from_config(
configuration_1)
self.assertNotEqual(instance_id_6, instance_id_7,
"A new instance id should be assigned here.")
config.MFLOW_NO_CLIENTS_TIMEOUT = old_timeout
instance_manager.stop_all_instances()
def test_get_invalid_instance_stream(self):
instance_manager = get_test_pipeline_manager_with_real_cam()
with self.assertRaisesRegex(
ValueError, "is not present on server and it is not a saved"):
instance_manager.get_instance_stream("simulation_sp1")
instance_manager.stop_all_instances()