def test_can_configure_stereo(self):
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world',
config={'stereo_offset': 1})
self.assertTrue(subject.is_stereo_available)
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world',
config={'stereo_offset': 0})
self.assertFalse(subject.is_stereo_available)
def test_can_configure_labels(self):
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world',
config={'provide_labels': True})
self.assertTrue(subject.is_labels_available)
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world',
config={'provide_labels': False})
self.assertFalse(subject.is_labels_available)
def test_get_next_image_cleans_up_image_files(self, mock_client, mock_cv2,
mock_os_remove, *_):
mock_cv2.imread.side_effect = (
lambda x, _=None: make_mock_image('object_mask')
if 'label' in x else make_mock_image(x))
filenames = {
'vget /camera/0/lit': '/tmp/001.png',
'vget /camera/0/depth': '/tmp/001.depth.png',
'vget /camera/0/object_mask': '/tmp/001.labels.png',
'vget /camera/0/normal': '/tmp/001.normal.png'
}
mock_client_instance = make_mock_unrealcv_client()
mock_client_instance.request.side_effect = (
lambda x: filenames[x]
if x in filenames else mock_unrealcv_request(x))
mock_client.return_value = mock_client_instance
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world',
config={
'provide_depth': True,
'provide_labels': True,
'provide_world_normals': True
})
subject.begin()
subject.get_next_image()
self.assertTrue(mock_os_remove.called)
for path in filenames.values():
self.assertIn(mock.call(path), mock_os_remove.call_args_list)
def test_begin_saves_settings(self, *_):
port = np.random.randint(0, 1000)
width = np.random.randint(10, 1000)
height = np.random.randint(10, 1000)
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world',
config={
'port': port,
'resolution': {
'width': width,
'height': height
},
})
mock_open = mock.mock_open()
with mock.patch('simulation.unrealcv.unrealcv_simulator.open',
mock_open,
create=True):
subject.begin()
self.assertTrue(mock_open.called)
self.assertEqual(mock.call('temp/test_project/unrealcv.ini', 'w'),
mock_open.call_args)
mock_file = mock_open()
self.assertTrue(mock_file.write.called)
file_contents = mock_file.write.call_args[0][0]
self.assertIn('Port={0}'.format(port), file_contents)
self.assertIn('Width={0}'.format(width), file_contents)
self.assertIn('Height={0}'.format(height), file_contents)
def test_get_next_image_returns_none_if_unstarted(self, *_):
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
result = subject.get_next_image()
self.assertEqual(2, len(result))
self.assertIsNone(result[0])
self.assertIsNone(result[1])
def test_begin_sets_camera_properties(self, mock_client, *_):
mock_client_instance = make_mock_unrealcv_client()
mock_client_instance.isconnected.return_value = True
mock_client.return_value = mock_client_instance
fov = np.random.uniform(1, 90)
focus_distance = np.random.uniform(0, 10000)
aperture = np.random.uniform(1, 24)
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world',
config={
'fov': fov,
'depth_of_field_enabled': True,
'focus_distance':
focus_distance,
'aperture': aperture,
})
subject.begin()
self.assertTrue(mock_client_instance.request.called)
self.assertIn(mock.call("vset /camera/0/fov {0}".format(fov)),
mock_client_instance.request.call_args_list)
self.assertIn(mock.call("vset /camera/0/fstop {0}".format(aperture)),
mock_client_instance.request.call_args_list)
self.assertIn(mock.call("vset /camera/0/enable-dof 1"),
mock_client_instance.request.call_args_list)
self.assertIn(mock.call("vset /camera/0/autofocus 0"),
mock_client_instance.request.call_args_list)
self.assertIn(
mock.call("vset /camera/0/focus-distance {0}".format(
focus_distance * 100)),
mock_client_instance.request.call_args_list)
def test_begin_starts_simulator(self, mock_subprocess, *_):
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
subject.begin()
self.assertTrue(mock_subprocess.Popen.called)
self.assertEqual(mock.call('temp/test_project/test.sh'),
mock_subprocess.Popen.call_args)
def test_begin_connects_to_simulator(self, mock_client, *_):
mock_client_instance = make_mock_unrealcv_client()
mock_client_instance.isconnected.return_value = False
mock_client.return_value = mock_client_instance
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
subject.begin()
self.assertTrue(mock_client_instance.connect.called)
def test_set_fstop(self, mock_client, mock_cv2, *_):
mock_cv2.imread.side_effect = make_mock_image
mock_client_instance = make_mock_unrealcv_client()
mock_client.return_value = mock_client_instance
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
subject.begin()
subject.set_fstop(22.23)
self.assertIn(mock.call("vset /camera/0/fstop 22.23"),
mock_client_instance.request.call_args_list)
def test_set_focus_distance(self, mock_client, mock_cv2, *_):
mock_cv2.imread.side_effect = make_mock_image
mock_client_instance = make_mock_unrealcv_client()
mock_client.return_value = mock_client_instance
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
subject.begin()
subject.set_focus_distance(19.043) # Distance is in meters
self.assertIn(mock.call("vset /camera/0/focus-distance 1904.3"),
mock_client_instance.request.call_args_list)
def test_get_next_image_returns_image_and_none(self, mock_client, mock_cv2,
*_):
mock_cv2.imread.side_effect = make_mock_image
mock_client.return_value = make_mock_unrealcv_client()
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
subject.begin()
result = subject.get_next_image()
self.assertEqual(2, len(result))
self.assertIsInstance(result[0], core.image.Image)
self.assertIsNone(result[1])
def test_get_next_image_captures_lit(self, mock_client, mock_cv2, *_):
mock_cv2.imread.side_effect = make_mock_image
mock_client_instance = make_mock_unrealcv_client()
mock_client.return_value = mock_client_instance
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
subject.begin()
mock_client_instance.request.called = False
subject.get_next_image()
self.assertTrue(mock_client_instance.request.called)
self.assertIn(mock.call("vget /camera/0/lit"),
mock_client_instance.request.call_args_list)
def make_instance(self, *args, **kwargs):
kwargs = du.defaults(
kwargs, {
'executable_path':
'temp/test_project/test-{0}.sh'.format(
np.random.randint(0, 1000)),
'world_name':
'sim_world_{0}'.format(np.random.randint(0, 1000)),
'environment_type':
imeta.EnvironmentType(np.random.randint(0, 4)),
'light_level':
imeta.LightingLevel(np.random.randint(0, 6)),
'time_of_day':
imeta.TimeOfDay(np.random.randint(0, 6)),
'config': {}
})
return uecvsim.UnrealCVSimulator(*args, **kwargs)
def test_set_camera_pose_handles_frame_conversion(self, mock_client,
mock_cv2, *_):
mock_cv2.imread.side_effect = make_mock_image
mock_client_instance = make_mock_unrealcv_client()
mock_client.return_value = mock_client_instance
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
subject.begin()
pose = tf.Transform((17, -21, 3), (0.1, 0.7, -0.3, 0.5))
ue_pose = uetf.transform_to_unreal(pose)
subject.set_camera_pose(pose)
self.assertEqual(subject.current_pose, pose)
self.assertIn(
mock.call(("vset /camera/0/location {0} {1} {2}".format(
ue_pose.location[0], ue_pose.location[1],
ue_pose.location[2]))),
mock_client_instance.request.call_args_list)
self.assertIn(
mock.call(("vset /camera/0/rotation {0} {1} {2}".format(
ue_pose.pitch, ue_pose.yaw, ue_pose.roll))),
mock_client_instance.request.call_args_list)
pose = tf.Transform((-175, 29, -870), (0.3, -0.2, -0.8, 0.6))
ue_pose = uetf.transform_to_unreal(pose)
subject.set_camera_pose(ue_pose)
self.assertTrue(
np.array_equal(subject.current_pose.location, pose.location))
self.assertTrue(
np.all(
np.isclose(subject.current_pose.rotation_quat(True),
pose.rotation_quat(True))))
self.assertIn(
mock.call(("vset /camera/0/location {0} {1} {2}".format(
ue_pose.location[0], ue_pose.location[1],
ue_pose.location[2]))),
mock_client_instance.request.call_args_list)
self.assertIn(
mock.call(("vset /camera/0/rotation {0} {1} {2}".format(
ue_pose.pitch, ue_pose.yaw, ue_pose.roll))),
mock_client_instance.request.call_args_list)
def test_begin_sets_quality_properties(self, mock_client, *_):
mock_client_instance = make_mock_unrealcv_client()
mock_client_instance.isconnected.return_value = True
mock_client.return_value = mock_client_instance
texture_mipmap_bias = np.random.randint(0, 10)
normal_maps_enabled = np.random.randint(0, 2)
roughness_enabled = np.random.randint(0, 2)
geometry_decimation = np.random.randint(0, 15)
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world',
config={
'texture_mipmap_bias':
texture_mipmap_bias,
'normal_maps_enabled':
bool(normal_maps_enabled),
'roughness_enabled':
bool(roughness_enabled),
'geometry_decimation':
geometry_decimation,
})
subject.begin()
self.assertTrue(mock_client_instance.request.called)
self.assertIn(
mock.call("vset /quality/texture-mipmap-bias {0}".format(
texture_mipmap_bias)),
mock_client_instance.request.call_args_list)
self.assertIn(
mock.call("vset /quality/normal-maps-enabled {0}".format(
normal_maps_enabled)),
mock_client_instance.request.call_args_list)
self.assertIn(
mock.call("vset /quality/roughness-enabled {0}".format(
roughness_enabled)),
mock_client_instance.request.call_args_list)
self.assertIn(
mock.call("vset /quality/geometry-decimation {0}".format(
geometry_decimation)),
mock_client_instance.request.call_args_list)
def test_begin_creates_client(self, mock_client, *_):
subject = uecvsim.UnrealCVSimulator('temp/test_project/test.sh',
'sim-world')
subject.begin()
self.assertTrue(mock_client.called)
self.assertEqual(mock_client.call_args, mock.call(('localhost', 9000)))
def do_imports(self, task_manager: batch_analysis.task_manager.TaskManager,
db_client: database.client.DatabaseClient):
"""
Import image sources for evaluation in this experiment
:param task_manager: The task manager, for creating import tasks
:param db_client: The database client, for saving declared objects too small to need a task
:return:
"""
# --------- SIMULATORS -----------
# Add simulators explicitly, they have different metadata, so we can't just search
for exe, world_name, environment_type, light_level, time_of_day in [
('/media/john/Storage/simulators/AIUE_V01_001/LinuxNoEditor/tempTest/Binaries/Linux/tempTest',
'AIUE_V01_001', imeta.EnvironmentType.INDOOR,
imeta.LightingLevel.WELL_LIT, imeta.TimeOfDay.DAY),
#(
# '/media/john/Storage/simulators/AIUE_V01_002/LinuxNoEditor/tempTest/Binaries/Linux/tempTest',
# 'AIUE_V01_002', imeta.EnvironmentType.INDOOR, imeta.LightingLevel.WELL_LIT,
# imeta.TimeOfDay.DAY
#),
('/media/john/Storage/simulators/AIUE_V01_005/LinuxNoEditor/tempTest/Binaries/Linux/tempTest',
'AIUE_V01_005', imeta.EnvironmentType.INDOOR,
imeta.LightingLevel.WELL_LIT, imeta.TimeOfDay.DAY)
]:
if world_name not in self._simulators:
simulator_id = dh.add_unique(
db_client.image_source_collection,
uecv_sim.UnrealCVSimulator(
executable_path=exe,
world_name=world_name,
environment_type=environment_type,
light_level=light_level,
time_of_day=time_of_day))
self._simulators[world_name] = simulator_id
self._set_property('simulators.{0}'.format(world_name),
simulator_id)
# --------- REAL WORLD DATASETS -----------
# Import KITTI dataset
for sequence_num in range(11):
path = os.path.expanduser(
os.path.join('~', 'datasets', 'KITTI', 'dataset'))
if os.path.isdir(path) and os.path.isdir(
os.path.join(path, 'sequences',
"{0:02}".format(sequence_num))):
task = task_manager.get_import_dataset_task(
module_name='dataset.kitti.kitti_loader',
path=path,
additional_args={'sequence_number': sequence_num},
num_cpus=1,
num_gpus=0,
memory_requirements='3GB',
expected_duration='12:00:00')
if task.is_finished:
trajectory_group = self._add_trajectory_group(
'KITTI trajectory {}'.format(sequence_num),
task.result)
self._update_trajectory_group(trajectory_group,
task_manager, db_client)
else:
task_manager.do_task(task)
# Import EuRoC datasets
for name, path in [
('EuRoC MH_01_easy',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'MH_01_easy'))),
('EuRoC MH_02_easy',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'MH_02_easy'))),
('EuRoC MH_02_medium',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'MH_03_medium'))),
('EuRoC MH_04_difficult',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'MH_04_difficult'))),
('EuRoC MH_05_difficult',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'MH_05_difficult'))),
('EuRoC V1_01_easy',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'V1_01_easy'))),
('EuRoC V1_02_medium',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'V1_02_medium'))),
('EuRoC V1_03_difficult',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'V1_03_difficult'))),
('EuRoC V2_01_easy',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'V2_01_easy'))),
('EuRoC V2_02_medium',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'V2_02_medium'))),
('EuRoC V2_03_difficult',
os.path.expanduser(
os.path.join('~', 'datasets', 'EuRoC', 'V2_03_difficult')))
]:
if os.path.isdir(path):
task = task_manager.get_import_dataset_task(
module_name='dataset.euroc.euroc_loader',
path=path,
num_cpus=1,
num_gpus=0,
memory_requirements='3GB',
expected_duration='4:00:00')
if task.is_finished:
trajectory_group = self._add_trajectory_group(
name, task.result)
self._update_trajectory_group(trajectory_group,
task_manager, db_client)
else:
task_manager.do_task(task)
# Import TUM datasets using the manager. Load all the TUM datasets we can
tum_manager = dataset.tum.tum_manager.TUMManager(
{name: True
for name in dataset.tum.tum_manager.dataset_names})
tum_manager.do_imports(
os.path.expanduser(os.path.join('~', 'datasets', 'TUM')),
task_manager)
for name, dataset_id in tum_manager.datasets:
trajectory_group = self._add_trajectory_group(
"TUM {}".format(name), dataset_id)
self._update_trajectory_group(trajectory_group, task_manager,
db_client)
# --------- SYSTEMS -----------
if self._libviso_system is None:
self._libviso_system = dh.add_unique(db_client.system_collection,
libviso2.LibVisOSystem())
self._set_property('libviso', self._libviso_system)
# ORBSLAM2 - Create 9 variants, with different procesing modes
settings_list = [(sensor_mode, n_features) for sensor_mode in {
orbslam2.SensorMode.STEREO, orbslam2.SensorMode.RGBD,
orbslam2.SensorMode.MONOCULAR
} for n_features in {1500}]
if len(self._orbslam_systems) < len(settings_list):
for sensor_mode, n_features in settings_list:
name = 'ORBSLAM2 {mode} ({features})'.format(
mode=sensor_mode.name.lower(), features=n_features)
vocab_path = os.path.join('systems', 'slam', 'ORBSLAM2',
'ORBvoc.txt')
if name not in self._orbslam_systems and os.path.isfile(
vocab_path):
orbslam_id = dh.add_unique(
db_client.system_collection,
orbslam2.ORBSLAM2(vocabulary_file=vocab_path,
mode=sensor_mode,
settings={
'ORBextractor': {
'nFeatures': n_features
}
}))
self._orbslam_systems[name] = orbslam_id
self._set_property('orbslam_systems.{}'.format(name),
orbslam_id)
# --------- BENCHMARKS -----------
# Create and store the benchmarks for camera trajectories
# Just using the default settings for now
if self._benchmark_rpe is None:
self._benchmark_rpe = dh.add_unique(
db_client.benchmarks_collection,
rpe.BenchmarkRPE(max_pairs=10000,
fixed_delta=False,
delta=1.0,
delta_unit='s',
offset=0,
scale_=1))
self._set_property('benchmark_rpe', self._benchmark_rpe)
if self._benchmark_ate is None:
self._benchmark_ate = dh.add_unique(
db_client.benchmarks_collection,
ate.BenchmarkATE(offset=0, max_difference=0.2, scale=1))
self._set_property('benchmark_ate', self._benchmark_ate)
if self._benchmark_trajectory_drift is None:
self._benchmark_trajectory_drift = dh.add_unique(
db_client.benchmarks_collection,
traj_drift.BenchmarkTrajectoryDrift(
segment_lengths=[100, 200, 300, 400, 500, 600, 700, 800],
step_size=10))
self._set_property('benchmark_trajectory_drift',
self._benchmark_trajectory_drift)
if self._benchmark_tracking is None:
self._benchmark_tracking = dh.add_unique(
db_client.benchmarks_collection,
tracking_benchmark.TrackingBenchmark(
initializing_is_lost=True))
self._set_property('benchmark_tracking', self._benchmark_tracking)
def do_imports(self, task_manager: batch_analysis.task_manager.TaskManager,
db_client: database.client.DatabaseClient):
"""
Import image sources for evaluation in this experiment
:param task_manager: The task manager, for creating import tasks
:param db_client: The database client, for saving declared objects too small to need a task
:return:
"""
# --------- SIMULATORS -----------
# Add simulators explicitly, they have different metadata, so we can't just search
for exe, world_name, environment_type, light_level, time_of_day in [
('/media/john/Storage/simulators/CorridorWorld/LinuxNoEditor/tempTest/Binaries/Linux/tempTest',
'CorridorWorld', imeta.EnvironmentType.OUTDOOR_LANDSCAPE,
imeta.LightingLevel.WELL_LIT, imeta.TimeOfDay.DAY)
]:
if world_name not in self._simulators:
simulator_id = dh.add_unique(
db_client.image_source_collection,
uecv_sim.UnrealCVSimulator(
executable_path=exe,
world_name=world_name,
environment_type=environment_type,
light_level=light_level,
time_of_day=time_of_day))
self._simulators[world_name] = simulator_id
self._set_property('simulators.{0}'.format(world_name),
simulator_id)
# --------- TRAJECTORY GROUPS -----------
for name, path in [
('forwards', get_forwards_trajectory()),
('upwards', get_upward_trajectory()),
('left', get_left_trajectory()),
('on the spot roll', get_on_the_spot_roll_trajectory()),
('on the spot pitch', get_on_the_spot_pitch_trajectory()),
('on the spot yaw', get_on_the_spot_yaw_trajectory()),
('circle roll', get_circle_roll_trajectory()),
('circle pitch', get_circle_pitch_trajectory()),
('circle yaw', get_circle_yaw_trajectory()),
]:
if name not in self._trajectory_groups:
# First, create the trajectory follow controller with the desired trajectory
controller = follow_cont.TrajectoryFollowController(
trajectory=path,
trajectory_source='custom {0}'.format(name),
sequence_type=core.sequence_type.ImageSequenceType.
SEQUENTIAL)
controller_id = dh.add_unique(
db_client.image_source_collection, controller)
# Then create a trajectory group for it
self._trajectory_groups[name] = TrajectoryGroup(
name=name,
controller_id=controller_id,
simulators={
'CorridorWorld': self._simulators['CorridorWorld']
})
self._set_property('trajectory_groups.{0}'.format(name),
self._trajectory_groups[name].serialize())
for group in self._trajectory_groups.values():
if group.do_imports(task_manager, db_client):
self._set_property('trajectory_groups.{0}'.format(group.name),
group.serialize())
# --------- SYSTEMS -----------
if self._libviso_system is None:
self._libviso_system = dh.add_unique(db_client.system_collection,
libviso2.LibVisOSystem())
self._set_property('libviso', self._libviso_system)
# ORBSLAM2 - Create orbslam systems in each sensor mode
for sensor_mode in {
orbslam2.SensorMode.STEREO, orbslam2.SensorMode.RGBD,
orbslam2.SensorMode.MONOCULAR
}:
name = 'ORBSLAM2 {mode}'.format(
mode=sensor_mode.name.lower()).replace('.', '-')
vocab_path = os.path.join('systems', 'slam', 'ORBSLAM2',
'ORBvoc.txt')
if name not in self._orbslam_systems and os.path.isfile(
vocab_path):
orbslam_id = dh.add_unique(
db_client.system_collection,
orbslam2.ORBSLAM2(
vocabulary_file=vocab_path,
mode=sensor_mode,
settings={'ORBextractor': {
'nFeatures': 1500
}}))
self._orbslam_systems[name] = orbslam_id
self._set_property('orbslam_systems.{}'.format(name),
orbslam_id)
# --------- BENCHMARKS -----------
# Create and store the benchmarks for camera trajectories
# Just using the default settings for now
if self._benchmark_rpe is None:
self._benchmark_rpe = dh.add_unique(
db_client.benchmarks_collection,
rpe.BenchmarkRPE(max_pairs=10000,
fixed_delta=False,
delta=1.0,
delta_unit='s',
offset=0,
scale_=1))
self._set_property('benchmark_rpe', self._benchmark_rpe)
if self._benchmark_ate is None:
self._benchmark_ate = dh.add_unique(
db_client.benchmarks_collection,
ate.BenchmarkATE(offset=0, max_difference=0.2, scale=1))
self._set_property('benchmark_ate', self._benchmark_ate)
if self._benchmark_trajectory_drift is None:
self._benchmark_trajectory_drift = dh.add_unique(
db_client.benchmarks_collection,
traj_drift.BenchmarkTrajectoryDrift(
segment_lengths=[100, 200, 300, 400, 500, 600, 700, 800],
step_size=10))
self._set_property('benchmark_trajectory_drift',
self._benchmark_trajectory_drift)
if self._benchmark_tracking is None:
self._benchmark_tracking = dh.add_unique(
db_client.benchmarks_collection,
tracking_benchmark.TrackingBenchmark(
initializing_is_lost=True))
self._set_property('benchmark_tracking', self._benchmark_tracking)