def make_instance(self, *args, **kwargs):
du.defaults(
kwargs, {
'benchmark_id': np.random.randint(0, 10),
'trial_result_ids': [bson.ObjectId() for _ in range(4)],
'frame_errors': {
time: [np.random.uniform(-100, 100) for _ in range(18)]
for time in range(100)
},
})
return feb.FrameErrorsResult(*args, **kwargs)
def make_instance(self, *args, **kwargs):
du.defaults(kwargs, {
'benchmark_id': np.random.randint(0, 10),
'trial_result_ids': [bson.ObjectId() for _ in range(4)],
'estimate_errors': {
bson.ObjectId(): [
[np.random.uniform(-100, 100) for _ in range(20)]
for _ in range(1000)
]
for _ in range(10)
},
})
return eeb.EstimateTrialErrorsResult(*args, **kwargs)
def export_data(self, db_client: arvet.database.client.DatabaseClient):
"""
Allow experiments to export some data, usually to file.
I'm currently using this to dump camera trajectories so I can build simulations around them,
but there will be other circumstances where we want to
:param db_client:
:return:
"""
# Save trajectory files for import into unreal
for trajectory_group in self._trajectory_groups.values():
data_helpers.dump_ue4_trajectory(
name=trajectory_group.name,
trajectory=traj_help.get_trajectory_for_image_source(
db_client, trajectory_group.reference_dataset))
# Group and print the trajectories for graphing
systems = du.defaults({'LIBVISO 2': self._libviso_system},
self._orbslam_systems)
for trajectory_group in self._trajectory_groups.values():
# Collect the trial results for each image source in this group
trial_results = {}
for system_name, system_id in systems.items():
for dataset_name, dataset_id in trajectory_group.datasets.items(
):
trial_result_list = self.get_trial_result(
system_id, dataset_id)
for idx, trial_result_id in enumerate(trial_result_list):
label = "{0} on {1} repeat {2}".format(
system_name, dataset_name, idx)
trial_results[label] = trial_result_id
data_helpers.export_trajectory_as_json(
trial_results, "Generated Data " + trajectory_group.name,
db_client)
def make_metadata(index=1, **kwargs):
kwargs = du.defaults(
kwargs, {
'img_hash':
b'\xf1\x9a\xe2|' +
np.random.randint(0, 0xFFFFFFFF).to_bytes(4, 'big'),
'source_type':
imeta.ImageSourceType.SYNTHETIC,
'camera_pose':
tf.Transform(
location=(1 + 100 * index, 2 + np.random.uniform(-1, 1), 3),
rotation=(4, 5, 6, 7 + np.random.uniform(-4, 4))),
'intrinsics':
cam_intr.CameraIntrinsics(800, 600, 550.2, 750.2, 400, 300),
'environment_type':
imeta.EnvironmentType.INDOOR_CLOSE,
'light_level':
imeta.LightingLevel.WELL_LIT,
'time_of_day':
imeta.TimeOfDay.DAY,
'lens_focal_distance':
5,
'aperture':
22,
'simulation_world':
'TestSimulationWorld',
'lighting_model':
imeta.LightingModel.LIT,
'texture_mipmap_bias':
1,
'normal_maps_enabled':
2,
'roughness_enabled':
True,
'geometry_decimation':
0.8,
'procedural_generation_seed':
16234,
'labelled_objects':
(imeta.LabelledObject(class_names=('car', ),
x=12,
y=144,
width=67,
height=43,
relative_pose=tf.Transform(
(12, 3, 4), (0.5, 0.1, 1, 1.7)),
instance_name='Car-002'),
imeta.LabelledObject(class_names=('cat', ),
x=125,
y=244,
width=117,
height=67,
relative_pose=tf.Transform(
(378, -1890, 38), (0.3, 1.12, 1.1, 0.2)),
instance_name='cat-090'))
})
return imeta.ImageMetadata(**kwargs)
def plot_component(ax, trajectories: typing.List[typing.Mapping[float, tf.Transform]],
get_value: typing.Callable[[tf.Transform], float], label: str = '', alpha: float = 1.0, **kwargs):
"""
Plot a particular trajectory component over time on an axis for a number of trajectories
:param ax: The axis on which to plot
:param trajectories: The list of trajectories to plot
:param get_value: A getter to retrieve the y value from the Transform object
:param label: The label
:param alpha: The total alpha value of the line, which will be divided amonst the given trajectories
:return:
"""
du.defaults(kwargs, {
'markersize': 2,
'marker': '.',
'linestyle': 'None'
})
for idx, traj in enumerate(trajectories):
x = sorted(traj.keys())
ax.plot(x, [get_value(traj[t]) for t in x], alpha=alpha / len(trajectories),
label="{0} {1}".format(label, idx), **kwargs)
def __init__(self, config):
self._node_id = config.get('node_id', type(self).__name__)
if config is not None and 'task_config' in config:
task_config = dict(config['task_config'])
else:
task_config = {}
# Default configuration. Also serves as an exemplar configuration argument
du.defaults(
task_config, {
'allow_import_dataset': True,
'allow_run_system': True,
'allow_measure': True,
'allow_trial_comparison': True
})
self._allow_import_dataset = bool(task_config['allow_import_dataset'])
self._allow_run_system = bool(task_config['allow_run_system'])
self._allow_measure = bool(task_config['allow_measure'])
self._allow_trial_comparison = bool(
task_config['allow_trial_comparison'])
def make_instance(self, *args, **kwargs):
kwargs = du.defaults(
kwargs, {
'systems': {
'ORBSLAM monocular': bson.ObjectId(),
'ORBSLAM stereo': bson.ObjectId(),
'LibVisO2': bson.ObjectId()
},
'simulators': {
'Block World': bson.ObjectId()
},
'trajectory_groups': {
'KITTI trajectory 1':
tg.TrajectoryGroup(
name='KITTI trajectory 1',
reference_id=bson.ObjectId(),
mappings=[('Block World', {
'location': [12, -63.2, 291.1],
'rotation': [-22, -214, 121]
})],
baseline_configuration={'test': bson.ObjectId()},
controller_id=bson.ObjectId(),
generated_datasets={
'Block World': {
'max_quality': bson.ObjectId(),
'min_quality': bson.ObjectId()
}
})
},
'benchmarks': {
'benchmark_rpe': bson.ObjectId(),
'benchmark_ate': bson.ObjectId(),
'benchmark_trajectory_drift': bson.ObjectId(),
'benchmark_tracking': bson.ObjectId(),
},
'trial_map': {
bson.ObjectId(): {
bson.ObjectId(): {
'trials': [
bson.ObjectId(),
bson.ObjectId(),
bson.ObjectId()
],
'results': {
bson.ObjectId(): bson.ObjectId()
}
}
}
},
'enabled': True
})
return bgde.GeneratedDataExperiment(*args, **kwargs)
def load_global_config(filename):
# Default global configuration.
# This is what you get if you don't have a configuration file.
config = {
'paths': ['~'],
'temp_folder': '/tmp',
'output_folder': '~',
'database': {},
'image_manager': {
'path': '~/arvet-imgs',
},
'task_config': {
'allow_generate_dataset': True,
'allow_import_dataset': True,
'allow_train_system': True,
'allow_run_system': True,
'allow_benchmark': True,
'allow_trial_comparison': True,
'allow_benchmark_comparison': True,
'allow_experiment_analysis': True
},
'job_system_config': {
'job_system': 'simple',
'node_id': 'unknown-job-system'
},
'logging': { # Default logging configuration
'version': 1,
'formatters': {
'f': {'format': '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'}
},
'handlers': {
'h': {'class': 'logging.StreamHandler', 'formatter': 'f', 'level': logging.DEBUG}
},
'root': {'handlers': ['h'], 'level': logging.DEBUG},
}
}
if os.path.isfile(filename):
loaded_config = None
repeat = 1
while loaded_config is None and repeat <= 3:
with open(filename, 'r') as config_file:
loaded_config = yaml.load(config_file, YamlLoader)
if loaded_config is None:
# Failed to load the config file for some reason, wait and try again
time.sleep(random.uniform(1, 3 * repeat))
repeat += 1
if loaded_config is not None:
config = du.defaults(loaded_config, config)
else:
# No global configuration file, create a default one
save_global_config(filename, config)
return config
def make_instance(self, *args, **kwargs):
kwargs = du.defaults(
kwargs, {
'libviso_system': bson.ObjectId(),
'orbslam_systems': {
'MONOCULAR': bson.ObjectId()
},
'simulators': {
'Block World': bson.ObjectId()
},
'trajectory_groups': {
'KITTI trajectory 1':
kitti_ex.TrajectoryGroup(
name='KITTI trajectory 1',
reference_id=bson.ObjectId(),
baseline_configuration={'test': bson.ObjectId()},
simulators={
'Block World': (bson.ObjectId(), {
'conf': bson.ObjectId()
})
},
controller_id=bson.ObjectId(),
generated_datasets={'Block World': bson.ObjectId()})
},
'benchmark_rpe': bson.ObjectId(),
'benchmark_ate': bson.ObjectId(),
'benchmark_trajectory_drift': bson.ObjectId(),
'benchmark_tracking': bson.ObjectId(),
'trial_map': {
bson.ObjectId(): {
bson.ObjectId(): bson.ObjectId()
}
},
'result_map': {
bson.ObjectId(): {
bson.ObjectId(): bson.ObjectId()
}
},
'enabled': True
})
return kitti_ex.KITTIGeneratedDataExperiment(*args, **kwargs)
def make_image(index=1, **kwargs):
kwargs = du.defaults(
kwargs, {
'pixels': np.random.uniform(0, 255, (32, 32, 3)),
'image_group': 'test',
'metadata': make_metadata(index),
'additional_metadata': {
'Source': 'Generated',
'Resolution': {
'width': 1280,
'height': 720
},
'Material Properties': {
'BaseMipMapBias': 0,
'RoughnessQuality': True
}
},
'depth': np.random.uniform(0, 1, (32, 32)),
'normals': np.random.uniform(0, 1, (32, 32, 3))
})
return im.Image(**kwargs)
def make_stereo_image(index=1, **kwargs):
kwargs = du.defaults(
kwargs, {
'pixels':
np.random.uniform(0, 255, (32, 32, 3)),
'right_pixels':
np.random.uniform(0, 255, (32, 32, 3)),
'image_group':
'test',
'metadata':
make_metadata(index),
'right_metadata':
make_metadata(
index,
camera_pose=tf.Transform(
location=(1 + 100 * index, 2 + np.random.uniform(-1, 1),
4),
rotation=(4, 5, 6, 7 + np.random.uniform(-4, 4))),
),
'additional_metadata': {
'Source': 'Generated',
'Resolution': {
'width': 1280,
'height': 720
},
'Material Properties': {
'BaseMipMapBias': 0,
'RoughnessQuality': True
}
},
'depth':
np.random.uniform(0, 1, (32, 32)),
'right_depth':
np.random.uniform(0, 1, (32, 32)),
'normals':
np.random.uniform(0, 1, (32, 32, 3)),
'right_normals':
np.random.uniform(0, 1, (32, 32, 3))
})
return im.StereoImage(**kwargs)
def make_instance(self, *args, **kwargs):
kwargs = du.defaults(
kwargs, {
'orbslam_mono': bson.ObjectId(),
'orbslam_stereo': bson.ObjectId(),
'datasets': {
'KITTI Trajectory 00': bson.ObjectId()
},
'benchmark_rpe': bson.ObjectId(),
'trial_map': {
bson.ObjectId(): {
bson.ObjectId(): bson.ObjectId()
}
},
'result_map': {
bson.ObjectId(): {
bson.ObjectId(): bson.ObjectId()
}
},
'enabled': True
})
return kitti_verify.OrbslamKITTIVerify(*args, **kwargs)
def export_data(self, db_client: arvet.database.client.DatabaseClient):
simulator_names = {v: k for k, v in self._simulators.items()}
systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems)
for trajectory_group in self._trajectory_groups.values():
# Collect all the image sources for this trajectory group
image_sources = {}
for simulator_id, dataset_id in trajectory_group.generated_datasets.items():
if simulator_id in simulator_names:
image_sources[simulator_names[simulator_id]] = dataset_id
else:
image_sources[simulator_id] = dataset_id
# Collect the trial results for each image source in this group
trial_results = {}
for system_name, system_id in systems.items():
for dataset_name, dataset_id in image_sources.items():
trial_result_list = self.get_trial_results(system_id, dataset_id)
for idx in range(len(trial_result_list)):
label = "{0} on {1} - repeat {2}".format(system_name, dataset_name, idx)
trial_results[label] = trial_result_list[idx]
data_helpers.export_trajectory_as_json(trial_results, "Simple Motion " + trajectory_group.name, db_client)
def _plot_relative_pose_error(self, db_client: arvet.database.client.DatabaseClient):
import matplotlib.pyplot as pyplot
logging.getLogger(__name__).info("Plotting relative pose error...")
# Map system ids and simulator ids to printable names
simulator_names = {v: k for k, v in self._simulators.items()}
systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems)
for trajectory_group in self._trajectory_groups.values():
# Collect all the image sources for this trajectory group
image_sources = {}
for simulator_id, dataset_id in trajectory_group.generated_datasets.items():
if simulator_id in simulator_names:
image_sources[simulator_names[simulator_id]] = dataset_id
else:
image_sources[simulator_id] = dataset_id
if len(image_sources) <= 1:
# Skip where we've only got one image source, it's not interesting.
continue
# Collect the results for each image source in this group
results = {}
for system_name, system_id in systems.items():
for dataset_name, dataset_id in image_sources.items():
trial_result_list = self.get_trial_results(system_id, dataset_id)
label = "{0} on {1}".format(system_name, dataset_name)
for idx in range(len(trial_result_list)):
result_id = self.get_benchmark_result(trial_result_list[idx], self._benchmark_rpe)
if result_id is not None:
if len(trial_result_list) > 1:
results[label + ' repeat {0}'.format(idx)] = result_id
else:
results[label] = result_id
if len(results) > 1:
figure = pyplot.figure(figsize=(14, 10), dpi=80)
figure.suptitle("Relative pose error for {0}".format(trajectory_group.name))
ax = figure.add_subplot(111)
ax.set_xlabel('time')
ax.set_ylabel('relative pose error')
# For each trial result
for label, result_id in results.items():
result = dh.load_object(db_client, db_client.results_collection, result_id)
if result is not None:
if result.success:
x = []
y = []
times = sorted(result.translational_error.keys())
for time in times:
error = result.translational_error[time]
if error < 100:
x.append(time - times[0])
y.append(error)
ax.plot(x, y, '-', label=label, alpha=0.7)
else:
print("Got failed result: {0}".format(result.reason))
logging.getLogger(__name__).info("... plotted rpe for {0}".format(trajectory_group.name))
ax.legend()
pyplot.tight_layout()
pyplot.subplots_adjust(top=0.95, right=0.99)
pyplot.show()
def _plot_trajectories(self, db_client: arvet.database.client.DatabaseClient):
"""
Plot the ground-truth and computed trajectories for each system for each trajectory.
This is important for validation
:param db_client:
:return:
"""
import matplotlib.pyplot as pyplot
# noinspection PyUnresolvedReferences
from mpl_toolkits.mplot3d import Axes3D
logging.getLogger(__name__).info("Plotting trajectories...")
# Map system ids and simulator ids to printable names
simulator_names = {v: k for k, v in self._simulators.items()}
systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems)
for trajectory_group in self._trajectory_groups.values():
# Collect all the image sources for this trajectory group
image_sources = {}
for simulator_id, dataset_id in trajectory_group.generated_datasets.items():
if simulator_id in simulator_names:
image_sources[simulator_names[simulator_id]] = dataset_id
else:
image_sources[simulator_id] = dataset_id
# Collect the trial results for each image source in this group
trial_results = {}
for system_name, system_id in systems.items():
for dataset_name, dataset_id in image_sources.items():
trial_result_list = self.get_trial_results(system_id, dataset_id)
label = "{0} on {1}".format(system_name, dataset_name)
for idx in range(len(trial_result_list)):
if len(trial_result_list) > 1:
trial_results[label + ' repeat {0}'.format(idx)] = trial_result_list[idx]
else:
trial_results[label] = trial_result_list[idx]
# Make sure we have at least one result to plot
if len(trial_results) >= 1:
figure = pyplot.figure(figsize=(14, 10), dpi=80)
figure.suptitle("Computed trajectories for {0}".format(trajectory_group.name))
ax = figure.add_subplot(111, projection='3d')
ax.set_xlabel('x-location')
ax.set_ylabel('y-location')
ax.set_zlabel('z-location')
figure = pyplot.figure(figsize=(14, 10), dpi=80)
figure.suptitle("Computed orientation for {0}".format(trajectory_group.name))
oax = figure.add_subplot(111, projection='3d')
oax.set_xlabel('x-location')
oax.set_ylabel('y-location')
oax.set_zlabel('z-location')
added_ground_truth = False
lowest = -0.001
highest = 0.001
cmap = pyplot.get_cmap('Set1')
colour_index = 0
# For each trial result
for label, trial_result_id in trial_results.items():
trial_result = dh.load_object(db_client, db_client.trials_collection, trial_result_id)
if trial_result is not None:
if trial_result.success:
if not added_ground_truth:
trajectory = trial_result.get_ground_truth_camera_poses()
lower, upper = data_helpers.plot_trajectory(ax, trajectory, 'ground truth trajectory',
style='k--')
lowest = min(lowest, lower)
highest = max(highest, upper)
added_ground_truth = True
trajectory = trial_result.get_computed_camera_poses()
lower, upper = data_helpers.plot_trajectory(ax, trajectory, label=label, style='-')
plot_forward(oax, trajectory, label=label, colors=[cmap(colour_index / 9, alpha=0.5)])
lowest = min(lowest, lower)
highest = max(highest, upper)
colour_index += 1
else:
print("Got failed trial: {0}".format(trial_result.reason))
logging.getLogger(__name__).info("... plotted trajectories for {0}".format(trajectory_group.name))
ax.legend()
ax.set_xlim(lowest, highest)
ax.set_ylim(lowest, highest)
ax.set_zlim(lowest, highest)
oax.legend()
oax.set_xlim(lowest, highest)
oax.set_ylim(lowest, highest)
oax.set_zlim(lowest, highest)
pyplot.tight_layout()
pyplot.subplots_adjust(top=0.95, right=0.99)
pyplot.show()
def schedule_generation(
self, simulators: typing.Mapping[str, bson.ObjectId],
quality_variations: typing.List[typing.Tuple[str, dict]],
task_manager: arvet.batch_analysis.task_manager.TaskManager,
db_client: arvet.database.client.DatabaseClient) -> bool:
"""
Do imports and dataset generation for this trajectory group.
Will create a controller, and then generate reduced quality synthetic datasets.
:param simulators: A Map of simulators, indexed by name
:param quality_variations: A list of names and quality variations
:param task_manager:
:param db_client:
:return: True if part of the group has changed, and it needs to be re-saved
"""
changed = False
# First, make a follow controller for the base dataset if we don't have one.
# This will be used to generate reduced-quality datasets following the same trajectory
# as the root dataset
if self.follow_controller_id is None:
self.follow_controller_id = follow_cont.create_follow_controller(
db_client,
self.reference_dataset,
sequence_type=sequence_type.ImageSequenceType.SEQUENTIAL)
changed = True
# Next, if we haven't already, compute baseline configuration from the reference dataset
if self.baseline_configuration is None or len(
self.baseline_configuration) == 0:
reference_dataset = dh.load_object(
db_client, db_client.image_source_collection,
self.reference_dataset)
if isinstance(reference_dataset,
arvet.core.image_collection.ImageCollection):
intrinsics = reference_dataset.get_camera_intrinsics()
self.baseline_configuration = {
# Simulation execution config
'stereo_offset': reference_dataset.get_stereo_baseline() \
if reference_dataset.is_stereo_available else 0,
'provide_rgb': True,
'provide_ground_truth_depth': False, # We don't care about this
'provide_labels': reference_dataset.is_labels_available,
'provide_world_normals': reference_dataset.is_normals_available,
# Depth settings
'provide_depth': reference_dataset.is_depth_available,
'depth_offset': reference_dataset.get_stereo_baseline() \
if reference_dataset.is_depth_available else 0,
'projector_offset': reference_dataset.get_stereo_baseline() \
if reference_dataset.is_depth_available else 0,
# Simulator camera settings, be similar to the reference dataset
'resolution': {'width': intrinsics.width, 'height': intrinsics.height},
'fov': max(intrinsics.horizontal_fov, intrinsics.vertical_fov),
'depth_of_field_enabled': False,
'focus_distance': None,
'aperture': 2.2,
# Quality settings - Maximum quality
'lit_mode': True,
'texture_mipmap_bias': 0,
'normal_maps_enabled': True,
'roughness_enabled': True,
'geometry_decimation': 0,
'depth_noise_quality': 1,
# Simulation server config
'host': 'localhost',
'port': 9000,
}
changed = True
# Then, for each simulator listed for this trajectory group
origin_counts = {}
for sim_name, origin in self.mappings:
# Count how many times each simulator is used, so we can assign a unique world name to each start point
if sim_name not in origin_counts:
origin_counts[sim_name] = 1
else:
origin_counts[sim_name] += 1
# Schedule generation of quality variations that don't exist yet
if sim_name in simulators:
# For every quality variation
for quality_name, config in quality_variations:
generate_dataset_task = task_manager.get_generate_dataset_task(
controller_id=self.follow_controller_id,
simulator_id=simulators[sim_name],
simulator_config=du.defaults(
{'origin': origin}, config,
self.baseline_configuration),
num_cpus=1,
num_gpus=0,
memory_requirements='3GB',
expected_duration='4:00:00')
if generate_dataset_task.is_finished:
world_name = "{0} {1}".format(sim_name,
origin_counts[sim_name])
if world_name not in self.generated_datasets:
self.generated_datasets[world_name] = {}
self.generated_datasets[world_name][
quality_name] = generate_dataset_task.result
changed = True
else:
task_manager.do_task(generate_dataset_task)
return changed
