def make_instance(self, *args, **kwargs): du.defaults( kwargs, { 'benchmark_id': np.random.randint(0, 10), 'trial_result_ids': [bson.ObjectId() for _ in range(4)], 'frame_errors': { time: [np.random.uniform(-100, 100) for _ in range(18)] for time in range(100) }, }) return feb.FrameErrorsResult(*args, **kwargs)
def make_instance(self, *args, **kwargs): du.defaults(kwargs, { 'benchmark_id': np.random.randint(0, 10), 'trial_result_ids': [bson.ObjectId() for _ in range(4)], 'estimate_errors': { bson.ObjectId(): [ [np.random.uniform(-100, 100) for _ in range(20)] for _ in range(1000) ] for _ in range(10) }, }) return eeb.EstimateTrialErrorsResult(*args, **kwargs)
def export_data(self, db_client: arvet.database.client.DatabaseClient): """ Allow experiments to export some data, usually to file. I'm currently using this to dump camera trajectories so I can build simulations around them, but there will be other circumstances where we want to :param db_client: :return: """ # Save trajectory files for import into unreal for trajectory_group in self._trajectory_groups.values(): data_helpers.dump_ue4_trajectory( name=trajectory_group.name, trajectory=traj_help.get_trajectory_for_image_source( db_client, trajectory_group.reference_dataset)) # Group and print the trajectories for graphing systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems) for trajectory_group in self._trajectory_groups.values(): # Collect the trial results for each image source in this group trial_results = {} for system_name, system_id in systems.items(): for dataset_name, dataset_id in trajectory_group.datasets.items( ): trial_result_list = self.get_trial_result( system_id, dataset_id) for idx, trial_result_id in enumerate(trial_result_list): label = "{0} on {1} repeat {2}".format( system_name, dataset_name, idx) trial_results[label] = trial_result_id data_helpers.export_trajectory_as_json( trial_results, "Generated Data " + trajectory_group.name, db_client)
def make_metadata(index=1, **kwargs): kwargs = du.defaults( kwargs, { 'img_hash': b'\xf1\x9a\xe2|' + np.random.randint(0, 0xFFFFFFFF).to_bytes(4, 'big'), 'source_type': imeta.ImageSourceType.SYNTHETIC, 'camera_pose': tf.Transform( location=(1 + 100 * index, 2 + np.random.uniform(-1, 1), 3), rotation=(4, 5, 6, 7 + np.random.uniform(-4, 4))), 'intrinsics': cam_intr.CameraIntrinsics(800, 600, 550.2, 750.2, 400, 300), 'environment_type': imeta.EnvironmentType.INDOOR_CLOSE, 'light_level': imeta.LightingLevel.WELL_LIT, 'time_of_day': imeta.TimeOfDay.DAY, 'lens_focal_distance': 5, 'aperture': 22, 'simulation_world': 'TestSimulationWorld', 'lighting_model': imeta.LightingModel.LIT, 'texture_mipmap_bias': 1, 'normal_maps_enabled': 2, 'roughness_enabled': True, 'geometry_decimation': 0.8, 'procedural_generation_seed': 16234, 'labelled_objects': (imeta.LabelledObject(class_names=('car', ), x=12, y=144, width=67, height=43, relative_pose=tf.Transform( (12, 3, 4), (0.5, 0.1, 1, 1.7)), instance_name='Car-002'), imeta.LabelledObject(class_names=('cat', ), x=125, y=244, width=117, height=67, relative_pose=tf.Transform( (378, -1890, 38), (0.3, 1.12, 1.1, 0.2)), instance_name='cat-090')) }) return imeta.ImageMetadata(**kwargs)
def plot_component(ax, trajectories: typing.List[typing.Mapping[float, tf.Transform]], get_value: typing.Callable[[tf.Transform], float], label: str = '', alpha: float = 1.0, **kwargs): """ Plot a particular trajectory component over time on an axis for a number of trajectories :param ax: The axis on which to plot :param trajectories: The list of trajectories to plot :param get_value: A getter to retrieve the y value from the Transform object :param label: The label :param alpha: The total alpha value of the line, which will be divided amonst the given trajectories :return: """ du.defaults(kwargs, { 'markersize': 2, 'marker': '.', 'linestyle': 'None' }) for idx, traj in enumerate(trajectories): x = sorted(traj.keys()) ax.plot(x, [get_value(traj[t]) for t in x], alpha=alpha / len(trajectories), label="{0} {1}".format(label, idx), **kwargs)
def __init__(self, config): self._node_id = config.get('node_id', type(self).__name__) if config is not None and 'task_config' in config: task_config = dict(config['task_config']) else: task_config = {} # Default configuration. Also serves as an exemplar configuration argument du.defaults( task_config, { 'allow_import_dataset': True, 'allow_run_system': True, 'allow_measure': True, 'allow_trial_comparison': True }) self._allow_import_dataset = bool(task_config['allow_import_dataset']) self._allow_run_system = bool(task_config['allow_run_system']) self._allow_measure = bool(task_config['allow_measure']) self._allow_trial_comparison = bool( task_config['allow_trial_comparison'])
def make_instance(self, *args, **kwargs): kwargs = du.defaults( kwargs, { 'systems': { 'ORBSLAM monocular': bson.ObjectId(), 'ORBSLAM stereo': bson.ObjectId(), 'LibVisO2': bson.ObjectId() }, 'simulators': { 'Block World': bson.ObjectId() }, 'trajectory_groups': { 'KITTI trajectory 1': tg.TrajectoryGroup( name='KITTI trajectory 1', reference_id=bson.ObjectId(), mappings=[('Block World', { 'location': [12, -63.2, 291.1], 'rotation': [-22, -214, 121] })], baseline_configuration={'test': bson.ObjectId()}, controller_id=bson.ObjectId(), generated_datasets={ 'Block World': { 'max_quality': bson.ObjectId(), 'min_quality': bson.ObjectId() } }) }, 'benchmarks': { 'benchmark_rpe': bson.ObjectId(), 'benchmark_ate': bson.ObjectId(), 'benchmark_trajectory_drift': bson.ObjectId(), 'benchmark_tracking': bson.ObjectId(), }, 'trial_map': { bson.ObjectId(): { bson.ObjectId(): { 'trials': [ bson.ObjectId(), bson.ObjectId(), bson.ObjectId() ], 'results': { bson.ObjectId(): bson.ObjectId() } } } }, 'enabled': True }) return bgde.GeneratedDataExperiment(*args, **kwargs)
def load_global_config(filename): # Default global configuration. # This is what you get if you don't have a configuration file. config = { 'paths': ['~'], 'temp_folder': '/tmp', 'output_folder': '~', 'database': {}, 'image_manager': { 'path': '~/arvet-imgs', }, 'task_config': { 'allow_generate_dataset': True, 'allow_import_dataset': True, 'allow_train_system': True, 'allow_run_system': True, 'allow_benchmark': True, 'allow_trial_comparison': True, 'allow_benchmark_comparison': True, 'allow_experiment_analysis': True }, 'job_system_config': { 'job_system': 'simple', 'node_id': 'unknown-job-system' }, 'logging': { # Default logging configuration 'version': 1, 'formatters': { 'f': {'format': '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'} }, 'handlers': { 'h': {'class': 'logging.StreamHandler', 'formatter': 'f', 'level': logging.DEBUG} }, 'root': {'handlers': ['h'], 'level': logging.DEBUG}, } } if os.path.isfile(filename): loaded_config = None repeat = 1 while loaded_config is None and repeat <= 3: with open(filename, 'r') as config_file: loaded_config = yaml.load(config_file, YamlLoader) if loaded_config is None: # Failed to load the config file for some reason, wait and try again time.sleep(random.uniform(1, 3 * repeat)) repeat += 1 if loaded_config is not None: config = du.defaults(loaded_config, config) else: # No global configuration file, create a default one save_global_config(filename, config) return config
def make_instance(self, *args, **kwargs): kwargs = du.defaults( kwargs, { 'libviso_system': bson.ObjectId(), 'orbslam_systems': { 'MONOCULAR': bson.ObjectId() }, 'simulators': { 'Block World': bson.ObjectId() }, 'trajectory_groups': { 'KITTI trajectory 1': kitti_ex.TrajectoryGroup( name='KITTI trajectory 1', reference_id=bson.ObjectId(), baseline_configuration={'test': bson.ObjectId()}, simulators={ 'Block World': (bson.ObjectId(), { 'conf': bson.ObjectId() }) }, controller_id=bson.ObjectId(), generated_datasets={'Block World': bson.ObjectId()}) }, 'benchmark_rpe': bson.ObjectId(), 'benchmark_ate': bson.ObjectId(), 'benchmark_trajectory_drift': bson.ObjectId(), 'benchmark_tracking': bson.ObjectId(), 'trial_map': { bson.ObjectId(): { bson.ObjectId(): bson.ObjectId() } }, 'result_map': { bson.ObjectId(): { bson.ObjectId(): bson.ObjectId() } }, 'enabled': True }) return kitti_ex.KITTIGeneratedDataExperiment(*args, **kwargs)
def make_image(index=1, **kwargs): kwargs = du.defaults( kwargs, { 'pixels': np.random.uniform(0, 255, (32, 32, 3)), 'image_group': 'test', 'metadata': make_metadata(index), 'additional_metadata': { 'Source': 'Generated', 'Resolution': { 'width': 1280, 'height': 720 }, 'Material Properties': { 'BaseMipMapBias': 0, 'RoughnessQuality': True } }, 'depth': np.random.uniform(0, 1, (32, 32)), 'normals': np.random.uniform(0, 1, (32, 32, 3)) }) return im.Image(**kwargs)
def make_stereo_image(index=1, **kwargs): kwargs = du.defaults( kwargs, { 'pixels': np.random.uniform(0, 255, (32, 32, 3)), 'right_pixels': np.random.uniform(0, 255, (32, 32, 3)), 'image_group': 'test', 'metadata': make_metadata(index), 'right_metadata': make_metadata( index, camera_pose=tf.Transform( location=(1 + 100 * index, 2 + np.random.uniform(-1, 1), 4), rotation=(4, 5, 6, 7 + np.random.uniform(-4, 4))), ), 'additional_metadata': { 'Source': 'Generated', 'Resolution': { 'width': 1280, 'height': 720 }, 'Material Properties': { 'BaseMipMapBias': 0, 'RoughnessQuality': True } }, 'depth': np.random.uniform(0, 1, (32, 32)), 'right_depth': np.random.uniform(0, 1, (32, 32)), 'normals': np.random.uniform(0, 1, (32, 32, 3)), 'right_normals': np.random.uniform(0, 1, (32, 32, 3)) }) return im.StereoImage(**kwargs)
def make_instance(self, *args, **kwargs): kwargs = du.defaults( kwargs, { 'orbslam_mono': bson.ObjectId(), 'orbslam_stereo': bson.ObjectId(), 'datasets': { 'KITTI Trajectory 00': bson.ObjectId() }, 'benchmark_rpe': bson.ObjectId(), 'trial_map': { bson.ObjectId(): { bson.ObjectId(): bson.ObjectId() } }, 'result_map': { bson.ObjectId(): { bson.ObjectId(): bson.ObjectId() } }, 'enabled': True }) return kitti_verify.OrbslamKITTIVerify(*args, **kwargs)
def export_data(self, db_client: arvet.database.client.DatabaseClient): simulator_names = {v: k for k, v in self._simulators.items()} systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems) for trajectory_group in self._trajectory_groups.values(): # Collect all the image sources for this trajectory group image_sources = {} for simulator_id, dataset_id in trajectory_group.generated_datasets.items(): if simulator_id in simulator_names: image_sources[simulator_names[simulator_id]] = dataset_id else: image_sources[simulator_id] = dataset_id # Collect the trial results for each image source in this group trial_results = {} for system_name, system_id in systems.items(): for dataset_name, dataset_id in image_sources.items(): trial_result_list = self.get_trial_results(system_id, dataset_id) for idx in range(len(trial_result_list)): label = "{0} on {1} - repeat {2}".format(system_name, dataset_name, idx) trial_results[label] = trial_result_list[idx] data_helpers.export_trajectory_as_json(trial_results, "Simple Motion " + trajectory_group.name, db_client)
def _plot_relative_pose_error(self, db_client: arvet.database.client.DatabaseClient): import matplotlib.pyplot as pyplot logging.getLogger(__name__).info("Plotting relative pose error...") # Map system ids and simulator ids to printable names simulator_names = {v: k for k, v in self._simulators.items()} systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems) for trajectory_group in self._trajectory_groups.values(): # Collect all the image sources for this trajectory group image_sources = {} for simulator_id, dataset_id in trajectory_group.generated_datasets.items(): if simulator_id in simulator_names: image_sources[simulator_names[simulator_id]] = dataset_id else: image_sources[simulator_id] = dataset_id if len(image_sources) <= 1: # Skip where we've only got one image source, it's not interesting. continue # Collect the results for each image source in this group results = {} for system_name, system_id in systems.items(): for dataset_name, dataset_id in image_sources.items(): trial_result_list = self.get_trial_results(system_id, dataset_id) label = "{0} on {1}".format(system_name, dataset_name) for idx in range(len(trial_result_list)): result_id = self.get_benchmark_result(trial_result_list[idx], self._benchmark_rpe) if result_id is not None: if len(trial_result_list) > 1: results[label + ' repeat {0}'.format(idx)] = result_id else: results[label] = result_id if len(results) > 1: figure = pyplot.figure(figsize=(14, 10), dpi=80) figure.suptitle("Relative pose error for {0}".format(trajectory_group.name)) ax = figure.add_subplot(111) ax.set_xlabel('time') ax.set_ylabel('relative pose error') # For each trial result for label, result_id in results.items(): result = dh.load_object(db_client, db_client.results_collection, result_id) if result is not None: if result.success: x = [] y = [] times = sorted(result.translational_error.keys()) for time in times: error = result.translational_error[time] if error < 100: x.append(time - times[0]) y.append(error) ax.plot(x, y, '-', label=label, alpha=0.7) else: print("Got failed result: {0}".format(result.reason)) logging.getLogger(__name__).info("... plotted rpe for {0}".format(trajectory_group.name)) ax.legend() pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) pyplot.show()
def _plot_trajectories(self, db_client: arvet.database.client.DatabaseClient): """ Plot the ground-truth and computed trajectories for each system for each trajectory. This is important for validation :param db_client: :return: """ import matplotlib.pyplot as pyplot # noinspection PyUnresolvedReferences from mpl_toolkits.mplot3d import Axes3D logging.getLogger(__name__).info("Plotting trajectories...") # Map system ids and simulator ids to printable names simulator_names = {v: k for k, v in self._simulators.items()} systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems) for trajectory_group in self._trajectory_groups.values(): # Collect all the image sources for this trajectory group image_sources = {} for simulator_id, dataset_id in trajectory_group.generated_datasets.items(): if simulator_id in simulator_names: image_sources[simulator_names[simulator_id]] = dataset_id else: image_sources[simulator_id] = dataset_id # Collect the trial results for each image source in this group trial_results = {} for system_name, system_id in systems.items(): for dataset_name, dataset_id in image_sources.items(): trial_result_list = self.get_trial_results(system_id, dataset_id) label = "{0} on {1}".format(system_name, dataset_name) for idx in range(len(trial_result_list)): if len(trial_result_list) > 1: trial_results[label + ' repeat {0}'.format(idx)] = trial_result_list[idx] else: trial_results[label] = trial_result_list[idx] # Make sure we have at least one result to plot if len(trial_results) >= 1: figure = pyplot.figure(figsize=(14, 10), dpi=80) figure.suptitle("Computed trajectories for {0}".format(trajectory_group.name)) ax = figure.add_subplot(111, projection='3d') ax.set_xlabel('x-location') ax.set_ylabel('y-location') ax.set_zlabel('z-location') figure = pyplot.figure(figsize=(14, 10), dpi=80) figure.suptitle("Computed orientation for {0}".format(trajectory_group.name)) oax = figure.add_subplot(111, projection='3d') oax.set_xlabel('x-location') oax.set_ylabel('y-location') oax.set_zlabel('z-location') added_ground_truth = False lowest = -0.001 highest = 0.001 cmap = pyplot.get_cmap('Set1') colour_index = 0 # For each trial result for label, trial_result_id in trial_results.items(): trial_result = dh.load_object(db_client, db_client.trials_collection, trial_result_id) if trial_result is not None: if trial_result.success: if not added_ground_truth: trajectory = trial_result.get_ground_truth_camera_poses() lower, upper = data_helpers.plot_trajectory(ax, trajectory, 'ground truth trajectory', style='k--') lowest = min(lowest, lower) highest = max(highest, upper) added_ground_truth = True trajectory = trial_result.get_computed_camera_poses() lower, upper = data_helpers.plot_trajectory(ax, trajectory, label=label, style='-') plot_forward(oax, trajectory, label=label, colors=[cmap(colour_index / 9, alpha=0.5)]) lowest = min(lowest, lower) highest = max(highest, upper) colour_index += 1 else: print("Got failed trial: {0}".format(trial_result.reason)) logging.getLogger(__name__).info("... plotted trajectories for {0}".format(trajectory_group.name)) ax.legend() ax.set_xlim(lowest, highest) ax.set_ylim(lowest, highest) ax.set_zlim(lowest, highest) oax.legend() oax.set_xlim(lowest, highest) oax.set_ylim(lowest, highest) oax.set_zlim(lowest, highest) pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) pyplot.show()
def schedule_generation( self, simulators: typing.Mapping[str, bson.ObjectId], quality_variations: typing.List[typing.Tuple[str, dict]], task_manager: arvet.batch_analysis.task_manager.TaskManager, db_client: arvet.database.client.DatabaseClient) -> bool: """ Do imports and dataset generation for this trajectory group. Will create a controller, and then generate reduced quality synthetic datasets. :param simulators: A Map of simulators, indexed by name :param quality_variations: A list of names and quality variations :param task_manager: :param db_client: :return: True if part of the group has changed, and it needs to be re-saved """ changed = False # First, make a follow controller for the base dataset if we don't have one. # This will be used to generate reduced-quality datasets following the same trajectory # as the root dataset if self.follow_controller_id is None: self.follow_controller_id = follow_cont.create_follow_controller( db_client, self.reference_dataset, sequence_type=sequence_type.ImageSequenceType.SEQUENTIAL) changed = True # Next, if we haven't already, compute baseline configuration from the reference dataset if self.baseline_configuration is None or len( self.baseline_configuration) == 0: reference_dataset = dh.load_object( db_client, db_client.image_source_collection, self.reference_dataset) if isinstance(reference_dataset, arvet.core.image_collection.ImageCollection): intrinsics = reference_dataset.get_camera_intrinsics() self.baseline_configuration = { # Simulation execution config 'stereo_offset': reference_dataset.get_stereo_baseline() \ if reference_dataset.is_stereo_available else 0, 'provide_rgb': True, 'provide_ground_truth_depth': False, # We don't care about this 'provide_labels': reference_dataset.is_labels_available, 'provide_world_normals': reference_dataset.is_normals_available, # Depth settings 'provide_depth': reference_dataset.is_depth_available, 'depth_offset': reference_dataset.get_stereo_baseline() \ if reference_dataset.is_depth_available else 0, 'projector_offset': reference_dataset.get_stereo_baseline() \ if reference_dataset.is_depth_available else 0, # Simulator camera settings, be similar to the reference dataset 'resolution': {'width': intrinsics.width, 'height': intrinsics.height}, 'fov': max(intrinsics.horizontal_fov, intrinsics.vertical_fov), 'depth_of_field_enabled': False, 'focus_distance': None, 'aperture': 2.2, # Quality settings - Maximum quality 'lit_mode': True, 'texture_mipmap_bias': 0, 'normal_maps_enabled': True, 'roughness_enabled': True, 'geometry_decimation': 0, 'depth_noise_quality': 1, # Simulation server config 'host': 'localhost', 'port': 9000, } changed = True # Then, for each simulator listed for this trajectory group origin_counts = {} for sim_name, origin in self.mappings: # Count how many times each simulator is used, so we can assign a unique world name to each start point if sim_name not in origin_counts: origin_counts[sim_name] = 1 else: origin_counts[sim_name] += 1 # Schedule generation of quality variations that don't exist yet if sim_name in simulators: # For every quality variation for quality_name, config in quality_variations: generate_dataset_task = task_manager.get_generate_dataset_task( controller_id=self.follow_controller_id, simulator_id=simulators[sim_name], simulator_config=du.defaults( {'origin': origin}, config, self.baseline_configuration), num_cpus=1, num_gpus=0, memory_requirements='3GB', expected_duration='4:00:00') if generate_dataset_task.is_finished: world_name = "{0} {1}".format(sim_name, origin_counts[sim_name]) if world_name not in self.generated_datasets: self.generated_datasets[world_name] = {} self.generated_datasets[world_name][ quality_name] = generate_dataset_task.result changed = True else: task_manager.do_task(generate_dataset_task) return changed