def run_task(self, db_client): import logging import traceback import util.database_helpers as dh import dataset.image_collection_builder as collection_builder # Try and import the desired loader module controller = dh.load_object(db_client, db_client.image_source_collection, self.controller_id) simulator = dh.load_object(db_client, db_client.image_source_collection, self.simulator_id, config=self.simulator_config) if simulator is None: logging.getLogger(__name__).error( "Could not deserialize simulator {0}".format( self.simulator_id)) self.mark_job_failed() elif controller is None: logging.getLogger(__name__).error( "Could not deserialize controller {0}".format( self.controller_id)) self.mark_job_failed() elif not controller.can_control_simulator(simulator): logging.getLogger(__name__).error( "Controller {0} can not control simulator {1}".format( self.controller_id, self.simulator_id)) self.mark_job_failed() else: controller.set_simulator(simulator) logging.getLogger(__name__).info( "Generating dataset from {0} using controller {1}".format( self.simulator_id, self.controller_id)) builder = collection_builder.ImageCollectionBuilder(db_client) try: builder.add_from_image_source(controller) dataset_id = builder.save() except Exception: dataset_id = None logging.getLogger(__name__).error( "Exception occurred while generating dataset from simulator {0} with controller {1}:\n{2}" .format(self.simulator_id, self.controller_id, traceback.format_exc())) if dataset_id is None: logging.getLogger(__name__).error( "Failed to generate dataset from simulator {0} with controller {1}: Dataset was null" .format(self.simulator_id, self.controller_id)) self.mark_job_failed() else: self.mark_job_complete(dataset_id) logging.getLogger(__name__).info( "Successfully generated dataset {0}".format(dataset_id))
def run_task(self, db_client): import logging import traceback import util.database_helpers as dh trial_result = dh.load_object(db_client, db_client.trials_collection, self.trial_result) benchmark = dh.load_object(db_client, db_client.benchmarks_collection, self.benchmark) if trial_result is None: logging.getLogger(__name__).error( "Could not deserialize trial result {0}".format( self.trial_result)) self.mark_job_failed() elif benchmark is None: logging.getLogger(__name__).error( "Could not deserialize benchmark {0}".format(self.benchmark)) self.mark_job_failed() elif not benchmark.is_trial_appropriate(trial_result): logging.getLogger(__name__).error( "Benchmark {0} cannot assess trial {1}".format( self.benchmark, self.trial_result)) self.mark_job_failed() else: logging.getLogger(__name__).info( "Benchmarking result {0} with benchmark {1}".format( self.trial_result, self.benchmark)) try: benchmark_result = benchmark.benchmark_results(trial_result) except Exception: logging.getLogger(__name__).error( "Exception while benchmarking {0} with benchmark {1}:\n{2}" .format(self.trial_result, self.benchmark, traceback.format_exc())) benchmark_result = None if benchmark_result is None: logging.getLogger(__name__).error( "Failed to benchmark {0} with {1}".format( self.trial_result, self.benchmark)) self.mark_job_failed() else: benchmark_result_id = db_client.results_collection.insert( benchmark_result.serialize()) logging.getLogger(__name__).info( "Successfully benchmarked trial {0} with benchmark {1}," "producing result {2}".format(self.trial_result, self.benchmark, benchmark_result_id)) self.mark_job_complete(benchmark_result_id)
def run_task(self, db_client): import logging import traceback import util.database_helpers as dh trainer = dh.load_object(db_client, db_client.trainer_collection, self.trainer) trainee = dh.load_object(db_client, db_client.trainee_collection, self.trainee) if trainer is None: logging.getLogger(__name__).error( "Could not deserialize trainer {0}".format(self.trainer)) self.mark_job_failed() elif trainee is None: logging.getLogger(__name__).error( "Could not deserialize trainee {0}".format(self.trainee)) self.mark_job_failed() elif not trainer.can_train_trainee(trainee): logging.getLogger(__name__).error( "Trainer {0} cannot train trainee {1}".format( self.trainer, self.trainee)) self.mark_job_failed() else: logging.getLogger(__name__).info( "Start training trainee {0} ({1}) with trainer {2} (3)".format( self.trainee, trainee.__module__ + '.' + trainee.__class__.__name__, self.trainer, trainer.__module__ + '.' + trainer.__class__.__name__)) try: system = trainer.train_vision_system(trainee) except Exception: logging.getLogger(__name__).error( "Error occurred while trainer {0} trains trainee {1}:\n{2}" .format(self.trainer, self.trainee, traceback.format_exc())) system = None if system is None: logging.getLogger(__name__).error( "Failed to train trainee {0} with trainer {1}".format( self.trainer, self.trainee)) self.mark_job_failed() else: system_id = db_client.system_collection.insert( system.serialize()) logging.getLogger(__name__).info( "Successfully trained system {0}".format(system_id)) self.mark_job_complete(system_id)
def main(*args): """ Run a particular task. :args: Only argument is the id of the task to run :return: """ if len(args) >= 1: task_id = bson.objectid.ObjectId(args[0]) config = global_conf.load_global_config('config.yml') if __name__ == '__main__': # Only configure the logging if this is the main function, don't reconfigure logging.config.dictConfig(config['logging']) db_client = database.client.DatabaseClient(config=config) task = dh.load_object(db_client, db_client.tasks_collection, task_id) if task is not None: try: task.run_task(db_client) except Exception: logging.getLogger(__name__).error( "Exception occurred while running {0}: {1}".format( type(task).__name__, traceback.format_exc())) task.mark_job_failed() task.save_updates(db_client.tasks_collection)
def main(do_imports: bool = True, schedule_tasks: bool = True, run_tasks: bool = True, experiment_ids: typing.List[str] = None): """ Schedule tasks for all experiments. We need to find a way of running this repeatedly as a daemon :param do_imports: Whether to do imports for all the experiments. Default true. :param schedule_tasks: Whether to schedule execution tasks for the experiments. Default true. :param experiment_ids: A limited set of experiments to schedule for. Default None, which is all experiments. :param run_tasks: Actually use the job system to execute scheduled tasks """ config = global_conf.load_global_config('config.yml') if __name__ == '__main__': logging.config.dictConfig(config['logging']) db_client = database.client.DatabaseClient(config=config) task_manager = batch_analysis.task_manager.TaskManager( db_client.tasks_collection, db_client, config) if do_imports or schedule_tasks: query = {'enabled': {'$ne': False}} if experiment_ids is not None and len(experiment_ids) > 0: query['_id'] = { '$in': [bson.ObjectId(id_) for id_ in experiment_ids] } experiment_ids = db_client.experiments_collection.find( query, {'_id': True}) logging.getLogger(__name__).info("Scheduling experiments...") for experiment_id in experiment_ids: experiment = dh.load_object(db_client, db_client.experiments_collection, experiment_id['_id']) if experiment is not None and experiment.enabled: logging.getLogger(__name__).info(" ... experiment {0}".format( experiment.identifier)) try: if do_imports: experiment.do_imports(task_manager, db_client) if schedule_tasks: experiment.schedule_tasks(task_manager, db_client) experiment.save_updates(db_client) except Exception: logging.getLogger(__name__).error( "Exception occurred during scheduling:\n{0}".format( traceback.format_exc())) if run_tasks: logging.getLogger(__name__).info("Running tasks...") job_system = job_system_factory.create_job_system(config=config) task_manager.schedule_tasks(job_system) # Actually run the queued jobs. job_system.run_queued_jobs()
def get(self, index): """ A getter for random access, since we're storing a list :param index: :return: """ if index in self._images: return dh.load_object(self._db_client, self._db_client.image_collection, self._images[index]) return None
def run_task(self, db_client): import logging import traceback import util.database_helpers as dbhelp system = dbhelp.load_object(db_client, db_client.system_collection, self.system) image_source = dbhelp.load_object(db_client, db_client.image_source_collection, self.image_source) if system is None: logging.getLogger(__name__).error("Could not deserialize system {0}".format(self.system)) self.mark_job_failed() elif image_source is None: logging.getLogger(__name__).error("Could not deserialize image source {0}".format(self.image_source)) self.mark_job_failed() elif not system.is_image_source_appropriate(image_source): logging.getLogger(__name__).error("Image source {0} is inappropriate for system {1}".format( self.image_source, self.system)) self.mark_job_failed() else: logging.getLogger(__name__).info("Start running system {0} ({1}) with image source {2}".format( self.system, system.__module__ + '.' + system.__class__.__name__, self.image_source)) try: trial_result = run_system_with_source(system, image_source) except Exception: logging.getLogger(__name__).error("Error occurred while running system {0} " "with image source {1}:\n{2}".format( self.system, self.image_source, traceback.format_exc())) trial_result = None if trial_result is None: logging.getLogger(__name__).error("Failed to system {0} with image source {1}.".format( self.system, self.image_source)) self.mark_job_failed() else: trial_result.save_data(db_client) trial_result_id = db_client.trials_collection.insert(trial_result.serialize()) logging.getLogger(__name__).info(("Successfully ran system {0} with image source {1}," "producing trial result {2}").format( self.system, self.image_source, trial_result_id)) self.mark_job_complete(trial_result_id)
def main(*args): config = global_conf.load_global_config("config.yml") if __name__ == '__main__': logging.config.dictConfig(config['logging']) db_client = database.client.DatabaseClient(config) experiment_ids = db_client.experiments_collection.find( {'enabled': { '$ne': False }}, {'_id': True}) for ex_id in experiment_ids: experiment = dh.load_object(db_client, db_client.experiments_collection, ex_id['_id']) if experiment is not None and experiment.enabled: experiment.plot_results(db_client)
def main(): """ Allow experiments to dump some data to file. This might be aggregate statistics, I'm currently using this for camera trajectories. :return: """ config = global_conf.load_global_config('config.yml') if __name__ == '__main__': logging.config.dictConfig(config['logging']) db_client = database.client.DatabaseClient(config=config) experiment_ids = db_client.experiments_collection.find( {'enabled': { '$ne': False }}, {'_id': True}) for ex_id in experiment_ids: experiment = dh.load_object(db_client, db_client.experiments_collection, ex_id['_id']) if experiment is not None and experiment.enabled: experiment.export_data(db_client)
def do_imports(self, task_manager: batch_analysis.task_manager.TaskManager, db_client: database.client.DatabaseClient) -> bool: """ Do imports and dataset generation for this trajectory group. Will create a controller, and then generate reduced quality synthetic datasets. :param task_manager: :param db_client: :return: True if part of the group has changed, and it needs to be re-saved """ changed = False # First, make a follow controller for the base dataset if we don't have one. # This will be used to generate reduced-quality datasets following the same trajectory # as the root dataset if self.follow_controller_id is None: self.follow_controller_id = follow_cont.create_follow_controller( db_client, self.reference_dataset, sequence_type=core.sequence_type.ImageSequenceType.SEQUENTIAL) changed = True # Next, if we haven't already, compute baseline configuration from the reference dataset if self.baseline_configuration is None or len( self.baseline_configuration) == 0: reference_dataset = dh.load_object( db_client, db_client.image_source_collection, self.reference_dataset) if isinstance(reference_dataset, core.image_collection.ImageCollection): intrinsics = reference_dataset.get_camera_intrinsics() self.baseline_configuration = { # Simulation execution config 'stereo_offset': reference_dataset.get_stereo_baseline() \ if reference_dataset.is_stereo_available else 0, 'provide_rgb': True, 'provide_depth': reference_dataset.is_depth_available, 'provide_labels': reference_dataset.is_labels_available, 'provide_world_normals': reference_dataset.is_normals_available, # Simulator camera settings, be similar to the reference dataset 'resolution': {'width': intrinsics.width, 'height': intrinsics.height}, 'fov': max(intrinsics.horizontal_fov, intrinsics.vertical_fov), 'depth_of_field_enabled': False, 'focus_distance': None, 'aperture': 2.2, # Quality settings - Maximum quality 'lit_mode': True, 'texture_mipmap_bias': 0, 'normal_maps_enabled': True, 'roughness_enabled': True, 'geometry_decimation': 0, 'depth_noise_quality': 1, # Simulation server config 'host': 'localhost', 'port': 9000, } changed = True # Then, for each combination of simulator id and config, generate a dataset for sim_name, (simulator_id, simulator_config) in self.simulators.items(): # Schedule generation of quality variations that don't exist yet if sim_name not in self.generated_datasets: generate_dataset_task = task_manager.get_generate_dataset_task( controller_id=self.follow_controller_id, simulator_id=simulator_id, simulator_config=du.defaults({}, simulator_config, self.baseline_configuration), num_cpus=1, num_gpus=0, memory_requirements='3GB', expected_duration='4:00:00') if generate_dataset_task.is_finished: self.generated_datasets[ sim_name] = generate_dataset_task.result changed = True else: task_manager.do_task(generate_dataset_task) return changed
def _plot_relative_pose_error(self, db_client: database.client.DatabaseClient): import matplotlib.pyplot as pyplot logging.getLogger(__name__).info("Plotting relative pose error...") # Map system ids and simulator ids to printable names simulator_names = {v: k for k, v in self._simulators.items()} systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems) for trajectory_group in self._trajectory_groups.values(): # Collect all the image sources for this trajectory group image_sources = { 'reference dataset': trajectory_group.reference_dataset } for simulator_id, dataset_id in trajectory_group.generated_datasets.items( ): if simulator_id in simulator_names: image_sources[simulator_names[simulator_id]] = dataset_id else: image_sources[simulator_id] = dataset_id if len(image_sources) <= 1: # Skip where we've only got one image source, it's not interesting. continue # Collect the results for each image source in this group results = {} styles = {} for system_name, system_id in systems.items(): for dataset_name, dataset_id in image_sources.items(): trial_result_id = self.get_trial_result( system_id, dataset_id) if trial_result_id is not None: result_id = self.get_benchmark_result( trial_result_id, self._benchmark_rpe) if result_id is not None: label = "{0} on {1}".format( system_name, dataset_name) results[label] = result_id styles[ label] = '-' if dataset_name == 'reference dataset' else '--' if len(results) > 1: figure = pyplot.figure(figsize=(14, 10), dpi=80) figure.suptitle("Relative pose error for {0}".format( trajectory_group.name)) ax = figure.add_subplot(111) ax.set_xlabel('time') ax.set_ylabel('relative pose error') # For each trial result for label, result_id in results.items(): result = dh.load_object(db_client, db_client.results_collection, result_id) if result is not None: if result.success: x = [] y = [] times = sorted(result.translational_error.keys()) for time in times: error = result.translational_error[time] if error < 100: x.append(time - times[0]) y.append(error) ax.plot(x, y, styles[label], label=label, alpha=0.7) else: print("Got failed result: {0}".format( result.reason)) logging.getLogger(__name__).info( "... plotted rpe for {0}".format(trajectory_group.name)) ax.legend() pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) pyplot.show()
def _plot_trajectories(self, db_client: database.client.DatabaseClient): """ Plot the ground-truth and computed trajectories for each system for each trajectory. This is important for validation :param db_client: :return: """ import matplotlib.pyplot as pyplot # noinspection PyUnresolvedReferences from mpl_toolkits.mplot3d import Axes3D logging.getLogger(__name__).info("Plotting trajectories...") # Map system ids and simulator ids to printable names simulator_names = {v: k for k, v in self._simulators.items()} systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems) for trajectory_group in self._trajectory_groups.values(): # Collect all the image sources for this trajectory group image_sources = { 'reference dataset': trajectory_group.reference_dataset } for simulator_id, dataset_id in trajectory_group.generated_datasets.items( ): if simulator_id in simulator_names: image_sources[simulator_names[simulator_id]] = dataset_id else: image_sources[simulator_id] = dataset_id if len(image_sources) <= 1: continue # Collect the trial results for each image source in this group trial_results = {} style = {} for system_name, system_id in systems.items(): for dataset_name, dataset_id in image_sources.items(): trial_result_id = self.get_trial_result( system_id, dataset_id) if trial_result_id is not None: label = "{0} on {1}".format(system_name, dataset_name) trial_results[label] = trial_result_id style[ label] = '--' if dataset_name == 'reference dataset' else '-' # Make sure we have at least one result to plot if len(trial_results) > 1: figure = pyplot.figure(figsize=(14, 10), dpi=80) figure.suptitle("Computed trajectories for {0}".format( trajectory_group.name)) ax = figure.add_subplot(111, projection='3d') ax.set_xlabel('x-location') ax.set_ylabel('y-location') ax.set_zlabel('z-location') ax.plot([0], [0], [0], 'ko', label='origin') added_ground_truth = False # For each trial result for label, trial_result_id in trial_results.items(): trial_result = dh.load_object(db_client, db_client.trials_collection, trial_result_id) if trial_result is not None: if trial_result.success: if not added_ground_truth: lower, upper = plot_trajectory( ax, trial_result.get_ground_truth_camera_poses( ), 'ground truth trajectory') mean = (upper + lower) / 2 lower = 1.2 * lower - mean upper = 1.2 * upper - mean ax.set_xlim(lower, upper) ax.set_ylim(lower, upper) ax.set_zlim(lower, upper) added_ground_truth = True plot_trajectory( ax, trial_result.get_computed_camera_poses(), label=label, style=style[label]) else: print("Got failed trial: {0}".format( trial_result.reason)) logging.getLogger(__name__).info( "... plotted trajectories for {0}".format( trajectory_group.name)) ax.legend() pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) pyplot.show()
def run_task(self, db_client): import logging import traceback import util.database_helpers as dh trial_result_1 = dh.load_object(db_client, db_client.trials_collection, self.trial_result1) trial_result_2 = dh.load_object(db_client, db_client.trials_collection, self.trial_result2) comparison_benchmark = dh.load_object(db_client, db_client.benchmarks_collection, self.comparison) if trial_result_1 is None: logging.getLogger(__name__).error( "Could not deserialize trial result {0}".format( self.trial_result1)) self.mark_job_failed() elif trial_result_2 is None: logging.getLogger(__name__).error( "Could not deserialize trial result {0}".format( self.trial_result2)) self.mark_job_failed() elif comparison_benchmark is None: logging.getLogger(__name__).error( "Could not deserialize comparison benchmark {0}".format( self.comparison)) self.mark_job_failed() elif (not comparison_benchmark.is_trial_appropriate(trial_result_1) or not comparison_benchmark.is_trial_appropriate(trial_result_2)): logging.getLogger(__name__).error( "Benchmark {0} is not appropriate for trial {1} or {2}".format( self.comparison, self.trial_result1, self.trial_result2)) self.mark_job_failed() else: logging.getLogger(__name__).info( "Comparing trial results {0} and {1} with comparison benchmark {2}" .format(self.trial_result1, self.trial_result2, self.comparison)) try: comparison_result = comparison_benchmark.compare_trial_results( trial_result_1, trial_result_2) except Exception: logging.getLogger(__name__).error( "Error occurred while comparing trials {0} and {1}" "with benchmark {2}:\n{3}".format(self.trial_result1, self.trial_result2, self.comparison, traceback.format_exc())) comparison_result = None if comparison_result is None: logging.getLogger(__name__).error( "Failed to compare trials {0} and {1}" "with benchmark {2}".format(self.trial_result1, self.trial_result2, self.comparison)) self.mark_job_failed() else: comparison_result.save_data(db_client) result_id = db_client.results_collection.insert( comparison_result.serialize()) logging.getLogger(__name__).info( "Successfully compared trials {0} and {1}," "producing result {2}".format(self.trial_result1, self.trial_result2, result_id)) self.mark_job_complete(result_id)
def plot_results(self, db_client): for result_id in self.results_ids: result = dh.load_object(db_client, db_client.results_collection, result_id) if result is not None: precision, recall, f1score, log_gt_area, iou = result.list_results( overlap_result.precision, overlap_result.recall, overlap_result.f1_score, lambda x: np.log(x['ground_truth_area']) if x['ground_truth_area'] > 0 else 0, lambda x: x['overlap'] / (x['bounding_box_area'] + x['ground_truth_area'] - x['overlap']) ) figure = pyplot.figure(figsize=(14, 10), dpi=80) ax_pr = figure.add_subplot(111) ax_pr.set_xlabel('precision') ax_pr.set_ylabel('recall') pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) figure = pyplot.figure(figsize=(14, 10), dpi=80) ax_p_area = figure.add_subplot(121) ax_p_area.set_xlabel('bounding box area') ax_p_area.set_ylabel('precision') ax_r_area = figure.add_subplot(122) ax_r_area.set_xlabel('bounding box area') ax_r_area.set_ylabel('recall') pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) fp_values = [] fn_values = [] false_labels = [] boxplot_stuff = [] boxplot_labels = [] for result_id in self.results_ids: s_result = db_client.results_collection.find_one({'_id': result_id}) result = db_client.deserialize_entity(s_result) name = self.get_name(result, db_client) precision, recall, f1score, gt_area, iou = result.list_results( overlap_result.precision, overlap_result.recall, overlap_result.f1_score, lambda x: np.log(x['ground_truth_area']) if x['ground_truth_area'] > 0 else 0, lambda x: x['overlap'] / (x['bounding_box_area'] + x['ground_truth_area'] - x['overlap'])) boxplot_stuff.append(iou) boxplot_labels.append(name) ax_pr.scatter(precision, recall, label=name) x, y = prune_zero_points(gt_area, precision) ax_p_area.scatter(x, y, label=name) x, y = prune_zero_points(gt_area, recall) ax_r_area.scatter(x, y, label=name) # Count false positives false_labels.append(name) false_positives = 0 false_negatives = 0 for bbox_results in result.overlaps.values(): for bbox_result in bbox_results: if bbox_result['ground_truth_area'] == 0: false_positives += 1 if bbox_result['bounding_box_area'] == 0: false_negatives += 1 fp_values.append(false_positives) fn_values.append(false_negatives) ax_pr.legend() ax_p_area.legend() # False positives x = np.arange(len(fp_values)) figure = pyplot.figure(figsize=(14, 10), dpi=80) ax_fp = figure.add_subplot(121) ax_fp.set_ylabel('false positives') ax_fp.set_xticklabels(false_labels) ax_fp.bar(x, fp_values, align='center') ax_fn = figure.add_subplot(122) ax_fn.set_ylabel('false negatives') ax_fn.set_xticklabels(false_labels) ax_fn.bar(x, fn_values, align='center') pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) figure = pyplot.figure(figsize=(14, 10), dpi=80) ax = figure.add_subplot(111) ax.set_xlabel('precision') ax.set_ylabel('Intersection over Union') ax.boxplot(boxplot_stuff, positions=list(range(len(self.results_ids)))) ax.set_xticks(list(range(len(self.results_ids)))) ax.set_xticklabels(boxplot_labels) pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) pyplot.show()
def schedule_all(self, task_manager: batch_analysis.task_manager.TaskManager, db_client: database.client.DatabaseClient, systems: typing.List[bson.ObjectId], image_sources: typing.List[bson.ObjectId], benchmarks: typing.List[bson.ObjectId]): """ Schedule all combinations of running some list of systems with some list of image sources, and then benchmarking the results with some list of benchmarks. Uses is_image_source_appropriate and is_benchmark_appropriate to filter. Created results can be retrieved with get_trial_result and get_benchmark_result. :param task_manager: The task manager to perform scheduling :param db_client: The database client, to load the systems, image sources, etc.. :param systems: The list of system ids to test :param image_sources: The list of image source ids to use :param benchmarks: The list of benchmark ids to measure the results :return: void """ # Trial results will be collected as we go trial_results = set() # For each image dataset, run libviso with that dataset, and store the result in the trial map for image_source_id in image_sources: image_source = dh.load_object(db_client, db_client.image_source_collection, image_source_id) if image_source is None: continue for system_id in systems: system = dh.load_object(db_client, db_client.system_collection, system_id) if system is not None and system.is_image_source_appropriate( image_source): task = task_manager.get_run_system_task( system_id=system.identifier, image_source_id=image_source.identifier, expected_duration='8:00:00', memory_requirements='12GB') if not task.is_finished: task_manager.do_task(task) else: trial_results.add(task.result) self.store_trial_result(system_id, image_source_id, task.result) # Benchmark trial results for trial_result_id in trial_results: trial_result = dh.load_object(db_client, db_client.trials_collection, trial_result_id) if trial_result is None: continue for benchmark_id in benchmarks: benchmark = dh.load_object(db_client, db_client.benchmarks_collection, benchmark_id) if benchmark is not None and benchmark.is_trial_appropriate( trial_result): task = task_manager.get_benchmark_task( trial_result_id=trial_result.identifier, benchmark_id=benchmark.identifier, expected_duration='6:00:00', memory_requirements='6GB') if not task.is_finished: task_manager.do_task(task) else: self.store_benchmark_result(trial_result_id, benchmark_id, task.result)
def _plot_trajectories(self, db_client: database.client.DatabaseClient): """ Plot the ground-truth and computed trajectories for each system for each trajectory. This is important for validation :param db_client: :return: """ import matplotlib.pyplot as pyplot # noinspection PyUnresolvedReferences from mpl_toolkits.mplot3d import Axes3D logging.getLogger(__name__).info("Plotting trajectories...") # Map system ids and simulator ids to printable names simulator_names = {v: k for k, v in self._simulators.items()} systems = du.defaults({'LIBVISO 2': self._libviso_system}, self._orbslam_systems) for trajectory_group in self._trajectory_groups.values(): # Collect all the image sources for this trajectory group image_sources = {} for simulator_id, dataset_id in trajectory_group.generated_datasets.items( ): if simulator_id in simulator_names: image_sources[simulator_names[simulator_id]] = dataset_id else: image_sources[simulator_id] = dataset_id # Collect the trial results for each image source in this group trial_results = {} for system_name, system_id in systems.items(): for dataset_name, dataset_id in image_sources.items(): trial_result_id = self.get_trial_result( system_id, dataset_id) if trial_result_id is not None: label = "{0} on {1}".format(system_name, dataset_name) trial_results[label] = trial_result_id # Make sure we have at least one result to plot if len(trial_results) >= 1: figure = pyplot.figure(figsize=(14, 10), dpi=80) figure.suptitle("Computed trajectories for {0}".format( trajectory_group.name)) ax = figure.add_subplot(111, projection='3d') ax.set_xlabel('x-location') ax.set_ylabel('y-location') ax.set_zlabel('z-location') figure = pyplot.figure(figsize=(14, 10), dpi=80) figure.suptitle("Computed orientation for {0}".format( trajectory_group.name)) oax = figure.add_subplot(111, projection='3d') oax.set_xlabel('x-location') oax.set_ylabel('y-location') oax.set_zlabel('z-location') added_ground_truth = False lowest = -0.001 highest = 0.001 cmap = pyplot.get_cmap('Set1') colour_index = 0 # For each trial result for label, trial_result_id in trial_results.items(): trial_result = dh.load_object(db_client, db_client.trials_collection, trial_result_id) if trial_result is not None: if trial_result.success: if not added_ground_truth: trajectory = trial_result.get_ground_truth_camera_poses( ) lower, upper = plot_trajectory( ax, trajectory, 'ground truth trajectory', style='k--') lowest = min(lowest, lower) highest = max(highest, upper) added_ground_truth = True trajectory = trial_result.get_computed_camera_poses( ) lower, upper = plot_trajectory(ax, trajectory, label=label, style='-') plot_forward( oax, trajectory, label=label, colors=[cmap(colour_index / 9, alpha=0.5)]) lowest = min(lowest, lower) highest = max(highest, upper) colour_index += 1 else: print("Got failed trial: {0}".format( trial_result.reason)) logging.getLogger(__name__).info( "... plotted trajectories for {0}".format( trajectory_group.name)) ax.legend() ax.set_xlim(lowest, highest) ax.set_ylim(lowest, highest) ax.set_zlim(lowest, highest) oax.legend() oax.set_xlim(lowest, highest) oax.set_ylim(lowest, highest) oax.set_zlim(lowest, highest) pyplot.tight_layout() pyplot.subplots_adjust(top=0.95, right=0.99) pyplot.show()