def draw_timing_plot(filename, keyframe_ids, pipelines_times, ylabel='Optimization time [s]', display_plot=False, display_x_label=True, latexify=True, fig_width=6, fig_height=3): """ Plots timing information for each pipeline contained in the list of dicts pipelines_times: - filename: where to save the figure. - pipelines_times: list of dicts of the form: [{ 'pipeline_name': pipeline_name, 'line_color': np.random.rand(3), 'line_style': '-', 'times': update_times }, ... ] - keyframe_ids: corresponds to the x ticks, so update_times and keyframe_ids must be the same length. - ylabel: Y label used for the plot. - display_plot: whether to display the plot or not. - display_x_label: whether to display the x label of the plot or not. - latexify: whether to use latex for the generation of the plot. """ if latexify: pt.latexify(fig_width, fig_height) fig = plt.figure(figsize=[fig_width, fig_height], dpi=1000) i = 0 for pipeline_time in pipelines_times: assert len(keyframe_ids) == len(pipeline_time['times']) plt.plot(keyframe_ids, pipeline_time['times'], linestyle=pipeline_time['line_style'], color=pipeline_time['line_color'], linewidth=0.5, label="$t_{" + pipeline_time['pipeline_name'] + "}^{opt}$") i = i + 1 plt.ylabel(ylabel) if display_x_label: plt.xlabel('Keyframe Index [-]') plt.xlim(min(keyframe_ids), max(keyframe_ids)) plt.ylim(bottom=0) plt.grid(axis='both', linestyle='--') plt.legend() # Create path to filename if it does not exist. evt.create_full_path_if_not_exists(filename) plt.savefig(filename, bbox_inches='tight', transparent=True, dpi=1000) if display_plot: plt.show()
def __init__(self, experiment_params, args, extra_flagfile_path=''): self.vocabulary_path = os.path.expandvars( experiment_params['vocabulary_path']) self.results_dir = os.path.expandvars(experiment_params['results_dir']) self.params_dir = os.path.expandvars(experiment_params['params_dir']) self.dataset_dir = os.path.expandvars(experiment_params['dataset_dir']) self.executable_path = os.path.expandvars( experiment_params['executable_path']) self.datasets_to_run = experiment_params['datasets_to_run'] self.verbose_vio = args.verbose_sparkvio self.extra_flagfile_path = extra_flagfile_path self.pipeline_output_dir = os.path.join(self.results_dir, "tmp_output/output/") evt.create_full_path_if_not_exists(self.pipeline_output_dir)
def save_results_to_file(self, results, title, dataset_pipeline_result_dir): """ Writes a result dictionary to file as a yaml file. Args: results: a dictionary containing ape, rpe rotation and rpe translation results and statistics. title: a string representing the filename without the '.yaml' extension. dataset_pipeline_result_dir: a string representing the filepath for the location to save the results file. """ results_file = os.path.join(dataset_pipeline_result_dir, title + '.yaml') evt.print_green("Saving analysis results to: %s" % results_file) evt.create_full_path_if_not_exists(results_file) with open(results_file, 'w') as outfile: outfile.write(yaml.dump(results, default_flow_style=False))
def write_vio_results_summary(results, vio_results_summary_path): # Get APE. ATE_mean = results['absolute_errors'].stats['mean'] ATE_rmse = results['absolute_errors'].stats['rmse'] # Get RPE for smallest segments. #assert(len(results['relative_errors']) > 0) #RPE_mean = results['relative_errors'][0]['mean'] #RPE_rmse = results['relative_errors'][0]['rmse'] # Generate path to summary if it does not exist. evt.create_full_path_if_not_exists(vio_results_summary_path) # Write to CSV file. with open(vio_results_summary_path, 'w') as vio_results_summary_file: log.info('Writing VIO summary results to: %s' % vio_results_summary_path) performance_metrics = ['ATE_mean', 'ATE_rmse'] writer = csv.DictWriter(vio_results_summary_file, fieldnames=performance_metrics) writer.writeheader() writer.writerow({'ATE_mean': ATE_mean, 'ATE_rmse': ATE_rmse})
def run_dataset(results_dir, params_dir, dataset_dir, dataset_properties, executable_path, run_pipeline, analyse_vio, plot, save_results, save_plots, save_boxplots, pipelines_to_run_list, initial_k, final_k, discard_n_start_poses=0, discard_n_end_poses=0, extra_flagfile_path='', verbose_sparkvio=False): """ Evaluates pipeline using Structureless(S), Structureless(S) + Projection(P), \ and Structureless(S) + Projection(P) + Regular(R) factors \ and then compiles a list of results """ dataset_name = dataset_properties['name'] dataset_segments = dataset_properties['segments'] ################### RUN PIPELINE ################################ pipeline_output_dir = os.path.join(results_dir, "tmp_output/output/") evt.create_full_path_if_not_exists(pipeline_output_dir) output_file = os.path.join(pipeline_output_dir, "output_posesVIO.csv") has_a_pipeline_failed = False if len(pipelines_to_run_list) == 0: log.warning("Not running pipeline...") for pipeline_type in pipelines_to_run_list: has_a_pipeline_failed = not process_vio( executable_path, dataset_dir, dataset_name, results_dir, params_dir, pipeline_output_dir, pipeline_type, dataset_segments, save_results, plot, save_plots, output_file, run_pipeline, analyse_vio, discard_n_start_poses, discard_n_end_poses, initial_k, final_k, extra_flagfile_path, verbose_sparkvio) # Save boxplots if save_boxplots: # TODO(Toni) is this really saving the boxplots? if not has_a_pipeline_failed: stats = dict() for pipeline_type in pipelines_to_run_list: results_dataset_dir = os.path.join(results_dir, dataset_name) results = os.path.join(results_dataset_dir, pipeline_type, "results.yaml") if not os.path.exists(results): raise Exception( "\033[91mCannot plot boxplots: missing results for %s pipeline \ and dataset: %s" % (pipeline_type, dataset_name) + "\033[99m \n \ Expected results here: %s" % results) try: stats[pipeline_type] = yaml.load(open(results, 'r'), Loader=yaml.Loader) except yaml.YAMLError as e: raise Exception("Error in results file: ", e) log.info("Check stats %s in %s" % (pipeline_type, results)) check_stats(stats[pipeline_type]) log.info("Drawing boxplots.") evt.draw_rpe_boxplots(results_dataset_dir, stats, len(dataset_segments)) else: log.warning( "A pipeline run has failed... skipping boxplot drawing.") if not has_a_pipeline_failed: evt.print_green("All pipeline runs were successful.") else: log.error("A pipeline has failed!") evt.print_green("Finished evaluation for dataset: " + dataset_name) return not has_a_pipeline_failed
def process_vio(executable_path, dataset_dir, dataset_name, results_dir, params_dir, pipeline_output_dir, pipeline_type, SEGMENTS, save_results, plot, save_plots, output_file, run_pipeline, analyse_vio, discard_n_start_poses, discard_n_end_poses, initial_k, final_k, extra_flagfile_path='', verbose_sparkvio=False): """ Args: - executable_path: path to the pipeline executable (i.e. `./build/spark_vio`). - dataset_dir: directory of the dataset, must contain traj_gt.csv (the ground truth trajectory for analysis to work). - dataset_name: specific dataset to run. - results_dir: directory where the results of the run will reside: - used as results_dir/dataset_name/S, results_dir/dataset_name/SP, results_dir/dataset_name/SPR - where each directory have traj_est.csv (the estimated trajectory), and plots if requested. - params_dir: directory where the parameters for each pipeline reside: - used as params_dir/S, params_dir/SP, params_dir/SPR. - pipeline_output_dir: where to store all output_* files produced by the pipeline. - pipeline_type: type of pipeline to process (1: S, 2: SP, 3: SPR). - SEGMENTS: segments for RPE boxplots. - save_results: saves APE, and RPE per segment results of the run. - plot: whether to plot the APE/RPE results or not. - save_plots: saves plots of APE/RPE. - output_file: the name of the trajectory estimate output of the vio which will then be copied as traj_est.csv. - run_pipeline: whether to run the VIO to generate a new traj_est.csv. - analyse_vio: whether to analyse traj_est.csv or not. - extra_flagfile_path: to be used in order to override other flags or add new ones. Useful for regression tests when the param to be regressed is a gflag. - verbose_sparkvio: whether to print the SparkVIO messages or not. This is useful for debugging, but too verbose when you want to see APE/RPE results. """ dataset_results_dir = os.path.join(results_dir, dataset_name) dataset_pipeline_result_dir = os.path.join(dataset_results_dir, pipeline_type) traj_ref_path = os.path.join(dataset_dir, dataset_name, "mav0/state_groundtruth_estimate0/data.csv" ) # TODO make it not specific to EUROC traj_es = os.path.join(dataset_results_dir, pipeline_type, "traj_es.csv") evt.create_full_path_if_not_exists(traj_es) if run_pipeline: evt.print_green("Run pipeline: %s" % pipeline_type) # The override flags are used by the regression tests. if run_vio(executable_path, dataset_dir, dataset_name, params_dir, pipeline_output_dir, pipeline_type, initial_k, final_k, extra_flagfile_path, verbose_sparkvio) == 0: evt.print_green("Successful pipeline run.") log.debug( "\033[1mCopying output file: \033[0m \n %s \n \033[1m to results file:\033[0m\n %s" % (output_file, traj_es)) copyfile(output_file, traj_es) output_destination_dir = os.path.join(dataset_pipeline_result_dir, "output") log.debug( "\033[1mMoving output dir:\033[0m \n %s \n \033[1m to destination:\033[0m \n %s" % (pipeline_output_dir, output_destination_dir)) try: evt.move_output_from_to(pipeline_output_dir, output_destination_dir) except: log.fatal( "\033[1mFailed copying output dir: \033[0m\n %s \n \033[1m to destination: %s \033[0m\n" % (pipeline_output_dir, output_destination_dir)) else: log.error("Pipeline failed on dataset: " + dataset_name) # Avoid writting results.yaml with analysis if the pipeline failed. log.info("Not writting results.yaml") return False if analyse_vio: log.debug( "\033[1mAnalysing dataset:\033[0m \n %s \n \033[1m for pipeline \033[0m %s." % (dataset_results_dir, pipeline_type)) evt.print_green("Starting analysis of pipeline: %s" % pipeline_type) run_analysis(traj_ref_path, traj_es, SEGMENTS, save_results, plot, save_plots, dataset_pipeline_result_dir, False, dataset_name, discard_n_start_poses, discard_n_end_poses) return True