def add_scores(metrics, scene, algo_dir, tgt_dir, scores, visualize): gt = scene.get_gt() algo_result = misc.get_algo_result_from_dir(algo_dir, scene) for metric in metrics: if visualize: score, vis = metric.get_score(algo_result, gt, scene, with_visualization=True) relative_fname = save_visualization(algo_result, vis, metric, scene, tgt_dir) metric_data = { "value": float(score), "visualization": { "thumb": relative_fname } } else: score = metric.get_score(algo_result, gt, scene) metric_data = {"value": float(score)} log.info("Score %5.2f for: %s, %s, Scale: %0.2f" % (score, metric.get_display_name(), scene.get_display_name(), scene.gt_scale)) scores[metric.get_id()] = metric_data return scores
def main(): parser = OptionParser([ ConverterOps(input_help="path to png disparity map", output_help="path to pfm disparity map") ]) image_path, config_path, pfm_path = parser.parse_args() from toolkit.scenes import PhotorealisticScene from toolkit.utils import log, file_io import numpy as np scene = PhotorealisticScene("demo", path_to_config=config_path) disp_map = file_io.read_file(image_path) log.info("Input range: [%0.1f, %0.1f]" % (np.min(disp_map), np.max(disp_map))) # scale from [MIN, MAX] to [disp_min, disp_max] disp_map = (scene.disp_max - scene.disp_min) * (disp_map - MIN) / ( MAX - MIN) + scene.disp_min log.info("Output range: [%0.1f, %0.1f]" % (np.min(disp_map), np.max(disp_map))) file_io.write_file(disp_map, pfm_path)
def main(): parser = OptionParser([SceneOps(), AlgorithmOps(with_gt=True), MetaAlgorithmOps(default=[])]) scenes, algorithms, meta_algorithms, compute_meta_algos = parser.parse_args() # delay imports to speed up usage response from toolkit import settings from toolkit.algorithms import MetaAlgorithm from toolkit.utils import log, misc, point_cloud if compute_meta_algos and meta_algorithms: MetaAlgorithm.prepare_meta_algorithms(meta_algorithms, algorithms, scenes) algorithms += meta_algorithms for scene in scenes: center_view = scene.get_center_view() for algorithm in algorithms: if algorithm.get_name() == "gt": disp_map = scene.get_gt() else: disp_map = misc.get_algo_result(algorithm, scene) log.info("Creating point cloud for scene '%s' with '%s' disparity map." % (scene.get_name(), algorithm.get_name())) pc = point_cloud.convert(scene, disp_map, center_view) file_name = "%s_%s.ply" % (scene.get_name(), algorithm.get_name()) file_path = op.join(settings.EVAL_PATH, "point_clouds", file_name) log.info("Saving point cloud to: %s" % file_path) point_cloud.save(pc, file_path)
def visualize_algo_result(scene, algo_dir, tgt_dir, add_pfms_to_result): algo_result = misc.get_algo_result_from_dir(algo_dir, scene) # visualize fig = init_figure() cm = plt.imshow(algo_result, **settings.disp_map_args(scene)) add_colorbar(cm, bins=8) # save fig relative_fname_thumb = get_relative_path(scene, "dispmap") fpath = op.normpath(op.join(tgt_dir, relative_fname_thumb)) plotting.save_tight_figure(fig, fpath, hide_frames=True, pad_inches=0.01) # path info height, width = np.shape(algo_result)[:2] disp_map_data = { "thumb": relative_fname_thumb, "channels": 3, "height": height, "width": width } # save raw disparity map if add_pfms_to_result and not scene.is_test(): relative_fname_raw = get_relative_path(scene, "dispmap", file_type="pfm") fpath_tgt = op.normpath(op.join(tgt_dir, relative_fname_raw)) fpath_src = misc.get_fname_algo_result(algo_dir, scene) log.info("Copying disp map file from %s to %s" % (fpath_src, fpath_tgt)) shutil.copyfile(fpath_src, fpath_tgt) disp_map_data["raw"] = relative_fname_raw return disp_map_data
def run_validation(submission_path): # delay imports to speed up usage response from toolkit.evaluations import submission_validation as validation is_unpacked = op.isdir(submission_path) try: if is_unpacked: unpacked_submission_path = submission_path else: # unpack zip archive from toolkit.utils.file_io import unzip tmp_dir = op.normpath(op.join(os.getcwd(), "../tmp")) try: log.info("Extracting archive.") submission_directory = op.splitext( op.basename(submission_path))[0] unpacked_submission_path = op.join(tmp_dir, submission_directory) unzip(submission_path, unpacked_submission_path) except IOError as e: log.error('Zip Error: %s.\nTerminated submission validation.' % e) # validate submission success, error_json = validation.validate_extracted_submission( unpacked_submission_path) # report results print_validation_results(success, error_json) finally: # clean up if not is_unpacked and op.isdir(tmp_dir): shutil.rmtree(tmp_dir)
def add_runtime(scene, algo_dir, scores, metrics): runtime_metrics = [m for m in metrics if "runtime" in m.get_id()] for metric in runtime_metrics: score = metric.get_score_from_dir(scene, algo_dir) scores[metric.get_id()] = {"value": score} log.info("Score %5.2f for: %s, %s, Scale: %0.2f" % (score, metric.get_display_name(), scene.get_display_name(), scene.gt_scale)) return scores
def write_file(data, tgt_file, **kwargs): check_dir_for_fname(tgt_file) if tgt_file.endswith('.png') or tgt_file.endswith('.jpg'): write_img(data, tgt_file, **kwargs) elif tgt_file.endswith('.json'): write_json(data, tgt_file) elif tgt_file.endswith('.pfm'): write_pfm(data, tgt_file, **kwargs) else: raise NotImplementedError('No support for file: %s' % tgt_file) log.info('Saved %s' % tgt_file)
def save_fig(fig, fig_name, dpi=150, bbox_inches='tight', hide_frames=False, remove_ticks=False, **kwargs): log.info("Saving figure...") if remove_ticks: remove_ticks_from_axes(fig.get_axes()) if hide_frames: remove_frames_from_axes(fig.get_axes()) file_io.check_dir_for_fname(fig_name) plt.savefig(fig_name, dpi=dpi, bbox_inches=bbox_inches, **kwargs) fig.clf() plt.close(fig) gc.collect() log.info('Saved: ' + fig_name)
def compute_scores(algorithms, scenes, thresholds=THRESHOLDS, penalize_missing_pixels=True): percentages_algo_thresh = np.full((len(algorithms), len(thresholds)), fill_value=np.nan) bad_pix_metric = BadPix() max_diff = np.max(thresholds) for idx_a, algorithm in enumerate(algorithms): combined_diffs = np.full(0, fill_value=np.nan) log.info('Computing BadPix scores for: %s' % algorithm.get_display_name()) for scene in scenes: gt = scene.get_gt() algo_result = misc.get_algo_result(algorithm, scene) diffs = np.abs(algo_result - gt) mask_valid = misc.get_mask_valid( algo_result) * misc.get_mask_valid(diffs) mask_eval = bad_pix_metric.get_evaluation_mask(scene) if penalize_missing_pixels: # penalize all invalid algorithm pixels with maximum error diffs[~mask_valid] = max_diff + 100 diffs = diffs[mask_eval] else: diffs = diffs[mask_eval * mask_valid] combined_diffs = np.concatenate((combined_diffs, diffs)) # compute BadPix score for each threshold for idx_t, t in enumerate(thresholds): bad_pix_metric.thresh = t bad_pix_score = bad_pix_metric.get_score_from_diffs(combined_diffs) percentages_algo_thresh[idx_a, idx_t] = 100 - bad_pix_score return percentages_algo_thresh
def plot_metric_rows(self, grids, algorithms, metrics, offset, fontsize): gt = self.get_gt() center_view = self.get_center_view() for idx_a, algorithm in enumerate(algorithms): log.info("Algorithm: %s" % algorithm) algo_result = misc.get_algo_result(algorithm, self) # add algorithm disparity map plt.sca(grids[0][idx_a + 1]) cm = plt.imshow(algo_result, **settings.disp_map_args(self)) plt.title(algorithm.get_display_name(), fontsize=fontsize) # add colorbar to last disparity map in row if idx_a == (len(algorithms) - 1): plotting.create_colorbar(cm, cax=grids[0].cbar_axes[0], colorbar_bins=7, fontsize=fontsize) # add algorithm metric visualizations for idx_m, metric in enumerate(metrics): log.info(metric.get_display_name()) mask = metric.get_evaluation_mask(self) plt.sca(grids[idx_m + offset + 1][idx_a + 1]) cm = self.plot_algo_vis_for_metric(metric, algo_result, gt, mask, self.hidden_gt(), fontsize) # add colorbar to last metric visualization in row if idx_a == len(algorithms) - 1: plotting.create_colorbar(cm, cax=grids[idx_m + offset + 1].cbar_axes[0], colorbar_bins=metric.colorbar_bins, fontsize=fontsize) # add mask visualizations as 1st column if idx_a == 0: plt.sca(grids[idx_m + offset + 1][0]) plotting.plot_img_with_transparent_mask(center_view, mask, alpha=0.7, color=settings.MASK_COLOR) plt.ylabel(metric.get_short_name(), fontsize=fontsize) plt.title("Region Mask", fontsize=fontsize)
def parse_args(self, args=None, namespace=None): # try to parse all provided arguments namespace = super(OptionParser, self).parse_args(args, namespace) # call action with default action if option string was not provided # e.g. collect default scenes if no "-s" or "--scenes" was provided for action in self.actions: if getattr(namespace, action.dest) is None: action.__call__(self, namespace, values=None) log.info("Command line arguments: ") [ log.info("%s: %s" % (a.dest.title(), getattr(namespace, a.dest))) for a in self.actions ] # return values in order of parser options values = [getattr(namespace, action.dest) for action in self.actions] if len(values) == 1: return values[0] return values
def plot_algo_overview(self, algorithms, subdir="algo_overview", fs=6): accv_metrics = [MSE(), BadPix(0.07), BumpinessPlanes(), BumpinessContinSurf(), Discontinuities(), FineFattening(), FineThinning()] metrics_low_res = [m for m in self.get_applicable_metrics_low_res() if m in accv_metrics] metrics_high_res = [m for m in self.get_applicable_metrics_high_res() if m in accv_metrics] # prepare figure rows = len(metrics_low_res + metrics_high_res) + 1 cols = len(algorithms) + 1 fig = plt.figure(figsize=(cols, rows*1.1)) grids = self._get_grids(fig, rows, cols, axes_pad=-0.2) # center view on top left grid cell self.set_high_gt_scale() plt.sca(grids[0][0]) plt.imshow(self.get_center_view()) plt.title("Center View", fontsize=fs) plt.ylabel("Disparity Map", fontsize=fs) # mask visualizations + algorithm disparity maps + metric visualizations log.info("Computing scores and visualizations for low resolution metrics.") self.set_low_gt_scale() self.plot_metric_rows(grids, algorithms, metrics_low_res, offset=0, fontsize=fs) log.info("Computing scores and visualizations for high resolution metrics.") self.set_high_gt_scale() self.plot_metric_rows(grids, algorithms, metrics_high_res, offset=len(metrics_low_res), fontsize=fs) # finalize figure for grid in grids: plotting.remove_ticks_from_axes(grid.axes_all) plotting.remove_frames_from_axes(grid.axes_all) plt.suptitle(self.get_display_name(), fontsize=fs+2) fig_path = plotting.get_path_to_figure("algo_overview_%s" % self.get_name(), subdir=subdir) plotting.save_fig(fig, fig_path, pad_inches=0.1)
def compare_relative_ranks(algorithms, scenes, metrics, all_but=0): scores_scenes_metrics_algos = misc.collect_scores(algorithms, scenes, metrics, masked=True) scores_metrics_algos = np.ma.median(scores_scenes_metrics_algos, axis=0) n_metrics = np.shape(scores_metrics_algos)[0] winners = dict() for idx_a1, algorithm1 in enumerate(algorithms): scores_a1 = scores_metrics_algos[:, idx_a1] worse_on_all_but_n = [] for idx_a2, algorithm2 in enumerate(algorithms): scores_a2 = scores_metrics_algos[:, idx_a2] n_better = np.sum(scores_a1 < scores_a2) if n_better == n_metrics - all_but: worse_on_all_but_n.append(algorithm2) if worse_on_all_but_n: winners[algorithm1] = worse_on_all_but_n n_winners = len(winners.keys()) log.info("%d Algorithm(s) better on all but %d score(s)." % (n_winners, all_but)) for idx_a, (algorithm, better_than) in enumerate(winners.items()): inferior_algorithms = ", ".join(a.get_display_name() for a in better_than) log.info( "%d) %s is better than: %s" % (idx_a + 1, algorithm.get_display_name(), inferior_algorithms)) return winners
def print_validation_results(success, error_json): if success: log.info("Yeah :) Congratulations, your submission archive is valid. " "Go ahead and submit it online!") else: error_messages = error_json["messages"] log.info('Validation found %d error(s).' % len(error_messages)) log.info( 'A detailed format description can be found in the SUBMISSION_INSTRUCTIONS. ' '\nThe identified problems are: ') for error in error_messages: log.error(error)
def main(): accv_algo_names = ["epi1", "epi2", "lf_occ", "lf", "mv"] parser = OptionParser( [AlgorithmOps(default=accv_algo_names), FigureOpsACCV16()]) algorithms, figure_options = parser.parse_args() # delay imports to speed up usage response from toolkit.evaluations import paper_accv_2016, error_heatmaps from toolkit.scenes import Backgammon, Dots, Pyramids, Stripes from toolkit.utils import log, misc if "heatmaps" in figure_options: log.info("Creating error heatmaps.") scenes = misc.get_stratified_scenes() + misc.get_training_scenes() error_heatmaps.plot(algorithms, scenes, subdir=SUBDIR) if "radar" in figure_options: log.info("Creating radar charts for stratified and training scenes.") paper_accv_2016.plot_radar_charts(algorithms, subdir=SUBDIR) if "backgammon" in figure_options: log.info("Creating special chart for backgammon scene.") Backgammon().plot_fattening_thinning(algorithms, subdir=SUBDIR) if "pyramids" in figure_options: log.info("Creating special chart for pyramids scene.") Pyramids().plot_algo_disp_vs_gt_disp(algorithms, subdir=SUBDIR) if "dots" in figure_options: log.info("Creating special chart for dots scene.") Dots().plot_error_vs_noise(algorithms, subdir=SUBDIR) if "stripes" in figure_options: log.info("Creating special chart for stripes scene.") Stripes().visualize_masks(subdir=SUBDIR) if "stratified" in figure_options: for scene in misc.get_stratified_scenes(): log.info("Creating metric visualizations for scene: %s." % scene.get_display_name()) scene.plot_algo_overview(algorithms, with_metric_vis=True, subdir=SUBDIR) if "training" in figure_options: for scene in misc.get_training_scenes(): log.info("Creating metric visualizations for scene: %s." % scene.get_display_name()) scene.plot_algo_overview(algorithms, subdir=SUBDIR)
def main(): figure_options = OptionParser([FigureOpsCVPR17()]).parse_args() # delay imports to speed up usage response from toolkit.algorithms import Algorithm, PerPixBest from toolkit.evaluations import paper_cvprw_2017 as cvprw from toolkit.utils import log, misc from toolkit.scenes import PhotorealisticScene # prepare scenes if USE_TEST_SCENE_GT: benchmark_scenes = misc.get_benchmark_scenes() else: benchmark_scenes = misc.get_stratified_scenes( ) + misc.get_training_scenes() # prepare algorithms fnames_baseline_algos = ["epi1", "epi2", "lf", "mv", "lf_occ26"] fnames_challenge_algos = [ "ober", "omg_occ", "ps_rf25", "rm3de", "sc_gc", "spo_lf4cv", "zctv1" ] fnames_other_submissions = ["obercross", "ofsy_330dnr2"] baseline_algorithms = Algorithm.initialize_algorithms( fnames_baseline_algos, is_baseline=True) challenge_algorithms = Algorithm.initialize_algorithms( fnames_challenge_algos) other_algorithms = Algorithm.initialize_algorithms( fnames_other_submissions) for algorithm in other_algorithms: algorithm.display_name = "'" + algorithm.display_name algorithms = sorted(baseline_algorithms) + sorted( other_algorithms) + sorted(challenge_algorithms) algorithms = Algorithm.set_colors(algorithms) # create figures if "normalsdemo" in figure_options: log.info("Creating normals demo figure with Sideboard scene.") scene = PhotorealisticScene("sideboard") cvprw.plot_normals_explanation(Algorithm("epi1"), scene, subdir=SUBDIR) if "radar" in figure_options: log.info("Creating radar charts.") cvprw.plot_radar_charts(algorithms, subdir=SUBDIR) if "badpix" in figure_options: log.info("Creating figures with BadPix series.") per_pix_best = PerPixBest() per_pix_best.compute_meta_results(algorithms, benchmark_scenes) cvprw.plot_bad_pix_series(algorithms + [per_pix_best], USE_TEST_SCENE_GT, subdir=SUBDIR) if "median" in figure_options: log.info("Creating median comparison figures.") cvprw.plot_median_diffs(algorithms, misc.get_stratified_scenes(), subdir=SUBDIR) cvprw.plot_median_diffs(algorithms, misc.get_training_scenes(), subdir=SUBDIR) if "normals" in figure_options: log.info("Creating surface normal figure with Cotton scene.") cvprw.plot_normal_maps(algorithms, PhotorealisticScene("cotton"), subdir=SUBDIR) if USE_TEST_SCENE_GT and "discont" in figure_options: log.info("Creating discontinuity figure with Bicycle scene.") cvprw.plot_discont_overview(algorithms, PhotorealisticScene("bicycle"), subdir=SUBDIR) if "accuracy" in figure_options: log.info("Creating high accuracy figure.") selection = [ "ofsy_330dnr2", "zctv1", "obercross", "ober", "sc_gc", "spo_lf4cv", "rm3de", "ps_rf25" ] high_accuracy_algorithms = [] # algorithms should be exactly in the order of 'selection' for algo_name in selection: high_accuracy_algorithms.append( [a for a in algorithms if a.get_name() == algo_name][0]) scenes = [PhotorealisticScene("cotton"), PhotorealisticScene("boxes")] cvprw.plot_high_accuracy(high_accuracy_algorithms, scenes, subdir=SUBDIR) if "scenes" in figure_options: log.info("Creating scene overview figure.") cvprw.plot_benchmark_scene_overview(misc.get_benchmark_scenes(), subdir=SUBDIR) if "difficulty" in figure_options: log.info("Creating scene difficulty figure.") cvprw.plot_scene_difficulty(benchmark_scenes, subdir=SUBDIR)
def validate_extracted_submission(submission_dir, data_path=None): log.info("Validating extracted submission: %s." % submission_dir) scene_names = [ s.get_name() for s in misc.get_benchmark_scenes(data_path=data_path) ] exp_height, exp_width = settings.HEIGHT, settings.WIDTH errors = [] # check disparity maps disp_maps_dir = op.normpath( op.join(submission_dir, settings.DIR_NAME_DISP_MAPS)) if not op.isdir(disp_maps_dir): errors.append('Could not find disparity map directory: "%s".' % settings.DIR_NAME_DISP_MAPS) else: log.info("Validating disparity map files.") for scene_name in scene_names: path_disp_maps = op.join(disp_maps_dir, "%s.pfm" % scene_name) relative_path_disp_maps = op.join(settings.DIR_NAME_DISP_MAPS, "%s.pfm" % scene_name) if not op.isfile(path_disp_maps): errors.append( 'Frame %s: Could not find disparity file: "%s".' % (scene_name, relative_path_disp_maps)) else: try: disp_map = file_io.read_pfm(path_disp_maps) height, width = np.shape(disp_map) if height != exp_height or width != exp_width: errors.append( "Frame %s, File %s: Resolution mismatch. " "Expected (%d, %d), got (%d, %d)." % (scene_name, relative_path_disp_maps, exp_height, exp_width, height, width)) except file_io.PFMExeption as e: errors.append("Frame %s, File %s, PFM Error: %s." % (scene_name, relative_path_disp_maps, e)) # check runtimes runtimes_dir = op.normpath(op.join(submission_dir, "runtimes")) if not op.isdir(runtimes_dir): errors.append('Could not find runtimes directory: "%s".' % settings.DIR_NAME_RUNTIMES) else: log.info("Validating runtime files.") for scene_name in scene_names: path_runtimes = op.join(runtimes_dir, "%s.txt" % scene_name) relative_path_runtimes = op.join(settings.DIR_NAME_RUNTIMES, "%s.txt" % scene_name) if not op.isfile(path_runtimes): errors.append('Frame %s: Could not find runtime file: "%s".' % (scene_name, relative_path_runtimes)) else: try: file_io.read_runtime(path_runtimes) except IOError as error: errors.append("Frame %s, File %s, Error: %s." % (scene_name, relative_path_runtimes, error)) success = not errors error_json = {"messages": errors} if success: log.info("Validated submission successfully :)") return success, error_json
def evaluate(evaluation_output_path, algorithm_input_path, scenes, metrics, visualize=False, add_to_existing_results=True, add_pfms_to_result=True): """ :param evaluation_output_path: target directory for all evaluation results :param algorithm_input_path: input directory for algorithm results, expected directories: runtimes, disp_maps :param scenes: scenes to be evaluated :param metrics: metrics to be evaluated :param visualize: whether to save visualizations (otherwise just the scores) :param add_to_existing_results: if set to True, will try to read results.json and add/replace entries, keeping existing scores of other scenes/metrics as is :param add_pfms_to_result: when executed on evaluation server, pfms are prepared for 3D point cloud view :return: success, {"messages": ["error 1", "error 2", ...]} """ log.info("Evaluating algorithm results in:\n %s" % algorithm_input_path) log.info("Writing results to:\n %s" % evaluation_output_path) log.info("Using ground truth data from:\n %s" % settings.DATA_PATH) log.info("Metrics:\n %s" % ", ".join(m.get_display_name() for m in metrics)) log.info("Scenes:\n %s" % ", ".join(s.get_display_name() for s in scenes)) file_name_results = op.join(evaluation_output_path, "results.json") admin_errors = [] eval_json = dict() if add_to_existing_results: try: eval_json = file_io.read_file(file_name_results) except IOError: pass # evaluate for scene in scenes: scene_data = eval_json.get(scene.get_name(), dict()) try: if visualize: log.info("Visualizing algorithm result on %s" % scene.get_display_name()) scene_data["algorithm_result"] = visualize_algo_result( scene, algorithm_input_path, evaluation_output_path, add_pfms_to_result) log.info("Processing scene: %s" % scene.get_display_name()) log.info("Using data from:\n %s" % scene.get_data_path()) scene_scores = compute_scores(scene, metrics, algorithm_input_path, evaluation_output_path, visualize) if add_to_existing_results: existing_scores = scene_data.get("scores", dict()) existing_scores.update(scene_scores) scene_scores = existing_scores scene_data["scores"] = scene_scores except IOError as e: admin_errors.append(e) log.error(e) continue eval_json[scene.get_name()] = scene_data # save json with scores and paths to visualizations file_io.write_file(eval_json, file_name_results) log.info("Done!") success = not admin_errors error_json = {"messages": admin_errors} return success, error_json
def plot_algo_overview(self, algorithms, with_metric_vis=True, subdir="algo_overview", fs=14): self.set_scale_for_algo_overview() metrics = self.get_scene_specific_metrics() n_metrics = len(metrics) if not with_metric_vis: rows, cols = 2 + n_metrics, len(algorithms) + 2 fig = plt.figure(figsize=(2.6 * len(algorithms), 4.9)) offset = 0 else: rows, cols = 2 + 2 * n_metrics, len(algorithms) + 2 fig = plt.figure(figsize=(2.6 * len(algorithms), rows + 3)) offset = n_metrics labelpad = -15 hscale, wscale = 7, 5 width_ratios = [wscale] * (len(algorithms) + 1) + [1] height_ratios = [hscale] * (rows - n_metrics) + [1] * n_metrics gs = gridspec.GridSpec(rows, cols, height_ratios=height_ratios, width_ratios=width_ratios) gt = self.get_gt() dummy = np.ones((self.get_height() / hscale, self.get_width())) cb_height, w = np.shape(gt) cb_width = w / float(wscale) # first column (gt, center view, ...) plt.subplot(gs[0]) plt.imshow(gt, **settings.disp_map_args(self)) plt.title("Ground Truth", fontsize=fs) plt.ylabel("Disparity Map", fontsize=fs) plt.subplot(gs[cols]) plt.imshow(self.get_center_view()) plt.ylabel("diff: GT - Algo", fontsize=fs) for idx_m, metric in enumerate(metrics): plt.subplot(gs[(2 + idx_m + offset) * cols]) plt.xlabel(metric.get_short_name(), labelpad=labelpad, fontsize=fs) plt.imshow(dummy, cmap="gray_r") # algorithm columns for idx_a, algorithm in enumerate(algorithms): log.info("Processing algorithm: %s" % algorithm) algo_result = misc.get_algo_result(algorithm, self) # algorithm disparities plt.subplot(gs[idx_a + 1]) plt.title(algorithm.get_display_name(), fontsize=fs) cm1 = plt.imshow(algo_result, **settings.disp_map_args(self)) # algorithm diff map plt.subplot(gs[cols + idx_a + 1]) cm2 = plt.imshow(gt - algo_result, **settings.diff_map_args()) # add colorbar if last column if idx_a == (len(algorithms) - 1): plotting.add_colorbar(gs[idx_a + 2], cm1, cb_height, cb_width, colorbar_bins=5, fontsize=fs - 4) plotting.add_colorbar(gs[cols + idx_a + 2], cm2, cb_height, cb_width, colorbar_bins=5, fontsize=fs - 4) # score and background color for metrics for idx_m, metric in enumerate(metrics): if with_metric_vis: plt.subplot(gs[(2 + idx_m) * cols + idx_a + 1]) score, vis = metric.get_score(algo_result, gt, self, with_visualization=True) cm3 = plt.imshow(vis, **settings.metric_args(metric)) if idx_a == 0: plt.ylabel(metric.get_short_name(), fontsize=fs) elif idx_a == (len(algorithms) - 1): plotting.add_colorbar( gs[(2 + idx_m) * cols + idx_a + 2], cm3, cb_height, cb_width, colorbar_bins=metric.colorbar_bins, fontsize=fs - 4) else: score = metric.get_score(algo_result, gt, self) plt.subplot(gs[(2 + idx_m + offset) * cols + idx_a + 1]) plt.imshow( dummy * score, **settings.score_color_args(vmin=metric.vmin, vmax=metric.vmax)) plt.xlabel(metric.format_score(score), labelpad=labelpad, fontsize=fs) fig_name = "algo_overview_" + self.get_name( ) + with_metric_vis * "_vis" fig_path = plotting.get_path_to_figure(fig_name, subdir=subdir) plotting.save_tight_figure(fig, fig_path, wspace=0.04, hide_frames=True)