def main(unused_argv): if not FLAGS.plot_dir: raise ValueError('apply_smurf needs plot directory.') if not tf.io.gfile.exists(FLAGS.plot_dir): print('Making new plot directory', FLAGS.plot_dir) tf.io.gfile.makedirs(FLAGS.plot_dir) gin.parse_config_files_and_bindings(FLAGS.config_file, FLAGS.gin_bindings) smurf = smurf_evaluator.build_network(batch_size=1) smurf.update_checkpoint_dir(FLAGS.checkpoint_dir) smurf.restore() for i, (image1, image2) in enumerate(get_image_iterator()): sys.stdout.write(':') sys.stdout.flush() flow_forward, occlusion, flow_backward = smurf.infer( image1, image2, input_height=FLAGS.height, input_width=FLAGS.width, infer_occlusion=True, infer_bw=True) occlusion = 1. - occlusion smurf_plotting.complete_paper_plot(plot_dir=FLAGS.plot_dir, index=i, image1=image1, image2=image2, flow_uv=flow_forward, ground_truth_flow_uv=None, flow_valid_occ=None, predicted_occlusion=occlusion, ground_truth_occlusion=None)
def evaluate( inference_fn, dataset, height, width, progress_bar=False, plot_dir='', num_plots=0, max_num_evals=10000, prefix='', has_occlusion=True, weights=None, ): """Evaluate an inference function for flow. Args: inference_fn: An inference function that produces a flow_field from two images, e.g. the infer method of SMURF. dataset: A dataset produced by the method above with for_eval=True. height: int, the height to which the images should be resized for inference. width: int, the width to which the images should be resized for inference. progress_bar: boolean, flag to indicate whether the function should print a progress_bar during evaluaton. plot_dir: string, optional path to a directory in which plots are saved (if num_plots > 0). num_plots: int, maximum number of qualitative results to plot for the evaluation. max_num_evals: int, maxmim number of evaluations. prefix: str, prefix to prepend to filenames for saved plots and for keys in results dictionary. has_occlusion: bool indicating whether or not the dataset includes ground truth occlusion. weights: unsupervised loss weights Returns: A dictionary of floats that represent different evaluation metrics. The keys of this dictionary are returned by the method list_eval_keys (see below). """ eval_start_in_s = time.time() it = tf.compat.v1.data.make_one_shot_iterator(dataset) epe_occ = [] # End point errors. errors_occ = [] inference_times = [] unsuper_losses = [] all_occlusion_results = defaultdict(lambda: defaultdict(int)) plot_count = 0 eval_count = -1 for test_batch in it: image_batch = test_batch['images'] flow_gt = test_batch['flow'] flow_valid = test_batch['flow_valid'] if has_occlusion: occ_mask_gt = test_batch['occlusions'] else: occ_mask_gt = tf.ones_like(flow_valid) if eval_count >= max_num_evals: break eval_count += 1 if eval_count >= max_num_evals: break if progress_bar: sys.stdout.write(':') sys.stdout.flush() f = lambda: inference_fn(image_batch[0], image_batch[1], input_height=height, input_width=width, infer_occlusion=True, infer_bw=True) inference_time_in_ms, (flow, soft_occlusion_mask, flow_bw) = smurf_utils.time_it( f, execute_once_before=eval_count == 1) inference_times.append(inference_time_in_ms) if not has_occlusion: best_thresh = .5 else: f_dict = compute_f_metrics(soft_occlusion_mask, occ_mask_gt) best_thresh = -1. best_f_score = -1. for thresh, metrics in f_dict.items(): precision = metrics['tp'] / (metrics['tp'] + metrics['fp'] + 1e-6) recall = metrics['tp'] / (metrics['tp'] + metrics['fn'] + 1e-6) f1 = 2 * precision * recall / (precision + recall + 1e-6) if f1 > best_f_score: best_thresh = thresh best_f_score = f1 all_occlusion_results[thresh]['tp'] += metrics['tp'] all_occlusion_results[thresh]['fp'] += metrics['fp'] all_occlusion_results[thresh]['tn'] += metrics['tn'] all_occlusion_results[thresh]['fn'] += metrics['fn'] final_flow = flow endpoint_error_occ = epe_elementwise(final_flow, flow_gt) * flow_valid outliers_occ = outliers_elementwise(final_flow, flow_gt) * flow_valid # NOTE(austinstone): The unsupervised loss function expects occluded areas # to be zeros, whereas the occlusion mask above returns 1s in areas # of occlusions. The unsupervised loss function below assumes some # parameters (smoothness edge info) is left at the default values. if weights is not None: inv_mask = tf.expand_dims(1. - soft_occlusion_mask, axis=0) occlusion_estimation_fn = lambda forward_flow, backward_flow: inv_mask loss_dict = smurf_utils.unsupervised_loss( images=tf.expand_dims(image_batch, axis=0), flows={ (0, 1, 'augmented-student'): [tf.expand_dims(flow, axis=0)], (1, 0, 'augmented-student'): [tf.expand_dims(flow_bw, axis=0)] }, weights=weights, occlusion_estimation_fn=occlusion_estimation_fn) loss_dict['total_loss'] = sum(loss_dict.values()) else: loss_dict = {} unsuper_losses.append(loss_dict) epe_occ.append(tf.reduce_mean(input_tensor=endpoint_error_occ)) errors_occ.append(tf.reduce_mean(input_tensor=outliers_occ)) if plot_dir and plot_count < num_plots: plot_count += 1 mask_thresh = tf.cast( tf.math.greater(soft_occlusion_mask, best_thresh), tf.float32) smurf_plotting.complete_paper_plot( plot_dir, plot_count, image_batch[0].numpy(), image_batch[1].numpy(), final_flow.numpy(), flow_gt.numpy(), np.ones_like(mask_thresh.numpy()), 1. - mask_thresh.numpy(), 1. - occ_mask_gt.numpy().astype('float32'), frame_skip=None) if progress_bar: sys.stdout.write('\n') sys.stdout.flush() fmax, best_thresh = get_fmax_and_best_thresh(all_occlusion_results) eval_stop_in_s = time.time() results = { 'occl-f-max': fmax, 'best-occl-thresh': best_thresh, 'EPE': np.mean(np.array(epe_occ)), 'ER': np.mean(np.array(errors_occ)), 'inf-time(ms)': np.mean(inference_times), 'eval-time(s)': eval_stop_in_s - eval_start_in_s, } for k in unsuper_losses[0].keys(): results.update({k: np.mean([l[k] for l in unsuper_losses])}) if prefix: return {prefix + '-' + k: v for k, v in results.items()} return results
def evaluate(inference_fn, dataset, height, width, progress_bar=False, plot_dir='', num_plots=0, prefix='kitti'): """Evaluate an iference function for flow with a kitti eval dataset. Args: inference_fn: An inference function that produces a flow_field from two images, e.g. the infer method of SMURF. dataset: A dataset produced by the method above with for_eval=True. height: int, the height to which the images should be resized for inference. width: int, the width to which the images should be resized for inference. progress_bar: boolean, flag to indicate whether the function should print a progress_bar during evaluaton. plot_dir: string, optional path to a directory in which plots are saved (if num_plots > 0). num_plots: int, maximum number of qualitative results to plot for the evaluation. Returns: A dictionary of floats that represent different evaluation metrics. The keys of this dictionary are returned by the method list_eval_keys (see below). """ eval_start_in_s = time.time() it = tf.compat.v1.data.make_one_shot_iterator(dataset) epe_occ = [] # End point errors. errors_occ = [] valid_occ = [] epe_noc = [] # End point errors. errors_noc = [] valid_noc = [] inference_times = [] all_occlusion_results = defaultdict(lambda: defaultdict(int)) for i, test_batch in enumerate(it): if progress_bar: sys.stdout.write(':') sys.stdout.flush() image_batch = test_batch['images'] flow_uv_occ = test_batch['flow_uv_occ'] flow_uv_noc = test_batch['flow_uv_noc'] flow_valid_occ = test_batch['flow_valid_occ'] flow_valid_noc = test_batch['flow_valid_noc'] flow_valid_occ = tf.cast(flow_valid_occ, 'float32') flow_valid_noc = tf.cast(flow_valid_noc, 'float32') f = lambda: inference_fn(image_batch[0], image_batch[1], input_height=height, input_width=width, infer_occlusion=True) inference_time_in_ms, (flow, soft_occlusion_mask) = smurf_utils.time_it( f, execute_once_before=i == 0) inference_times.append(inference_time_in_ms) occ_mask_gt = flow_valid_occ - flow_valid_noc f_dict = data_utils.compute_f_metrics( soft_occlusion_mask * flow_valid_occ, occ_mask_gt * flow_valid_occ) best_thresh = -1. best_f_score = -1. for thresh, metrics in f_dict.items(): precision = metrics['tp'] / (metrics['tp'] + metrics['fp'] + 1e-6) recall = metrics['tp'] / (metrics['tp'] + metrics['fn'] + 1e-6) f1 = 2 * precision * recall / (precision + recall + 1e-6) if f1 > best_f_score: best_thresh = thresh best_f_score = f1 all_occlusion_results[thresh]['tp'] += metrics['tp'] all_occlusion_results[thresh]['fp'] += metrics['fp'] all_occlusion_results[thresh]['tn'] += metrics['tn'] all_occlusion_results[thresh]['fn'] += metrics['fn'] mask_thresh = tf.cast( tf.math.greater(soft_occlusion_mask, best_thresh), tf.float32) # Image coordinates are swapped in labels final_flow = flow[Ellipsis, ::-1] endpoint_error_occ = data_utils.epe_elementwise( final_flow, flow_uv_occ) outliers_occ = data_utils.outliers_elementwise(final_flow, flow_uv_occ) endpoint_error_noc = data_utils.epe_elementwise( final_flow, flow_uv_noc) outliers_noc = data_utils.outliers_elementwise(final_flow, flow_uv_noc) epe_occ.append( tf.reduce_sum(input_tensor=flow_valid_occ * endpoint_error_occ)) errors_occ.append( tf.reduce_sum(input_tensor=flow_valid_occ * outliers_occ)) valid_occ.append(tf.reduce_sum(input_tensor=flow_valid_occ)) epe_noc.append( tf.reduce_sum(input_tensor=flow_valid_noc * endpoint_error_noc)) errors_noc.append( tf.reduce_sum(input_tensor=flow_valid_noc * outliers_noc)) valid_noc.append(tf.reduce_sum(input_tensor=flow_valid_noc)) if plot_dir and i < num_plots: smurf_plotting.complete_paper_plot(plot_dir, i, image_batch[0].numpy(), image_batch[1].numpy(), final_flow.numpy(), flow_uv_occ.numpy(), flow_valid_occ.numpy(), (1. - mask_thresh).numpy(), (1. - occ_mask_gt).numpy(), frame_skip=None) if progress_bar: sys.stdout.write('\n') sys.stdout.flush() fmax, best_thresh = data_utils.get_fmax_and_best_thresh( all_occlusion_results) eval_stop_in_s = time.time() results = { prefix + '-occl-f-max': fmax, prefix + '-best-occl-thresh': best_thresh, prefix + '-EPE(occ)': np.clip(np.mean(np.array(epe_occ) / np.array(valid_occ)), 0.0, 50.0), prefix + '-ER(occ)': np.mean(np.array(errors_occ) / np.array(valid_occ)), prefix + '-EPE(noc)': np.clip(np.mean(np.array(epe_noc) / np.array(valid_noc)), 0.0, 50.0), prefix + '-ER(noc)': np.mean(np.array(errors_noc) / np.array(valid_noc)), prefix + '-inf-time(ms)': np.mean(inference_times), prefix + '-eval-time(s)': eval_stop_in_s - eval_start_in_s, } return results