Esempio n. 1
0
def evaluate(
    inference_fn,
    dataset,
    height,
    width,
    progress_bar=False,
    plot_dir='',
    num_plots=0,
    max_num_evals=10000,
    prefix='',
    has_occlusion=True,
):
    """Evaluate an inference function for flow.

  Args:
    inference_fn: An inference function that produces a flow_field from two
      images, e.g. the infer method of UFlow.
    dataset: A dataset produced by the method above with for_eval=True.
    height: int, the height to which the images should be resized for inference.
    width: int, the width to which the images should be resized for inference.
    progress_bar: boolean, flag to indicate whether the function should print a
      progress_bar during evaluaton.
    plot_dir: string, optional path to a directory in which plots are saved (if
      num_plots > 0).
    num_plots: int, maximum number of qualitative results to plot for the
      evaluation.
    max_num_evals: int, maxmim number of evaluations.
    prefix: str, prefix to prepend to filenames for saved plots and for keys in
      results dictionary.
    has_occlusion: bool indicating whether or not the dataset includes ground
      truth occlusion.

  Returns:
    A dictionary of floats that represent different evaluation metrics. The keys
    of this dictionary are returned by the method list_eval_keys (see below).
  """

    eval_start_in_s = time.time()

    it = tf.compat.v1.data.make_one_shot_iterator(dataset)
    epe_occ = []  # End point errors.
    errors_occ = []
    inference_times = []
    all_occlusion_results = defaultdict(lambda: defaultdict(int))

    plot_count = 0
    eval_count = -1
    for test_batch in it:

        if eval_count >= max_num_evals:
            break

        eval_count += 1
        if eval_count >= max_num_evals:
            break

        if progress_bar:
            sys.stdout.write(':')
            sys.stdout.flush()

        if has_occlusion:
            (image_batch, flow_gt, _, occ_mask_gt) = test_batch
        else:
            (image_batch, flow_gt, _) = test_batch
            occ_mask_gt = tf.ones_like(flow_gt[Ellipsis, -1:])
        # pylint:disable=cell-var-from-loop
        # pylint:disable=g-long-lambda
        f = lambda: inference_fn(image_batch[0],
                                 image_batch[1],
                                 input_height=height,
                                 input_width=width,
                                 infer_occlusion=True)
        inference_time_in_ms, (flow,
                               soft_occlusion_mask) = uflow_utils.time_it(
                                   f, execute_once_before=eval_count == 1)
        inference_times.append(inference_time_in_ms)

        if not has_occlusion:
            best_thresh = .5
        else:
            f_dict = compute_f_metrics(soft_occlusion_mask, occ_mask_gt)
            best_thresh = -1.
            best_f_score = -1.
            for thresh, metrics in f_dict.items():
                precision = metrics['tp'] / (metrics['tp'] + metrics['fp'] +
                                             1e-6)
                recall = metrics['tp'] / (metrics['tp'] + metrics['fn'] + 1e-6)
                f1 = 2 * precision * recall / (precision + recall + 1e-6)
                if f1 > best_f_score:
                    best_thresh = thresh
                    best_f_score = f1
                all_occlusion_results[thresh]['tp'] += metrics['tp']
                all_occlusion_results[thresh]['fp'] += metrics['fp']
                all_occlusion_results[thresh]['tn'] += metrics['tn']
                all_occlusion_results[thresh]['fn'] += metrics['fn']

        final_flow = flow
        endpoint_error_occ = tf.reduce_sum(input_tensor=(final_flow -
                                                         flow_gt)**2,
                                           axis=-1,
                                           keepdims=True)**0.5
        gt_flow_abs = tf.reduce_sum(input_tensor=flow_gt**2,
                                    axis=-1,
                                    keepdims=True)**0.5
        outliers_occ = tf.cast(
            tf.logical_and(endpoint_error_occ > 3.,
                           endpoint_error_occ > 0.05 * gt_flow_abs), 'float32')
        epe_occ.append(tf.reduce_mean(input_tensor=endpoint_error_occ))
        errors_occ.append(tf.reduce_mean(input_tensor=outliers_occ))

        if plot_dir and plot_count < num_plots:
            plot_count += 1
            mask_thresh = tf.cast(
                tf.math.greater(soft_occlusion_mask, best_thresh), tf.float32)
            uflow_plotting.complete_paper_plot(
                plot_dir,
                plot_count,
                image_batch[0].numpy(),
                image_batch[1].numpy(),
                final_flow.numpy(),
                flow_gt.numpy(),
                np.ones_like(mask_thresh.numpy()),
                1. - mask_thresh.numpy(),
                1. - occ_mask_gt.numpy().astype('float32'),
                frame_skip=None)
    if progress_bar:
        sys.stdout.write('\n')
        sys.stdout.flush()

    fmax, best_thresh = get_fmax_and_best_thresh(all_occlusion_results)
    eval_stop_in_s = time.time()

    results = {
        'occl-f-max': fmax,
        'best-occl-thresh': best_thresh,
        'EPE': np.mean(np.array(epe_occ)),
        'ER': np.mean(np.array(errors_occ)),
        'inf-time(ms)': np.mean(inference_times),
        'eval-time(s)': eval_stop_in_s - eval_start_in_s
    }
    if prefix:
        return {prefix + '-' + k: v for k, v in results.items()}
    return results
Esempio n. 2
0
def evaluate(inference_fn,
             dataset,
             height,
             width,
             progress_bar=False,
             plot_dir='',
             num_plots=0,
             prefix='kitti'):
  """Evaluate an iference function for flow with a kitti eval dataset.

  Args:
    inference_fn: An inference function that produces a flow_field from two
      images, e.g. the infer method of UFlow.
    dataset: A dataset produced by the method above with for_eval=True.
    height: int, the height to which the images should be resized for inference.
    width: int, the width to which the images should be resized for inference.
    progress_bar: boolean, flag to indicate whether the function should print a
      progress_bar during evaluaton.
    plot_dir: string, optional path to a directory in which plots are saved (if
      num_plots > 0).
    num_plots: int, maximum number of qualitative results to plot for the
      evaluation.
    prefix: str to prefix evaluation keys with in the returned dictionary.

  Returns:
    A dictionary of floats that represent different evaluation metrics. The keys
    of this dictionary are returned by the method list_eval_keys (see below).
  """

  eval_start_in_s = time.time()

  it = tf.compat.v1.data.make_one_shot_iterator(dataset)
  epe_occ = []  # End point errors.
  errors_occ = []
  valid_occ = []
  epe_noc = []  # End point errors.
  errors_noc = []
  valid_noc = []
  inference_times = []
  all_occlusion_results = defaultdict(lambda: defaultdict(int))

  for i, test_batch in enumerate(it):

    if progress_bar:
      sys.stdout.write(':')
      sys.stdout.flush()
    (image_batch, flow_uv_occ, flow_uv_noc, flow_valid_occ,
     flow_valid_noc) = test_batch

    flow_valid_occ = tf.cast(flow_valid_occ, 'float32')
    flow_valid_noc = tf.cast(flow_valid_noc, 'float32')

    # pylint:disable=cell-var-from-loop
    f = lambda: inference_fn(
        image_batch[0],
        image_batch[1],
        input_height=height,
        input_width=width,
        infer_occlusion=True)
    inference_time_in_ms, (flow, soft_occlusion_mask) = uflow_utils.time_it(
        f, execute_once_before=i == 0)
    inference_times.append(inference_time_in_ms)

    occ_mask_gt = flow_valid_occ - flow_valid_noc
    f_dict = data_utils.compute_f_metrics(soft_occlusion_mask * flow_valid_occ,
                                          occ_mask_gt * flow_valid_occ)
    best_thresh = -1.
    best_f_score = -1.
    for thresh, metrics in f_dict.items():
      precision = metrics['tp'] / (metrics['tp'] + metrics['fp'] + 1e-6)
      recall = metrics['tp'] / (metrics['tp'] + metrics['fn'] + 1e-6)
      f1 = 2 * precision * recall / (precision + recall + 1e-6)
      if f1 > best_f_score:
        best_thresh = thresh
        best_f_score = f1
      all_occlusion_results[thresh]['tp'] += metrics['tp']
      all_occlusion_results[thresh]['fp'] += metrics['fp']
      all_occlusion_results[thresh]['tn'] += metrics['tn']
      all_occlusion_results[thresh]['fn'] += metrics['fn']

    mask_thresh = tf.cast(
        tf.math.greater(soft_occlusion_mask, best_thresh), tf.float32)
    # Image coordinates are swapped in labels
    final_flow = flow[Ellipsis, ::-1]

    endpoint_error_occ = tf.reduce_sum(
        input_tensor=(final_flow - flow_uv_occ)**2, axis=-1, keepdims=True)**0.5
    gt_flow_abs = tf.reduce_sum(
        input_tensor=flow_uv_occ**2, axis=-1, keepdims=True)**0.5
    outliers_occ = tf.cast(
        tf.logical_and(endpoint_error_occ > 3.,
                       endpoint_error_occ > 0.05 * gt_flow_abs), 'float32')

    endpoint_error_noc = tf.reduce_sum(
        input_tensor=(final_flow - flow_uv_noc)**2, axis=-1, keepdims=True)**0.5
    gt_flow_abs = tf.reduce_sum(
        input_tensor=flow_uv_noc**2, axis=-1, keepdims=True)**0.5
    outliers_noc = tf.cast(
        tf.logical_and(endpoint_error_noc > 3.,
                       endpoint_error_noc > 0.05 * gt_flow_abs), 'float32')

    epe_occ.append(
        tf.reduce_sum(input_tensor=flow_valid_occ * endpoint_error_occ))
    errors_occ.append(tf.reduce_sum(input_tensor=flow_valid_occ * outliers_occ))
    valid_occ.append(tf.reduce_sum(input_tensor=flow_valid_occ))

    epe_noc.append(
        tf.reduce_sum(input_tensor=flow_valid_noc * endpoint_error_noc))
    errors_noc.append(tf.reduce_sum(input_tensor=flow_valid_noc * outliers_noc))
    valid_noc.append(tf.reduce_sum(input_tensor=flow_valid_noc))

    if plot_dir and i < num_plots:
      uflow_plotting.complete_paper_plot(
          plot_dir,
          i,
          image_batch[0].numpy(),
          image_batch[1].numpy(),
          final_flow.numpy(),
          flow_uv_occ.numpy(),
          flow_valid_occ.numpy(), (1. - mask_thresh).numpy(),
          (1. - occ_mask_gt).numpy(),
          frame_skip=None)
  if progress_bar:
    sys.stdout.write('\n')
    sys.stdout.flush()

  fmax, best_thresh = data_utils.get_fmax_and_best_thresh(all_occlusion_results)
  eval_stop_in_s = time.time()

  results = {
      prefix + '-occl-f-max':
          fmax,
      prefix + '-best-occl-thresh':
          best_thresh,
      prefix + '-EPE(occ)':
          np.clip(np.mean(np.array(epe_occ) / np.array(valid_occ)), 0.0, 50.0),
      prefix + '-ER(occ)':
          np.mean(np.array(errors_occ) / np.array(valid_occ)),
      prefix + '-EPE(noc)':
          np.clip(np.mean(np.array(epe_noc) / np.array(valid_noc)), 0.0, 50.0),
      prefix + '-ER(noc)':
          np.mean(np.array(errors_noc) / np.array(valid_noc)),
      prefix + '-inf-time(ms)':
          np.mean(inference_times),
      prefix + '-eval-time(s)':
          eval_stop_in_s - eval_start_in_s,
  }
  return results