def main(unused_argv):
if not FLAGS.plot_dir:
raise ValueError('apply_smurf needs plot directory.')
if not tf.io.gfile.exists(FLAGS.plot_dir):
print('Making new plot directory', FLAGS.plot_dir)
tf.io.gfile.makedirs(FLAGS.plot_dir)
gin.parse_config_files_and_bindings(FLAGS.config_file, FLAGS.gin_bindings)
smurf = smurf_evaluator.build_network(batch_size=1)
smurf.update_checkpoint_dir(FLAGS.checkpoint_dir)
smurf.restore()
for i, (image1, image2) in enumerate(get_image_iterator()):
sys.stdout.write(':')
sys.stdout.flush()
flow_forward, occlusion, flow_backward = smurf.infer(
image1,
image2,
input_height=FLAGS.height,
input_width=FLAGS.width,
infer_occlusion=True,
infer_bw=True)
occlusion = 1. - occlusion
smurf_plotting.complete_paper_plot(plot_dir=FLAGS.plot_dir,
index=i,
image1=image1,
image2=image2,
flow_uv=flow_forward,
ground_truth_flow_uv=None,
flow_valid_occ=None,
predicted_occlusion=occlusion,
ground_truth_occlusion=None)
def evaluate(
inference_fn,
dataset,
height,
width,
progress_bar=False,
plot_dir='',
num_plots=0,
max_num_evals=10000,
prefix='',
has_occlusion=True,
weights=None,
):
"""Evaluate an inference function for flow.
Args:
inference_fn: An inference function that produces a flow_field from two
images, e.g. the infer method of SMURF.
dataset: A dataset produced by the method above with for_eval=True.
height: int, the height to which the images should be resized for inference.
width: int, the width to which the images should be resized for inference.
progress_bar: boolean, flag to indicate whether the function should print a
progress_bar during evaluaton.
plot_dir: string, optional path to a directory in which plots are saved (if
num_plots > 0).
num_plots: int, maximum number of qualitative results to plot for the
evaluation.
max_num_evals: int, maxmim number of evaluations.
prefix: str, prefix to prepend to filenames for saved plots and for keys in
results dictionary.
has_occlusion: bool indicating whether or not the dataset includes ground
truth occlusion.
weights: unsupervised loss weights
Returns:
A dictionary of floats that represent different evaluation metrics. The keys
of this dictionary are returned by the method list_eval_keys (see below).
"""
eval_start_in_s = time.time()
it = tf.compat.v1.data.make_one_shot_iterator(dataset)
epe_occ = [] # End point errors.
errors_occ = []
inference_times = []
unsuper_losses = []
all_occlusion_results = defaultdict(lambda: defaultdict(int))
plot_count = 0
eval_count = -1
for test_batch in it:
image_batch = test_batch['images']
flow_gt = test_batch['flow']
flow_valid = test_batch['flow_valid']
if has_occlusion:
occ_mask_gt = test_batch['occlusions']
else:
occ_mask_gt = tf.ones_like(flow_valid)
if eval_count >= max_num_evals:
break
eval_count += 1
if eval_count >= max_num_evals:
break
if progress_bar:
sys.stdout.write(':')
sys.stdout.flush()
f = lambda: inference_fn(image_batch[0],
image_batch[1],
input_height=height,
input_width=width,
infer_occlusion=True,
infer_bw=True)
inference_time_in_ms, (flow, soft_occlusion_mask,
flow_bw) = smurf_utils.time_it(
f, execute_once_before=eval_count == 1)
inference_times.append(inference_time_in_ms)
if not has_occlusion:
best_thresh = .5
else:
f_dict = compute_f_metrics(soft_occlusion_mask, occ_mask_gt)
best_thresh = -1.
best_f_score = -1.
for thresh, metrics in f_dict.items():
precision = metrics['tp'] / (metrics['tp'] + metrics['fp'] +
1e-6)
recall = metrics['tp'] / (metrics['tp'] + metrics['fn'] + 1e-6)
f1 = 2 * precision * recall / (precision + recall + 1e-6)
if f1 > best_f_score:
best_thresh = thresh
best_f_score = f1
all_occlusion_results[thresh]['tp'] += metrics['tp']
all_occlusion_results[thresh]['fp'] += metrics['fp']
all_occlusion_results[thresh]['tn'] += metrics['tn']
all_occlusion_results[thresh]['fn'] += metrics['fn']
final_flow = flow
endpoint_error_occ = epe_elementwise(final_flow, flow_gt) * flow_valid
outliers_occ = outliers_elementwise(final_flow, flow_gt) * flow_valid
# NOTE(austinstone): The unsupervised loss function expects occluded areas
# to be zeros, whereas the occlusion mask above returns 1s in areas
# of occlusions. The unsupervised loss function below assumes some
# parameters (smoothness edge info) is left at the default values.
if weights is not None:
inv_mask = tf.expand_dims(1. - soft_occlusion_mask, axis=0)
occlusion_estimation_fn = lambda forward_flow, backward_flow: inv_mask
loss_dict = smurf_utils.unsupervised_loss(
images=tf.expand_dims(image_batch, axis=0),
flows={
(0, 1, 'augmented-student'):
[tf.expand_dims(flow, axis=0)],
(1, 0, 'augmented-student'):
[tf.expand_dims(flow_bw, axis=0)]
},
weights=weights,
occlusion_estimation_fn=occlusion_estimation_fn)
loss_dict['total_loss'] = sum(loss_dict.values())
else:
loss_dict = {}
unsuper_losses.append(loss_dict)
epe_occ.append(tf.reduce_mean(input_tensor=endpoint_error_occ))
errors_occ.append(tf.reduce_mean(input_tensor=outliers_occ))
if plot_dir and plot_count < num_plots:
plot_count += 1
mask_thresh = tf.cast(
tf.math.greater(soft_occlusion_mask, best_thresh), tf.float32)
smurf_plotting.complete_paper_plot(
plot_dir,
plot_count,
image_batch[0].numpy(),
image_batch[1].numpy(),
final_flow.numpy(),
flow_gt.numpy(),
np.ones_like(mask_thresh.numpy()),
1. - mask_thresh.numpy(),
1. - occ_mask_gt.numpy().astype('float32'),
frame_skip=None)
if progress_bar:
sys.stdout.write('\n')
sys.stdout.flush()
fmax, best_thresh = get_fmax_and_best_thresh(all_occlusion_results)
eval_stop_in_s = time.time()
results = {
'occl-f-max': fmax,
'best-occl-thresh': best_thresh,
'EPE': np.mean(np.array(epe_occ)),
'ER': np.mean(np.array(errors_occ)),
'inf-time(ms)': np.mean(inference_times),
'eval-time(s)': eval_stop_in_s - eval_start_in_s,
}
for k in unsuper_losses[0].keys():
results.update({k: np.mean([l[k] for l in unsuper_losses])})
if prefix:
return {prefix + '-' + k: v for k, v in results.items()}
return results
def evaluate(inference_fn,
dataset,
height,
width,
progress_bar=False,
plot_dir='',
num_plots=0,
prefix='kitti'):
"""Evaluate an iference function for flow with a kitti eval dataset.
Args:
inference_fn: An inference function that produces a flow_field from two
images, e.g. the infer method of SMURF.
dataset: A dataset produced by the method above with for_eval=True.
height: int, the height to which the images should be resized for inference.
width: int, the width to which the images should be resized for inference.
progress_bar: boolean, flag to indicate whether the function should print a
progress_bar during evaluaton.
plot_dir: string, optional path to a directory in which plots are saved
(if num_plots > 0).
num_plots: int, maximum number of qualitative results to plot for the
evaluation.
Returns:
A dictionary of floats that represent different evaluation metrics. The keys
of this dictionary are returned by the method list_eval_keys (see below).
"""
eval_start_in_s = time.time()
it = tf.compat.v1.data.make_one_shot_iterator(dataset)
epe_occ = [] # End point errors.
errors_occ = []
valid_occ = []
epe_noc = [] # End point errors.
errors_noc = []
valid_noc = []
inference_times = []
all_occlusion_results = defaultdict(lambda: defaultdict(int))
for i, test_batch in enumerate(it):
if progress_bar:
sys.stdout.write(':')
sys.stdout.flush()
image_batch = test_batch['images']
flow_uv_occ = test_batch['flow_uv_occ']
flow_uv_noc = test_batch['flow_uv_noc']
flow_valid_occ = test_batch['flow_valid_occ']
flow_valid_noc = test_batch['flow_valid_noc']
flow_valid_occ = tf.cast(flow_valid_occ, 'float32')
flow_valid_noc = tf.cast(flow_valid_noc, 'float32')
f = lambda: inference_fn(image_batch[0],
image_batch[1],
input_height=height,
input_width=width,
infer_occlusion=True)
inference_time_in_ms, (flow,
soft_occlusion_mask) = smurf_utils.time_it(
f, execute_once_before=i == 0)
inference_times.append(inference_time_in_ms)
occ_mask_gt = flow_valid_occ - flow_valid_noc
f_dict = data_utils.compute_f_metrics(
soft_occlusion_mask * flow_valid_occ, occ_mask_gt * flow_valid_occ)
best_thresh = -1.
best_f_score = -1.
for thresh, metrics in f_dict.items():
precision = metrics['tp'] / (metrics['tp'] + metrics['fp'] + 1e-6)
recall = metrics['tp'] / (metrics['tp'] + metrics['fn'] + 1e-6)
f1 = 2 * precision * recall / (precision + recall + 1e-6)
if f1 > best_f_score:
best_thresh = thresh
best_f_score = f1
all_occlusion_results[thresh]['tp'] += metrics['tp']
all_occlusion_results[thresh]['fp'] += metrics['fp']
all_occlusion_results[thresh]['tn'] += metrics['tn']
all_occlusion_results[thresh]['fn'] += metrics['fn']
mask_thresh = tf.cast(
tf.math.greater(soft_occlusion_mask, best_thresh), tf.float32)
# Image coordinates are swapped in labels
final_flow = flow[Ellipsis, ::-1]
endpoint_error_occ = data_utils.epe_elementwise(
final_flow, flow_uv_occ)
outliers_occ = data_utils.outliers_elementwise(final_flow, flow_uv_occ)
endpoint_error_noc = data_utils.epe_elementwise(
final_flow, flow_uv_noc)
outliers_noc = data_utils.outliers_elementwise(final_flow, flow_uv_noc)
epe_occ.append(
tf.reduce_sum(input_tensor=flow_valid_occ * endpoint_error_occ))
errors_occ.append(
tf.reduce_sum(input_tensor=flow_valid_occ * outliers_occ))
valid_occ.append(tf.reduce_sum(input_tensor=flow_valid_occ))
epe_noc.append(
tf.reduce_sum(input_tensor=flow_valid_noc * endpoint_error_noc))
errors_noc.append(
tf.reduce_sum(input_tensor=flow_valid_noc * outliers_noc))
valid_noc.append(tf.reduce_sum(input_tensor=flow_valid_noc))
if plot_dir and i < num_plots:
smurf_plotting.complete_paper_plot(plot_dir,
i,
image_batch[0].numpy(),
image_batch[1].numpy(),
final_flow.numpy(),
flow_uv_occ.numpy(),
flow_valid_occ.numpy(),
(1. - mask_thresh).numpy(),
(1. - occ_mask_gt).numpy(),
frame_skip=None)
if progress_bar:
sys.stdout.write('\n')
sys.stdout.flush()
fmax, best_thresh = data_utils.get_fmax_and_best_thresh(
all_occlusion_results)
eval_stop_in_s = time.time()
results = {
prefix + '-occl-f-max':
fmax,
prefix + '-best-occl-thresh':
best_thresh,
prefix + '-EPE(occ)':
np.clip(np.mean(np.array(epe_occ) / np.array(valid_occ)), 0.0, 50.0),
prefix + '-ER(occ)':
np.mean(np.array(errors_occ) / np.array(valid_occ)),
prefix + '-EPE(noc)':
np.clip(np.mean(np.array(epe_noc) / np.array(valid_noc)), 0.0, 50.0),
prefix + '-ER(noc)':
np.mean(np.array(errors_noc) / np.array(valid_noc)),
prefix + '-inf-time(ms)':
np.mean(inference_times),
prefix + '-eval-time(s)':
eval_stop_in_s - eval_start_in_s,
}
return results