def eval_ap(outputs,
global_step,
output_dir,
metric_summary,
N,
num_classes=4):
"""Processes the collected outputs to compute AP for action prediction.
Args:
outputs : [logits, labels]
global_step : global_step.
output_dir : where to store results.
metric_summary : summary object to add summaries to.
N : number of outputs to process.
num_classes : number of classes to compute AP over, and to reshape tensors.
"""
if N >= 0:
outputs = outputs[:N]
logits = np.concatenate(map(lambda x: x[0], outputs), axis=0).reshape(
(-1, num_classes))
labels = np.concatenate(map(lambda x: x[1], outputs), axis=0).reshape(
(-1, num_classes))
aps = []
for i in range(logits.shape[1]):
ap, rec, prec = utils.calc_pr(labels[:, i], logits[:, i])
ap = ap[0]
tf_utils.add_value_to_summary(metric_summary,
'aps/ap_{:d}: '.format(i), ap)
aps.append(ap)
return aps
def eval_ap(outputs, global_step, output_dir, metric_summary, N, num_classes=4):
"""Processes the collected outputs to compute AP for action prediction.
Args:
outputs : [logits, labels]
global_step : global_step.
output_dir : where to store results.
metric_summary : summary object to add summaries to.
N : number of outputs to process.
num_classes : number of classes to compute AP over, and to reshape tensors.
"""
if N >= 0:
outputs = outputs[:N]
logits = np.concatenate(map(lambda x: x[0], outputs), axis=0).reshape((-1, num_classes))
labels = np.concatenate(map(lambda x: x[1], outputs), axis=0).reshape((-1, num_classes))
aps = []
for i in range(logits.shape[1]):
ap, rec, prec = utils.calc_pr(labels[:,i], logits[:,i])
ap = ap[0]
tf_utils.add_value_to_summary(metric_summary, 'aps/ap_{:d}: '.format(i), ap)
aps.append(ap)
return aps
def eval_dist(outputs, global_step, output_dir, metric_summary, N):
"""Processes the collected outputs during validation to
1. Plot the distance over time curve.
2. Compute mean and median distances.
3. Plots histogram of end distances.
Args:
outputs : [locs, goal_loc, gt_dist_to_goal].
global_step : global_step.
output_dir : where to store results.
metric_summary : summary object to add summaries to.
N : number of outputs to process.
"""
SUCCESS_THRESH = 3
if N >= 0:
outputs = outputs[:N]
# Plot distance at time t.
d_at_t = []
for i in range(len(outputs)):
locs, goal_loc, gt_dist_to_goal = outputs[i]
d_at_t.append(gt_dist_to_goal[:, :, 0] * 1)
# Plot the distance.
fig, axes = utils.subplot(plt, (1, 1), (5, 5))
d_at_t = np.concatenate(d_at_t, axis=0)
axes.plot(np.arange(d_at_t.shape[1]), np.mean(d_at_t, axis=0), 'r.')
axes.set_xlabel('time step')
axes.set_ylabel('dist to next goal')
axes.grid('on')
file_name = os.path.join(output_dir,
'dist_at_t_{:d}.png'.format(global_step))
with fu.fopen(file_name, 'w') as f:
fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
file_name = os.path.join(output_dir,
'dist_at_t_{:d}.pkl'.format(global_step))
utils.save_variables(file_name, [d_at_t], ['d_at_t'], overwrite=True)
plt.close(fig)
# Plot the trajectories and the init_distance and final distance.
d_inits = []
d_ends = []
for i in range(len(outputs)):
locs, goal_loc, gt_dist_to_goal = outputs[i]
d_inits.append(gt_dist_to_goal[:, 0, 0] * 1)
d_ends.append(gt_dist_to_goal[:, -1, 0] * 1)
# Plot the distance.
fig, axes = utils.subplot(plt, (1, 1), (5, 5))
d_inits = np.concatenate(d_inits, axis=0)
d_ends = np.concatenate(d_ends, axis=0)
axes.plot(d_inits + np.random.rand(*(d_inits.shape)) - 0.5,
d_ends + np.random.rand(*(d_ends.shape)) - 0.5,
'.',
mec='red',
mew=1.0)
axes.set_xlabel('init dist')
axes.set_ylabel('final dist')
axes.grid('on')
axes.axis('equal')
title_str = 'mean: {:0.1f}, 50: {:0.1f}, 75: {:0.2f}, s: {:0.1f}'
title_str = title_str.format(np.mean(d_ends), np.median(d_ends),
np.percentile(d_ends, q=75),
100 * (np.mean(d_ends <= SUCCESS_THRESH)))
axes.set_title(title_str)
file_name = os.path.join(output_dir, 'dist_{:d}.png'.format(global_step))
with fu.fopen(file_name, 'w') as f:
fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
file_name = os.path.join(output_dir, 'dist_{:d}.pkl'.format(global_step))
utils.save_variables(file_name, [d_inits, d_ends], ['d_inits', 'd_ends'],
overwrite=True)
plt.close(fig)
# Plot the histogram of the end_distance.
with plt.style.context('seaborn-white'):
d_ends_ = np.sort(d_ends)
d_inits_ = np.sort(d_inits)
leg = []
fig, ax = utils.subplot(plt, (1, 1), (5, 5))
ax.grid('on')
ax.set_xlabel('Distance from goal')
ax.xaxis.label.set_fontsize(16)
ax.set_ylabel('Fraction of data')
ax.yaxis.label.set_fontsize(16)
ax.plot(d_ends_, np.arange(d_ends_.size) * 1. / d_ends_.size, 'r')
ax.plot(d_inits_, np.arange(d_inits_.size) * 1. / d_inits_.size, 'k')
leg.append('Final')
leg.append('Init')
ax.legend(leg, fontsize='x-large')
ax.set_axis_on()
title_str = 'mean: {:0.1f}, 50: {:0.1f}, 75: {:0.2f}, s: {:0.1f}'
title_str = title_str.format(np.mean(d_ends), np.median(d_ends),
np.percentile(d_ends, q=75),
100 * (np.mean(d_ends <= SUCCESS_THRESH)))
ax.set_title(title_str)
file_name = os.path.join(output_dir,
'dist_hist_{:d}.png'.format(global_step))
with fu.fopen(file_name, 'w') as f:
fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
# Log distance metrics.
tf_utils.add_value_to_summary(metric_summary, 'dists/success_init: ',
100 * (np.mean(d_inits <= SUCCESS_THRESH)))
tf_utils.add_value_to_summary(metric_summary, 'dists/success_end: ',
100 * (np.mean(d_ends <= SUCCESS_THRESH)))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (75): ',
np.percentile(d_inits, q=75))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (75): ',
np.percentile(d_ends, q=75))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (median): ',
np.median(d_inits))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (median): ',
np.median(d_ends))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (mean): ',
np.mean(d_inits))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (mean): ',
np.mean(d_ends))
return np.median(d_inits), np.median(d_ends), np.mean(d_inits), np.mean(d_ends), \
np.percentile(d_inits, q=75), np.percentile(d_ends, q=75), \
100*(np.mean(d_inits) <= SUCCESS_THRESH), 100*(np.mean(d_ends) <= SUCCESS_THRESH)
def eval_dist(outputs, global_step, output_dir, metric_summary, N):
"""Processes the collected outputs during validation to
1. Plot the distance over time curve.
2. Compute mean and median distances.
3. Plots histogram of end distances.
Args:
outputs : [locs, goal_loc, gt_dist_to_goal].
global_step : global_step.
output_dir : where to store results.
metric_summary : summary object to add summaries to.
N : number of outputs to process.
"""
SUCCESS_THRESH = 3
if N >= 0:
outputs = outputs[:N]
# Plot distance at time t.
d_at_t = []
for i in range(len(outputs)):
locs, goal_loc, gt_dist_to_goal = outputs[i]
d_at_t.append(gt_dist_to_goal[:,:,0]*1)
# Plot the distance.
fig, axes = utils.subplot(plt, (1,1), (5,5))
d_at_t = np.concatenate(d_at_t, axis=0)
axes.plot(np.arange(d_at_t.shape[1]), np.mean(d_at_t, axis=0), 'r.')
axes.set_xlabel('time step')
axes.set_ylabel('dist to next goal')
axes.grid('on')
file_name = os.path.join(output_dir, 'dist_at_t_{:d}.png'.format(global_step))
with fu.fopen(file_name, 'w') as f:
fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
file_name = os.path.join(output_dir, 'dist_at_t_{:d}.pkl'.format(global_step))
utils.save_variables(file_name, [d_at_t], ['d_at_t'], overwrite=True)
plt.close(fig)
# Plot the trajectories and the init_distance and final distance.
d_inits = []
d_ends = []
for i in range(len(outputs)):
locs, goal_loc, gt_dist_to_goal = outputs[i]
d_inits.append(gt_dist_to_goal[:,0,0]*1)
d_ends.append(gt_dist_to_goal[:,-1,0]*1)
# Plot the distance.
fig, axes = utils.subplot(plt, (1,1), (5,5))
d_inits = np.concatenate(d_inits, axis=0)
d_ends = np.concatenate(d_ends, axis=0)
axes.plot(d_inits+np.random.rand(*(d_inits.shape))-0.5,
d_ends+np.random.rand(*(d_ends.shape))-0.5, '.', mec='red', mew=1.0)
axes.set_xlabel('init dist'); axes.set_ylabel('final dist');
axes.grid('on'); axes.axis('equal');
title_str = 'mean: {:0.1f}, 50: {:0.1f}, 75: {:0.2f}, s: {:0.1f}'
title_str = title_str.format(
np.mean(d_ends), np.median(d_ends), np.percentile(d_ends, q=75),
100*(np.mean(d_ends <= SUCCESS_THRESH)))
axes.set_title(title_str)
file_name = os.path.join(output_dir, 'dist_{:d}.png'.format(global_step))
with fu.fopen(file_name, 'w') as f:
fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
file_name = os.path.join(output_dir, 'dist_{:d}.pkl'.format(global_step))
utils.save_variables(file_name, [d_inits, d_ends], ['d_inits', 'd_ends'],
overwrite=True)
plt.close(fig)
# Plot the histogram of the end_distance.
with plt.style.context('seaborn-white'):
d_ends_ = np.sort(d_ends)
d_inits_ = np.sort(d_inits)
leg = [];
fig, ax = utils.subplot(plt, (1,1), (5,5))
ax.grid('on')
ax.set_xlabel('Distance from goal'); ax.xaxis.label.set_fontsize(16);
ax.set_ylabel('Fraction of data'); ax.yaxis.label.set_fontsize(16);
ax.plot(d_ends_, np.arange(d_ends_.size)*1./d_ends_.size, 'r')
ax.plot(d_inits_, np.arange(d_inits_.size)*1./d_inits_.size, 'k')
leg.append('Final'); leg.append('Init');
ax.legend(leg, fontsize='x-large');
ax.set_axis_on()
title_str = 'mean: {:0.1f}, 50: {:0.1f}, 75: {:0.2f}, s: {:0.1f}'
title_str = title_str.format(
np.mean(d_ends), np.median(d_ends), np.percentile(d_ends, q=75),
100*(np.mean(d_ends <= SUCCESS_THRESH)))
ax.set_title(title_str)
file_name = os.path.join(output_dir, 'dist_hist_{:d}.png'.format(global_step))
with fu.fopen(file_name, 'w') as f:
fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
# Log distance metrics.
tf_utils.add_value_to_summary(metric_summary, 'dists/success_init: ',
100*(np.mean(d_inits <= SUCCESS_THRESH)))
tf_utils.add_value_to_summary(metric_summary, 'dists/success_end: ',
100*(np.mean(d_ends <= SUCCESS_THRESH)))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (75): ',
np.percentile(d_inits, q=75))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (75): ',
np.percentile(d_ends, q=75))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (median): ',
np.median(d_inits))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (median): ',
np.median(d_ends))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (mean): ',
np.mean(d_inits))
tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (mean): ',
np.mean(d_ends))
return np.median(d_inits), np.median(d_ends), np.mean(d_inits), np.mean(d_ends), \
np.percentile(d_inits, q=75), np.percentile(d_ends, q=75), \
100*(np.mean(d_inits) <= SUCCESS_THRESH), 100*(np.mean(d_ends) <= SUCCESS_THRESH)