def GenerateSummaries(self, name):
"""Generate an image summary for max recall by number of points by class."""
image_summaries = self._GenerateCumulativeSummaries(name)
p = self.params
legend_names = []
for j in p.metadata.EvalClassIndices():
legend_names.append(p.metadata.ClassNames()[j])
num_points_bins = self._values.shape[0]
ys = np.zeros(shape=(num_points_bins,
len(p.metadata.EvalClassIndices())),
dtype=np.float32)
for num_points in self._precision_recall:
ys[num_points, :] = _FindMaximumRecall(
self._precision_recall[num_points])
def _Setter(fig, axes):
"""Configure the plot for max recall versus number of points."""
axes.grid(b=True)
fontsize = 14
for i, j in enumerate(p.metadata.EvalClassIndices()):
for n, x in enumerate(self._values):
h = self._histogram[n][j]
y = min(ys[n][i] + 0.03, 1.0)
if h > 0:
text_label = '{} {}s'.format(
h, legend_names[i].lower()[:3])
axes.text(x,
y,
text_label,
fontdict={'fontsize': fontsize - 2})
axes.set_xlabel('number of points', fontsize=fontsize)
axes.set_xticks(self._values)
axes.set_ylabel('maximum recall', fontsize=fontsize)
axes.set_ylim([-0.01, 1.05])
axes.set_xlim([(1.0 / 1.3) * self._values[0],
1.3 * self._values[-1]])
axes.set_yticks(np.arange(0.0, 1.05, 0.1))
axes.set_xscale('log')
axes.legend([name.lower() for name in legend_names],
numpoints=1,
loc='upper left')
fig.tight_layout()
tag_str = '{}/recall_by_num_points'.format(name)
image_summary = plot.Curve(name=tag_str,
figsize=(10, 8),
xs=self._values,
ys=ys,
setter=_Setter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
image_summaries.append(image_summary)
return image_summaries
def GenerateSummaries(self, name):
"""Generate an image summary for AP versus rotation by class."""
p = self.params
legend_names = []
for j in p.metadata.EvalClassIndices():
legend_names.append(p.metadata.ClassNames()[j])
num_rotations = self._values.shape[0]
rotation_in_degrees = self._values * 180.0 / np.pi
ys = np.zeros(shape=(num_rotations,
len(p.metadata.EvalClassIndices())),
dtype=np.float32)
for rotation in self._precision_recall:
ys[rotation, :] = _FindMaximumRecall(
self._precision_recall[rotation])
def _Setter(fig, axes):
"""Configure the plot for max recall versus distance."""
axes.grid(b=True)
fontsize = 14
for i, j in enumerate(p.metadata.EvalClassIndices()):
for r, x in enumerate(rotation_in_degrees):
h = self._histogram[r][j]
y = min(ys[r][i] + 0.03, 1.0)
text_label = '{} {}s'.format(h,
legend_names[i].lower()[:3])
axes.text(x,
y,
text_label,
fontdict={'fontsize': fontsize - 2})
axes.set_xlabel('rotation (degrees)', fontsize=fontsize)
bin_width = (p.metadata.MaximumRotation() /
float(self.NumBinsOfHistogram()))
axes.set_xticks(
np.arange(0.0,
p.metadata.MaximumRotation() + bin_width, bin_width)
* 180.0 / np.pi)
axes.set_ylabel('maximum recall', fontsize=fontsize)
axes.set_ylim([-0.01, 1.05])
axes.set_yticks(np.arange(0.0, 1.05, 0.1))
axes.set_xlim([0.0, 180.0])
axes.legend([name.lower() for name in legend_names],
numpoints=1,
loc='upper right')
fig.tight_layout()
tag_str = '{}/recall_by_rotation'.format(name)
image_summary = plot.Curve(name=tag_str,
figsize=(10, 8),
xs=rotation_in_degrees,
ys=ys,
setter=_Setter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
return [image_summary]
def GenerateSummaries(self, name):
"""Generate an image summary for AP versus distance by class."""
num_distances = self._values.shape[0]
p = self.params
ys = np.zeros(shape=(num_distances,
len(p.metadata.EvalClassIndices())),
dtype=np.float32)
legend_names = []
for i, j in enumerate(p.metadata.EvalClassIndices()):
legend_names.append(p.metadata.ClassNames()[j])
for distance in self._average_precisions:
v = self._average_precisions[distance][i]
if np.isnan(v):
v = 0.0
ys[distance, i] = v
def _Setter(fig, axes):
"""Configure the plot for mAP versus distance."""
axes.grid(b=False)
fontsize = 14
for i, j in enumerate(p.metadata.EvalClassIndices()):
for d, x in enumerate(self._values):
h = self._histogram[d][j]
y = min(ys[d][i] + 0.03, 1.0)
if h > 0:
text_label = '{} {}s'.format(
h, legend_names[i].lower()[:3])
axes.text(x,
y,
text_label,
fontdict={'fontsize': fontsize - 2})
axes.set_xlabel('distance (world coordinates)', fontsize=fontsize)
axes.set_xticks(
np.arange(
0.0,
p.metadata.MaximumDistance() +
p.metadata.DistanceBinWidth(),
p.metadata.DistanceBinWidth()))
axes.set_ylabel('average precision (AP)', fontsize=fontsize)
axes.set_ylim([-0.02, 1.05])
axes.set_yticks(np.arange(0.0, 1.05, 0.1))
axes.legend([name.lower() for name in legend_names],
numpoints=1,
loc=3)
fig.tight_layout()
tag_str = '{}/AP_by_distance'.format(name)
image_summary = plot.Curve(name=tag_str,
figsize=(10, 8),
xs=self._values,
ys=ys,
setter=_Setter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
return [image_summary]
def Summary(self, name):
"""Generate tf summaries for calibration.
Args:
name: str, name of summary.
Returns:
list of tf.Summary
"""
summaries = []
for class_id in self._metadata.EvalClassIndices():
classname = self._metadata.ClassNames()[class_id]
tag_str = '{}/{}/calibration'.format(name, classname)
if classname not in self._calibration_by_class:
continue
# Extract the data.
mean_predicted_accuracy = self._calibration_by_class[classname][
0, :]
mean_empirical_accuracy = self._calibration_by_class[classname][
1, :]
num_examples_per_bin = self._calibration_by_class[classname][-1, :]
total_examples = np.sum(num_examples_per_bin)
legend = ['%s (%d)' % (classname, total_examples)]
def _CalibrationSetter(fig, axes):
"""Configure the plot for calibration."""
ticks = np.arange(0, 1.05, 0.1)
axes.grid(b=False)
axes.set_xlabel('Predicted accuracy')
axes.set_xticks(ticks)
axes.set_ylabel('Empirical accuracy')
axes.set_yticks(ticks)
axes.legend(legend, numpoints=1) # pylint: disable=cell-var-from-loop
fig.tight_layout()
calibration_curve_summary = plot.Curve(name=tag_str,
figsize=(10, 8),
xs=mean_predicted_accuracy,
ys=mean_empirical_accuracy,
setter=_CalibrationSetter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
ece = ExpectedCalibrationError(mean_predicted_accuracy,
mean_empirical_accuracy,
num_examples_per_bin)
ece_summary = tf.Summary(value=[
tf.Summary.Value(tag='{}/{}/calibration_ece'.format(
name, classname),
simple_value=ece)
])
summaries.extend([calibration_curve_summary, ece_summary])
return summaries
def GenerateSummaries(self, name):
"""Generate an image summary for PR by difficulty and for calibration.
Args:
name: str, name of summary.
Returns:
list of summaries
"""
legend = {}
p = self.params
for class_id in p.metadata.EvalClassIndices():
legend[class_id] = []
for difficulty, i in p.metadata.DifficultyLevels().items():
num_objects = self._histogram[i][class_id]
legend[class_id].append('%s (%d)' % (difficulty, num_objects))
image_summaries = []
for i, j in enumerate(p.metadata.EvalClassIndices()):
def _PRSetter(fig, axes):
"""Configure the plot for precision recall."""
ticks = np.arange(0, 1.05, 0.1)
axes.grid(b=False)
axes.set_xlabel('Recall')
axes.set_xticks(ticks)
axes.set_ylabel('Precision')
axes.set_yticks(ticks)
axes.legend(legend[j], numpoints=1) # pylint: disable=cell-var-from-loop
fig.tight_layout()
classname = p.metadata.ClassNames()[j]
# Generate Precision-Recall curves.
rs = []
ps = []
for difficulty in p.metadata.DifficultyLevels():
ps += [self._precision_recall[difficulty][i][:, 0]]
rs += [self._precision_recall[difficulty][i][:, 1]]
tag_str = '{}/{}/PR'.format(name, classname)
image_summary = plot.Curve(
name=tag_str,
figsize=(10, 8),
xs=rs[0],
ys=np.array(ps).T,
setter=_PRSetter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
image_summaries.append(image_summary)
for difficulty, c in self._calibration.items():
calibration_summaries = c.Summary(difficulty)
for calibration_summary in calibration_summaries:
image_summaries.append(calibration_summary)
return image_summaries
def _GenerateCumulativeSummaries(self, name):
"""Generate an image summary for CDF of a variate."""
xs = []
ys = []
num_zeros = []
legend_names = []
min_value = 5.0
p = self.params
for i, j in enumerate(p.metadata.EvalClassIndices()):
legend_names.append(p.metadata.ClassNames()[j])
if len(self._cumulative_distribution[j]) > min_value:
self._cumulative_distribution[j].sort()
x = np.array(self._cumulative_distribution[j])
nonzeros = np.flatnonzero(x)
cdf = np.arange(x.size).astype(np.float) / x.size
xs.append(x)
ys.append(cdf)
num_zeros.append(x.size - nonzeros.size)
else:
xs.append(None)
ys.append(None)
num_zeros.append(None)
image_summaries = []
for i, j in enumerate(p.metadata.EvalClassIndices()):
classname = p.metadata.ClassNames()[j]
def _Setter(fig, axes):
"""Configure the plot for CDF of the variate."""
axes.grid(b=False)
fontsize = 14
axes.set_ylim([0, 1.05])
axes.set_xlim([1.0, 11500])
axes.set_ylabel('cumulative distribution', fontsize=fontsize)
axes.set_xlabel('number of points', fontsize=fontsize)
axes.set_xscale('log')
legend_text = '{} {}s ({} contain zero points)'.format(
xs[i].size,
p.metadata.ClassNames()[j].lower(), num_zeros[i])
axes.legend({legend_text}, loc='upper left')
fig.tight_layout()
if xs[i] is not None:
tag_str = '{}/{}/cdf_of_num_points'.format(name, classname)
image_summary = plot.Curve(
name=tag_str,
figsize=(10, 8),
xs=xs[i],
ys=ys[i],
setter=_Setter,
marker='',
linestyle='-',
linewidth=2,
alpha=0.5)
image_summaries.append(image_summary)
return image_summaries
def GenerateSummaries(self, name):
"""Generate an image summary for precision recall by difficulty."""
p = self.params
image_summaries = []
for i, class_index in enumerate(p.metadata.EvalClassIndices()):
def _Setter(fig, axes):
"""Configure the plot for precision recall."""
ticks = np.arange(0, 1.05, 0.1)
axes.grid(b=False)
axes.set_xlabel('Recall')
axes.set_xticks(ticks)
axes.set_ylabel('Precision')
axes.set_yticks(ticks)
# TODO(vrv): Add legend indicating number of objects in breakdown.
fig.tight_layout()
classname = p.metadata.ClassNames()[class_index]
for breakdown_name in p.breakdown_list:
# 'ONE_SHARD' breakdowns are the overall metrics (not sliced up)
# So we should never skip this.
if 'ONE_SHARD' in breakdown_name:
breakdown_str = breakdown_name.replace('ONE_SHARD_', '')
tag_str = '{}/{}/{}/PR'.format(name, classname,
breakdown_str)
# Otherwise check that the class we are looking at is in the breakdown.
elif classname.lower() in breakdown_name.lower():
tag_str = '{}/{}/{}/PR'.format(name, classname,
breakdown_name)
else:
continue
ps = [self._precision_recall[breakdown_name][i][:, 0]]
rs = [self._precision_recall[breakdown_name][i][:, 1]]
image_summary = plot.Curve(name=tag_str,
figsize=(10, 8),
xs=rs[0],
ys=np.array(ps).T,
setter=_Setter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
image_summaries.append(image_summary)
return image_summaries
def GenerateSummaries(self, name):
"""Generate an image summary for precision recall by difficulty."""
p = self.params
image_summaries = []
for i, j in enumerate(p.metadata.EvalClassIndices()):
def _Setter(fig, axes):
"""Configure the plot for precision recall."""
ticks = np.arange(0, 1.05, 0.1)
axes.grid(b=False)
axes.set_xlabel('Recall')
axes.set_xticks(ticks)
axes.set_ylabel('Precision')
axes.set_yticks(ticks)
# TODO(vrv): Add legend indicating number of objects in breakdown.
fig.tight_layout()
classname = p.metadata.ClassNames()[j]
for k, breakdown_str in enumerate(p.breakdown_list):
# Skip adding entries for breakdowns that are in a different class.
#
# The first breakdown is the overall one, so never skip it.
if k > 0 and classname.lower() not in breakdown_str.lower():
continue
ps = [self._precision_recall[breakdown_str][i][:, 0]]
rs = [self._precision_recall[breakdown_str][i][:, 1]]
if k == 0:
# Overall class PR.
tag_str = '{}/{}/PR'.format(name, classname)
else:
tag_str = '{}/{}/{}/PR'.format(name, classname,
breakdown_str)
image_summary = plot.Curve(name=tag_str,
figsize=(10, 8),
xs=rs[0],
ys=np.array(ps).T,
setter=_Setter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
image_summaries.append(image_summary)
return image_summaries
def Summary(self, name):
def _Setter(fig, axes):
# 20 ticks betweein 0 and 1.
ticks = np.arange(0, 1.05, 0.05)
axes.grid(b=True)
axes.set_xlabel(self._plot_labels[0])
axes.set_xticks(ticks)
axes.set_ylabel(self._plot_labels[1])
axes.set_yticks(ticks)
fig.tight_layout()
xs, ys, _ = self._curve_fn(
self._label, self._prob, sample_weight=self._weight)
if self._mode == 'pr':
# Swap because sklearn returns <'precision', 'recall'>.
xs, ys = ys, xs
ret = plot.Curve(name=name, figsize=(12, 12), xs=xs, ys=ys, setter=_Setter)
ret.value.add(tag=name, simple_value=self.value)
return ret
def GenerateSummaries(self, name):
"""Generate an image summary for PR by difficulty and for calibration.
Args:
name: str, name of summary.
Returns:
list of summaries
"""
legend = {}
p = self.params
for class_id in p.metadata.EvalClassIndices():
legend[class_id] = []
for difficulty, i in p.metadata.DifficultyLevels().items():
num_objects = self._histogram[i][class_id]
legend[class_id].append('%s (%d)' % (difficulty, num_objects))
summaries = []
for i, j in enumerate(p.metadata.EvalClassIndices()):
def _PRSetter(fig, axes):
"""Configure the plot for precision recall."""
ticks = np.arange(0, 1.05, 0.1)
axes.grid(b=False)
axes.set_xlabel('Recall')
axes.set_xticks(ticks)
axes.set_ylabel('Precision')
axes.set_yticks(ticks)
# pylint: disable=undefined-loop-variable
axes.legend(legend[j], numpoints=1) # pylint: disable=cell-var-from-loop
# pylint: enable=undefined-loop-variable
fig.tight_layout()
classname = p.metadata.ClassNames()[j]
# Generate Precision-Recall curves.
rs = []
ps = []
for difficulty in p.metadata.DifficultyLevels():
ps += [self._precision_recall[difficulty][i][:, 0]]
rs += [self._precision_recall[difficulty][i][:, 1]]
tag_str = '{}/{}/PR'.format(name, classname)
image_summary = plot.Curve(name=tag_str,
figsize=(10, 8),
xs=rs[0],
ys=np.array(ps).T,
setter=_PRSetter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
summaries.append(image_summary)
for difficulty, c in self._calibration.items():
# Note that we only generate a calibration for a single difficulty level.
calibration_summaries = c.Summary(name)
for calibration_summary in calibration_summaries:
summaries.append(calibration_summary)
# Generate scalar summaries for the various recalls for each difficulty.
for difficulty in p.metadata.DifficultyLevels():
max_recall = _FindMaximumRecall(self._precision_recall[difficulty])
for i, j in enumerate(p.metadata.EvalClassIndices()):
classname = p.metadata.ClassNames()[j]
summary = tf.Summary(value=[
tf.Summary.Value(tag='{}/{}/max_recall_{}'.format(
name, classname, difficulty),
simple_value=max_recall[i])
])
summaries.append(summary)
for precision_level in p.metadata.RecallAtPrecision():
recall_at_precision = _FindRecallAtGivenPrecision(
self._precision_recall[difficulty], precision_level)
for i, j in enumerate(p.metadata.EvalClassIndices()):
classname = p.metadata.ClassNames()[j]
summary = tf.Summary(value=[
tf.Summary.Value(tag='{}/{}/recall_{}_{}'.format(
name, classname, int(precision_level *
100), difficulty),
simple_value=recall_at_precision[i])
])
summaries.append(summary)
return summaries
def GenerateSummaries(self, name):
"""Generate an image summary for AP versus rotation by class."""
num_rotations = self._values.shape[0]
p = self.params
rotation_in_degrees = self._values * 180.0 / np.pi
ys = np.zeros(
shape=(num_rotations, len(p.metadata.EvalClassIndices())),
dtype=np.float32)
legend_names = []
for i, j in enumerate(p.metadata.EvalClassIndices()):
legend_names.append(p.metadata.ClassNames()[j])
for rotation in self._average_precisions:
v = self._average_precisions[rotation][i]
if np.isnan(v):
v = 0.0
ys[rotation, i] = v
def _Setter(fig, axes):
"""Configure the plot for mAP versus distance."""
axes.grid(b=False)
fontsize = 14
for i, j in enumerate(p.metadata.EvalClassIndices()):
for r, x in enumerate(rotation_in_degrees):
h = self._histogram[r][j]
y = min(ys[r][i] + 0.03, 1.0)
# TODO(shlens): Only display car currently for visual clarity
# but relax this soon.
if h > 0 and p.metadata.ClassNames()[j] in ['Car', 'Vehicle']:
text_label = '{} {}s'.format(h, legend_names[i].lower()[:3])
axes.text(x, y, text_label, fontdict={'fontsize': fontsize - 2})
axes.set_xlabel('rotation (degrees)', fontsize=fontsize)
bin_width = (
p.metadata.MaximumRotation() / float(self.NumBinsOfHistogram()))
axes.set_xticks(
np.arange(0.0,
p.metadata.MaximumRotation() + bin_width, bin_width) *
180.0 / np.pi)
axes.set_ylabel('average precision (AP)', fontsize=fontsize)
axes.set_ylim([-0.02, 1.05])
axes.set_yticks(np.arange(0.0, 1.05, 0.1))
axes.set_xlim([0.0, 180.0])
axes.legend([name.lower() for name in legend_names],
numpoints=1,
loc='upper right')
fig.tight_layout()
tag_str = '{}/AP_by_rotation'.format(name)
image_summary = plot.Curve(
name=tag_str,
figsize=(10, 8),
xs=rotation_in_degrees,
ys=ys,
setter=_Setter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
return [image_summary]
def GenerateSummaries(self, name):
"""Generate an image summary for max recall by number of points by class."""
image_summaries = self._GenerateCumulativeSummaries(name)
p = self.params
num_points_bins = self._values.shape[0]
ys = np.zeros(
shape=(num_points_bins, len(p.metadata.EvalClassIndices())),
dtype=np.float32)
# The method for computing precision-recall inserts precision = 0.0
# when a particular recall value has not been achieved. The maximum
# recall value is therefore the highest recall value when the associated
# precision > 0.
valid_precisions = {}
for num_points in self._precision_recall:
valid_precisions[num_points] = (
self._precision_recall[num_points][:, :, 0] > 0.0)
legend_names = []
for i, j in enumerate(p.metadata.EvalClassIndices()):
legend_names.append(p.metadata.ClassNames()[j])
for num_points in self._precision_recall:
valid_precisions_indices = valid_precisions[num_points][i]
if not np.any(valid_precisions_indices):
v = 0.0
else:
v = np.max(self._precision_recall[num_points]
[i, valid_precisions_indices, 1])
ys[num_points, i] = v
def _Setter(fig, axes):
"""Configure the plot for max recall versus number of points."""
axes.grid(b=True)
fontsize = 14
for i, j in enumerate(p.metadata.EvalClassIndices()):
for n, x in enumerate(self._values):
h = self._histogram[n][j]
y = min(ys[n][i] + 0.03, 1.0)
if h > 0:
text_label = '{} {}s'.format(h, legend_names[i].lower()[:3])
axes.text(x, y, text_label, fontdict={'fontsize': fontsize - 2})
axes.set_xlabel('number of points', fontsize=fontsize)
axes.set_xticks(self._values)
axes.set_ylabel('maximum recall', fontsize=fontsize)
axes.set_ylim([-0.01, 1.05])
axes.set_xlim([(1.0 / 1.3) * self._values[0], 1.3 * self._values[-1]])
axes.set_yticks(np.arange(0.0, 1.05, 0.1))
axes.set_xscale('log')
axes.legend([name.lower() for name in legend_names],
numpoints=1,
loc='upper left')
fig.tight_layout()
tag_str = '{}/recall_by_num_points'.format(name)
image_summary = plot.Curve(
name=tag_str,
figsize=(10, 8),
xs=self._values,
ys=ys,
setter=_Setter,
marker='.',
markersize=14,
linestyle='-',
linewidth=2,
alpha=0.5)
image_summaries.append(image_summary)
return image_summaries