def test_serialize_tf_linspace_numpy(self):
# Should be subsumed by `test_serialize_numpy_scalars`; separate
# test because it's a common use case.
hparams = {
"f_default": tf.linspace(1.0, 2.0, 5).numpy()[0],
"f32": tf.cast(tf.linspace(1.0, 2.0, 5), tf.float32).numpy()[0],
"f64": tf.cast(tf.linspace(1.0, 2.0, 5), tf.float64).numpy()[0],
}
hp.hparams_pb(hparams)
def when_nonsingular():
bucket_width = range_ / tf.cast(bucket_count, tf.float64)
offsets = data - min_
bucket_indices = tf.cast(tf.floor(offsets / bucket_width),
dtype=tf.int32)
clamped_indices = tf.minimum(bucket_indices, bucket_count - 1)
one_hots = tf.one_hot(clamped_indices, depth=bucket_count)
bucket_counts = tf.cast(tf.reduce_sum(one_hots, axis=0),
dtype=tf.float64)
edges = tf.lin_space(min_, max_, bucket_count + 1)
left_edges = edges[:-1]
right_edges = edges[1:]
return tf.transpose(
tf.stack([left_edges, right_edges, bucket_counts]))
def when_singular():
center = min_
bucket_starts = tf.stack([center - 0.5])
bucket_ends = tf.stack([center + 0.5])
bucket_counts = tf.stack([tf.cast(tf.size(data), tf.float64)])
return tf.transpose(
tf.stack([bucket_starts, bucket_ends, bucket_counts]))
def op(name,
data,
display_name=None,
description=None,
collections=None):
"""Create a legacy scalar summary op.
Arguments:
name: A unique name for the generated summary node.
data: A real numeric rank-0 `Tensor`. Must have `dtype` castable
to `float32`.
display_name: Optional name for this summary in TensorBoard, as a
constant `str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
constant `str`. Markdown is supported. Defaults to empty.
collections: Optional list of graph collections keys. The new
summary op is added to these collections. Defaults to
`[Graph Keys.SUMMARIES]`.
Returns:
A TensorFlow summary op.
"""
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description)
with tf.name_scope(name):
with tf.control_dependencies([tf.assert_scalar(data)]):
return tf.summary.tensor_summary(name='scalar_summary',
tensor=tf.cast(data, tf.float32),
collections=collections,
summary_metadata=summary_metadata)
def scalar(name, tensor, tag=None, description=None, step=None):
"""Create a scalar summary op.
Arguments:
name: A name for the generated summary node.
tensor: A real numeric rank-0 `Tensor`. Must have `dtype` castable
to `float32`.
tag: Optional rank-0 string `Tensor` to identify this summary in
TensorBoard. Defaults to the generated name of this op.
description: Optional long-form description for this summary, as a
constant `str`. Markdown is supported. Defaults to empty.
step: Optional `int64` monotonic step variable, which defaults
to `tf.train.get_global_step`.
Returns:
A TensorFlow summary op.
"""
# TODO(nickfelt): make tag param work
summary_metadata = metadata.create_summary_metadata(
display_name=None, description=description)
with tf.name_scope(name, values=[tensor, tag, step]) as scope:
with tf.control_dependencies([tf.assert_scalar(tensor)]):
return tf.contrib.summary.generic(name=scope,
tensor=tf.cast(tensor, tf.float32),
metadata=summary_metadata,
step=step)
def test_new_style_audio(self):
audio = tf.reshape(tf.linspace(0.0, 100.0, 4 * 10 * 2), (4, 10, 2))
op = audio_summary.op('k488',
tf.cast(audio, tf.float32),
sample_rate=44100,
display_name='Piano Concerto No.23',
description='In **A major**.')
value = self._value_from_op(op)
assert value.HasField('tensor'), value
self._assert_noop(value)
def _buckets(data, bucket_count=None):
"""Create a TensorFlow op to group data into histogram buckets.
Arguments:
data: A `Tensor` of any shape. Must be castable to `float64`.
bucket_count: Optional positive `int` or scalar `int32` `Tensor`.
Returns:
A `Tensor` of shape `[k, 3]` and type `float64`. The `i`th row is
a triple `[left_edge, right_edge, count]` for a single bucket.
The value of `k` is either `bucket_count` or `1` or `0`.
"""
if bucket_count is None:
bucket_count = DEFAULT_BUCKET_COUNT
with tf.name_scope('buckets', values=[data, bucket_count]), \
tf.control_dependencies([tf.assert_scalar(bucket_count),
tf.assert_type(bucket_count, tf.int32)]):
data = tf.reshape(data, shape=[-1]) # flatten
data = tf.cast(data, tf.float64)
is_empty = tf.equal(tf.size(data), 0)
def when_empty():
return tf.constant([], shape=(0, 3), dtype=tf.float64)
def when_nonempty():
min_ = tf.reduce_min(data)
max_ = tf.reduce_max(data)
range_ = max_ - min_
is_singular = tf.equal(range_, 0)
def when_nonsingular():
bucket_width = range_ / tf.cast(bucket_count, tf.float64)
offsets = data - min_
bucket_indices = tf.cast(tf.floor(offsets / bucket_width),
dtype=tf.int32)
clamped_indices = tf.minimum(bucket_indices, bucket_count - 1)
one_hots = tf.one_hot(clamped_indices, depth=bucket_count)
bucket_counts = tf.cast(tf.reduce_sum(one_hots, axis=0),
dtype=tf.float64)
edges = tf.lin_space(min_, max_, bucket_count + 1)
left_edges = edges[:-1]
right_edges = edges[1:]
return tf.transpose(
tf.stack([left_edges, right_edges, bucket_counts]))
def when_singular():
center = min_
bucket_starts = tf.stack([center - 0.5])
bucket_ends = tf.stack([center + 0.5])
bucket_counts = tf.stack([tf.cast(tf.size(data), tf.float64)])
return tf.transpose(
tf.stack([bucket_starts, bucket_ends, bucket_counts]))
return tf.cond(is_singular, when_singular, when_nonsingular)
return tf.cond(is_empty, when_empty, when_nonempty)
def compute_and_check_summary_pb(self,
name,
images,
max_outputs=3,
images_tensor=None,
feed_dict=None):
"""Use both `op` and `pb` to get a summary, asserting equality.
Returns:
a `Summary` protocol buffer
"""
if images_tensor is None:
images_tensor = tf.cast(tf.constant(images), tf.uint8)
op = summary.op(name, images_tensor, max_outputs=max_outputs)
pb = summary.pb(name, images, max_outputs=max_outputs)
pb_via_op = self.pb_via_op(op, feed_dict=feed_dict)
self.assertProtoEquals(pb, pb_via_op)
return pb
def _create_tensor_summary(name,
true_positive_counts,
false_positive_counts,
true_negative_counts,
false_negative_counts,
precision,
recall,
num_thresholds=None,
display_name=None,
description=None,
collections=None):
"""A private helper method for generating a tensor summary.
We use a helper method instead of having `op` directly call `raw_data_op`
to prevent the scope of `raw_data_op` from being embedded within `op`.
Arguments are the same as for raw_data_op.
Returns:
A tensor summary that collects data for PR curves.
"""
# Store the number of thresholds within the summary metadata because
# that value is constant for all pr curve summaries with the same tag.
summary_metadata = metadata.create_summary_metadata(
display_name=display_name if display_name is not None else name,
description=description or '',
num_thresholds=num_thresholds)
# Store values within a tensor. We store them in the order:
# true positives, false positives, true negatives, false
# negatives, precision, and recall.
combined_data = tf.stack([
tf.cast(true_positive_counts, tf.float32),
tf.cast(false_positive_counts, tf.float32),
tf.cast(true_negative_counts, tf.float32),
tf.cast(false_negative_counts, tf.float32),
tf.cast(precision, tf.float32),
tf.cast(recall, tf.float32)
])
return tf.summary.tensor_summary(name='pr_curves',
tensor=combined_data,
collections=collections,
summary_metadata=summary_metadata)
def run_sobel(logdir, verbose=False):
"""Run a Sobel edge detection demonstration.
See the summary description for more details.
Arguments:
logdir: Directory into which to write event logs.
verbose: Boolean; whether to log any output.
"""
if verbose:
tf.logging.info('--- Starting run: sobel')
tf.reset_default_graph()
tf.set_random_seed(0)
image = get_image(verbose=verbose)
kernel_radius = tf.placeholder(shape=(), dtype=tf.int32)
with tf.name_scope('horizontal_kernel'):
kernel_side_length = kernel_radius * 2 + 1
# Drop off influence for pixels further away from the center.
weighting_kernel = (
1.0 - tf.abs(tf.linspace(-1.0, 1.0, num=kernel_side_length)))
differentiation_kernel = tf.linspace(-1.0, 1.0, num=kernel_side_length)
horizontal_kernel = tf.matmul(tf.expand_dims(weighting_kernel, 1),
tf.expand_dims(differentiation_kernel, 0))
with tf.name_scope('vertical_kernel'):
vertical_kernel = tf.transpose(horizontal_kernel)
float_image = tf.cast(image, tf.float32)
dx = convolve(float_image, horizontal_kernel, name='convolve_dx')
dy = convolve(float_image, vertical_kernel, name='convolve_dy')
gradient_magnitude = tf.norm([dx, dy], axis=0, name='gradient_magnitude')
with tf.name_scope('normalized_gradient'):
normalized_gradient = gradient_magnitude / tf.reduce_max(gradient_magnitude)
with tf.name_scope('output_image'):
output_image = tf.cast(255 * normalized_gradient, tf.uint8)
summ = image_summary.op(
'sobel', tf.stack([output_image]),
display_name='Sobel edge detection',
description=(u'Demonstration of [Sobel edge detection]. The step '
'parameter adjusts the radius of the kernel. '
'The kernel can be of arbitrary size, and considers '
u'nearby pixels with \u2113\u2082-linear falloff.\n\n'
# (that says ``$\ell_2$-linear falloff'')
'Edge detection is done on a per-channel basis, so '
'you can observe which edges are “mostly red '
'edges,” for instance.\n\n'
'For practical edge detection, a small kernel '
'(usually not more than more than *r*=2) is best.\n\n'
'[Sobel edge detection]: %s\n\n'
"%s"
% ('https://en.wikipedia.org/wiki/Sobel_operator',
IMAGE_CREDIT)))
with tf.Session() as sess:
sess.run(image.initializer)
writer = tf.summary.FileWriter(os.path.join(logdir, 'sobel'))
writer.add_graph(sess.graph)
for step in xrange(8):
if verbose:
tf.logging.info("--- sobel: step: %s" % step)
feed_dict = {kernel_radius: step}
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
s = sess.run(summ, feed_dict=feed_dict,
options=run_options, run_metadata=run_metadata)
writer.add_summary(s, global_step=step)
writer.add_run_metadata(run_metadata, 'step_%04d' % step)
writer.close()
def run_box_to_gaussian(logdir, verbose=False):
"""Run a box-blur-to-Gaussian-blur demonstration.
See the summary description for more details.
Arguments:
logdir: Directory into which to write event logs.
verbose: Boolean; whether to log any output.
"""
if verbose:
tf.logging.info('--- Starting run: box_to_gaussian')
tf.reset_default_graph()
tf.set_random_seed(0)
image = get_image(verbose=verbose)
blur_radius = tf.placeholder(shape=(), dtype=tf.int32)
with tf.name_scope('filter'):
blur_side_length = blur_radius * 2 + 1
pixel_filter = tf.ones((blur_side_length, blur_side_length))
pixel_filter = (pixel_filter
/ tf.cast(tf.size(pixel_filter), tf.float32)) # normalize
iterations = 4
images = [tf.cast(image, tf.float32) / 255.0]
for _ in xrange(iterations):
images.append(convolve(images[-1], pixel_filter))
with tf.name_scope('convert_to_uint8'):
images = tf.stack(
[tf.cast(255 * tf.clip_by_value(image_, 0.0, 1.0), tf.uint8)
for image_ in images])
summ = image_summary.op(
'box_to_gaussian', images, max_outputs=iterations,
display_name='Gaussian blur as a limit process of box blurs',
description=('Demonstration of forming a Gaussian blur by '
'composing box blurs, each of which can be expressed '
'as a 2D convolution.\n\n'
'A Gaussian blur is formed by convolving a Gaussian '
'kernel over an image. But a Gaussian kernel is '
'itself the limit of convolving a constant kernel '
'with itself many times. Thus, while applying '
'a box-filter convolution just once produces '
'results that are noticeably different from those '
'of a Gaussian blur, repeating the same convolution '
'just a few times causes the result to rapidly '
'converge to an actual Gaussian blur.\n\n'
'Here, the step value controls the blur radius, '
'and the image sample controls the number of times '
'that the convolution is applied (plus one). '
'So, when *sample*=1, the original image is shown; '
'*sample*=2 shows a box blur; and a hypothetical '
'*sample*=∞ would show a true Gaussian blur.\n\n'
'This is one ingredient in a recipe to compute very '
'fast Gaussian blurs. The other pieces require '
'special treatment for the box blurs themselves '
'(decomposition to dual one-dimensional box blurs, '
'each of which is computed with a sliding window); '
'we don’t perform those optimizations here.\n\n'
'[Here are some slides describing the full process.]'
'(%s)\n\n'
'%s'
% ('http://elynxsdk.free.fr/ext-docs/Blur/Fast_box_blur.pdf',
IMAGE_CREDIT)))
with tf.Session() as sess:
sess.run(image.initializer)
writer = tf.summary.FileWriter(os.path.join(logdir, 'box_to_gaussian'))
writer.add_graph(sess.graph)
for step in xrange(8):
if verbose:
tf.logging.info('--- box_to_gaussian: step: %s' % step)
feed_dict = {blur_radius: step}
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
s = sess.run(summ, feed_dict=feed_dict,
options=run_options, run_metadata=run_metadata)
writer.add_summary(s, global_step=step)
writer.add_run_metadata(run_metadata, 'step_%04d' % step)
writer.close()
def start_runs(logdir,
steps,
run_name,
thresholds,
mask_every_other_prediction=False):
"""Generate a PR curve with precision and recall evenly weighted.
Arguments:
logdir: The directory into which to store all the runs' data.
steps: The number of steps to run for.
run_name: The name of the run.
thresholds: The number of thresholds to use for PR curves.
mask_every_other_prediction: Whether to mask every other prediction by
alternating weights between 0 and 1.
"""
tf.reset_default_graph()
tf.set_random_seed(42)
# Create a normal distribution layer used to generate true color labels.
distribution = tf.distributions.Normal(loc=0., scale=142.)
# Sample the distribution to generate colors. Lets generate different numbers
# of each color. The first dimension is the count of examples.
# The calls to sample() are given fixed random seed values that are "magic"
# in that they correspond to the default seeds for those ops when the PR
# curve test (which depends on this code) was written. We've pinned these
# instead of continuing to use the defaults since the defaults are based on
# node IDs from the sequence of nodes added to the graph, which can silently
# change when this code or any TF op implementations it uses are modified.
# TODO(nickfelt): redo the PR curve test to avoid reliance on random seeds.
# Generate reds.
number_of_reds = 100
true_reds = tf.clip_by_value(
tf.concat([
255 - tf.abs(distribution.sample([number_of_reds, 1], seed=11)),
tf.abs(distribution.sample([number_of_reds, 2], seed=34))
],
axis=1), 0, 255)
# Generate greens.
number_of_greens = 200
true_greens = tf.clip_by_value(
tf.concat([
tf.abs(distribution.sample([number_of_greens, 1], seed=61)),
255 - tf.abs(distribution.sample([number_of_greens, 1], seed=82)),
tf.abs(distribution.sample([number_of_greens, 1], seed=105))
],
axis=1), 0, 255)
# Generate blues.
number_of_blues = 150
true_blues = tf.clip_by_value(
tf.concat([
tf.abs(distribution.sample([number_of_blues, 2], seed=132)),
255 - tf.abs(distribution.sample([number_of_blues, 1], seed=153))
],
axis=1), 0, 255)
# Assign each color a vector of 3 booleans based on its true label.
labels = tf.concat([
tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)),
tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)),
tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)),
],
axis=0)
# We introduce 3 normal distributions. They are used to predict whether a
# color falls under a certain class (based on distances from corners of the
# color triangle). The distributions vary per color. We have the distributions
# narrow over time.
initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)]
iteration = tf.placeholder(tf.int32, shape=[])
red_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[0] - iteration,
dtype=tf.float32))
green_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[1] - iteration,
dtype=tf.float32))
blue_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[2] - iteration,
dtype=tf.float32))
# Make predictions (assign 3 probabilities to each color based on each color's
# distance to each of the 3 corners). We seek double the area in the right
# tail of the normal distribution.
examples = tf.concat([true_reds, true_greens, true_blues], axis=0)
probabilities_colors_are_red = (1 - red_predictor.cdf(
tf.norm(examples - tf.constant([255., 0, 0]), axis=1))) * 2
probabilities_colors_are_green = (1 - green_predictor.cdf(
tf.norm(examples - tf.constant([0, 255., 0]), axis=1))) * 2
probabilities_colors_are_blue = (1 - blue_predictor.cdf(
tf.norm(examples - tf.constant([0, 0, 255.]), axis=1))) * 2
predictions = (probabilities_colors_are_red,
probabilities_colors_are_green,
probabilities_colors_are_blue)
# This is the crucial piece. We write data required for generating PR curves.
# We create 1 summary per class because we create 1 PR curve per class.
for i, color in enumerate(('red', 'green', 'blue')):
description = (
'The probabilities used to create this PR curve are '
'generated from a normal distribution. Its standard '
'deviation is initially %0.0f and decreases over time.' %
initial_standard_deviations[i])
weights = None
if mask_every_other_prediction:
# Assign a weight of 0 to every even-indexed prediction. Odd-indexed
# predictions are assigned a default weight of 1.
consecutive_indices = tf.reshape(tf.range(tf.size(predictions[i])),
tf.shape(predictions[i]))
weights = tf.cast(consecutive_indices % 2, dtype=tf.float32)
summary.op(name=color,
labels=labels[:, i],
predictions=predictions[i],
num_thresholds=thresholds,
weights=weights,
display_name='classifying %s' % color,
description=description)
merged_summary_op = tf.summary.merge_all()
events_directory = os.path.join(logdir, run_name)
sess = tf.Session()
writer = tf.summary.FileWriter(events_directory, sess.graph)
for step in xrange(steps):
feed_dict = {
iteration: step,
}
merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict)
writer.add_summary(merged_summary, step)
writer.close()
def op(name,
labels,
predictions,
num_thresholds=None,
weights=None,
display_name=None,
description=None,
collections=None):
"""Create a PR curve summary op for a single binary classifier.
Computes true/false positive/negative values for the given `predictions`
against the ground truth `labels`, against a list of evenly distributed
threshold values in `[0, 1]` of length `num_thresholds`.
Each number in `predictions`, a float in `[0, 1]`, is compared with its
corresponding boolean label in `labels`, and counts as a single tp/fp/tn/fn
value at each threshold. This is then multiplied with `weights` which can be
used to reweight certain values, or more commonly used for masking values.
Args:
name: A tag attached to the summary. Used by TensorBoard for organization.
labels: The ground truth values. A Tensor of `bool` values with arbitrary
shape.
predictions: A float32 `Tensor` whose values are in the range `[0, 1]`.
Dimensions must match those of `labels`.
num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to
compute PR metrics for. Should be `>= 2`. This value should be a
constant integer value, not a Tensor that stores an integer.
weights: Optional float32 `Tensor`. Individual counts are multiplied by this
value. This tensor must be either the same shape as or broadcastable to
the `labels` tensor.
display_name: Optional name for this summary in TensorBoard, as a
constant `str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
constant `str`. Markdown is supported. Defaults to empty.
collections: Optional list of graph collections keys. The new
summary op is added to these collections. Defaults to
`[Graph Keys.SUMMARIES]`.
Returns:
A summary operation for use in a TensorFlow graph. The float32 tensor
produced by the summary operation is of dimension (6, num_thresholds). The
first dimension (of length 6) is of the order: true positives,
false positives, true negatives, false negatives, precision, recall.
"""
if num_thresholds is None:
num_thresholds = _DEFAULT_NUM_THRESHOLDS
if weights is None:
weights = 1.0
dtype = predictions.dtype
with tf.name_scope(name, values=[labels, predictions, weights]):
tf.assert_type(labels, tf.bool)
# We cast to float to ensure we have 0.0 or 1.0.
f_labels = tf.cast(labels, dtype)
# Ensure predictions are all in range [0.0, 1.0].
predictions = tf.minimum(1.0, tf.maximum(0.0, predictions))
# Get weighted true/false labels.
true_labels = f_labels * weights
false_labels = (1.0 - f_labels) * weights
# Before we begin, flatten predictions.
predictions = tf.reshape(predictions, [-1])
# Shape the labels so they are broadcast-able for later multiplication.
true_labels = tf.reshape(true_labels, [-1, 1])
false_labels = tf.reshape(false_labels, [-1, 1])
# To compute TP/FP/TN/FN, we are measuring a binary classifier
# C(t) = (predictions >= t)
# at each threshold 't'. So we have
# TP(t) = sum( C(t) * true_labels )
# FP(t) = sum( C(t) * false_labels )
#
# But, computing C(t) requires computation for each t. To make it fast,
# observe that C(t) is a cumulative integral, and so if we have
# thresholds = [t_0, ..., t_{n-1}]; t_0 < ... < t_{n-1}
# where n = num_thresholds, and if we can compute the bucket function
# B(i) = Sum( (predictions == t), t_i <= t < t{i+1} )
# then we get
# C(t_i) = sum( B(j), j >= i )
# which is the reversed cumulative sum in tf.cumsum().
#
# We can compute B(i) efficiently by taking advantage of the fact that
# our thresholds are evenly distributed, in that
# width = 1.0 / (num_thresholds - 1)
# thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0]
# Given a prediction value p, we can map it to its bucket by
# bucket_index(p) = floor( p * (num_thresholds - 1) )
# so we can use tf.scatter_add() to update the buckets in one pass.
# Compute the bucket indices for each prediction value.
bucket_indices = tf.cast(tf.floor(predictions * (num_thresholds - 1)),
tf.int32)
# Bucket predictions.
tp_buckets = tf.reduce_sum(
tf.one_hot(bucket_indices, depth=num_thresholds) * true_labels,
axis=0)
fp_buckets = tf.reduce_sum(
tf.one_hot(bucket_indices, depth=num_thresholds) * false_labels,
axis=0)
# Set up the cumulative sums to compute the actual metrics.
tp = tf.cumsum(tp_buckets, reverse=True, name='tp')
fp = tf.cumsum(fp_buckets, reverse=True, name='fp')
# fn = sum(true_labels) - tp
# = sum(tp_buckets) - tp
# = tp[0] - tp
# Similarly,
# tn = fp[0] - fp
tn = fp[0] - fp
fn = tp[0] - tp
precision = tp / tf.maximum(_MINIMUM_COUNT, tp + fp)
recall = tp / tf.maximum(_MINIMUM_COUNT, tp + fn)
return _create_tensor_summary(name, tp, fp, tn, fn, precision, recall,
num_thresholds, display_name,
description, collections)
