def _full_hessian(self, grads, predictions):
"""Prepares hessians for full-hessian multiclass strategy."""
# Because of
# https://github.com/tensorflow/tensorflow/issues/675, we can't just
# compute the full hessian with a single call to gradients, but instead
# must compute it row-by-row.
gradients_list = array_ops.unstack(
grads, num=self._learner_config.num_classes, axis=1)
hessian_rows = []
for row in range(self._learner_config.num_classes):
# If current row is i, K is number of classes,each row returns a tensor of
# size batch_size x K representing for each example dx_i dx_1, dx_i dx_2
# etc dx_i dx_K
hessian_row = gradients_impl.gradients(
gradients_list[row],
predictions,
name="Hessian_%d" % row,
colocate_gradients_with_ops=False,
gate_gradients=0,
aggregation_method=None)
# hessian_row is of dimension 1, batch_size, K, => trim first dimension
# to get batch_size x K
hessian_row = array_ops.squeeze(array_ops.unstack(hessian_row), [0])
hessian_rows.append(hessian_row)
return hessian_rows
def testAggregate(self):
a = array_ops.constant([3., 4.])
b = array_ops.constant([5., 6.])
hint = op_hint.OpHint("agg")
a0, a1 = array_ops.unstack(a)
b0, b1 = array_ops.unstack(b)
a0 = hint.add_input(a0, tag="c", aggregate=op_hint.OpHint.AGGREGATE_STACK)
b0 = hint.add_input(b0, tag="n", aggregate=op_hint.OpHint.AGGREGATE_STACK)
a1 = hint.add_input(a1, tag="c", aggregate=op_hint.OpHint.AGGREGATE_STACK)
b1 = hint.add_input(b1, tag="n", aggregate=op_hint.OpHint.AGGREGATE_STACK)
c0 = math_ops.add(a0, b0, name="addleft")
c1 = math_ops.add(a1, b1, name="addright")
c0 = hint.add_output(
c0, tag="out", aggregate=op_hint.OpHint.AGGREGATE_STACK)
c1 = hint.add_output(
c1, tag="out", aggregate=op_hint.OpHint.AGGREGATE_STACK)
curr = array_ops.stack([c0, c1])
output = array_ops.identity(curr, name="FINAL_OUTPUT")
with self.cached_session() as sess:
stubbed_graphdef = op_hint.convert_op_hints_to_stubs(
graph_def=sess.graph_def)
self.assertEqual(
self._getGraphOpTypes(
stubbed_graphdef,
output_nodes=[op_hint._tensor_name_base(output.name)]),
set(["agg", "Const", "Identity"]))
def _diagonal_hessian(self, grads, predictions):
"""Prepares hessians for diagonal-hessian multiclass mode."""
diag_hessian_list = []
gradients_list = array_ops.unstack(
grads, num=self._learner_config.num_classes, axis=1)
for row, row_grads in enumerate(gradients_list):
# If current row is i, K is number of classes,each row returns a tensor of
# size batch_size x K representing for each example dx_i dx_1, dx_1 dx_2
# etc dx_i dx_K
hessian_row = gradients_impl.gradients(
row_grads,
predictions,
name="Hessian_%d" % row,
colocate_gradients_with_ops=False,
gate_gradients=0,
aggregation_method=None)
# hessian_row is of dimension 1, batch_size, K, => trim first dimension
# to get batch_size x K
hessian_row = array_ops.squeeze(array_ops.unstack(hessian_row), [0])
# Get dx_i^2 for the whole batch.
elem = array_ops.transpose(hessian_row)[row]
diag_hessian_list.append(elem)
return diag_hessian_list
def _prepare_inputs_for_rnn(sequence_features, context_features, num_unroll):
"""Prepares features batched by the SQSS for input to a state-saving RNN.
Args:
sequence_features: A dict of sequence feature name to `Tensor`, with
tensors of shape `[batch_size, num_unroll, ...]` and type float32.
context_features: A dict of context feature name to `Tensor`, with
tensors of shape `[batch_size, 1, ...]` and type float32.
num_unroll: Python integer, how many time steps to unroll at a time.
The input sequences of length `k` are then split into `k / num_unroll`
many segments.
Returns:
features_by_time: A list of length `num_unroll` with `Tensor` entries of
shape `[batch_size, len(sequence_features) + len(context_features)]` of
type float32. Features are stored in lexicographic order by their
corresponding feature dict keys, first in the `sequence_features` and
then in the `context_features` dicts. Context features are copied into
each time step.
"""
def _tile(feature):
return array_ops.squeeze(
array_ops.tile(array_ops.expand_dims(feature, 1), [1, num_unroll, 1]),
axis=2)
sequence_features = [sequence_features[k] for k in sorted(sequence_features)]
if not context_features:
return array_ops.unstack(array_ops.stack(sequence_features, 2), axis=1)
context_features = [
_tile(context_features[k]) for k in sorted(context_features)
]
return array_ops.unstack(
array_ops.stack(sequence_features + context_features, 2), axis=1)
def _add_comparison_summary(gan_model, reconstructions):
image_list = (array_ops.unstack(gan_model.generator_inputs[:1]) +
array_ops.unstack(gan_model.generated_data[:1]) +
array_ops.unstack(reconstructions[:1]))
summary.image(
'image_comparison', eval_utils.image_reshaper(
image_list, num_cols=len(image_list)), max_outputs=1)
def seq2seq_inputs(x, y, input_length, output_length, sentinel=None, name=None):
"""Processes inputs for Sequence to Sequence models.
Args:
x: Input Tensor [batch_size, input_length, embed_dim].
y: Output Tensor [batch_size, output_length, embed_dim].
input_length: length of input x.
output_length: length of output y.
sentinel: optional first input to decoder and final output expected.
If sentinel is not provided, zeros are used. Due to fact that y is not
available in sampling time, shape of sentinel will be inferred from x.
name: Operation name.
Returns:
Encoder input from x, and decoder inputs and outputs from y.
"""
with ops.name_scope(name, "seq2seq_inputs", [x, y]):
in_x = array_ops.unstack(x, axis=1)
y = array_ops.unstack(y, axis=1)
if not sentinel:
# Set to zeros of shape of y[0], using x for batch size.
sentinel_shape = array_ops.stack(
[array_ops.shape(x)[0], y[0].get_shape()[1]])
sentinel = array_ops.zeros(sentinel_shape)
sentinel.set_shape(y[0].get_shape())
in_y = [sentinel] + y
out_y = y + [sentinel]
return in_x, in_y, out_y
def __call__(self,
inputs,
initial_state=None,
dtype=None,
sequence_length=None,
scope=None):
is_list = isinstance(inputs, list)
if self._use_dynamic_rnn:
if is_list:
inputs = array_ops.stack(inputs)
outputs, state = rnn.dynamic_rnn(
self._cell,
inputs,
sequence_length=sequence_length,
initial_state=initial_state,
dtype=dtype,
time_major=True,
scope=scope)
if is_list:
# Convert outputs back to list
outputs = array_ops.unstack(outputs)
else: # non-dynamic rnn
if not is_list:
inputs = array_ops.unstack(inputs)
outputs, state = contrib_rnn.static_rnn(self._cell,
inputs,
initial_state=initial_state,
dtype=dtype,
sequence_length=sequence_length,
scope=scope)
if not is_list:
# Convert outputs back to tensor
outputs = array_ops.stack(outputs)
return outputs, state
def testCannotInferNumFromNoneShape(self):
x = array_ops.placeholder(np.float32, shape=(None,))
with self.assertRaisesRegexp(ValueError,
r'Cannot infer num from shape \(\?,\)'):
array_ops.unpack(x)
with self.assertRaisesRegexp(ValueError,
r'Cannot infer num from shape \(\?,\)'):
array_ops.unstack(x)
def testCannotInferNumFromUnknownShape(self):
x = array_ops.placeholder(np.float32)
with self.assertRaisesRegexp(ValueError,
r'Cannot infer num from shape <unknown>'):
array_ops.unpack(x)
with self.assertRaisesRegexp(ValueError,
r'Cannot infer num from shape <unknown>'):
array_ops.unstack(x)
def add_image_comparison_summaries(gan_model, num_comparisons=2,
display_diffs=False):
"""Adds image summaries to compare triplets of images.
The first image is the generator input, the second is the generator output,
and the third is the real data. This style of comparison is useful for
image translation problems, where the generator input is a corrupted image,
the generator output is the reconstruction, and the real data is the target.
Args:
gan_model: A GANModel tuple.
num_comparisons: The number of image triplets to display.
display_diffs: Also display the difference between generated and target.
Raises:
ValueError: If real data, generated data, and generator inputs aren't
images.
ValueError: If the generator input, real, and generated data aren't all the
same size.
"""
if isinstance(gan_model, namedtuples.CycleGANModel):
saved_params = locals()
saved_params.pop('gan_model', None)
with ops.name_scope('cyclegan_x2y_image_comparison_summaries'):
add_image_comparison_summaries(gan_model.model_x2y, **saved_params)
with ops.name_scope('cyclegan_y2x_image_comparison_summaries'):
add_image_comparison_summaries(gan_model.model_y2x, **saved_params)
return
_assert_is_image(gan_model.generator_inputs)
_assert_is_image(gan_model.generated_data)
_assert_is_image(gan_model.real_data)
gan_model.generated_data.shape.assert_is_compatible_with(
gan_model.generator_inputs.shape)
gan_model.real_data.shape.assert_is_compatible_with(
gan_model.generated_data.shape)
image_list = []
image_list.extend(
array_ops.unstack(gan_model.generator_inputs[:num_comparisons]))
image_list.extend(
array_ops.unstack(gan_model.generated_data[:num_comparisons]))
image_list.extend(array_ops.unstack(gan_model.real_data[:num_comparisons]))
if display_diffs:
generated_list = array_ops.unstack(
gan_model.generated_data[:num_comparisons])
real_list = array_ops.unstack(gan_model.real_data[:num_comparisons])
diffs = [
math_ops.abs(math_ops.to_float(generated) - math_ops.to_float(real)) for
generated, real in zip(generated_list, real_list)]
image_list.extend(diffs)
# Reshape image and display.
summary.image(
'image_comparison',
eval_utils.image_reshaper(image_list, num_cols=num_comparisons),
max_outputs=1)
def _prepare_inputs_for_rnn(sequence_features, context_features,
sequence_feature_columns, num_unroll):
"""Prepares features batched by the SQSS for input to a state-saving RNN.
Args:
sequence_features: A dict of sequence feature name to `Tensor` or
`SparseTensor`, with `Tensor`s of shape `[batch_size, num_unroll, ...]`
or `SparseTensors` of dense shape `[batch_size, num_unroll, d]`.
context_features: A dict of context feature name to `Tensor`, with
tensors of shape `[batch_size, 1, ...]` and type float32.
sequence_feature_columns: An iterable containing all the feature columns
describing sequence features. All items in the set should be instances
of classes derived from `FeatureColumn`.
num_unroll: Python integer, how many time steps to unroll at a time.
The input sequences of length `k` are then split into `k / num_unroll`
many segments.
Returns:
features_by_time: A list of length `num_unroll` with `Tensor` entries of
shape `[batch_size, sum(sequence_features dimensions) +
sum(context_features dimensions)]` of type float32.
Context features are copied into each time step.
"""
def _tile(feature):
return array_ops.squeeze(
array_ops.tile(array_ops.expand_dims(feature, 1), [1, num_unroll, 1]),
axis=2)
for feature in sequence_features.values():
if isinstance(feature, sparse_tensor.SparseTensor):
# Explicitly set dense_shape's shape to 3 ([batch_size, num_unroll, d])
# since it can't be statically inferred.
feature.dense_shape.set_shape([3])
sequence_features = layers.sequence_input_from_feature_columns(
columns_to_tensors=sequence_features,
feature_columns=sequence_feature_columns,
weight_collections=None,
scope=None)
# Explicitly set shape along dimension 1 to num_unroll for the unstack op.
sequence_features.set_shape([None, num_unroll, None])
if not context_features:
return array_ops.unstack(sequence_features, axis=1)
# TODO(jtbates): Call layers.input_from_feature_columns for context features.
context_features = [
_tile(context_features[k]) for k in sorted(context_features)
]
return array_ops.unstack(
array_ops.concat(
[sequence_features, array_ops.stack(context_features, 2)], axis=2),
axis=1)
def test_axis(self):
unpack_lt = ops.unpack(self.original_lt, axis_name='z')[0]
golden_lt = core.LabeledTensor(
array_ops.unstack(
self.original_lt.tensor, axis=2)[0], [self.a0, self.a1, self.a3])
self.assertLabeledTensorsEqual(unpack_lt, golden_lt)
def transition_power_noise_accumulator(self, num_steps):
"""Computes power sums in closed form."""
def _pack_and_reshape(*values):
return array_ops.reshape(
array_ops.stack(axis=1, values=values),
array_ops.concat(values=[array_ops.shape(num_steps), [2, 2]], axis=0))
num_steps = math_ops.cast(num_steps, self.dtype)
noise_transitions = num_steps - 1
noise_transform = ops.convert_to_tensor(self.get_noise_transform(),
self.dtype)
noise_covariance_transformed = math_ops.matmul(
math_ops.matmul(noise_transform,
self.state_transition_noise_covariance),
noise_transform,
adjoint_b=True)
# Un-packing the transformed noise as:
# [[a b]
# [c d]]
a, b, c, d = array_ops.unstack(
array_ops.reshape(noise_covariance_transformed, [-1, 4]), axis=1)
sum_of_first_n = noise_transitions * (noise_transitions + 1) / 2
sum_of_first_n_squares = sum_of_first_n * (2 * noise_transitions + 1) / 3
return _pack_and_reshape(
num_steps * a + sum_of_first_n * (b + c) + sum_of_first_n_squares * d,
num_steps * b + sum_of_first_n * d,
num_steps * c + sum_of_first_n * d,
num_steps * d)
def testInferNum(self):
with self.test_session():
for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2):
x = array_ops.placeholder(np.float32, shape=shape)
cs = array_ops.unstack(x)
self.assertEqual(type(cs), list)
self.assertEqual(len(cs), shape[0])
def Loop(cell, w, i):
x = array_ops.unstack(i, self.NUM_UNROLL)
m = array_ops.zeros_like(x[0])
c = array_ops.zeros_like(x[0])
for i in range(self.NUM_UNROLL):
m, c = cell(x[i], m, c, w)
return m
def scatter(self, indices, value, name=None):
"""See TensorArray."""
del name # unused in Eager
ta = self.identity()
for index, val in zip(indices.numpy(), array_ops.unstack(value)):
_eager_write_no_copy(ta._implementation, index, val) # pylint: disable=protected-access
return ta
def sequence_softmax(inputs, noutput, scope=None, name=None, linear_name=None):
"""Run a softmax layer over all the time steps of an input sequence.
Args:
inputs: (length, batch_size, depth) tensor
noutput: output depth
scope: optional scope name
name: optional name for output tensor
linear_name: name for linear (pre-softmax) output
Returns:
A tensor of size (length, batch_size, noutput).
"""
length, _, ninputs = _shape(inputs)
inputs_u = array_ops.unstack(inputs)
output_u = []
with variable_scope.variable_scope(scope, "SequenceSoftmax", [inputs]):
initial_w = random_ops.truncated_normal([0 + ninputs, noutput], stddev=0.1)
initial_b = constant_op.constant(0.1, shape=[noutput])
w = variables.model_variable("weights", initializer=initial_w)
b = variables.model_variable("biases", initializer=initial_b)
for i in xrange(length):
with variable_scope.variable_scope(scope, "SequenceSoftmaxStep",
[inputs_u[i]]):
# TODO(tmb) consider using slim.fully_connected(...,
# activation_fn=tf.nn.softmax)
linear = nn_ops.xw_plus_b(inputs_u[i], w, b, name=linear_name)
output = nn_ops.softmax(linear)
output_u += [output]
outputs = array_ops.stack(output_u, name=name)
return outputs
def sequence_to_final(inputs, noutput, scope=None, name=None, reverse=False):
"""Run an LSTM across all steps and returns only the final state.
Args:
inputs: (length, batch_size, depth) tensor
noutput: size of output vector
scope: optional scope name
name: optional name for output tensor
reverse: run in reverse
Returns:
Batch of size (batch_size, noutput).
"""
with variable_scope.variable_scope(scope, "SequenceToFinal", [inputs]):
length, batch_size, _ = _shape(inputs)
lstm = core_rnn_cell_impl.BasicLSTMCell(noutput, state_is_tuple=False)
state = array_ops.zeros([batch_size, lstm.state_size])
inputs_u = array_ops.unstack(inputs)
if reverse:
inputs_u = list(reversed(inputs_u))
for i in xrange(length):
if i > 0:
variable_scope.get_variable_scope().reuse_variables()
output, state = lstm(inputs_u[i], state)
outputs = array_ops.reshape(output, [batch_size, noutput], name=name)
return outputs
def ndlstm_base_unrolled(inputs, noutput, scope=None, reverse=False):
"""Run an LSTM, either forward or backward.
This is a 1D LSTM implementation using unrolling and the TensorFlow
LSTM op.
Args:
inputs: input sequence (length, batch_size, ninput)
noutput: depth of output
scope: optional scope name
reverse: run LSTM in reverse
Returns:
Output sequence (length, batch_size, noutput)
"""
with variable_scope.variable_scope(scope, "SeqLstmUnrolled", [inputs]):
length, batch_size, _ = _shape(inputs)
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(noutput, state_is_tuple=False)
state = array_ops.zeros([batch_size, lstm_cell.state_size])
output_u = []
inputs_u = array_ops.unstack(inputs)
if reverse:
inputs_u = list(reversed(inputs_u))
for i in xrange(length):
if i > 0:
variable_scope.get_variable_scope().reuse_variables()
output, state = lstm_cell(inputs_u[i], state)
output_u += [output]
if reverse:
output_u = list(reversed(output_u))
outputs = array_ops.stack(output_u)
return outputs
def grow_tree_from_accumulated_summaries_fn():
"""Updates the tree with the best layer from accumulated summaries."""
# Take out the accumulated summaries from the accumulator and grow.
stats_summary_list = array_ops.unstack(
summary_accumulator.take_grad(1), axis=0)
grow_op = grow_tree_from_stats_summaries(stats_summary_list)
return grow_op
def _get_permutations(num_results, dims, seed=None):
"""Uniform iid sample from the space of permutations.
Draws a sample of size `num_results` from the group of permutations of degrees
specified by the `dims` tensor. These are packed together into one tensor
such that each row is one sample from each of the dimensions in `dims`. For
example, if dims = [2,3] and num_results = 2, the result is a tensor of shape
[2, 2 + 3] and the first row of the result might look like:
[1, 0, 2, 0, 1]. The first two elements are a permutation over 2 elements
while the next three are a permutation over 3 elements.
Args:
num_results: A positive scalar `Tensor` of integral type. The number of
draws from the discrete uniform distribution over the permutation groups.
dims: A 1D `Tensor` of the same dtype as `num_results`. The degree of the
permutation groups from which to sample.
seed: (Optional) Python integer to seed the random number generator.
Returns:
permutations: A `Tensor` of shape `[num_results, sum(dims)]` and the same
dtype as `dims`.
"""
sample_range = math_ops.range(num_results)
def generate_one(d):
fn = lambda _: random_ops.random_shuffle(math_ops.range(d), seed=seed)
return functional_ops.map_fn(fn, sample_range)
return array_ops.concat([generate_one(d) for d in array_ops.unstack(dims)],
axis=-1)
def _fn(x):
"""MADE parameterized via `masked_autoregressive_default_template`."""
# TODO(b/67594795): Better support of dynamic shape.
input_depth = x.shape.with_rank_at_least(1)[-1].value
if input_depth is None:
raise NotImplementedError(
"Rightmost dimension must be known prior to graph execution.")
input_shape = (np.int32(x.shape.as_list()) if x.shape.is_fully_defined()
else array_ops.shape(x))
for i, units in enumerate(hidden_layers):
x = masked_dense(
inputs=x,
units=units,
num_blocks=input_depth,
exclusive=True if i == 0 else False,
activation=activation,
*args,
**kwargs)
x = masked_dense(
inputs=x,
units=(1 if shift_only else 2) * input_depth,
num_blocks=input_depth,
activation=None,
*args,
**kwargs)
if shift_only:
x = array_ops.reshape(x, shape=input_shape)
return x, None
x = array_ops.reshape(
x, shape=array_ops.concat([input_shape, [2]], axis=0))
shift, log_scale = array_ops.unstack(x, num=2, axis=-1)
which_clip = (math_ops.clip_by_value if log_scale_clip_gradient
else _clip_by_value_preserve_grad)
log_scale = which_clip(log_scale, log_scale_min_clip, log_scale_max_clip)
return shift, log_scale
def testBatch(self):
# Build an arbitrary RGB image
np.random.seed(7)
batch_size = 5
shape = (batch_size, 2, 7, 3)
for nptype in self.float_types:
inp = GenerateNumpyRandomRGB(shape).astype(nptype)
# Convert to HSV and back, as a batch and individually
with self.test_session() as sess:
batch0 = array_ops.placeholder(nptype, shape=shape)
with self.test_scope():
batch1 = image_ops.rgb_to_hsv(batch0)
batch2 = image_ops.hsv_to_rgb(batch1)
split0 = array_ops.unstack(batch0)
with self.test_scope():
split1 = list(map(image_ops.rgb_to_hsv, split0))
split2 = list(map(image_ops.hsv_to_rgb, split1))
join1 = array_ops.stack(split1)
join2 = array_ops.stack(split2)
batch1, batch2, join1, join2 = sess.run([batch1, batch2, join1, join2],
{batch0: inp})
# Verify that processing batch elements together is the same as separate
self.assertAllClose(batch1, join1)
self.assertAllClose(batch2, join2)
self.assertAllCloseAccordingToType(
batch2, inp, bfloat16_atol=0.03, half_rtol=0.02)
def _generate_optimization_test_cases():
def base_dataset_factory():
return dataset_ops.Dataset.from_tensors(np.random.rand(10, 3)).repeat(5)
rand_val = np.random.rand(1, 1, 1, 1, 1, 1)
test_cases = [
("Basic", lambda x: (x, x + 1), base_dataset_factory),
("Const", lambda x: 2, base_dataset_factory),
# Math ops exercise broadcasting capabilities
("Add", lambda x: x + rand_val, base_dataset_factory),
("Cast", lambda x: math_ops.cast(x, dtypes.float64),
base_dataset_factory),
("Reshape", lambda x: array_ops.reshape(x, (-1, 30)),
base_dataset_factory),
("Unpack", array_ops.unstack, base_dataset_factory),
("UnpackNegativeAxis", lambda x: array_ops.unstack(x, axis=-1),
base_dataset_factory),
]
return [{
"testcase_name":
x[0] + "Parallel" if num_parallel_calls is not None else x[0],
"map_fn":
x[1],
"base_dataset_factory":
x[2],
"num_parallel_calls":
num_parallel_calls
} for x in test_cases for num_parallel_calls in (None, 12)]
def test(self):
unpack_lt = ops.unpack(self.original_lt)[0]
golden_lt = core.LabeledTensor(
array_ops.unstack(self.original_lt.tensor)[0],
[self.a1, self.a2, self.a3])
self.assertLabeledTensorsEqual(unpack_lt, golden_lt)
def testAxis0Default(self):
a = constant_op.constant([[1, 2, 3], [4, 5, 6]], name='a')
unstacked = self.evaluate(array_ops.unstack(a))
self.assertEqual(len(unstacked), 2)
self.assertAllEqual(unstacked[0], [1, 2, 3])
self.assertAllEqual(unstacked[1], [4, 5, 6])
def unpack(labeled_tensor, axis_name=None, name=None):
"""Unpack the tensor.
See tf.unpack.
Args:
labeled_tensor: The input tensor.
axis_name: Optional name of axis to unpack. By default, the first axis is
used.
name: Optional op name.
Returns:
The list of unpacked LabeledTensors.
Raises:
ValueError: If `axis_name` is not an axis on the input.
"""
with ops.name_scope(name, 'lt_unpack', [labeled_tensor]) as scope:
labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor)
axis_names = list(labeled_tensor.axes.keys())
if axis_name is None:
axis_name = axis_names[0]
if axis_name not in axis_names:
raise ValueError('%s not in %s' % (axis_name, axis_names))
axis = axis_names.index(axis_name)
unpack_ops = array_ops.unstack(labeled_tensor.tensor, axis=axis, name=scope)
axes = [a for i, a in enumerate(labeled_tensor.axes.values()) if i != axis]
return [core.LabeledTensor(t, axes) for t in unpack_ops]
def LSTMLoop10(weights, inp):
x = array_ops.unstack(inp, self.NUM_UNROLL)
m = array_ops.zeros_like(x[0])
c = array_ops.zeros_like(x[0])
assert self.NUM_UNROLL % 10 == 0
for i in range(0, self.NUM_UNROLL, 10):
m, c = Loop10(weights, m, c, *x[i:i + 10])
return m
def _move_tensors(tensors, device):
"""Moves a list of tensors to a device by concatenating/splitting them."""
# Reset the device setting to avoid weird interactions with device merging
# logic.
with ops.device(None):
if all(tensor.shape == tensor_shape.scalar() for tensor in tensors):
with ops.device(tensors[0].device):
values = array_ops.stack(tensors)
with ops.device(device):
return array_ops.unstack(values)
else:
with ops.device(tensors[0].device):
sizes = array_ops.stack([array_ops.size(tensor) for tensor in tensors])
values = array_ops.concat(tensors, axis=0)
with ops.device(device):
sizes = array_ops.unstack(sizes)
return list(array_ops.split(values, sizes))
def scatter(self, indices, value, name=None):
"""See TensorArray."""
del name # not meaningful when executing eagerly.
if isinstance(indices, ops.EagerTensor):
indices = indices.numpy()
for index, val in zip(indices, array_ops.unstack(value)):
self._write(index, val) # pylint: disable=protected-access
return self.parent()
def interpolate_trilinear(grid,
query_points,
name='interpolate_trilinear',
indexing='ijk'):
"""Similar to Matlab's interp2 function but for 3D.
Finds values for query points on a grid using trilinear interpolation.
Args:
grid: a 5-D float `Tensor` of shape `[batch, height, width, depth, channels]`.
query_points: a 3-D float `Tensor` of N points with shape `[batch, N, 3]`.
name: a name for the operation (optional).
indexing: whether the query points are specified as row and column (ijk),
or Cartesian coordinates (xyz).
Returns:
values: a 3-D `Tensor` with shape `[batch, N, channels]`
Raises:
ValueError: if the indexing mode is invalid, or if the shape of the inputs
invalid.
"""
DEBUG = 0
if indexing != 'ijk' and indexing != 'xyz':
raise ValueError('Indexing mode must be \'ijk\' or \'xyz\'')
with ops.name_scope(name):
grid = ops.convert_to_tensor(grid)
query_points = ops.convert_to_tensor(query_points)
shape = array_ops.unstack(array_ops.shape(grid))
if len(shape) != 5:
msg = 'Grid must be 5 dimensional. Received: '
raise ValueError(msg + str(shape))
batch_size, height, width, depth, channels = shape
grid_type = grid.dtype
query_type = query_points.dtype
query_shape = array_ops.unstack(array_ops.shape(query_points))
if len(query_shape) != 3:
msg = ('Query points must be 3 dimensional. Received: ')
raise ValueError(msg + str(query_shape))
_, num_queries, _ = query_shape
alphas = []
floors = []
ceils = []
index_order = [0, 1, 2] if indexing == 'ijk' else [1, 0, 2]
unstacked_query_points = array_ops.unstack(query_points, axis=2)
for dim in index_order:
with ops.name_scope('dim-' + str(dim)):
queries = unstacked_query_points[dim]
size_in_indexing_dimension = shape[
dim + 1] # because batch size is the first index in shape
# max_floor is size_in_indexing_dimension - 2 so that max_floor + 1
# is still a valid index into the grid.
max_floor = math_ops.cast(size_in_indexing_dimension - 2,
query_type)
min_floor = constant_op.constant(0.0, dtype=query_type)
floor = math_ops.minimum(
math_ops.maximum(min_floor, math_ops.floor(queries)),
max_floor)
int_floor = math_ops.cast(floor, dtypes.int32)
if DEBUG:
int_floor = K.print_tensor(int_floor,
message='int_floor is:')
floors.append(int_floor)
ceil = int_floor + 1
if DEBUG: ceil = K.print_tensor(ceil, message='ceil is:')
ceils.append(ceil)
# alpha has the same type as the grid, as we will directly use alpha
# when taking linear combinations of pixel values from the image.
alpha = math_ops.cast(queries - floor, grid_type)
min_alpha = constant_op.constant(0.0, dtype=grid_type)
max_alpha = constant_op.constant(1.0, dtype=grid_type)
alpha = math_ops.minimum(math_ops.maximum(min_alpha, alpha),
max_alpha)
# Expand alpha to [b, n, 1] so we can use broadcasting
# (since the alpha values don't depend on the channel).
alpha = array_ops.expand_dims(alpha, 2)
if DEBUG: alpha = K.print_tensor(alpha, message='alpha is:')
alphas.append(alpha)
K.print_tensor(alpha)
flattened_grid = array_ops.reshape(
grid, [batch_size * height * width * depth, channels])
batch_offsets = array_ops.reshape(
math_ops.range(batch_size) * height * width * depth,
[batch_size, 1])
# This wraps array_ops.gather. We reshape the image data such that the
# batch, y, x and z coordinates are pulled into the first dimension.
# Then we gather. Finally, we reshape the output back. It's possible this
# code would be made simpler by using array_ops.gather_nd.
# def gather(y_coords, x_coords, z_coords, name):
# with ops.name_scope('gather-' + name):
# # map a 3d coordinates to a single number
# # https://stackoverflow.com/questions/10903149/how-do-i-compute-the-linear-index-of-a-3d-coordinate-and-vice-versa
# linear_coordinates = batch_offsets + x_coords*(width) + y_coords *(width*height) + z_coords
# gathered_values = array_ops.gather(flattened_grid, linear_coordinates)
# return array_ops.reshape(gathered_values,
# [batch_size, num_queries, channels])
def gather(i_coords, j_coords, k_coords, name):
with ops.name_scope('gather-' + name):
# map the indices of a matrix to a 1-dim array
#https://stackoverflow.com/questions/14015556/how-to-map-the-indexes-of-a-matrix-to-a-1-dimensional-array-c/27084843
linear_coordinates = batch_offsets + i_coords * (
width * height) + j_coords * (width) + k_coords
gathered_values = array_ops.gather(flattened_grid,
linear_coordinates)
return array_ops.reshape(gathered_values,
[batch_size, num_queries, channels])
# grab the pixel values in the 8 corners around each query point
# coordinates: y x z (rows(height), columns(width), depth) # yxz -> ijk
fy_fx_fz = gather(floors[0], floors[1], floors[2], 'fy_fx_fz') # 000
fy_fx_cz = gather(floors[0], floors[1], ceils[2], 'fy_fx_cz') # 001
fy_cx_fz = gather(floors[0], ceils[1], floors[2], 'fy_cx_fz') # 010
fy_cx_cz = gather(floors[0], ceils[1], ceils[2], 'fy_cx_cz') # 011
cy_fx_fz = gather(ceils[0], floors[1], floors[2], 'cy_fx_fz') # 100
cy_fx_cz = gather(ceils[0], floors[1], ceils[2], 'cy_fx_cz') # 101
cy_cx_fz = gather(ceils[0], ceils[1], floors[2], 'cy_cx_fz') # 110
cy_cx_cz = gather(ceils[0], ceils[1], ceils[2], 'cy_cx_cz') # 111
# now, do the actual interpolation
with ops.name_scope('interpolate'):
# we perform 4 linear interpolation to compute a, b, c, and d using alpha[1],
# then we compute e and f by interpolating a, b, c and d using alpha[0],
# and finally we find our sample point by interpolating e and f using alpha[2]
# with ops.name_scope('alpha-a'): a = alphas[1] * (cy_cx_fz - cy_fx_fz) + cy_fx_fz
# with ops.name_scope('alpha-b'): b = alphas[1] * (fy_cx_fz - fy_fx_fz) + fy_fx_fz
# with ops.name_scope('alpha-c'): c = alphas[1] * (cy_cx_cz - cy_fx_cz) + cy_fx_cz
# with ops.name_scope('alpha-d'): d = alphas[1] * (fy_cx_cz - fy_fx_cz) + fy_fx_cz
# with ops.name_scope('alpha-e'): e = alphas[0] * (b - a) + a
# with ops.name_scope('alpha-f'): f = alphas[0] * (d - c) + c
# with ops.name_scope('alpha-g'): g = alphas[2] * (f - e) + e
# 000 is at the top left
with ops.name_scope('alpha-a'):
a = alphas[1] * (fy_cx_fz - fy_fx_fz) + fy_fx_fz
with ops.name_scope('alpha-b'):
b = alphas[1] * (cy_cx_fz - cy_fx_fz) + cy_fx_fz
with ops.name_scope('alpha-c'):
c = alphas[1] * (fy_cx_cz - fy_fx_cz) + fy_fx_cz
with ops.name_scope('alpha-d'):
d = alphas[1] * (cy_cx_cz - cy_fx_cz) + cy_fx_cz
with ops.name_scope('alpha-e'):
e = alphas[0] * (b - a) + a
with ops.name_scope('alpha-f'):
f = alphas[0] * (d - c) + c
with ops.name_scope('alpha-g'):
g = alphas[2] * (f - e) + e
return g
def call(self,
inputs,
initial_state=None,
dtype=None,
sequence_length=None):
"""Run this LSTM on inputs, starting from the given state.
Args:
inputs: `3-D` tensor with shape `[time_len, batch_size, input_size]`.
initial_state: a tuple `(initial_cell_state, initial_output)` with tensors
of shape `[batch_size, self._num_units]`. If this is not provided, the
cell is expected to create a zero initial state of type `dtype`.
dtype: The data type for the initial state and expected output. Required
if `initial_state` is not provided or RNN state has a heterogeneous
dtype.
sequence_length: Specifies the length of each sequence in inputs. An
`int32` or `int64` vector (tensor) size `[batch_size]`, values in `[0,
time_len).`
Defaults to `time_len` for each element.
Returns:
A pair containing:
- Output: A `3-D` tensor of shape `[time_len, batch_size, output_size]`
or a list of time_len tensors of shape `[batch_size, output_size]`,
to match the type of the `inputs`.
- Final state: a tuple `(cell_state, output)` matching `initial_state`.
Raises:
ValueError: in case of shape mismatches
"""
is_list = isinstance(inputs, list)
if is_list:
inputs = array_ops.stack(inputs)
inputs_shape = inputs.get_shape().with_rank(3)
if not inputs_shape[2]:
raise ValueError("Expecting inputs_shape[2] to be set: %s" %
inputs_shape)
batch_size = inputs_shape[1].value
if batch_size is None:
batch_size = array_ops.shape(inputs)[1]
time_len = inputs_shape[0].value
if time_len is None:
time_len = array_ops.shape(inputs)[0]
# Provide default values for initial_state and dtype
if initial_state is None:
if dtype is None:
raise ValueError(
"Either initial_state or dtype needs to be specified")
z = array_ops.zeros(array_ops.stack([batch_size, self.num_units]),
dtype=dtype)
initial_state = z, z
else:
if len(initial_state) != 2:
raise ValueError(
"Expecting initial_state to be a tuple with length 2 or None"
)
if dtype is None:
dtype = initial_state[0].dtype
# create the actual cell
if sequence_length is not None:
sequence_length = ops.convert_to_tensor(sequence_length)
initial_cell_state, initial_output = initial_state # pylint: disable=unpacking-non-sequence
cell_states, outputs = self._call_cell(inputs, initial_cell_state,
initial_output, dtype,
sequence_length)
if sequence_length is not None:
# Mask out the part beyond sequence_length
mask = array_ops.transpose(
array_ops.sequence_mask(sequence_length, time_len,
dtype=dtype), [1, 0])
mask = array_ops.tile(array_ops.expand_dims(mask, [-1]),
[1, 1, self.num_units])
outputs *= mask
# Prepend initial states to cell_states and outputs for indexing to work
# correctly,since we want to access the last valid state at
# sequence_length - 1, which can even be -1, corresponding to the
# initial state.
mod_cell_states = array_ops.concat(
[array_ops.expand_dims(initial_cell_state, [0]), cell_states],
0)
mod_outputs = array_ops.concat(
[array_ops.expand_dims(initial_output, [0]), outputs], 0)
final_cell_state = self._gather_states(mod_cell_states,
sequence_length, batch_size)
final_output = self._gather_states(mod_outputs, sequence_length,
batch_size)
else:
# No sequence_lengths used: final state is the last state
final_cell_state = cell_states[-1]
final_output = outputs[-1]
if is_list:
# Input was a list, so return a list
outputs = array_ops.unstack(outputs)
final_state = rnn_cell_impl.LSTMStateTuple(final_cell_state,
final_output)
return outputs, final_state
def frechet_classifier_distance(real_images,
generated_images,
classifier_fn,
num_batches=1):
"""Classifier distance for evaluating a generative model.
This is based on the Frechet Inception distance, but for an arbitrary
classifier.
This technique is described in detail in https://arxiv.org/abs/1706.08500.
Given two Gaussian distribution with means m and m_w and covariance matrices
C and C_w, this function calcuates
|m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2))
which captures how different the distributions of real images and generated
images (or more accurately, their visual features) are. Note that unlike the
Inception score, this is a true distance and utilizes information about real
world images.
Note that when computed using sample means and sample covariance matrices,
Frechet distance is biased. It is more biased for small sample sizes. (e.g.
even if the two distributions are the same, for a small sample size, the
expected Frechet distance is large). It is important to use the same
sample size to compute frechet classifier distance when comparing two
generative models.
NOTE: This function consumes images, computes their activations, and then
computes the classifier score. If you would like to precompute many
activations for real and generated images for large batches, please use
frechet_clasifier_distance_from_activations(), which this method also uses.
Args:
real_images: Real images to use to compute Frechet Inception distance.
generated_images: Generated images to use to compute Frechet Inception
distance.
classifier_fn: A function that takes images and produces activations
based on a classifier.
num_batches: Number of batches to split images in to in order to
efficiently run them through the classifier network.
Returns:
The Frechet Inception distance. A floating-point scalar of the same type
as the output of `classifier_fn`.
"""
real_images_list = array_ops.split(real_images,
num_or_size_splits=num_batches)
generated_images_list = array_ops.split(generated_images,
num_or_size_splits=num_batches)
imgs = array_ops.stack(real_images_list + generated_images_list)
# Compute the activations using the memory-efficient `map_fn`.
activations = functional_ops.map_fn(fn=classifier_fn,
elems=imgs,
parallel_iterations=1,
back_prop=False,
swap_memory=True,
name='RunClassifier')
# Split the activations by the real and generated images.
real_a, gen_a = array_ops.split(activations, [num_batches, num_batches], 0)
# Ensure the activations have the right shapes.
real_a = array_ops.concat(array_ops.unstack(real_a), 0)
gen_a = array_ops.concat(array_ops.unstack(gen_a), 0)
return frechet_classifier_distance_from_activations(real_a, gen_a)
def hessians(ys,
xs,
name="hessians",
colocate_gradients_with_ops=False,
gate_gradients=False,
aggregation_method=None):
"""Constructs the Hessian of sum of `ys` with respect to `x` in `xs`.
`hessians()` adds ops to the graph to output the Hessian matrix of `ys`
with respect to `xs`. It returns a list of `Tensor` of length `len(xs)`
where each tensor is the Hessian of `sum(ys)`. This function currently
only supports evaluating the Hessian with respect to (a list of) one-
dimensional tensors.
The Hessian is a matrix of second-order partial derivatives of a scalar
tensor (see https://en.wikipedia.org/wiki/Hessian_matrix for more details).
Args:
ys: A `Tensor` or list of tensors to be differentiated.
xs: A `Tensor` or list of tensors to be used for differentiation.
name: Optional name to use for grouping all the gradient ops together.
defaults to 'hessians'.
colocate_gradients_with_ops: See `gradients()` documentation for details.
gate_gradients: See `gradients()` documentation for details.
aggregation_method: See `gradients()` documentation for details.
Returns:
A list of Hessian matrices of `sum(y)` for each `x` in `xs`.
Raises:
LookupError: if one of the operations between `xs` and `ys` does not
have a registered gradient function.
ValueError: if the arguments are invalid or not supported. Currently,
this function only supports one-dimensional `x` in `xs`.
"""
xs = _AsList(xs)
kwargs = {
'colocate_gradients_with_ops': colocate_gradients_with_ops,
'gate_gradients': gate_gradients,
'aggregation_method': aggregation_method
}
# Compute a hessian matrix for each x in xs
hessians = []
for i, x in enumerate(xs):
# Check dimensions
ndims = x.get_shape().ndims
if ndims is None:
raise ValueError(
'Cannot compute Hessian because the dimensionality of '
'element number %d of `xs` cannot be determined' % i)
elif ndims != 1:
raise ValueError(
'Computing hessians is currently only supported for '
'one-dimensional tensors. Element number %d of `xs` has '
'%d dimensions.' % (i, ndims))
with ops.name_scope(name + '_first_derivative'):
# Compute the partial derivatives of the input with respect to all
# elements of `x`
_gradients = gradients(ys, x, **kwargs)[0]
# Unpack the gradients into a list so we can take derivatives with
# respect to each element
_gradients = array_ops.unstack(_gradients)
with ops.name_scope(name + '_second_derivative'):
# Compute the partial derivatives with respect to each element of the list
_hess = [
gradients(_gradient, x, **kwargs)[0]
for _gradient in _gradients
]
# Pack the list into a matrix and add to the list of hessians
hessians.append(array_ops.stack(_hess, name=name))
return hessians
def testUnknownShapeOkWithNum(self):
x = array_ops.placeholder(np.float32)
array_ops.unstack(x, num=2)
def calibration_layer(uncalibrated_tensor,
num_keypoints,
keypoints_initializers=None,
keypoints_initializer_fns=None,
bound=False,
monotonic=None,
missing_input_values=None,
missing_output_values=None,
l1_reg=None,
l2_reg=None,
l1_laplacian_reg=None,
l2_laplacian_reg=None,
name=None):
"""Creates a calibration layer for uncalibrated values.
Returns a calibrated tensor of the same shape as the uncalibrated continuous
signals passed in, and a list of projection ops, that must be applied at
each step (or every so many steps) to project the model to a feasible space:
used for bounding the outputs or for imposing monotonicity -- the list will be
empty if bound and monotonic are not set.
Args:
uncalibrated_tensor: Tensor of shape [batch_size, ...] with uncalibrated
values.
num_keypoints: Number of keypoints to use. Either a scalar value that
will be used for every uncalibrated signal, or a list of n values,
per uncalibrated signal -- uncalibrated is first flattened (
see tf.contrib.layers.flatten) to [batch_size, n], and there should
be one value in the list per n. If a value of the list is 0 or None
the correspondent signal won't be calibrated.
keypoints_initializers: For evaluation or inference (or when resuming
training from a checkpoint) the values will be loaded from disk, so they
don't need to be given (leave it as None).
Otherwise provide either a tuple of two tensors of shape [num_keypoints],
or a list of n pairs of tensors, each of shape [num_keypoints]. In this
list there should be one pair per uncalibrated signal, just like
num_keypoints above. Notice that num_keypoints can be different per
signal.
keypoints_initializer_fns: Like keypoints_initializers but using lambda
initializers. They should be compatible with tf.get_variable. If this is
set, then keypoints_initializers must be None.
bound: boolean whether output of calibration must be bound. Alternatively
a list of n booleans, one per uncalibrated value, like num_keypoints
above.
monotonic: whether calibration is monotonic: None or 0 means no
monotonicity. Positive or negative values mean increasing or decreasing
monotonicity respectively. Alternatively a list of n monotonic values,
one per uncalibrated value, like num_keypoints above.
missing_input_values: If set, and if the input has this value it is assumed
to be missing and the output will either be calibrated to some value
between `[calibration_output_min, calibration_output_max]` or set to a
fixed value set by missing_output_value. Limitation: it only works for
scalars. Either one value for all inputs, or a list with one value per
uncalibrated value.
missing_output_values: Requires missing_input_value also to be set. If set
if will convert missing input to this value. Either one value for all
outputs, or a list with one value per uncalibrated value.
l1_reg: (list of floats or float) l1 regularization amount.
If float, then same value is applied to all dimensions.
l2_reg: (list of floats or float) l2 regularization amount.
If float, then same value is applied to all dimensions.
l1_laplacian_reg: (list of floats or float) l1 laplacian
regularization amount. If float, then same value is applied to all
dimensions.
l2_laplacian_reg: (list of floats or float) l2 laplacian
regularization amount. If float, then same value is applied to all
dimensions.
name: Name scope for operations.
Returns:
A tuple of:
* calibrated tensor of shape [batch_size, ...], the same shape as
uncalibrated.
* list of projection ops, that must be applied at each step (or every so
many steps) to project the model to a feasible space: used for bounding
the outputs or for imposing monotonicity. Empty if none are requested.
* None or tensor with regularization loss.
Raises:
ValueError: If dimensions don't match.
"""
with ops.name_scope(name or 'calibration_layer'):
# Flattening uncalibrated tensor [batch_Size, k1, k2, ..., kn] to
# [batch_size, k1 * k2 * ... * kn].
uncalibrated_shape = uncalibrated_tensor.get_shape().as_list()
n = 1
for non_batch_dim in uncalibrated_shape[1:]:
n *= non_batch_dim
flat_uncalibrated = array_ops.reshape(
uncalibrated_tensor, shape=[-1, n], name='flat_uncalibrated')
num_keypoints = tools.cast_to_list(num_keypoints, n, 'num_keypoints')
keypoints_initializers = tools.cast_to_list(keypoints_initializers, n,
'keypoints_initializers')
keypoints_initializer_fns = tools.cast_to_list(keypoints_initializer_fns, n,
'keypoints_initializer_fns')
bound = tools.cast_to_list(bound, n, 'bound')
monotonic = tools.cast_to_list(monotonic, n, 'monotonic')
missing_input_values = tools.cast_to_list(missing_input_values, n,
'missing_input_values')
missing_output_values = tools.cast_to_list(missing_output_values, n,
'missing_output_values')
l1_regs = tools.cast_to_list(l1_reg, n, 'calibration_l1_reg')
l2_regs = tools.cast_to_list(l2_reg, n, 'calibration_l2_reg')
l1_laplacian_regs = tools.cast_to_list(l1_laplacian_reg, n,
'calibration_l1_laplacian_reg')
l2_laplacian_regs = tools.cast_to_list(l2_laplacian_reg, n,
'calibration_l2_laplacian_reg')
signal_names = ['signal_%d' % ii for ii in range(n)]
uncalibrated_splits = array_ops.unstack(flat_uncalibrated, axis=1)
calibrated_splits = []
projection_ops = []
total_regularization = None
for ii in range(n):
if not num_keypoints[ii]:
# No calibration for this signal.
calibrated_splits += [uncalibrated_splits[ii]]
else:
calibrated, projection, reg = one_dimensional_calibration_layer(
uncalibrated_splits[ii],
num_keypoints[ii],
signal_name=signal_names[ii],
keypoints_initializers=keypoints_initializers[ii],
keypoints_initializer_fns=keypoints_initializer_fns[ii],
bound=bound[ii],
monotonic=monotonic[ii],
missing_input_value=missing_input_values[ii],
missing_output_value=missing_output_values[ii],
l1_reg=l1_regs[ii],
l2_reg=l2_regs[ii],
l1_laplacian_reg=l1_laplacian_regs[ii],
l2_laplacian_reg=l2_laplacian_regs[ii])
calibrated_splits += [calibrated]
if projection is not None:
projection_ops += [projection]
total_regularization = tools.add_if_not_none(total_regularization, reg)
flat_calibrated = array_ops.stack(
calibrated_splits, axis=1, name='stack_calibrated')
reshaped_calibrated = array_ops.reshape(
flat_calibrated,
shape=array_ops.shape(uncalibrated_tensor),
name='reshape_calibrated')
return reshaped_calibrated, projection_ops, total_regularization
def input_calibration_layer(columns_to_tensors,
num_keypoints,
feature_columns=None,
keypoints_initializers=None,
keypoints_initializer_fns=None,
bound=False,
monotonic=None,
missing_input_values=None,
missing_output_values=None,
l1_reg=None,
l2_reg=None,
l1_laplacian_reg=None,
l2_laplacian_reg=None,
dtype=dtypes.float32):
"""Creates a calibration layer for the given input and feature_columns.
Returns a tensor with the calibrated values of the given features, a list
of the names of the features in the order they feature in the returned, and
a list of projection ops, that must be applied at each step (or every so many
steps) to project the model to a feasible space: used for bounding the outputs
or for imposing monotonic -- the list will be empty if bound and
monotonic are not set.
Args:
columns_to_tensors: A mapping from feature name to tensors. 'string' key
means a base feature (not-transformed). If feature_columns is not set
these are the features calibrated. Otherwise the transformed
feature_columns are the ones calibrated.
num_keypoints: Number of keypoints to use. Either a single int, or a dict
mapping feature names to num_keypoints. If a value of the dict is 0 or
None the correspondent feature won't be calibrated.
feature_columns: Optional. If set to a set of FeatureColumns, these will
be the features used and calibrated.
keypoints_initializers: For evaluation or inference (or when resuming
training from a checkpoint) the values will be loaded from disk, so they
don't need to be given (leave it as None).
Either a tuple of two tensors of shape [num_keypoints], or a dict mapping
feature names to pair of tensors of shape [num_keypoints[feature_name]].
See load_keypoints_from_quantiles or uniform_keypoints_for_signal on how
to generate these (module keypoints_initialization).
keypoints_initializer_fns: Like keypoints_initializers but using lambda
initializers. They should be compatible with tf.get_variable. If this is
set, then keypoints_initializers must be None.
bound: boolean whether output of calibration must be bound. Alternatively
a dict mapping feature name to boundness.
monotonic: whether calibration has to be kept monotonic: None or 0 means
no monotonic. Positive or negative values mean increasing or decreasing
monotonic respectively. Alternatively a dict mapping feature name
to monotonic.
missing_input_values: If set, and if the input has this value it is assumed
to be missing and the output will either be calibrated to some value
between `[calibration_output_min, calibration_output_max]` or set to a
fixed value set by missing_output_value. Limitation: it only works for
scalars. Either one value for all inputs, or a dict mapping feature name
to missing_input_value for the respective feature.
missing_output_values: Requires missing_input_value also to be set. If set
if will convert missing input to this value. Either one value for all
inputs, or a dict mapping feature name to missing_input_value for the
respective feature.
l1_reg: ({feature_name: float} dict or float) l1 regularization amount.
If float, then same value is applied to all features.
l2_reg: ({feature_name: float} dict or float) l2 regularization amount.
If float, then same value is applied to all features.
l1_laplacian_reg: ({feature_name: float} dict or float) l1 laplacian
regularization amount. If float, then same value is applied to all
features.
l2_laplacian_reg: ({feature_name: float} dict or float) l2 laplacian
regularization amount. If float, then same value is applied to all
features.
dtype: If any of the scalars are not given as tensors, they are converted
to tensors with this dtype.
Returns:
A tuple of:
* calibrated tensor of shape [batch_size, sum(features dimensions)].
* list of the feature names in the order they feature in the calibrated
tensor. A name may appear more than once if the feature is
multi-dimension (for instance a multi-dimension embedding)
* list of projection ops, that must be applied at each step (or every so
many steps) to project the model to a feasible space: used for bounding
the outputs or for imposing monotonicity. Empty if none are requested.
* None or tensor with regularization loss.
Raises:
ValueError: if dtypes are incompatible.
"""
with ops.name_scope('input_calibration_layer'):
feature_names = tools.get_sorted_feature_names(columns_to_tensors,
feature_columns)
num_keypoints = tools.cast_to_dict(num_keypoints, feature_names,
'num_keypoints')
bound = tools.cast_to_dict(bound, feature_names, 'bound')
monotonic = tools.cast_to_dict(monotonic, feature_names, 'monotonic')
keypoints_initializers = tools.cast_to_dict(
keypoints_initializers, feature_names, 'keypoints_initializers')
keypoints_initializer_fns = tools.cast_to_dict(
keypoints_initializer_fns, feature_names, 'keypoints_initializer_fns')
missing_input_values = tools.cast_to_dict(
missing_input_values, feature_names, 'missing_input_values')
missing_output_values = tools.cast_to_dict(
missing_output_values, feature_names, 'missing_output_values')
l1_regs = tools.cast_to_dict(l1_reg, feature_names, 'calibration_l1_reg')
l2_regs = tools.cast_to_dict(l2_reg, feature_names, 'calibration_l2_reg')
l1_laplacian_regs = tools.cast_to_dict(l1_laplacian_reg, feature_names,
'calibration_l1_laplacian_reg')
l2_laplacian_regs = tools.cast_to_dict(l2_laplacian_reg, feature_names,
'calibration_l2_laplacian_reg')
per_dimension_feature_names = []
# Get uncalibrated tensors, either from columns_to_tensors, or using
# feature_columns.
if feature_columns is None:
uncalibrated_features = [
columns_to_tensors[name] for name in feature_names
]
else:
transformed_columns_to_tensors = columns_to_tensors.copy()
dict_feature_columns = {f_col.name: f_col for f_col in feature_columns}
uncalibrated_features = [
tools.input_from_feature_column(transformed_columns_to_tensors,
dict_feature_columns[name], dtype)
for name in feature_names
]
projection_ops = []
calibrated_splits = []
total_regularization = None
for feature_idx in range(len(feature_names)):
name = feature_names[feature_idx]
uncalibrated_feature = uncalibrated_features[feature_idx]
if uncalibrated_feature.shape.ndims == 1:
feature_dim = 1
uncalibrated_splits = [uncalibrated_feature]
elif uncalibrated_feature.shape.ndims == 2:
feature_dim = uncalibrated_feature.shape.dims[1].value
uncalibrated_splits = array_ops.unstack(uncalibrated_feature, axis=1)
else:
raise ValueError(
'feature {}: it has rank {}, but only ranks 1 or 2 are '
'supported; feature shape={}'.format(
name, uncalibrated_feature.shape.ndims,
uncalibrated_feature.shape))
missing_input_value = missing_input_values[name]
missing_output_value = missing_output_values[name]
l1_reg = l1_regs[name]
l2_reg = l2_regs[name]
l1_laplacian_reg = l1_laplacian_regs[name]
l2_laplacian_reg = l2_laplacian_regs[name]
# FutureWork: make the interpolation ops handle multi-dimension values,
# so this step is not needed.
for dim_idx in range(feature_dim):
per_dimension_feature_names += [name]
split_name = name
if feature_dim > 1:
split_name = '{}_dim_{}'.format(name, dim_idx)
uncalibrated = uncalibrated_splits[dim_idx]
if not num_keypoints[name]:
# No calibration for this feature:
calibrated_splits += [uncalibrated]
if (missing_input_value is not None or
missing_output_value is not None):
raise ValueError(
'feature %s: cannot handle missing values if feature is not '
'calibrated, missing_input_value=%s, missing_output_value=%s' %
(name, missing_input_value, missing_output_value))
else:
calibrated, projection, reg = one_dimensional_calibration_layer(
uncalibrated,
num_keypoints[name],
signal_name=split_name,
keypoints_initializers=keypoints_initializers[name],
keypoints_initializer_fns=keypoints_initializer_fns[name],
bound=bound[name],
monotonic=monotonic[name],
missing_input_value=missing_input_value,
missing_output_value=missing_output_value,
l1_reg=l1_reg,
l2_reg=l2_reg,
l1_laplacian_reg=l1_laplacian_reg,
l2_laplacian_reg=l2_laplacian_reg)
calibrated_splits += [calibrated]
if projection is not None:
projection_ops += [projection]
total_regularization = tools.add_if_not_none(total_regularization,
reg)
all_calibrated = array_ops.stack(
calibrated_splits, axis=1, name='stack_calibrated')
return (all_calibrated, per_dimension_feature_names, projection_ops,
total_regularization)
def _PackGrad(op, grad):
"""Gradient for pack op."""
return array_ops.unstack(grad,
num=op.get_attr("N"),
axis=op.get_attr("axis"))
def extract_features(features, feature_columns):
"""Extracts columns from a dictionary of features.
Args:
features: `dict` of `Tensor` objects.
feature_columns: A list of feature_columns.
Returns:
Seven values:
- A list of all feature column names.
- A list of dense floats.
- A list of sparse float feature indices.
- A list of sparse float feature values.
- A list of sparse float feature shapes.
- A list of sparse int feature indices.
- A list of sparse int feature values.
- A list of sparse int feature shapes.
Raises:
ValueError: if features is not valid.
"""
if not features:
raise ValueError("Features dictionary must be specified.")
# Make a shallow copy of features to ensure downstream usage
# is unaffected by modifications in the model function.
features = copy.copy(features)
if feature_columns:
scope = "gbdt"
with variable_scope.variable_scope(scope):
feature_columns = list(feature_columns)
transformed_features = {}
for fc in feature_columns:
# pylint: disable=protected-access
if isinstance(fc, feature_column_lib._EmbeddingColumn):
# pylint: enable=protected-access
transformed_features[fc.name] = fc_core.input_layer(
features, [fc], weight_collections=[scope])
else:
result = feature_column_ops.transform_features(
features, [fc])
if len(result) > 1:
raise ValueError(
"Unexpected number of output features")
transformed_features[fc.name] = result[list(
result.keys())[0]]
features = transformed_features
dense_float_names = []
dense_floats = []
sparse_float_names = []
sparse_float_indices = []
sparse_float_values = []
sparse_float_shapes = []
sparse_int_names = []
sparse_int_indices = []
sparse_int_values = []
sparse_int_shapes = []
for key in sorted(features.keys()):
tensor = features[key]
if isinstance(tensor, sparse_tensor.SparseTensor):
if tensor.values.dtype == dtypes.float32:
sparse_float_names.append(key)
sparse_float_indices.append(tensor.indices)
sparse_float_values.append(tensor.values)
sparse_float_shapes.append(tensor.dense_shape)
elif tensor.values.dtype == dtypes.int64:
sparse_int_names.append(key)
sparse_int_indices.append(tensor.indices)
sparse_int_values.append(tensor.values)
sparse_int_shapes.append(tensor.dense_shape)
else:
raise ValueError(
"Unsupported sparse feature %s with dtype %s." %
(tensor.indices.name, tensor.dtype))
else:
if tensor.dtype == dtypes.float32:
if len(tensor.shape) > 1 and tensor.shape[1] > 1:
unstacked = array_ops.unstack(tensor, axis=1)
for i in range(len(unstacked)):
dense_float_names.append(_FEATURE_NAME_TEMPLATE %
(key, i))
dense_floats.append(
array_ops.reshape(unstacked[i], [-1, 1]))
else:
dense_float_names.append(key)
dense_floats.append(tensor)
else:
raise ValueError(
"Unsupported dense feature %s with dtype %s." %
(tensor.name, tensor.dtype))
# Feature columns are logically organized into incrementing slots starting
# from dense floats, then sparse floats then sparse ints.
fc_names = (dense_float_names + sparse_float_names + sparse_int_names)
return (fc_names, dense_floats, sparse_float_indices, sparse_float_values,
sparse_float_shapes, sparse_int_indices, sparse_int_values,
sparse_int_shapes)
def _groupwise_dnn_v2(features, labels, mode, params, config):
"""Defines the dnn for groupwise scoring functions."""
with ops.name_scope('transform'):
context_features, per_example_features = _call_transform_fn(
features, mode, params)
def _score_fn(context_features, group_features, reuse):
with variable_scope.variable_scope('group_score', reuse=reuse):
return group_score_fn(context_features, group_features, mode,
params, config)
# Scatter/Gather per-example scores through groupwise comparison. Each
# instance in a mini-batch will form a number of groups. Each groups of
# examples are scored by 'score_fn' and socres for individual examples
# accumulated over groups.
with ops.name_scope('groupwise_dnn_v2'):
with ops.name_scope('infer_sizes'):
if labels is not None:
batch_size, list_size = array_ops.unstack(
array_ops.shape(labels))
is_valid = utils.is_label_valid(labels)
else:
# Infer batch_size and list_size from a feature.
example_tensor_shape = array_ops.shape(
next(six.itervalues(per_example_features)))
batch_size = example_tensor_shape[0]
list_size = example_tensor_shape[1]
is_valid = utils.is_label_valid(
array_ops.ones([batch_size, list_size]))
if batch_size is None or list_size is None:
raise ValueError('Invalid batch_size=%s or list_size=%s' %
(batch_size, list_size))
# For each example feature, assume the shape is [batch_size, list_size,
# feature_size], the groups are formed along the 2nd dim. Each group has a
# 'group_size' number of indices in [0, list_size). Based on these
# indices, we can gather the example feature into a sub-tensor for each
# group. The total number of groups we have for a mini-batch is batch_size
# * num_groups. Inside each group, we have a 'group_size' number of
# examples.
indices, mask = _form_group_indices_nd(
is_valid,
group_size,
shuffle=(mode != model_fn.ModeKeys.PREDICT))
num_groups = array_ops.shape(mask)[1]
with ops.name_scope('group_features'):
# For context features, We have shape [batch_size * num_groups, ...].
large_batch_context_features = {}
for name, value in six.iteritems(context_features):
# [batch_size, 1, ...].
value = array_ops.expand_dims(value, axis=1)
# [batch_size, num_groups, ...]. ,就是tile
value = array_ops.gather(value,
array_ops.zeros([num_groups],
dtypes.int32),
axis=1)
# [batch_size * num_groups, ...]
large_batch_context_features[
name] = utils.reshape_first_ndims(
value, 2, [batch_size * num_groups])
# For example feature, we have shape [batch_size * num_groups,
# group_size, ...].
large_batch_group_features = {}
for name, value in six.iteritems(per_example_features):
# [batch_size, num_groups, group_size, ...].
value = array_ops.gather_nd(value, indices)
# [batch_size * num_groups, group_size, ...].
large_batch_group_features[
name] = utils.reshape_first_ndims(
value, 3, [batch_size * num_groups, group_size])
# Do the inference and get scores for the large batch.
# [batch_size * num_groups, group_size].
scores = _score_fn(large_batch_context_features,
large_batch_group_features,
reuse=False)
with ops.name_scope('accumulate_scores'):
scores = array_ops.reshape(
scores, [batch_size, num_groups, group_size])
# Reset invalid scores to 0 based on mask.
scores = array_ops.where(
array_ops.gather(array_ops.expand_dims(mask, 2),
array_ops.zeros([group_size],
dtypes.int32),
axis=2), scores,
array_ops.zeros_like(scores))
# [batch_size, num_groups, group_size].
list_scores = array_ops.scatter_nd(indices, scores,
[batch_size, list_size])
# Use average.
list_scores /= math_ops.to_float(group_size)
if mode == model_fn.ModeKeys.PREDICT:
return list_scores
else:
features.update(context_features)
features.update(per_example_features)
return list_scores
def testUnpack_Axis0(self): inputs = np.random.rand(3, 4, 7) tf_vals = array_ops.unstack(inputs) c_vals = [tensor_util.constant_value(x) for x in tf_vals] self.assertAllClose(inputs, c_vals)
def call(self, inputs, states, training=None):
# previous memory
h_tm1_tmp = states[0] if nest.is_sequence(states) else states
dp_mask = self.get_dropout_mask_for_cell(inputs, training, count=3)
rec_dp_mask = self.get_recurrent_dropout_mask_for_cell(
h_tm1_tmp, training, count=3)
if self.state_quantizer:
h_tm1 = self.state_quantizer_internal(h_tm1_tmp)
else:
h_tm1 = h_tm1_tmp
if self.kernel_quantizer:
quantized_kernel = self.kernel_quantizer_internal(self.kernel)
else:
quantized_kernel = self.kernel
if self.recurrent_quantizer:
quantized_recurrent = self.recurrent_quantizer_internal(self.recurrent_kernel)
else:
quantized_recurrent = self.kernel
if self.use_bias:
if self.bias_quantizer:
quantized_bias = self.bias_quantizer_internal(self.bias)
else:
quantized_bias = self.bias
if not self.reset_after:
input_bias, recurrent_bias = quantized_bias, None
else:
input_bias, recurrent_bias = array_ops.unstack(quantized_bias)
if self.implementation == 1:
if 0. < self.dropout < 1.:
inputs_z = inputs * dp_mask[0]
inputs_r = inputs * dp_mask[1]
inputs_h = inputs * dp_mask[2]
else:
inputs_z = inputs
inputs_r = inputs
inputs_h = inputs
x_z = K.dot(inputs_z, quantized_kernel[:, :self.units])
x_r = K.dot(inputs_r, quantized_kernel[:, self.units:self.units * 2])
x_h = K.dot(inputs_h, quantized_kernel[:, self.units * 2:])
if self.use_bias:
x_z = K.bias_add(x_z, input_bias[:self.units])
x_r = K.bias_add(x_r, input_bias[self.units: self.units * 2])
x_h = K.bias_add(x_h, input_bias[self.units * 2:])
if 0. < self.recurrent_dropout < 1.:
h_tm1_z = h_tm1 * rec_dp_mask[0]
h_tm1_r = h_tm1 * rec_dp_mask[1]
h_tm1_h = h_tm1 * rec_dp_mask[2]
else:
h_tm1_z = h_tm1
h_tm1_r = h_tm1
h_tm1_h = h_tm1
recurrent_z = K.dot(h_tm1_z, quantized_recurrent[:, :self.units])
recurrent_r = K.dot(h_tm1_r,
quantized_recurrent[:, self.units:self.units * 2])
if self.reset_after and self.use_bias:
recurrent_z = K.bias_add(recurrent_z, recurrent_bias[:self.units])
recurrent_r = K.bias_add(recurrent_r,
recurrent_bias[self.units:self.units * 2])
z = self.recurrent_activation(x_z + recurrent_z)
r = self.recurrent_activation(x_r + recurrent_r)
# reset gate applied after/before matrix multiplication
if self.reset_after:
recurrent_h = K.dot(h_tm1_h, quantized_recurrent[:, self.units * 2:])
if self.use_bias:
recurrent_h = K.bias_add(recurrent_h, recurrent_bias[self.units * 2:])
recurrent_h = r * recurrent_h
else:
recurrent_h = K.dot(r * h_tm1_h,
quantized_recurrent[:, self.units * 2:])
hh = self.activation(x_h + recurrent_h)
else:
if 0. < self.dropout < 1.:
inputs = inputs * dp_mask[0]
# inputs projected by all gate matrices at once
matrix_x = K.dot(inputs, quantized_kernel)
if self.use_bias:
# biases: bias_z_i, bias_r_i, bias_h_i
matrix_x = K.bias_add(matrix_x, input_bias)
x_z, x_r, x_h = array_ops.split(matrix_x, 3, axis=-1)
if self.reset_after:
# hidden state projected by all gate matrices at once
matrix_inner = K.dot(h_tm1, quantized_recurrent)
if self.use_bias:
matrix_inner = K.bias_add(matrix_inner, recurrent_bias)
else:
# hidden state projected separately for update/reset and new
matrix_inner = K.dot(h_tm1, quantized_recurrent[:, :2 * self.units])
recurrent_z, recurrent_r, recurrent_h = array_ops.split(
matrix_inner, [self.units, self.units, -1], axis=-1)
z = self.recurrent_activation(x_z + recurrent_z)
r = self.recurrent_activation(x_r + recurrent_r)
if self.reset_after:
recurrent_h = r * recurrent_h
else:
recurrent_h = K.dot(r * h_tm1,
quantized_recurrent[:, 2 * self.units:])
hh = self.activation(x_h + recurrent_h)
# previous and candidate state mixed by update gate
h = z * h_tm1 + (1 - z) * hh
return h, [h]
def testInferNum(self):
for shape in (2, ), (3, ), (2, 3), (3, 2), (4, 3, 2):
x = array_ops.placeholder(np.float32, shape=shape)
cs = array_ops.unstack(x)
self.assertEqual(type(cs), list)
self.assertEqual(len(cs), shape[0])
def run_test_sample_consistent_log_prob(self,
sess_run_fn,
dist,
num_samples=int(1e5),
num_threshold=int(1e3),
seed=42,
rtol=1e-2,
atol=0.):
"""Tests that sample/log_prob are consistent with each other.
"Consistency" means that `sample` and `log_prob` correspond to the same
distribution.
Note: this test only verifies a necessary condition for consistency--it does
does not verify sufficiency hence does not prove `sample`, `log_prob` truly
are consistent.
Args:
sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and
returning a list of results after running one "step" of TensorFlow
computation, typically set to `sess.run`.
dist: Distribution instance or object which implements `sample`,
`log_prob`, `event_shape_tensor` and `batch_shape_tensor`.
num_samples: Python `int` scalar indicating the number of Monte-Carlo
samples to draw from `dist`.
num_threshold: Python `int` scalar indicating the number of samples a
bucket must contain before being compared to the probability.
Default value: 1e3; must be at least 1.
Warning, set too high will cause test to falsely pass but setting too
low will cause the test to falsely fail.
seed: Python `int` indicating the seed to use when sampling from `dist`.
In general it is not recommended to use `None` during a test as this
increases the likelihood of spurious test failure.
rtol: Python `float`-type indicating the admissible relative error between
analytical and sample statistics.
atol: Python `float`-type indicating the admissible absolute error between
analytical and sample statistics.
Raises:
ValueError: if `num_threshold < 1`.
"""
if num_threshold < 1:
raise ValueError(
"num_threshold({}) must be at least 1.".format(num_threshold))
# Histogram only supports vectors so we call it once per batch coordinate.
y = dist.sample(num_samples, seed=seed)
y = array_ops.reshape(y, shape=[num_samples, -1])
batch_size = math_ops.reduce_prod(dist.batch_shape_tensor())
batch_dims = array_ops.shape(dist.batch_shape_tensor())[0]
edges_expanded_shape = 1 + array_ops.pad([-2],
paddings=[[0, batch_dims]])
for b, x in enumerate(array_ops.unstack(y, axis=1)):
counts, edges = self.histogram(x)
edges = array_ops.reshape(edges, edges_expanded_shape)
probs = math_ops.exp(dist.log_prob(edges))
probs = array_ops.reshape(probs, shape=[-1, batch_size])[:, b]
[counts_, probs_] = sess_run_fn([counts, probs])
valid = counts_ > num_threshold
probs_ = probs_[valid]
counts_ = counts_[valid]
self.assertAllClose(probs_,
counts_ / num_samples,
rtol=rtol,
atol=atol)
def testCannotInferNumFromNoneShape(self):
x = array_ops.placeholder(np.float32, shape=(None, ))
with self.assertRaisesRegexp(ValueError,
r'Cannot infer num from shape \(\?,\)'):
array_ops.unstack(x)
def testExecuteListOutputLen0(self):
empty = constant_op.constant([], dtype=dtypes.int32)
result = array_ops.unstack(empty, 0)
self.assertTrue(isinstance(result, list))
self.assertEqual(0, len(result))
def testCannotInferNumFromUnknownShape(self):
x = array_ops.placeholder(np.float32)
with self.assertRaisesRegexp(ValueError,
r'Cannot infer num from shape <unknown>'):
array_ops.unstack(x)
def testBasicRNNFusedWrapper(self):
"""This test checks that using a wrapper for BasicRNN works as expected."""
with self.test_session() as sess:
initializer = init_ops.random_uniform_initializer(-0.01,
0.01,
seed=19890212)
cell = core_rnn_cell_impl.BasicRNNCell(10)
batch_size = 5
input_size = 20
timelen = 15
inputs = constant_op.constant(
np.random.randn(timelen, batch_size, input_size))
with variable_scope.variable_scope("basic",
initializer=initializer):
unpacked_inputs = array_ops.unstack(inputs)
outputs, state = core_rnn.static_rnn(cell,
unpacked_inputs,
dtype=dtypes.float64)
packed_outputs = array_ops.stack(outputs)
basic_vars = [
v for v in variables.trainable_variables()
if v.name.startswith("basic/")
]
sess.run([variables.global_variables_initializer()])
basic_outputs, basic_state = sess.run([packed_outputs, state])
basic_grads = sess.run(
gradients_impl.gradients(packed_outputs, inputs))
basic_wgrads = sess.run(
gradients_impl.gradients(packed_outputs, basic_vars))
with variable_scope.variable_scope("fused_static",
initializer=initializer):
fused_cell = fused_rnn_cell.FusedRNNCellAdaptor(
core_rnn_cell_impl.BasicRNNCell(10))
outputs, state = fused_cell(inputs, dtype=dtypes.float64)
fused_static_vars = [
v for v in variables.trainable_variables()
if v.name.startswith("fused_static/")
]
sess.run([variables.global_variables_initializer()])
fused_static_outputs, fused_static_state = sess.run(
[outputs, state])
fused_static_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
fused_static_wgrads = sess.run(
gradients_impl.gradients(outputs, fused_static_vars))
self.assertAllClose(basic_outputs, fused_static_outputs)
self.assertAllClose(basic_state, fused_static_state)
self.assertAllClose(basic_grads, fused_static_grads)
for basic, fused in zip(basic_wgrads, fused_static_wgrads):
self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2)
with variable_scope.variable_scope("fused_dynamic",
initializer=initializer):
fused_cell = fused_rnn_cell.FusedRNNCellAdaptor(
core_rnn_cell_impl.BasicRNNCell(10), use_dynamic_rnn=True)
outputs, state = fused_cell(inputs, dtype=dtypes.float64)
fused_dynamic_vars = [
v for v in variables.trainable_variables()
if v.name.startswith("fused_dynamic/")
]
sess.run([variables.global_variables_initializer()])
fused_dynamic_outputs, fused_dynamic_state = sess.run(
[outputs, state])
fused_dynamic_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
fused_dynamic_wgrads = sess.run(
gradients_impl.gradients(outputs, fused_dynamic_vars))
self.assertAllClose(basic_outputs, fused_dynamic_outputs)
self.assertAllClose(basic_state, fused_dynamic_state)
self.assertAllClose(basic_grads, fused_dynamic_grads)
for basic, fused in zip(basic_wgrads, fused_dynamic_wgrads):
self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2)
def call(self, inputs, states, training=None):
h_tm1 = states[0] # previous memory
dp_mask = self.get_dropout_mask_for_cell(inputs, training, count=2)
rec_dp_mask = self.get_recurrent_dropout_mask_for_cell(h_tm1,
training,
count=2)
if self.use_bias:
if not self.reset_after:
input_bias, recurrent_bias = self.bias, None
else:
input_bias, recurrent_bias = array_ops.unstack(self.bias)
if self.implementation == 1:
if 0. < self.dropout < 1.:
inputs_z = inputs * dp_mask[0]
# inputs_r = inputs * dp_mask[1]
inputs_h = inputs * dp_mask[1]
else:
inputs_z = inputs
# inputs_r = inputs
inputs_h = inputs
x_z = K.dot(inputs_z, self.kernel[:, :self.units])
# x_r = K.dot(inputs_r, self.kernel[:, self.units:self.units * 2])
x_h = K.dot(inputs_h, self.kernel[:, self.units:])
# print(x_h.shape)
if self.use_bias:
x_z = K.bias_add(x_z, input_bias[:self.units])
x_h = K.bias_add(x_h, input_bias[self.units:self.units * 2])
# x_h = K.bias_add(x_h, input_bias[self.units:])
if 0. < self.recurrent_dropout < 1.:
h_tm1_z = h_tm1 * rec_dp_mask[0]
# h_tm1_r = h_tm1 * rec_dp_mask[1]
h_tm1_h = h_tm1 * rec_dp_mask[1]
else:
h_tm1_z = h_tm1
# h_tm1_r = h_tm1
h_tm1_h = h_tm1
recurrent_z = K.dot(h_tm1_z, self.recurrent_kernel[:, :self.units])
# recurrent_r = K.dot(h_tm1_r,
# self.recurrent_kernel[:, self.units:self.units * 2])
if self.reset_after and self.use_bias:
recurrent_z = K.bias_add(recurrent_z,
recurrent_bias[:self.units])
# recurrent_r = K.bias_add(recurrent_r,
# recurrent_bias[self.units:self.units * 2])
z = self.recurrent_activation(x_z + recurrent_z)
# r = self.recurrent_activation(x_r + recurrent_r)
# reset gate applied after/before matrix multiplication
if self.reset_after:
recurrent_h = K.dot(h_tm1_h,
self.recurrent_kernel[:, self.units:])
if self.use_bias:
recurrent_h = K.bias_add(recurrent_h,
recurrent_bias[self.units:])
recurrent_h = z * recurrent_h
else:
recurrent_h = K.dot(z * h_tm1_h,
self.recurrent_kernel[:, self.units:])
# recurrent_h = z * (K.dot(h_tm1_h,
# self.recurrent_kernel[:, self.units:]))
hh = self.activation(x_h + recurrent_h)
else:
if 0. < self.dropout < 1.:
inputs *= dp_mask[0]
# inputs projected by all gate matrices at once
matrix_x = K.dot(inputs, self.kernel)
if self.use_bias:
# biases: bias_z_i, bias_r_i, bias_h_i
matrix_x = K.bias_add(matrix_x, input_bias)
x_z = matrix_x[:, :self.units]
# x_r = matrix_x[:, self.units: 2 * self.units]
x_h = matrix_x[:, self.units:]
if 0. < self.recurrent_dropout < 1.:
h_tm1 *= rec_dp_mask[0]
if self.reset_after:
# hidden state projected by all gate matrices at once
matrix_inner = K.dot(h_tm1, self.recurrent_kernel)
if self.use_bias:
matrix_inner = K.bias_add(matrix_inner, recurrent_bias)
else:
# hidden state projected separately for update/reset and new
matrix_inner = K.dot(h_tm1,
self.recurrent_kernel[:, :self.units])
recurrent_z = matrix_inner[:, :self.units]
# recurrent_r = matrix_inner[:, self.units:2 * self.units]
z = self.recurrent_activation(x_z + recurrent_z)
# r = self.recurrent_activation(x_r + recurrent_r)
if self.reset_after:
# recurrent_h = z[:self.units-6] * matrix_inner[:, self.units:2*self.units-6].extend((1-z)[self.units-6:]* matrix_inner[:, 2*self.units-6:2*self.units])
recurrent_h = z * matrix_inner[:, self.units:]
else:
recurrent_h = K.dot(z * h_tm1,
self.recurrent_kernel[:, self.units:])
# recurrent_h = z * (K.dot(h_tm1,
# self.recurrent_kernel[:, self.units:]))
hh = self.activation(x_h + recurrent_h)
# previous and candidate state mixed by update gate
if self.mgu_n:
h = z * h_tm1 + (1 - z) * hh
else:
h = (1 - z) * h_tm1 + z * hh
return h, [h]
def testTimeReversedFusedRNN(self):
with self.test_session() as sess:
initializer = init_ops.random_uniform_initializer(-0.01,
0.01,
seed=19890213)
fw_cell = core_rnn_cell_impl.BasicRNNCell(10)
bw_cell = core_rnn_cell_impl.BasicRNNCell(10)
batch_size = 5
input_size = 20
timelen = 15
inputs = constant_op.constant(
np.random.randn(timelen, batch_size, input_size))
# test bi-directional rnn
with variable_scope.variable_scope("basic",
initializer=initializer):
unpacked_inputs = array_ops.unstack(inputs)
outputs, fw_state, bw_state = core_rnn.static_bidirectional_rnn(
fw_cell, bw_cell, unpacked_inputs, dtype=dtypes.float64)
packed_outputs = array_ops.stack(outputs)
basic_vars = [
v for v in variables.trainable_variables()
if v.name.startswith("basic/")
]
sess.run([variables.global_variables_initializer()])
basic_outputs, basic_fw_state, basic_bw_state = sess.run(
[packed_outputs, fw_state, bw_state])
basic_grads = sess.run(
gradients_impl.gradients(packed_outputs, inputs))
basic_wgrads = sess.run(
gradients_impl.gradients(packed_outputs, basic_vars))
with variable_scope.variable_scope("fused",
initializer=initializer):
fused_cell = fused_rnn_cell.FusedRNNCellAdaptor(
core_rnn_cell_impl.BasicRNNCell(10))
fused_bw_cell = fused_rnn_cell.TimeReversedFusedRNN(
fused_rnn_cell.FusedRNNCellAdaptor(
core_rnn_cell_impl.BasicRNNCell(10)))
fw_outputs, fw_state = fused_cell(inputs,
dtype=dtypes.float64,
scope="fw")
bw_outputs, bw_state = fused_bw_cell(inputs,
dtype=dtypes.float64,
scope="bw")
outputs = array_ops.concat([fw_outputs, bw_outputs], 2)
fused_vars = [
v for v in variables.trainable_variables()
if v.name.startswith("fused/")
]
sess.run([variables.global_variables_initializer()])
fused_outputs, fused_fw_state, fused_bw_state = sess.run(
[outputs, fw_state, bw_state])
fused_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
fused_wgrads = sess.run(
gradients_impl.gradients(outputs, fused_vars))
self.assertAllClose(basic_outputs, fused_outputs)
self.assertAllClose(basic_fw_state, fused_fw_state)
self.assertAllClose(basic_bw_state, fused_bw_state)
self.assertAllClose(basic_grads, fused_grads)
for basic, fused in zip(basic_wgrads, fused_wgrads):
self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2)
def loop_fn(i):
x_i = array_ops.gather(x, i)
return array_ops.unstack(x_i, 4,
axis=-1), array_ops.unstack(x_i,
3,
axis=1)
def __call__(self, inputs, state, scope=None):
'''
To support parallelization, multiple sequences from a batch are
passed to RNNCell objects. This method takes in a tensor of shape
[BxL1], (where B = batch size and L1 = size of an individual element of
an input sequence) and produces an output [BxL2].
The __call__ method is called for each element in the sequence of
length T.
Arguments:
inputs - A tensor of input values [BxL1] from some piece of the
sequence being fed into the entire recurrent unit.
state - A tuple of tensor values consisting of the recurrent state
from the previous timestep. The recurrent state consists of:
1. The memory matrix
2. The read vector from the previous timestep
3. The write vector from the previous timestep
Outputs:
reads - Tensor of values that were read from the memory matrix.
state_tuple - Tuple containing tensors relating to the recurrent
state.
'''
M = self.M
N = self.N
S = self.shift_range
with vs.variable_scope(scope or 'ntm_cell'):
#write_head, read_head = array_ops.split(inputs, [3*M+S+3, M+S+3],
# axis=1)
mem_prev = array_ops.stack(state[0:-2], axis=1)
w_read_prev = state[-2]
w_write_prev = state[-1]
write_pieces, read_pieces = self.head_pieces(inputs, (N, M), S)
w_write = generate_address(write_pieces[0:5], w_write_prev,
mem_prev, N, S)
w_read = generate_address(read_pieces, w_read_prev, mem_prev, N, S)
erase = array_ops.expand_dims(write_pieces[-1], axis=2)
add = array_ops.expand_dims(write_pieces[-2], axis=2)
w_write_ = array_ops.expand_dims(w_write, axis=2)
erase_box = \
math_ops.matmul(w_write_,
array_ops.transpose(erase, perm=[0, 2, 1]))
add_box = \
math_ops.matmul(w_write_,
array_ops.transpose(add, perm=[0, 2, 1]))
mem_new = mem_prev * (1. - erase_box) + add_box
read_w_ = array_ops.expand_dims(w_read, axis=1)
reads = array_ops.squeeze(math_ops.matmul(read_w_, mem_new))
state_tuple = tuple(array_ops.unstack(mem_new, axis=1)) + \
(w_read, w_write)
return reads, state_tuple
def scatter(self, indices, value, name=None):
"""See TensorArray."""
del name # not meaningful when executing eagerly.
for index, val in zip(indices.numpy(), array_ops.unstack(value)):
self._write(index, val) # pylint: disable=protected-access
return self.parent()
def testConcat(self):
c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32)
l0 = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape())
l1 = list_ops.tensor_list_from_tensor([-1.0],
element_shape=scalar_shape())
l_batch_0 = array_ops.stack([l0, l1])
l_batch_1 = array_ops.stack([l1, l0])
l_concat_01 = list_ops.tensor_list_concat_lists(
l_batch_0, l_batch_1, element_dtype=dtypes.float32)
l_concat_10 = list_ops.tensor_list_concat_lists(
l_batch_1, l_batch_0, element_dtype=dtypes.float32)
l_concat_00 = list_ops.tensor_list_concat_lists(
l_batch_0, l_batch_0, element_dtype=dtypes.float32)
l_concat_11 = list_ops.tensor_list_concat_lists(
l_batch_1, l_batch_1, element_dtype=dtypes.float32)
expected_00 = [[1.0, 2.0, 1.0, 2.0], [-1.0, -1.0]]
expected_01 = [[1.0, 2.0, -1.0], [-1.0, 1.0, 2.0]]
expected_10 = [[-1.0, 1.0, 2.0], [1.0, 2.0, -1.0]]
expected_11 = [[-1.0, -1.0], [1.0, 2.0, 1.0, 2.0]]
for i, (concat, expected) in enumerate(
zip([l_concat_00, l_concat_01, l_concat_10, l_concat_11],
[expected_00, expected_01, expected_10, expected_11])):
splitted = array_ops.unstack(concat)
splitted_stacked_ret = self.evaluate(
(list_ops.tensor_list_stack(splitted[0], dtypes.float32),
list_ops.tensor_list_stack(splitted[1], dtypes.float32)))
print("Test concat %d: %s, %s, %s, %s" %
(i, expected[0], splitted_stacked_ret[0], expected[1],
splitted_stacked_ret[1]))
self.assertAllClose(expected[0], splitted_stacked_ret[0])
self.assertAllClose(expected[1], splitted_stacked_ret[1])
# Concatenating mismatched shapes fails.
with self.assertRaises((errors.InvalidArgumentError, ValueError)):
self.evaluate(
list_ops.tensor_list_concat_lists(
l_batch_0,
list_ops.empty_tensor_list(scalar_shape(), dtypes.float32),
element_dtype=dtypes.float32))
with self.assertRaisesRegexp(
errors.InvalidArgumentError,
"element shapes are not identical at index 0"):
l_batch_of_vec_tls = array_ops.stack(
[list_ops.tensor_list_from_tensor([[1.0]], element_shape=[1])
] * 2)
self.evaluate(
list_ops.tensor_list_concat_lists(
l_batch_0,
l_batch_of_vec_tls,
element_dtype=dtypes.float32))
with self.assertRaisesRegexp(errors.InvalidArgumentError,
r"input_b\[0\].dtype != element_dtype."):
l_batch_of_int_tls = array_ops.stack([
list_ops.tensor_list_from_tensor([1],
element_shape=scalar_shape())
] * 2)
self.evaluate(
list_ops.tensor_list_concat_lists(
l_batch_0,
l_batch_of_int_tls,
element_dtype=dtypes.float32))
def frechet_classifier_distance(real_images,
generated_images,
classifier_fn,
num_batches=1):
"""Classifier distance for evaluating a generative model.
This is based on the Frechet Inception distance, but for an arbitrary
classifier.
This technique is described in detail in https://arxiv.org/abs/1706.08500.
Given two Gaussian distribution with means m and m_w and covariance matrices
C and C_w, this function calcuates
|m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2))
which captures how different the distributions of real images and generated
images (or more accurately, their visual features) are. Note that unlike the
Inception score, this is a true distance and utilizes information about real
world images.
Note that when computed using sample means and sample covariance matrices,
Frechet distance is biased. It is more biased for small sample sizes. (e.g.
even if the two distributions are the same, for a small sample size, the
expected Frechet distance is large). It is important to use the same
sample size to compute frechet classifier distance when comparing two
generative models.
Args:
real_images: Real images to use to compute Frechet Inception distance.
generated_images: Generated images to use to compute Frechet Inception
distance.
classifier_fn: A function that takes images and produces activations
based on a classifier.
num_batches: Number of batches to split images in to in order to
efficiently run them through the classifier network.
Returns:
The Frechet Inception distance. A floating-point scalar.
"""
real_images_list = array_ops.split(real_images,
num_or_size_splits=num_batches)
generated_images_list = array_ops.split(generated_images,
num_or_size_splits=num_batches)
imgs = array_ops.stack(real_images_list + generated_images_list)
# Compute the activations using the memory-efficient `map_fn`.
activations = functional_ops.map_fn(fn=classifier_fn,
elems=imgs,
parallel_iterations=1,
back_prop=False,
swap_memory=True,
name='RunClassifier')
# Split the activations by the real and generated images.
real_a, gen_a = array_ops.split(activations, [num_batches, num_batches], 0)
# Ensure the activations have the right shapes.
real_a = array_ops.concat(array_ops.unstack(real_a), 0)
gen_a = array_ops.concat(array_ops.unstack(gen_a), 0)
real_a.shape.assert_has_rank(2)
gen_a.shape.assert_has_rank(2)
# Compute mean and covariance matrices of activations.
m = math_ops.reduce_mean(real_a, 0)
m_v = math_ops.reduce_mean(gen_a, 0)
num_examples = math_ops.to_float(array_ops.shape(real_a)[0])
# sigma = (1 / (n - 1)) * (X - mu) (X - mu)^T
sigma = math_ops.matmul(real_a - m, real_a - m,
transpose_a=True) / (num_examples - 1)
sigma_v = math_ops.matmul(gen_a - m_v, gen_a - m_v,
transpose_a=True) / (num_examples - 1)
# Find the Tr(sqrt(sigma sigma_v)) component of FID
sqrt_trace_component = trace_sqrt_product(sigma, sigma_v)
# Compute the two components of FID.
# First the covariance component.
# Here, note that trace(A + B) = trace(A) + trace(B)
trace = math_ops.trace(sigma + sigma_v) - 2.0 * sqrt_trace_component
# Next the distance between means.
mean = math_ops.square(linalg_ops.norm(m - m_v)) # This uses the L2 norm.
fid = trace + mean
return fid
def testZeroLengthDim(self):
with self.test_session():
x = array_ops.zeros(shape=(0, 1, 2))
y = array_ops.unstack(x, axis=1)[0].eval()
self.assertEqual(y.shape, (0, 2))
def _block_lstm(seq_len_max,
x,
w,
b,
cs_prev=None,
h_prev=None,
wci=None,
wcf=None,
wco=None,
forget_bias=None,
cell_clip=None,
use_peephole=None,
name=None):
r"""TODO(williamchan): add doc.
Args:
seq_len_max: A `Tensor` of type `int64`.
x: A list of at least 1 `Tensor` objects of the same type in: `float32`.
w: A `Tensor`. Must have the same type as `x`.
b: A `Tensor`. Must have the same type as `x`.
cs_prev: A `Tensor`. Must have the same type as `x`.
h_prev: A `Tensor`. Must have the same type as `x`.
wci: A `Tensor`. Must have the same type as `x`.
wcf: A `Tensor`. Must have the same type as `x`.
wco: A `Tensor`. Must have the same type as `x`.
forget_bias: An optional `float`. Defaults to `1`.
cell_clip: An optional `float`. Defaults to `-1` (no clipping).
use_peephole: An optional `bool`. Defaults to `False`.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (i, cs, f, o, ci, co, h).
i: A list with the same number of `Tensor` objects as `x` of `Tensor`
objects of the same type as x.
cs: A list with the same number of `Tensor` objects as `x` of `Tensor`
objects of the same type as x.
f: A list with the same number of `Tensor` objects as `x` of `Tensor`
objects of the same type as x.
o: A list with the same number of `Tensor` objects as `x` of `Tensor`
objects of the same type as x.
ci: A list with the same number of `Tensor` objects as `x` of `Tensor`
objects of the same type as x.
co: A list with the same number of `Tensor` objects as `x` of `Tensor`
objects of the same type as x.
h: A list with the same number of `Tensor` objects as `x` of `Tensor`
objects of the same type as x.
Raises:
ValueError: If `b` does not have a valid shape.
"""
batch_size = x[0].get_shape().with_rank(2)[0].value
cell_size4 = b.get_shape().with_rank(1)[0].value
if cell_size4 is None:
raise ValueError("`b` shape must not be None.")
cell_size = cell_size4 / 4
zero_state = None
if cs_prev is None or h_prev is None:
zero_state = array_ops.constant(0,
dtype=dtypes.float32,
shape=[batch_size, cell_size])
if cs_prev is None:
cs_prev = zero_state
if h_prev is None:
h_prev = zero_state
if wci is None:
wci = array_ops.constant(0, dtype=dtypes.float32, shape=[cell_size])
wcf = wci
wco = wci
# pylint: disable=protected-access
i, cs, f, o, ci, co, h = gen_lstm_ops.block_lstm(
seq_len_max=seq_len_max,
x=array_ops.stack(x),
cs_prev=cs_prev,
h_prev=h_prev,
w=w,
wci=wci,
wcf=wcf,
wco=wco,
b=b,
forget_bias=forget_bias,
cell_clip=cell_clip if cell_clip is not None else -1,
name=name,
use_peephole=use_peephole)
return array_ops.unstack(i), array_ops.unstack(cs), array_ops.unstack(
f), array_ops.unstack(o), array_ops.unstack(ci), array_ops.unstack(
co), array_ops.unstack(h)
def testAxisOutOfNegativeRange(self):
a = constant_op.constant([[1, 2, 3], [4, 5, 6]], name='a')
with self.assertRaisesRegexp(ValueError,
r'axis = -3 not in \[-2, 2\)'):
array_ops.unstack(a, axis=-3)
def standard_gru(inputs, init_h, kernel, recurrent_kernel, bias, activation,
recurrent_activation, time_major):
"""GRU with standard kernel implementation.
This implementation can be run on all types of hardware.
This implementation lifts out all the layer weights and make them function
parameters. It has same number of tensor input params as the CuDNN
counterpart. The RNN step logic has been simplified, eg dropout and mask is
removed since CuDNN implementation does not support that.
Arguments:
inputs: input tensor of GRU layer.
init_h: initial state tensor for the cell output.
kernel: weights for cell kernel.
recurrent_kernel: weights for cell recurrent kernel.
bias: weights for cell kernel bias and recurrent bias. The bias contains the
combined input_bias and recurrent_bias.
activation: Activation function to use for output.
recurrent_activation: Activation function to use for hidden recurrent state.
time_major: boolean, whether the inputs are in the format of
[time, batch, feature] or [batch, time, feature].
Returns:
last_output: output tensor for the last timestep, which has shape
[batch, units].
outputs: output tensor for all timesteps, which has shape
[batch, time, units].
state_0: the cell output, which has same shape as init_h.
runtime: constant string tensor which indicate real runtime hardware. This
value is for testing purpose and should be used by user.
"""
input_shape = K.int_shape(inputs)
timesteps = input_shape[0] if time_major else input_shape[1]
input_bias, recurrent_bias = array_ops.unstack(bias)
def step(cell_inputs, cell_states):
"""Step function that will be used by Keras RNN backend."""
h_tm1 = cell_states[0]
# inputs projected by all gate matrices at once
matrix_x = K.dot(cell_inputs, kernel)
matrix_x = K.bias_add(matrix_x, input_bias)
x_z, x_r, x_h = array_ops.split(matrix_x, 3, axis=1)
# hidden state projected by all gate matrices at once
matrix_inner = K.dot(h_tm1, recurrent_kernel)
matrix_inner = K.bias_add(matrix_inner, recurrent_bias)
recurrent_z, recurrent_r, recurrent_h = array_ops.split(matrix_inner,
3,
axis=1)
z = recurrent_activation(x_z + recurrent_z)
r = recurrent_activation(x_r + recurrent_r)
hh = activation(x_h + r * recurrent_h)
# previous and candidate state mixed by update gate
h = z * h_tm1 + (1 - z) * hh
return h, [h]
last_output, outputs, new_states = K.rnn(step,
inputs, [init_h],
constants=None,
unroll=False,
time_major=time_major,
input_length=timesteps)
return last_output, outputs, new_states[0], _runtime('cpu')
def add_image_comparison_summaries(gan_model,
num_comparisons=2,
display_diffs=False):
"""Adds image summaries to compare triplets of images.
The first image is the generator input, the second is the generator output,
and the third is the real data. This style of comparison is useful for
image translation problems, where the generator input is a corrupted image,
the generator output is the reconstruction, and the real data is the target.
Args:
gan_model: A GANModel tuple.
num_comparisons: The number of image triplets to display.
display_diffs: Also display the difference between generated and target.
Raises:
ValueError: If real data, generated data, and generator inputs aren't
images.
ValueError: If the generator input, real, and generated data aren't all the
same size.
"""
if isinstance(gan_model, namedtuples.CycleGANModel):
saved_params = locals()
saved_params.pop('gan_model', None)
with ops.name_scope('cyclegan_x2y_image_comparison_summaries'):
add_image_comparison_summaries(gan_model.model_x2y, **saved_params)
with ops.name_scope('cyclegan_y2x_image_comparison_summaries'):
add_image_comparison_summaries(gan_model.model_y2x, **saved_params)
return
_assert_is_image(gan_model.generator_inputs)
_assert_is_image(gan_model.generated_data)
_assert_is_image(gan_model.real_data)
gan_model.generated_data.shape.assert_is_compatible_with(
gan_model.generator_inputs.shape)
gan_model.real_data.shape.assert_is_compatible_with(
gan_model.generated_data.shape)
image_list = []
image_list.extend(
array_ops.unstack(gan_model.generator_inputs[:num_comparisons]))
image_list.extend(
array_ops.unstack(gan_model.generated_data[:num_comparisons]))
image_list.extend(array_ops.unstack(gan_model.real_data[:num_comparisons]))
if display_diffs:
generated_list = array_ops.unstack(
gan_model.generated_data[:num_comparisons])
real_list = array_ops.unstack(gan_model.real_data[:num_comparisons])
diffs = [
math_ops.abs(
math_ops.to_float(generated) - math_ops.to_float(real))
for generated, real in zip(generated_list, real_list)
]
image_list.extend(diffs)
# Reshape image and display.
summary.image('image_comparison',
eval_utils.image_reshaper(image_list,
num_cols=num_comparisons),
max_outputs=1)
