def test_binary_cwise_ops(self):
logical_ops = [
math_ops.logical_and, math_ops.logical_or, math_ops.logical_xor
]
bool_ops = [
math_ops.less, math_ops.less_equal, math_ops.greater,
math_ops.greater_equal, math_ops.equal, math_ops.not_equal
]
float_ops = [
math_ops.add, math_ops.subtract, math_ops.multiply, math_ops.divide,
math_ops.maximum, math_ops.minimum
]
for op in logical_ops + bool_ops + float_ops:
x = random_ops.random_uniform([7, 3, 5])
y = random_ops.random_uniform([3, 5])
if op in logical_ops:
x = x > 0
y = y > 0
# pylint: disable=cell-var-from-loop
def loop_fn(i):
x1 = array_ops.gather(x, i)
y1 = array_ops.gather(y, i)
return op(x, y), op(x1, y), op(x, y1), op(x1, y1), op(x1, x1)
# pylint: enable=cell-var-from-loop
dtype = dtypes.float32 if op in float_ops else dtypes.bool
self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtype] * 5)
def testLargeCase(self):
shape = [32, 512, 256, 1]
predictions = random_ops.random_uniform(
shape, 0.0, 1.0, dtype=dtypes_lib.float32)
labels = math_ops.greater(random_ops.random_uniform(shape, 0.0, 1.0), 0.5)
result, update_op = metric_ops.precision_recall_at_equal_thresholds(
labels=labels, predictions=predictions, num_thresholds=201)
# Run many updates, enough to cause highly inaccurate values if the
# code used float32 for accumulation.
num_updates = 71
with self.test_session() as sess:
sess.run(variables.local_variables_initializer())
for _ in xrange(num_updates):
sess.run(update_op)
prdata = sess.run(result)
# Since we use random values, we won't know the tp/fp/tn/fn values, but
# tp and fp at threshold 0 should be the total number of positive and
# negative labels, hence their sum should be total number of pixels.
expected_value = 1.0 * np.product(shape) * num_updates
got_value = prdata.tp[0] + prdata.fp[0]
# They should be at least within 1.
self.assertNear(got_value, expected_value, 1.0)
def _testKLPenaltyBoth(self, layer_class):
def _make_normal(dtype, *args): # pylint: disable=unused-argument
return normal_lib.Normal(
loc=dtype.as_numpy_dtype(0.), scale=dtype.as_numpy_dtype(1.))
with self.test_session():
layer = layer_class(
filters=2,
kernel_size=3,
bias_posterior_fn=prob_layers_util.default_mean_field_normal_fn(),
bias_prior_fn=_make_normal)
if layer_class == prob_layers_lib.Conv1DVariational:
inputs = random_ops.random_uniform([2, 3, 1], seed=1)
elif layer_class == prob_layers_lib.Conv2DVariational:
inputs = random_ops.random_uniform([2, 3, 3, 1], seed=1)
elif layer_class == prob_layers_lib.Conv3DVariational:
inputs = random_ops.random_uniform([2, 3, 3, 3, 1], seed=1)
# No keys.
losses = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)
self.assertEqual(len(losses), 0)
self.assertListEqual(layer.losses, losses)
_ = layer(inputs)
# Yes keys.
losses = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)
self.assertEqual(len(losses), 2)
self.assertListEqual(layer.losses, losses)
def testGradientFloat16(self):
with self.test_session(use_gpu=True) as sess:
# Randomly construct a 1D shape from [1, 40)
shape = random_ops.random_uniform(
[1], minval=1, maxval=40, dtype=dtypes.int32)
# Construct the fp32 graph and its gradient.
x = random_ops.random_uniform(shape, minval=-1, maxval=1, name="x")
y1 = nn_ops.relu(x, name="relu_fp32")
l1 = nn_ops.l2_loss(y1)
dx_f32 = gradients_impl.gradients(l1, x)
# Construct the fp16 graph and its gradient.
# It starts with the same x, in fp32. But before it reaches Relu, it is
# cast into fp16. So during backprop, the gradient computation is in fp16.
x2 = math_ops.cast(x, dtype=dtypes.float16, name="cast")
y2 = nn_ops.relu(x2, name="relu_fp16")
l2 = nn_ops.l2_loss(y2)
dx_f16 = gradients_impl.gradients(l2, x)
# Repeat the experiment for 100 times. All tensor shapes and its tensor
# values are randomly generated for each run.
for _ in xrange(100):
dx_f32_v, dx_f16_v = sess.run([dx_f32, dx_f16])
self.assertAllClose(dx_f32_v, dx_f16_v, atol=3e-4)
def benchmarkBatchSelect(self):
for (m, n, use_gpu) in itertools.product([1000, 10000, 100000],
[10, 100, 1000], [False, True]):
name = "m_%d_n_%d_use_gpu_%s" % (m, n, use_gpu)
device = "/%s:0" % ("gpu" if use_gpu else "cpu")
with ops.Graph().as_default():
with ops.device(device):
x_gen = random_ops.random_uniform([m, n], dtype=dtypes.float32)
y_gen = random_ops.random_uniform([m, n], dtype=dtypes.float32)
c_gen = random_ops.random_uniform([m], dtype=dtypes.float32) <= 0.5
x = resource_variable_ops.ResourceVariable(x_gen)
y = resource_variable_ops.ResourceVariable(y_gen)
c = resource_variable_ops.ResourceVariable(c_gen)
op = array_ops.where(c, x, y)
with session.Session(config=benchmark.benchmark_config()) as sess:
x.initializer.run()
y.initializer.run()
c.initializer.run()
r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
# approximate size of output: m*n*2 floats for each axis.
gb_processed = m * n * 8 / 1.0e9
throughput = gb_processed / r["wall_time"]
print("Benchmark: %s \t wall_time: %0.03g s \t "
"Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
sys.stdout.flush()
def testExtractPointwiseConv2dPatches(self):
with ops.Graph().as_default(), self.test_session() as sess:
batch_size = 10
image_height = image_width = 8
in_channels = out_channels = 3
kernel_height = kernel_width = 1
strides = [1, 1, 1, 1]
padding = 'VALID'
images = random_ops.random_uniform(
[batch_size, image_height, image_width, in_channels], seed=0)
kernel_shape = [kernel_height, kernel_width, in_channels, out_channels]
kernel = random_ops.random_uniform(kernel_shape, seed=1)
# Ensure shape matches expectation.
patches = utils.extract_pointwise_conv2d_patches(images, kernel_shape)
self.assertEqual(patches.shape.as_list(), [
batch_size, image_height, image_width, kernel_height, kernel_width,
in_channels
])
# Ensure extract...patches() + matmul() and conv2d() implementation
# give the same answer.
outputs = nn_ops.conv2d(images, kernel, strides, padding)
patches_flat = array_ops.reshape(
patches, [-1, kernel_height * kernel_width * in_channels])
kernel_flat = array_ops.reshape(kernel, [-1, out_channels])
outputs_flat = math_ops.matmul(patches_flat, kernel_flat)
outputs_, outputs_flat_ = sess.run([outputs, outputs_flat])
self.assertAllClose(outputs_.flatten(), outputs_flat_.flatten())
def testHasBias(self):
with tf_ops.Graph().as_default():
inputs = random_ops.random_uniform(
[self.batch_size, self.height, self.width, self.in_channels])
outputs_grads = [
random_ops.random_uniform([
self.batch_size, self.height // self.strides[1],
self.width // self.strides[2], self.out_channels
]) for _ in range(3)
]
factor = ff.ConvDiagonalFactor(
inputs,
outputs_grads,
self.kernel_shape,
self.strides,
self.padding,
data_format=self.data_format,
has_bias=True)
factor.instantiate_cov_variables()
# Ensure shape accounts for bias.
self.assertEqual([
self.kernel_height * self.kernel_width * self.in_channels + 1,
self.out_channels
],
factor.get_cov_var().shape.as_list())
# Ensure update op doesn't crash.
cov_update_op = factor.make_covariance_update_op(0.0)
with self.test_session() as sess:
sess.run(tf_variables.global_variables_initializer())
sess.run(cov_update_op)
def testInit(self):
with tf_ops.Graph().as_default():
inputs = random_ops.random_uniform(
[self.batch_size, self.height, self.width, self.in_channels])
outputs_grads = [
random_ops.random_uniform([
self.batch_size, self.height // self.strides[1],
self.width // self.strides[2], self.out_channels
]) for _ in range(3)
]
factor = ff.ConvDiagonalFactor(
inputs,
outputs_grads,
self.kernel_shape,
self.strides,
self.padding,
data_format=self.data_format)
factor.instantiate_cov_variables()
# Ensure covariance matrix's shape makes sense.
self.assertEqual([
self.kernel_height * self.kernel_width * self.in_channels,
self.out_channels
],
factor.get_cov_var().shape.as_list())
def model_fn():
"""Mnist model with synthetic input."""
data_format = 'channels_last'
input_shape = [28, 28, 1]
l = keras.layers
max_pool = l.MaxPooling2D((2, 2), (2, 2),
padding='same',
data_format=data_format)
model = keras.Sequential([
l.Reshape(target_shape=input_shape, input_shape=(28 * 28,)),
l.Conv2D(
32,
5,
padding='same',
data_format=data_format,
activation=nn.relu), max_pool,
l.Conv2D(
64,
5,
padding='same',
data_format=data_format,
activation=nn.relu), max_pool,
l.Flatten(),
l.Dense(1024, activation=nn.relu),
l.Dropout(0.4),
l.Dense(10)
])
image = random_ops.random_uniform([2, 28, 28])
label = random_ops.random_uniform([2, 1], maxval=10, dtype=dtypes.int32)
logits = model(image, training=True)
loss = losses.sparse_softmax_cross_entropy(labels=label, logits=logits)
optimizer = adam.AdamOptimizer(learning_rate=1e-4)
train_op = optimizer.minimize(loss,
training_util.get_or_create_global_step())
return train_op
def input_fn():
start = random_ops.random_uniform(
(), minval=0, maxval=sequence_length, dtype=dtypes.int32, seed=seed)
# Concatenate lyrics_list so inputs and labels wrap when start > 0.
lyrics_list_concat = lyrics_list + lyrics_list
inputs_dense = array_ops.slice(lyrics_list_concat, [start],
[sequence_length])
indices = array_ops.constant(
[[i, 0] for i in range(sequence_length)], dtype=dtypes.int64)
dense_shape = [sequence_length, 1]
inputs = sparse_tensor.SparseTensor(
indices=indices, values=inputs_dense, dense_shape=dense_shape)
table = lookup.string_to_index_table_from_tensor(
mapping=list(vocab), default_value=-1, name='lookup')
labels = table.lookup(
array_ops.slice(lyrics_list_concat, [start + 1], [sequence_length]))
input_key = string_ops.string_join([
'key_', string_ops.as_string(
random_ops.random_uniform(
(),
minval=0,
maxval=10000000,
dtype=dtypes.int32,
seed=seed))
])
return {'lyrics': inputs, input_key_column_name: input_key}, labels
def test_while_jacobian(self):
x = random_ops.random_uniform([1, 3])
y = random_ops.random_uniform([3, 3])
# out = x @ y @ y @ y @ y, where @ is matmul operator.
_, out = control_flow_ops.while_loop(
lambda i, _: i < 4, lambda i, out: (i + 1, math_ops.matmul(out, y)),
[0, x])
def loop_fn(i):
out_i = array_ops.gather(out, i, axis=1)
return array_ops.reshape(gradient_ops.gradients(out_i, x)[0], [-1])
out = pfor_control_flow_ops.pfor(loop_fn, iters=3)
# The above code does not work with tf.while_loop instead of pfor. So we
# manually compute the expected output here.
# Note that gradient of output w.r.t is (y @ y @ y @ y)^T.
expected_output = y
for _ in range(3):
expected_output = math_ops.matmul(expected_output, y)
expected_output = array_ops.transpose(expected_output, [1, 0])
with session.Session() as sess:
out, expected = sess.run([out, expected_output])
self.assertAllClose(expected, out)
def _test_unary_cwise_ops(self, ops, is_complex):
for op in ops:
with backprop.GradientTape(persistent=True) as g:
x = random_ops.random_uniform([3, 5])
g.watch(x)
if is_complex:
y = random_ops.random_uniform([3, 5])
g.watch(y)
x = math_ops.complex(x, y)
# pylint: disable=cell-var-from-loop
output_dtypes = []
def loop_fn(i):
with g:
x1 = array_ops.gather(x, i)
y1 = op(x1)
outputs = [op(x), y1]
if y1.dtype == dtypes.float32:
loss = math_ops.reduce_sum(y1 * y1)
else:
loss = None
if loss is not None:
grad = g.gradient(loss, x1)
if grad is not None:
outputs.append(grad)
del output_dtypes[:]
output_dtypes.extend([t.dtype for t in outputs])
return outputs
# pylint: enable=cell-var-from-loop
self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
def testImportGraphWithFunctionTwice(self):
g = ops.Graph()
with g.as_default():
@function.Defun()
def Add2(x, y):
return math_ops.add(x, y)
x = array_ops.placeholder(dtype=dtypes.float32, name="x")
y = array_ops.placeholder(dtype=dtypes.float32, name="y")
_ = Add2(x, y, name="z") # pylint: disable=unexpected-keyword-arg
gdef = g.as_graph_def()
x = random_ops.random_uniform(dtype=dtypes.float32, shape=())
y = random_ops.random_uniform(dtype=dtypes.float32, shape=())
input_map = {"x:0": x, "y:0": y}
with ops.name_scope("first"):
z1 = importer.import_graph_def(gdef, return_elements=["z:0"],
input_map=input_map)[0]
with ops.name_scope("second"):
z2 = importer.import_graph_def(gdef, return_elements=["z:0"],
input_map=input_map)[0]
with self.test_session() as sess:
z1_val, z2_val = sess.run((z1, z2))
self.assertAllEqual(z1_val, z2_val)
def random_uniform(shape, minval=None, maxval=None, dtype=dtypes.float32, seed=None):
"""Tensor with (possibly complex) Uniform entries.
Samples are distributed like
```
Uniform[minval, maxval], if dtype is real,
X + iY, where X, Y ~ Uniform[minval, maxval], if dtype is complex.
```
Args:
shape: `TensorShape` or Python list. Shape of the returned tensor.
minval: `0-D` `Tensor` giving the minimum values.
maxval: `0-D` `Tensor` giving the maximum values.
dtype: `TensorFlow` `dtype` or Python dtype
seed: Python integer seed for the RNG.
Returns:
`Tensor` with desired shape and dtype.
"""
dtype = dtypes.as_dtype(dtype)
with ops.name_scope("random_uniform"):
samples = random_ops.random_uniform(shape, dtype=dtype.real_dtype, minval=minval, maxval=maxval, seed=seed)
if dtype.is_complex:
if seed is not None:
seed += 12345
more_samples = random_ops.random_uniform(
shape, dtype=dtype.real_dtype, minval=minval, maxval=maxval, seed=seed
)
samples = math_ops.complex(samples, more_samples)
return samples
def testConstraints(self):
# Conv1D
k_constraint = lambda x: x / math_ops.reduce_sum(x)
b_constraint = lambda x: x / math_ops.reduce_max(x)
conv1d = conv_layers.Conv1D(2, 3,
kernel_constraint=k_constraint,
bias_constraint=b_constraint)
inputs = random_ops.random_uniform((5, 3, 5), seed=1)
conv1d(inputs)
self.assertEqual(conv1d.kernel_constraint, k_constraint)
self.assertEqual(conv1d.bias_constraint, b_constraint)
# Conv2D
k_constraint = lambda x: x / math_ops.reduce_sum(x)
b_constraint = lambda x: x / math_ops.reduce_max(x)
conv2d = conv_layers.Conv2D(2, 3,
kernel_constraint=k_constraint,
bias_constraint=b_constraint)
inputs = random_ops.random_uniform((5, 3, 3, 5), seed=1)
conv2d(inputs)
self.assertEqual(conv2d.kernel_constraint, k_constraint)
self.assertEqual(conv2d.bias_constraint, b_constraint)
# Conv3D
k_constraint = lambda x: x / math_ops.reduce_sum(x)
b_constraint = lambda x: x / math_ops.reduce_max(x)
conv3d = conv_layers.Conv3D(2, 3,
kernel_constraint=k_constraint,
bias_constraint=b_constraint)
inputs = random_ops.random_uniform((5, 3, 3, 3, 5), seed=1)
conv3d(inputs)
self.assertEqual(conv3d.kernel_constraint, k_constraint)
self.assertEqual(conv3d.bias_constraint, b_constraint)
def make_relaxed_categorical(batch_shape, num_classes, dtype=dtypes.float32):
logits = random_ops.random_uniform(
list(batch_shape) + [num_classes], -10, 10, dtype=dtype) - 50.
temperatures = random_ops.random_uniform(
list(batch_shape), 0.1, 10, dtype=dtypes.float32)
return relaxed_onehot_categorical.RelaxedOneHotCategorical(
temperatures, logits, dtype=dtype)
def testCustomGrad(self):
def fn(a, b, c):
return core_layers.dense(a, 10, use_bias=False) + math_ops.matmul(b, c)
def grad_fn(inputs, trainable_variables, unused_outputs,
unused_grad_outputs):
grad_inputs = [
array_ops.ones_like(t) * (i + 1.) for i, t in enumerate(inputs)
]
grad_vars = [
array_ops.ones_like(t) * (i + len(inputs) + 1.)
for i, t in enumerate(trainable_variables)
]
return grad_inputs, grad_vars
a = random_ops.random_uniform([11, 6])
b = random_ops.random_uniform([11, 7])
c = random_ops.random_uniform([7, 10])
w = random_ops.random_uniform([6, 10])
out = rev_block_lib._fn_with_custom_grad(grad_fn)(fn)(a, b, c)
loss = math_ops.reduce_mean(out)
grads = gradients_impl.gradients(
loss, [a, b, c, variables.trainable_variables()[0]])
expected_grads = [
array_ops.ones_like(t) * (i + 1.) for i, t in enumerate([a, b, c, w])
]
with self.test_session() as sess:
sess.run(variables.global_variables_initializer())
g_val, eg_val = sess.run([grads, expected_grads])
for g1, g2 in zip(g_val, eg_val):
self.assertAllClose(g1, g2)
def testVirtualCluster(self):
with ops.Graph().as_default() as g:
a = random_ops.random_uniform(shape=())
b = random_ops.random_uniform(shape=())
c = a + b
train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
train_op.append(c)
mg = meta_graph.create_meta_graph_def(graph=g)
grappler_item = item.Item(mg)
device_properties = device_properties_pb2.DeviceProperties(
type='GPU',
frequency=1000,
num_cores=60,
environment={
'architecture': '7'
})
named_device = device_properties_pb2.NamedDevice(
properties=device_properties, name='/GPU:0')
grappler_cluster = cluster.Cluster(devices=[named_device])
op_perfs, run_time, _ = grappler_cluster.MeasureCosts(grappler_item)
self.assertGreater(run_time, 0)
self.assertEqual(len(op_perfs), 15)
estimated_perf = grappler_cluster.EstimatePerformance(named_device)
self.assertEqual(7680.0, estimated_perf)
def test_good_kernel_approximation_multiple_inputs(self, initializer, scale,
exact_kernel_fn):
# Parameters.
input_dim = 5
output_dim = 5000
x_rows = 20
y_rows = 30
random_seed.set_random_seed(1234)
x = random_ops.random_uniform(shape=(x_rows, input_dim), maxval=1.0)
y = random_ops.random_uniform(shape=(y_rows, input_dim), maxval=1.0)
rff_layer = kernel_layers.RandomFourierFeatures(
output_dim=output_dim,
kernel_initializer=initializer,
scale=scale,
name='random_fourier_features')
# The shapes of output_x and output_y are (x_rows, output_dim) and
# (y_rows, output_dim) respectively.
output_x = math.sqrt(2.0 / output_dim) * rff_layer.apply(x)
output_y = math.sqrt(2.0 / output_dim) * rff_layer.apply(y)
approx_kernel_matrix = kernelized_utils.inner_product(output_x, output_y)
exact_kernel_matrix = exact_kernel_fn(x, y)
self._assert_all_close(approx_kernel_matrix, exact_kernel_matrix, atol=0.1)
def build_graph(device, n, m, k, transpose_a, transpose_b, dtype):
"""Build a graph containing a sequence of matmul operations.
Args:
device: String, the device to run on.
n: tensor A's first dimension size.
m: tensor A's second dimension size.
k: tensor B's second dimension size.
transpose_a: boolean value to show if tensor A is transposed.
transpose_b: boolean value to show if tensor B is transposed.
dtype: numpy data type of the input tensor.
Returns:
A matmul operation to run()
"""
with ops.device('%s' % device):
if not transpose_a:
x = variables.VariableV1(random_ops.random_uniform([n, m], dtype=dtype),
use_resource=False)
else:
x = variables.VariableV1(random_ops.random_uniform([m, n], dtype=dtype),
use_resource=False)
if not transpose_b:
y = variables.VariableV1(random_ops.random_uniform([m, k], dtype=dtype),
use_resource=False)
else:
y = variables.VariableV1(random_ops.random_uniform([k, m], dtype=dtype),
use_resource=False)
z = math_ops.matmul(x, y, transpose_a=transpose_a, transpose_b=transpose_b)
return control_flow_ops.group(z)
def testAttentionCellWrapperCorrectResult(self):
num_units = 4
attn_length = 6
batch_size = 2
expected_output = np.array(
[[1.068372, 0.45496, -0.678277, 0.340538],
[1.018088, 0.378983, -0.572179, 0.268591]],
dtype=np.float32)
expected_state = np.array(
[[0.74946702, 0.34681597, 0.26474735, 1.06485605, 0.38465962,
0.11420801, 0.10272158, 0.30925757, 0.63899988, 0.7181077,
0.47534478, 0.33715725, 0.58086717, 0.49446869, 0.7641536,
0.12814975, 0.92231739, 0.89857256, 0.21889746, 0.38442063,
0.53481543, 0.8876909, 0.45823169, 0.5905602, 0.78038228,
0.56501579, 0.03971386, 0.09870267, 0.8074435, 0.66821432,
0.99211812, 0.12295902, 1.14606023, 0.34370938, -0.79251152,
0.51843399],
[0.5179342, 0.48682183, -0.25426468, 0.96810579, 0.28809637,
0.13607743, -0.11446252, 0.26792109, 0.78047138, 0.63460857,
0.49122369, 0.52007174, 0.73000264, 0.66986895, 0.73576689,
0.86301267, 0.87887371, 0.35185754, 0.93417215, 0.64732957,
0.63173044, 0.66627824, 0.53644657, 0.20477486, 0.98458421,
0.38277245, 0.03746676, 0.92510188, 0.57714164, 0.84932971,
0.36127412, 0.12125921, 1.1362772, 0.34361625, -0.78150457,
0.70582712]],
dtype=np.float32)
seed = 12345
random_seed.set_random_seed(seed)
for state_is_tuple in [False, True]:
with session.Session() as sess:
with variable_scope.variable_scope(
"state_is_tuple", reuse=state_is_tuple,
initializer=init_ops.glorot_uniform_initializer()):
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(
num_units, state_is_tuple=state_is_tuple)
cell = rnn_cell.AttentionCellWrapper(
lstm_cell, attn_length, state_is_tuple=state_is_tuple)
zeros1 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 1)
zeros2 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 2)
zeros3 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 3)
attn_state_zeros = random_ops.random_uniform(
(batch_size, attn_length * num_units), 0.0, 1.0, seed=seed + 4)
zero_state = ((zeros1, zeros2), zeros3, attn_state_zeros)
if not state_is_tuple:
zero_state = array_ops.concat([
zero_state[0][0], zero_state[0][1], zero_state[1], zero_state[2]
], 1)
inputs = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 5)
output, state = cell(inputs, zero_state)
if state_is_tuple:
state = array_ops.concat(
[state[0][0], state[0][1], state[1], state[2]], 1)
sess.run(variables.global_variables_initializer())
self.assertAllClose(sess.run(output), expected_output)
self.assertAllClose(sess.run(state), expected_state)
def testNoCSE(self):
shape = [2, 3, 4]
for dtype in dtypes.float16, dtypes.float32, dtypes.int32:
with self.test_session(use_gpu=True):
rnd1 = random_ops.random_uniform(shape, 0, 17, dtype=dtype)
rnd2 = random_ops.random_uniform(shape, 0, 17, dtype=dtype)
diff = (rnd2 - rnd1).eval()
self.assertTrue(np.linalg.norm(diff) > 0.1)
def _testOneSimpleTraining(self, rnn_mode, num_layers, num_units, input_size,
batch_size, seq_length, dir_count, dropout,
tolerance):
# Gradient checking runs two forward ops with almost the same input. Need to
# make sure the drop patterns across the two runs are the same.
old_env_state = os.environ.get("TF_CUDNN_RESET_RND_GEN_STATE", str(False))
os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = str(True)
has_input_c = (rnn_mode == "lstm")
random_seed.set_random_seed(1234)
model = self._CreateModel(rnn_mode, num_layers, num_units, input_size,
dropout)
params_size_t = model.params_size()
input_data = variables.Variable(
random_ops.random_uniform([seq_length, batch_size, input_size]))
input_h = variables.Variable(
random_ops.random_uniform(
[num_layers * dir_count, batch_size, num_units]))
params = variables.Variable(
random_ops.random_uniform([params_size_t]), validate_shape=False)
if has_input_c:
input_c = variables.Variable(
random_ops.random_uniform(
[num_layers * dir_count, batch_size, num_units]))
output, output_h, output_c = model(
input_data=input_data,
input_h=input_h,
input_c=input_c,
params=params)
else:
output, output_h = model(
input_data=input_data, input_h=input_h, params=params)
output_sum = math_ops.reduce_sum(output)
output_h_sum = math_ops.reduce_sum(output_h)
total_sum = output_sum + output_h_sum
if has_input_c:
output_c_sum = math_ops.reduce_sum(output_c)
total_sum += output_c_sum
with self.test_session(use_gpu=True) as sess:
params_size_v = sess.run(params_size_t)
inputs_and_shapes = [
(input_data, [seq_length, batch_size, input_size]),
(input_h, [num_layers * dir_count, batch_size, num_units]),
(params, [params_size_v]),
]
if has_input_c:
inputs_and_shapes.append(
(input_c, [num_layers * dir_count, batch_size, num_units]),)
sess.run(variables.global_variables_initializer())
all_inputs = [entry[0] for entry in inputs_and_shapes]
all_shapes = [entry[1] for entry in inputs_and_shapes]
err = gradient_checker.compute_gradient_error(all_inputs, all_shapes,
total_sum, [1])
self.assertLess(err, tolerance)
os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = old_env_state
def testAttentionCellWrapperCorrectResult(self):
num_units = 4
attn_length = 6
batch_size = 2
expected_output = np.array(
[[0.955392, 0.408507, -0.60122, 0.270718],
[0.903681, 0.331165, -0.500238, 0.224052]],
dtype=np.float32)
expected_state = np.array(
[[0.81331915, 0.32036272, 0.28079176, 1.08888793, 0.41264394,
0.1062041, 0.10444493, 0.32050529, 0.64655536, 0.70794445,
0.51896095, 0.31809306, 0.58086717, 0.49446869, 0.7641536,
0.12814975, 0.92231739, 0.89857256, 0.21889746, 0.38442063,
0.53481543, 0.8876909, 0.45823169, 0.5905602, 0.78038228,
0.56501579, 0.03971386, 0.09870267, 0.8074435, 0.66821432,
0.99211812, 0.12295902, 1.01412082, 0.33123279, -0.71114945,
0.40583119],
[0.59962207, 0.42597458, -0.22491696, 0.98063421, 0.32548007,
0.11623692, -0.10100613, 0.27708149, 0.76956916, 0.6360054,
0.51719815, 0.50458527, 0.73000264, 0.66986895, 0.73576689,
0.86301267, 0.87887371, 0.35185754, 0.93417215, 0.64732957,
0.63173044, 0.66627824, 0.53644657, 0.20477486, 0.98458421,
0.38277245, 0.03746676, 0.92510188, 0.57714164, 0.84932971,
0.36127412, 0.12125921, 0.99780077, 0.31886846, -0.67595094,
0.56531656]],
dtype=np.float32)
seed = 12345
random_seed.set_random_seed(seed)
for state_is_tuple in [False, True]:
with session.Session() as sess:
with variable_scope.variable_scope(
"state_is_tuple", reuse=state_is_tuple):
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(
num_units, state_is_tuple=state_is_tuple)
cell = rnn_cell.AttentionCellWrapper(
lstm_cell, attn_length, state_is_tuple=state_is_tuple)
zeros1 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 1)
zeros2 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 2)
zeros3 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 3)
attn_state_zeros = random_ops.random_uniform(
(batch_size, attn_length * num_units), 0.0, 1.0, seed=seed + 4)
zero_state = ((zeros1, zeros2), zeros3, attn_state_zeros)
if not state_is_tuple:
zero_state = array_ops.concat_v2([
zero_state[0][0], zero_state[0][1], zero_state[1], zero_state[2]
], 1)
inputs = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 5)
output, state = cell(inputs, zero_state)
if state_is_tuple:
state = array_ops.concat_v2(
[state[0][0], state[0][1], state[1], state[2]], 1)
sess.run(variables.global_variables_initializer())
self.assertAllClose(sess.run(output), expected_output)
self.assertAllClose(sess.run(state), expected_state)
def test_pack(self):
x = random_ops.random_uniform([3, 2, 3])
y = random_ops.random_uniform([2, 3])
def loop_fn(i):
x1 = array_ops.gather(x, i)
return array_ops.stack([x1, y], axis=-1)
self._test_loop_fn(loop_fn, 1)
def testErrorOnClosedOverTensor(self):
x = random_ops.random_uniform((4, 8))
y = random_ops.random_uniform((4, 8))
z = x * y
with self.assertRaisesWithPredicateMatch(ValueError, "closes over"):
@rev_block_lib.recompute_grad
def fn_with_capture(a): # pylint: disable=unused-variable
return a * z
def __init__(self):
# used for multiply benchmarks
self._m_2 = random_ops.random_uniform([2])
# used for matmul benchmarks
self._m_2_by_2 = random_ops.random_uniform((2, 2))
self._m_100_by_784 = random_ops.random_uniform((100, 784))
self._num_iters_2_by_2 = 30000
self._num_iters_100_by_784 = 1000
def testFastpathExecute_AddNCorrectResponse(self):
ctx = context.context()
a_2_by_2 = random_ops.random_uniform((2, 2))
b_2_by_2 = random_ops.random_uniform((2, 2))
self.assertAllClose(
math_ops.add_n([a_2_by_2, b_2_by_2]),
pywrap_tensorflow.TFE_Py_FastPathExecute(ctx._handle, ctx.device_name,
"AddN", None, None,
[a_2_by_2, b_2_by_2]))
def testActivation(self):
dense = core_layers.Dense(2, activation=nn_ops.relu, name='dense1')
inputs = random_ops.random_uniform((5, 3), seed=1)
outputs = dense(inputs)
self.assertEqual(outputs.op.name, 'dense1/Relu')
dense = core_layers.Dense(2, name='dense2')
inputs = random_ops.random_uniform((5, 3), seed=1)
outputs = dense(inputs)
self.assertEqual(outputs.op.name, 'dense2/BiasAdd')
def test_tanh_axpy(self):
a = constant_op.constant(3.)
x = random_ops.random_uniform([4, 5])
y = random_ops.random_uniform([6, 5])
n = x.shape[0]
def loop_fn(i):
return math_ops.tanh(a * array_ops.gather(x, i) + array_ops.gather(y, i))
self._test_loop_fn(loop_fn, n)
def worker_train_fn(): x = random_ops.random_uniform((2, 10)) y = random_ops.random_uniform((10, 2)) return math_ops.reduce_mean(math_ops.matmul(x, y))
def normal_function(): x = random_ops.random_uniform((2, 10)) y = random_ops.random_uniform((10, 2)) return math_ops.reduce_mean(math_ops.matmul(x, y))
def training_graph(self,
input_data,
input_labels,
num_trainers=1,
trainer_id=0,
**tree_kwargs):
"""Constructs a TF graph for training a random forest.
Args:
input_data: A tensor or dict of string->Tensor for input data.
input_labels: A tensor or placeholder for labels associated with
input_data.
num_trainers: Number of parallel trainers to split trees among.
trainer_id: Which trainer this instance is.
**tree_kwargs: Keyword arguments passed to each tree's training_graph.
Returns:
The last op in the random forest training graph.
Raises:
NotImplementedError: If trying to use bagging with sparse features.
"""
processed_dense_features, processed_sparse_features, data_spec = (
data_ops.ParseDataTensorOrDict(input_data))
if input_labels is not None:
labels = data_ops.ParseLabelTensorOrDict(input_labels)
data_spec = data_spec or self.get_default_data_spec(input_data)
tree_graphs = []
trees_per_trainer = self.params.num_trees / num_trainers
tree_start = int(trainer_id * trees_per_trainer)
tree_end = int((trainer_id + 1) * trees_per_trainer)
for i in range(tree_start, tree_end):
with ops.device(self.variables.device_dummies[i].device):
seed = self.params.base_random_seed
if seed != 0:
seed += i
# If using bagging, randomly select some of the input.
tree_data = processed_dense_features
tree_labels = labels
if self.params.bagging_fraction < 1.0:
# TODO(gilberth): Support bagging for sparse features.
if processed_sparse_features is not None:
raise NotImplementedError(
'Bagging not supported with sparse features.')
# TODO(thomaswc): This does sampling without replacement. Consider
# also allowing sampling with replacement as an option.
batch_size = array_ops.strided_slice(
array_ops.shape(processed_dense_features), [0], [1])
r = random_ops.random_uniform(batch_size, seed=seed)
mask = math_ops.less(
r,
array_ops.ones_like(r) * self.params.bagging_fraction)
gather_indices = array_ops.squeeze(array_ops.where(mask),
squeeze_dims=[1])
# TODO(thomaswc): Calculate out-of-bag data and labels, and store
# them for use in calculating statistics later.
tree_data = array_ops.gather(processed_dense_features,
gather_indices)
tree_labels = array_ops.gather(labels, gather_indices)
if self.params.bagged_features:
if processed_sparse_features is not None:
raise NotImplementedError(
'Feature bagging not supported with sparse features.'
)
tree_data = self._bag_features(i, tree_data)
tree_graphs.append(self.trees[i].training_graph(
tree_data,
tree_labels,
seed,
data_spec=data_spec,
sparse_features=processed_sparse_features,
**tree_kwargs))
return control_flow_ops.group(*tree_graphs, name='train')
def _testOneSimpleTraining(self, rnn_mode, num_layers, num_units,
input_size, batch_size, seq_length, dir_count,
dropout, tolerance):
# Gradient checking runs two forward ops with almost the same input. Need to
# make sure the drop patterns across the two runs are the same.
old_env_state = os.environ.get("TF_CUDNN_RESET_RND_GEN_STATE",
str(False))
os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = str(True)
has_input_c = (rnn_mode == cudnn_rnn_ops.CUDNN_LSTM)
random_seed.set_random_seed(1234)
model = self._CreateModel(rnn_mode,
num_layers,
num_units,
input_size,
dropout=dropout)
params_size_t = model.params_size()
input_data = variables.Variable(
random_ops.random_uniform([seq_length, batch_size, input_size]))
input_h = variables.Variable(
random_ops.random_uniform(
[num_layers * dir_count, batch_size, num_units]))
params = variables.Variable(random_ops.random_uniform([params_size_t]),
validate_shape=False)
if has_input_c:
input_c = variables.Variable(
random_ops.random_uniform(
[num_layers * dir_count, batch_size, num_units]))
output, output_h, output_c = model(input_data=input_data,
input_h=input_h,
input_c=input_c,
params=params)
else:
output, output_h = model(input_data=input_data,
input_h=input_h,
params=params)
output_sum = math_ops.reduce_sum(output)
output_h_sum = math_ops.reduce_sum(output_h)
total_sum = output_sum + output_h_sum
if has_input_c:
output_c_sum = math_ops.reduce_sum(output_c)
total_sum += output_c_sum
with self.test_session(use_gpu=True) as sess:
params_size_v = sess.run(params_size_t)
inputs_and_shapes = [
(input_data, [seq_length, batch_size, input_size]),
(input_h, [num_layers * dir_count, batch_size, num_units]),
(params, [params_size_v]),
]
if has_input_c:
inputs_and_shapes.append(
(input_c, [num_layers * dir_count, batch_size, num_units
]), )
sess.run(variables.global_variables_initializer())
all_inputs = [entry[0] for entry in inputs_and_shapes]
all_shapes = [entry[1] for entry in inputs_and_shapes]
err = gradient_checker.compute_gradient_error(
all_inputs, all_shapes, total_sum, [1])
self.assertLess(err, tolerance)
os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = old_env_state
def testCreateAveragePooling2D(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) layer = pooling_layers.AveragePooling2D([2, 2], strides=2) output = layer.apply(images) self.assertListEqual(output.get_shape().as_list(), [5, 3, 4, 4])
def testCreateMaxPooling2DIntegerPoolSize(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) layer = pooling_layers.MaxPooling2D(2, strides=2) output = layer.apply(images) self.assertListEqual(output.get_shape().as_list(), [5, 3, 4, 4])
def rejection_sample(tensors, accept_prob_fn, batch_size, queue_threads=1,
enqueue_many=False, prebatch_capacity=16,
prebatch_threads=1, runtime_checks=False, name=None):
"""Stochastically creates batches by rejection sampling.
Each list of non-batched tensors is evaluated by `accept_prob_fn`, to produce
a scalar tensor between 0 and 1. This tensor corresponds to the probability of
being accepted. When `batch_size` tensor groups have been accepted, the batch
queue will return a mini-batch.
Args:
tensors: List of tensors for data. All tensors are either one item or a
batch, according to enqueue_many.
accept_prob_fn: A python lambda that takes a non-batch tensor from each
item in `tensors`, and produces a scalar tensor.
batch_size: Size of batch to be returned.
queue_threads: The number of threads for the queue that will hold the final
batch.
enqueue_many: Bool. If true, interpret input tensors as having a batch
dimension.
prebatch_capacity: Capacity for the large queue that is used to convert
batched tensors to single examples.
prebatch_threads: Number of threads for the large queue that is used to
convert batched tensors to single examples.
runtime_checks: Bool. If true, insert runtime checks on the output of
`accept_prob_fn`. Using `True` might have a performance impact.
name: Optional prefix for ops created by this function.
Raises:
ValueError: enqueue_many is True and labels doesn't have a batch
dimension, or if enqueue_many is False and labels isn't a scalar.
ValueError: enqueue_many is True, and batch dimension on data and labels
don't match.
ValueError: if a zero initial probability class has a nonzero target
probability.
Returns:
A list of tensors of the same length as `tensors`, with batch dimension
`batch_size`.
Example:
# Get tensor for a single data and label example.
data, label = data_provider.Get(['data', 'label'])
# Get stratified batch according to data tensor.
accept_prob_fn = lambda x: (tf.tanh(x[0]) + 1) / 2
data_batch = tf.contrib.training.rejection_sample(
[data, label], accept_prob_fn, 16)
# Run batch through network.
...
"""
with variable_scope.variable_scope(name, 'rejection_sample', tensors):
tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensors)
# Reduce the case of a batched example to that of a batch of a single
# example by taking a batch of size one.
if enqueue_many:
# Validate that batch dimension of the input is consistent.
tensor_list = _verify_data_inputs(tensor_list)
# Make a single queue to hold input examples. Reshape output so examples
# don't have singleton batch dimension.
batched = input_ops.batch(tensor_list,
batch_size=1,
num_threads=prebatch_threads,
capacity=prebatch_capacity,
enqueue_many=True)
tensor_list = [array_ops.squeeze(x, [0]) for x in batched]
# Set up a queue containing batches that have the distribution.
cur_prob = accept_prob_fn(tensor_list)
if runtime_checks:
cur_prob = array_ops.identity(control_flow_ops.with_dependencies(
[check_ops.assert_less_equal(0.0, cur_prob),
check_ops.assert_less_equal(cur_prob, 1.0)],
cur_prob), name='prob_with_checks')
keep_input = random_ops.random_uniform([]) < cur_prob
return _conditional_batch(
tensor_list, keep_input, batch_size, num_threads=queue_threads)
def stratified_sample(tensors, labels, target_probs, batch_size,
init_probs=None, enqueue_many=False, queue_capacity=16,
threads_per_queue=1, name=None):
"""Stochastically creates batches based on per-class probabilities.
This method discards examples. Internally, it creates one queue to amortize
the cost of disk reads, and one queue to hold the properly-proportioned
batch. See `stratified_sample_unknown_dist` for a function that performs
stratified sampling with one queue per class and doesn't require knowing the
class data-distribution ahead of time.
Args:
tensors: List of tensors for data. All tensors are either one item or a
batch, according to enqueue_many.
labels: Tensor for label of data. Label is a single integer or a batch,
depending on enqueue_many. It is not a one-hot vector.
target_probs: Target class proportions in batch. An object whose type has a
registered Tensor conversion function.
batch_size: Size of batch to be returned.
init_probs: Class proportions in the data. An object whose type has a
registered Tensor conversion function, or `None` for estimating the
initial distribution.
enqueue_many: Bool. If true, interpret input tensors as having a batch
dimension.
queue_capacity: Capacity of the large queue that holds input examples.
threads_per_queue: Number of threads for the large queue that holds input
examples and for the final queue with the proper class proportions.
name: Optional prefix for ops created by this function.
Raises:
ValueError: enqueue_many is True and labels doesn't have a batch
dimension, or if enqueue_many is False and labels isn't a scalar.
ValueError: enqueue_many is True, and batch dimension on data and labels
don't match.
ValueError: if probs don't sum to one.
ValueError: if a zero initial probability class has a nonzero target
probability.
TFAssertion: if labels aren't integers in [0, num classes).
Returns:
(data_batch, label_batch), where data_batch is a list of tensors of the same
length as `tensors`
Example:
# Get tensor for a single data and label example.
data, label = data_provider.Get(['data', 'label'])
# Get stratified batch according to per-class probabilities.
target_probs = [...distribution you want...]
[data_batch], labels = tf.contrib.training.stratified_sample(
[data], label, target_probs)
# Run batch through network.
...
"""
with ops.name_scope(name, 'stratified_sample', tensors + [labels]):
tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensors)
labels = ops.convert_to_tensor(labels)
target_probs = ops.convert_to_tensor(target_probs, dtype=dtypes.float32)
# Reduce the case of a single example to that of a batch of size 1.
if not enqueue_many:
tensor_list = [array_ops.expand_dims(tensor, 0) for tensor in tensor_list]
labels = array_ops.expand_dims(labels, 0)
# If `init_probs` is `None`, set up online estimation of data distribution.
if init_probs is None:
# We use `target_probs` to get the number of classes, so its shape must be
# fully defined at graph construction time.
target_probs.get_shape().assert_is_fully_defined()
init_probs = _estimate_data_distribution(
labels, target_probs.get_shape().num_elements())
else:
init_probs = ops.convert_to_tensor(init_probs, dtype=dtypes.float32)
# Validate that input is consistent.
tensor_list, labels, [init_probs, target_probs] = _verify_input(
tensor_list, labels, [init_probs, target_probs])
# Check that all zero initial probabilities also have zero target
# probabilities.
assert_op = control_flow_ops.Assert(
math_ops.reduce_all(math_ops.logical_or(
math_ops.not_equal(init_probs, 0),
math_ops.equal(target_probs, 0))),
['All classes with zero initial probability must also have zero target '
'probability: ', init_probs, target_probs])
init_probs = control_flow_ops.with_dependencies([assert_op], init_probs)
# Calculate acceptance sampling probabilities.
accept_probs = _calculate_acceptance_probabilities(init_probs, target_probs)
proportion_rejected = math_ops.reduce_sum((1 - accept_probs) * init_probs)
accept_probs = control_flow_ops.cond(
math_ops.less(proportion_rejected, .5),
lambda: accept_probs,
lambda: logging_ops.Print( # pylint: disable=g-long-lambda
accept_probs, [accept_probs],
message='Proportion of examples rejected by sampler is high.',
first_n=10))
# Make a single queue to hold input examples. Reshape output so examples
# don't have singleton batch dimension.
batched = input_ops.batch(tensor_list + [labels],
batch_size=1,
num_threads=threads_per_queue,
capacity=queue_capacity,
enqueue_many=True)
val_list = [array_ops.squeeze(x, [0]) for x in batched[:-1]]
label = array_ops.squeeze(batched[-1], [0])
# Set up second queue containing batches that have the desired class
# proportions.
cur_prob = array_ops.gather(accept_probs, label)
keep_input = random_ops.random_uniform([]) < cur_prob
batched = _conditional_batch(
val_list + [label],
keep_input,
batch_size,
num_threads=threads_per_queue)
return batched[:-1], batched[-1]
def randomize_splits(inputs):
return random_ops.random_uniform(inputs.shape,
maxval=2,
dtype=dtypes.int32)
def _Weights(self): dims = self.LSTM_DIMS return random_ops.random_uniform([2 * dims, 4 * dims], -1, 1, seed=123456)
def testCallTensorDot(self):
dense = core_layers.Dense(2, activation=nn_ops.relu, name='my_dense')
inputs = random_ops.random_uniform((5, 4, 3), seed=1)
outputs = dense(inputs)
self.assertListEqual([5, 4, 2], outputs.get_shape().as_list())
def _Input(self):
return random_ops.random_uniform(
[self.NUM_UNROLL, self.BATCH_SIZE, self.LSTM_DIMS], seed=654321)
def __call__(self, shape, dtype=None, partition_info=None):
if dtype is None:
dtype = self.dtype
return random_ops.random_uniform(shape, self.minval, self.maxval,
dtype, seed=self.seed)
def batch_noise(s, inner_seed): shape = convert_to_batch_shape(s) return random_ops.random_uniform(shape, seed=inner_seed, dtype=dtype)
def filter(self, input_tensor=None, **kwargs):
"""
Filter new feature keys by probability before training.
Prevent unpopular features from affecting training.
Args:
**kwargs: keyword arguments, including
input_tensor: SparseTensor or DenseTensor.
Feature keys need to filter.
probability: float. Filtering new feature keys by
probability, and permitting old keys.
Returns:
Tensor that are filtered for training.
"""
if input_tensor is None:
raise KeyError("filter method expects parameter `input_tensor`.")
elif isinstance(input_tensor, ops.Tensor):
input_type = "DenseTensor"
values = input_tensor
elif isinstance(input_tensor, sparse_tensor.SparseTensor):
input_type = "SparseTensor"
values = input_tensor.values
indices = input_tensor.indices
else:
raise TypeError("input_tensor must be " \
"either a SparseTensor or dense Tensor.")
if 'probability' in kwargs:
probability = kwargs['probability']
else:
raise KeyError("filter method expects parameter `probability`.")
if not isinstance(probability, float):
raise TypeError("probability must be a float.")
if probability < 0.0 or probability > 1.0:
raise ValueError("probability value must be in [0.0, 1.0].")
idx = 0
status_values = array_ops.reshape(values, (-1, ))
partition_index = \
self.var.partition_fn(status_values, self.var.shard_num)
partitioned_values_list, partitioned_indices_list = \
dynamic_embedding_ops._partition(status_values,
partition_index,
self.var.shard_num)
fv_list = []
for idx, dev in enumerate(self.tstp_var.devices):
with ops.device(dev):
feature_status = \
self.tstp_var.tables[idx].lookup(
partitioned_values_list[idx],
dynamic_default_values=self.default_tstp,
)
sub_fv = array_ops.reshape(feature_status, (-1, ))
fv_list.append(sub_fv)
total_fv = dynamic_embedding_ops._stitch(fv_list,
partitioned_indices_list)
value_size = array_ops.size(values)
old_prob = array_ops.ones(value_size)
new_prob = array_ops.fill([value_size], probability)
random_prob = random_ops.random_uniform([value_size], maxval=1.0)
condition = math_ops.greater(total_fv, self.default_tstp)
total_prob = array_ops.where(condition, old_prob, new_prob)
total_mask = math_ops.greater_equal(total_prob, random_prob)
filter_values = array_ops.boolean_mask(values, total_mask)
if input_type == "DenseTensor":
filter_tensor = filter_values
elif input_type == "SparseTensor":
filter_indices = array_ops.boolean_mask(indices, total_mask)
filter_tensor = sparse_tensor.SparseTensor(
indices=filter_indices,
values=filter_values,
dense_shape=input_tensor.dense_shape)
return filter_tensor
def dataset_fn(): data = random_ops.random_uniform((10, 10)) dataset = dataset_ops.DatasetV2.from_tensors([data]).repeat() return dataset
def _sample_n(self, n, seed=None):
shape = array_ops.concat([[n], self.batch_shape_tensor()], 0)
samples = random_ops.random_uniform(shape=shape,
dtype=self.dtype,
seed=seed)
return self.low + self.range() * samples
def _train_fn_internal(self, iterator, iterator2): x = math_ops.matmul(array_ops.squeeze(next(iterator)), self.w) x = math_ops.matmul(array_ops.squeeze(next(iterator2)), x) x = math_ops.matmul(random_ops.random_uniform((10, 10)), x) self.w.assign_add(x)
def testInvalidDataFormat(self):
height, width = 7, 9
images = random_ops.random_uniform((5, height, width, 3), seed=1)
with self.assertRaisesRegex(ValueError, 'data_format'):
pooling_layers.max_pooling2d(images, 3, strides=2, data_format='invalid')
def test_rotate_static_shape(self):
image = array_ops.diag([1., 2., 3.])
result = image_ops.rotate(image,
random_ops.random_uniform((), -1, 1),
interpolation="BILINEAR")
self.assertEqual(image.get_shape(), result.get_shape())
def wasserstein_gradient_penalty(
real_data,
generated_data,
generator_inputs,
discriminator_fn,
discriminator_scope,
epsilon=1e-10,
target=1.0,
one_sided=False,
weights=1.0,
scope=None,
loss_collection=ops.GraphKeys.LOSSES,
reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
add_summaries=False):
"""The gradient penalty for the Wasserstein discriminator loss.
See `Improved Training of Wasserstein GANs`
(https://arxiv.org/abs/1704.00028) for more details.
Args:
real_data: Real data.
generated_data: Output of the generator.
generator_inputs: Exact argument to pass to the generator, which is used as
optional conditioning to the discriminator.
discriminator_fn: A discriminator function that conforms to TF-GAN API.
discriminator_scope: If not `None`, reuse discriminators from this scope.
epsilon: A small positive number added for numerical stability when
computing the gradient norm.
target: Optional Python number or `Tensor` indicating the target value of
gradient norm. Defaults to 1.0.
one_sided: If `True`, penalty proposed in https://arxiv.org/abs/1709.08894
is used. Defaults to `False`.
weights: Optional `Tensor` whose rank is either 0, or the same rank as
`real_data` and `generated_data`, and must be broadcastable to them (i.e.,
all dimensions must be either `1`, or the same as the corresponding
dimension).
scope: The scope for the operations performed in computing the loss.
loss_collection: collection to which this loss will be added.
reduction: A `tf.compat.v1.losses.Reduction` to apply to loss.
add_summaries: Whether or not to add summaries for the loss.
Returns:
A loss Tensor. The shape depends on `reduction`.
Raises:
ValueError: If the rank of data Tensors is unknown.
"""
with ops.name_scope(scope, 'wasserstein_gradient_penalty',
(real_data, generated_data)) as scope:
real_data = ops.convert_to_tensor(real_data)
generated_data = ops.convert_to_tensor(generated_data)
if real_data.shape.ndims is None:
raise ValueError('`real_data` can\'t have unknown rank.')
if generated_data.shape.ndims is None:
raise ValueError('`generated_data` can\'t have unknown rank.')
differences = generated_data - real_data
batch_size = differences.shape.dims[0].value or array_ops.shape(
differences)[0]
alpha_shape = [batch_size] + [1] * (differences.shape.ndims - 1)
alpha = random_ops.random_uniform(shape=alpha_shape)
interpolates = real_data + (alpha * differences)
with ops.name_scope(
None): # Clear scope so update ops are added properly.
# Reuse variables if variables already exists.
with variable_scope.variable_scope(
discriminator_scope,
'gpenalty_dscope',
reuse=variable_scope.AUTO_REUSE):
disc_interpolates = discriminator_fn(interpolates,
generator_inputs)
if isinstance(disc_interpolates, tuple):
# ACGAN case: disc outputs more than one tensor
disc_interpolates = disc_interpolates[0]
gradients = gradients_impl.gradients(disc_interpolates,
interpolates)[0]
gradient_squares = math_ops.reduce_sum(
math_ops.square(gradients),
axis=list(range(1, gradients.shape.ndims)))
# Propagate shape information, if possible.
if isinstance(batch_size, int):
gradient_squares.set_shape([batch_size] +
gradient_squares.shape.as_list()[1:])
# For numerical stability, add epsilon to the sum before taking the square
# root. Note tf.norm does not add epsilon.
slopes = math_ops.sqrt(gradient_squares + epsilon)
penalties = slopes / target - 1.0
if one_sided:
penalties = math_ops.maximum(0., penalties)
penalties_squared = math_ops.square(penalties)
penalty = losses.compute_weighted_loss(penalties_squared,
weights,
scope=scope,
loss_collection=loss_collection,
reduction=reduction)
if add_summaries:
summary.scalar('gradient_penalty_loss', penalty)
return penalty
def test_transform_static_output_shape(self):
image = constant_op.constant([[1., 2.], [3., 4.]])
result = image_ops.transform(image,
random_ops.random_uniform([8], -1, 1),
output_shape=constant_op.constant([3, 5]))
self.assertAllEqual([3, 5], result.get_shape())
def arbitrary_style_image_inputs(style_dataset_file,
batch_size=None,
image_size=None,
center_crop=True,
shuffle=True,
augment_style_images=False,
random_style_image_size=False,
min_rand_image_size=128,
max_rand_image_size=300):
"""Loads a batch of random style image given the path of tfrecord dataset.
This method does not return pre-compute Gram matrices for the images like
style_image_inputs. But it can provide data augmentation. If
augment_style_images is equal to True, then style images will randomly
modified (eg. changes in brightness, hue or saturation) for data
augmentation. If random_style_image_size is set to True then all images
in one batch will be resized to a random size.
Args:
style_dataset_file: str, path to the tfrecord dataset of style files.
batch_size: int. If provided, batches style images. Defaults to None.
image_size: int. The images will be resized bilinearly so that the smallest
side has size image_size. Defaults to None.
center_crop: bool. If True, center-crops to [image_size, image_size].
Defaults to False.
shuffle: bool, whether to shuffle style files at random. Defaults to False.
augment_style_images: bool. Wheather to augment style images or not.
random_style_image_size: bool. If this value is True, then all the style
images in one batch will be resized to a random size between
min_rand_image_size and max_rand_image_size.
min_rand_image_size: int. If random_style_image_size is True, this value
specifies the minimum image size.
max_rand_image_size: int. If random_style_image_size is True, this value
specifies the maximum image size.
Returns:
4-D tensor of shape [1, ?, ?, 3] with values in [0, 1] for the style
image (with random changes for data augmentation if
augment_style_image_size is set to true), and 0-D tensor for the style
label, 4-D tensor of shape [1, ?, ?, 3] with values in [0, 1] for the style
image without random changes for data augmentation.
Raises:
ValueError: if center cropping is requested but no image size is provided,
or if batch size is specified but center-cropping or
augment-style-images is not requested,
or if both augment-style-images and center-cropping are requested.
"""
if center_crop and image_size is None:
raise ValueError('center-cropping requires specifying the image size.')
if center_crop and augment_style_images:
raise ValueError(
'When augment_style_images is true images will be randomly cropped.'
)
if batch_size is not None and not center_crop and not augment_style_images:
raise ValueError(
'batching requires same image sizes (Set center-cropping or '
'augment_style_images to true)')
with tf.name_scope('style_image_processing'):
# Force all input processing onto CPU in order to reserve the GPU for the
# forward inference and back-propagation.
with tf.device('/cpu:0'):
filename_queue = tf.train.string_input_producer(
[style_dataset_file],
shuffle=False,
capacity=1,
name='filename_queue')
if shuffle:
examples_queue = tf.RandomShuffleQueue(
capacity=64,
min_after_dequeue=32,
dtypes=[tf.string],
name='random_examples_queue')
else:
examples_queue = tf.FIFOQueue(capacity=64,
dtypes=[tf.string],
name='fifo_examples_queue')
reader = tf.TFRecordReader()
_, value = reader.read(filename_queue)
enqueue_ops = [examples_queue.enqueue([value])]
tf.train.queue_runner.add_queue_runner(
tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops))
example_serialized = examples_queue.dequeue()
features = tf.parse_single_example(
example_serialized,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string)
})
image = tf.image.decode_jpeg(features['image_raw'])
image.set_shape([None, None, 3])
label = features['label']
if image_size is not None:
image_channels = image.shape[2].value
if augment_style_images:
image_orig = image
image = tf.image.random_brightness(image, max_delta=0.8)
image = tf.image.random_saturation(image,
lower=0.5,
upper=1.5)
image = tf.image.random_hue(image, max_delta=0.2)
image = tf.image.random_flip_left_right(image)
image = tf.image.random_flip_up_down(image)
random_larger_image_size = random_ops.random_uniform(
[],
minval=image_size + 2,
maxval=image_size + 200,
dtype=dtypes.int32)
image = _aspect_preserving_resize(
image, random_larger_image_size)
image = tf.random_crop(
image, size=[image_size, image_size, image_channels])
image.set_shape([image_size, image_size, image_channels])
image_orig = _aspect_preserving_resize(
image_orig, image_size + 2)
image_orig = _central_crop([image_orig], image_size,
image_size)[0]
image_orig.set_shape([image_size, image_size, 3])
elif center_crop:
image = _aspect_preserving_resize(image, image_size + 2)
image = _central_crop([image], image_size, image_size)[0]
image.set_shape([image_size, image_size, image_channels])
image_orig = image
else:
image = _aspect_preserving_resize(image, image_size)
image_orig = image
image = tf.to_float(image) / 255.0
image_orig = tf.to_float(image_orig) / 255.0
if batch_size is None:
image = tf.expand_dims(image, 0)
else:
[image, image_orig,
label] = tf.train.batch([image, image_orig, label],
batch_size=batch_size)
if random_style_image_size:
# Selects a random size for the style images and resizes all the images
# in the batch to that size.
image = _aspect_preserving_resize(
image,
random_ops.random_uniform([],
minval=min_rand_image_size,
maxval=max_rand_image_size,
dtype=dtypes.int32))
return image, label, image_orig
def test_batch_jacobian_bad_shapes(self):
x = random_ops.random_uniform([2, 2])
y = random_ops.random_uniform([3, 2])
with self.assertRaisesRegexp(ValueError,
"Need first dimension of output"):
gradients.batch_jacobian(y, x, use_pfor=True)
def _normal_function(self):
x = random_ops.random_uniform((2, 10))
y = random_ops.random_uniform((10, 2))
self.iteration.assign_add(1.0)
return math_ops.reduce_mean(math_ops.matmul(x, y))
def __init__(self):
self.filters = variables.Variable(
random_ops.random_uniform(shape=(2, 3, 3, 2),
minval=-1.,
maxval=1.))
def error_function():
x = random_ops.random_uniform((2, 10))
y = random_ops.random_uniform((10, 2))
check_ops.assert_non_positive_v2(
math_ops.reduce_sum(math_ops.matmul(x, y)))
return x
def test_binary_cwise_ops(self):
logical_ops = [
math_ops.logical_and,
math_ops.logical_or,
math_ops.logical_xor
]
# Wrapper functions restricting the range of inputs of zeta and polygamma.
def safe_polygamma(x, y):
return math_ops.polygamma(
math_ops.round(clip_ops.clip_by_value(y, 1, 10)),
x * x + 1)
def safe_zeta(x, y):
return math_ops.zeta(x * x + 1, y * y)
float_ops = [
math_ops.add,
math_ops.add_v2,
math_ops.atan2,
math_ops.complex,
math_ops.div,
math_ops.divide,
math_ops.div_no_nan,
math_ops.equal,
math_ops.floor_mod,
math_ops.greater,
math_ops.greater_equal,
math_ops.igamma,
math_ops.igammac,
math_ops.igamma_grad_a,
math_ops.less,
math_ops.less_equal,
math_ops.maximum,
math_ops.minimum,
math_ops.mod,
math_ops.multiply,
math_ops.not_equal,
math_ops.pow,
math_ops.squared_difference,
math_ops.subtract,
math_ops.truncate_mod,
safe_polygamma,
safe_zeta,
]
# FloorDiv fails on XLA due floor's discontinuities exacerbating small
# division differences.
if not test_util.is_xla_enabled():
float_ops += [math_ops.floor_div]
for op in logical_ops + float_ops:
x = random_ops.random_uniform([7, 3, 5])
y = random_ops.random_uniform([3, 5])
if op in logical_ops:
x = x > 0
y = y > 0
output_dtypes = []
# pylint: disable=cell-var-from-loop
def loop_fn(i):
x1 = array_ops.gather(x, i)
y1 = array_ops.gather(y, i)
outputs = [op(x, y), op(x1, y), op(x, y1), op(x1, y1), op(x1, x1)]
del output_dtypes[:]
output_dtypes.extend([t.dtype for t in outputs])
return outputs
# pylint: enable=cell-var-from-loop
self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
def test_indexed_slice(self):
inp = random_ops.random_uniform([3, 2])
output = nn.embedding_lookup(inp, [0, 2])
pfor_jacobian = gradients.jacobian(output, inp, use_pfor=True)
while_jacobian = gradients.jacobian(output, inp, use_pfor=False)
self.run_and_assert_equal(while_jacobian, pfor_jacobian)
def rng(dtype):
dtype = dtypes.as_dtype(dtype)
return random_ops.random_uniform(shape=[2],
dtype=dtype,
maxval=dtype.max)
