def testReadSnapshotParallelAfterWrite(self):
with compat.forward_compatibility_horizon(2019, 8, 16):
self.setUpTFRecord(10, 4000)
filenames = self.test_filenames
expected = [
b"Record %d of file %d" % (r, f) # pylint:disable=g-complex-comprehension
for f in range(0, 10) for r in range(0, 4000)
]
tmpdir = self.makeSnapshotDirectory()
dataset = core_readers._TFRecordDataset(filenames)
dataset = dataset.apply(
snapshot.snapshot(tmpdir,
shard_size_bytes=1024 * 1024,
num_reader_threads=2,
reader_buffer_size=10))
self.assertDatasetProduces(dataset,
expected,
assert_items_equal=True)
# remove the original files and try to read the data back only from
# snapshot.
self.removeTFRecords()
dataset2 = core_readers._TFRecordDataset(filenames)
dataset2 = dataset2.apply(
snapshot.snapshot(tmpdir,
shard_size_bytes=1024 * 1024,
num_reader_threads=2,
reader_buffer_size=10))
self.assertDatasetProduces(dataset2,
expected,
assert_items_equal=True)
def test_conv_bn_dropout(self):
"""Test dropout precision of convolution batch norm graph."""
with compat.forward_compatibility_horizon(2019, 6, 7):
if test.is_gpu_available(cuda_only=True):
random_seed.set_random_seed(0)
x = _input([2, 8, 8, 1])
y = _conv_bn(x)
y = nn.dropout(y, rate=0.5)
y = math_ops.add(y, 1, name='addition')
y = _conv_bn(y)
y = array_ops.identity(y)
optimizer = gradient_descent.GradientDescentOptimizer(
learning_rate=0.01)
g = optimizer.compute_gradients(y, [x])
output = (y, g)
output_val_ref, output_val, cost_graph = self._run(output)
node_map = _build_node_map(cost_graph.node)
self._assert_output_fp16(node_map, 'Conv2D')
self._assert_output_fp16(node_map, 'FusedBatchNormV3')
# We do not assert dropout's dtype because we do not want to rely on the
# node names of dropout's internal implementation.
self._assert_output_fp16(node_map, 'addition')
self._assert_output_fp16(node_map, 'Conv2D_1')
output_val_ref, output_val, cost_graph = self._run(output)
self.assertAllClose(output_val_ref,
output_val,
atol=2e-3,
rtol=2e-3)
def test_stft_and_inverse_stft(self, signal_length, frame_length,
frame_step, fft_length, np_rtype, tol):
"""Test that spectral_ops.stft/inverse_stft match a NumPy implementation."""
# Enable float64 support for RFFTs.
with compat.forward_compatibility_horizon(2019, 10, 13):
signal = np.random.random(signal_length).astype(np_rtype)
self._compare(signal, frame_length, frame_step, fft_length, tol)
def testBatchDimsMatchesPythonBatching(self, params_shape, indices_shape,
batch_dims, axis, output_shape):
"""Checks that batch_dims matches multiple calls to tf.gather()."""
# Generate a `params` tensor with the indicated shape.
params_size = np.prod(params_shape)
params = np.reshape(np.arange(params_size), params_shape)
# Generate an `indices` tensor with the indicated shape, where each index
# is within the appropriate range.
indices_size = np.prod(indices_shape)
indices = np.reshape(np.arange(indices_size), indices_shape)
indices = indices % params_shape[axis]
# Perform repeated (batched) gather operations with numpy, to find the
# expected result.
expected = self._batchNumpyGather(params, indices, axis, batch_dims)
# On Windows, we get an exception if we pass in the transformed numpy
# arrays ("Failed to convert numpy ndarray to a Tensor (Unsupported
# feed type)."); so convert them back to lists before calling tf.gather.
params = params.tolist()
indices = indices.tolist()
result = array_ops.gather(params, indices, axis=axis, batch_dims=batch_dims)
self.assertAllEqual(output_shape, result.shape.as_list())
self.assertAllEqual(expected, result)
with compat.forward_compatibility_horizon(2019, 8, 11):
result = array_ops.gather(
params, indices, axis=axis, batch_dims=batch_dims)
self.assertAllEqual(output_shape, result.shape.as_list())
self.assertAllEqual(expected, result)
def testNMS128From1024(self):
with compat.forward_compatibility_horizon(2018, 8, 8):
num_boxes = 1024
boxes_np = np.random.normal(50, 10, (num_boxes, 4)).astype("f4")
scores_np = np.random.normal(0.5, 0.1, (num_boxes,)).astype("f4")
max_output_size = 128
iou_threshold_np = np.array(0.5, dtype=np.float32)
score_threshold_np = np.array(0.0, dtype=np.float32)
with self.cached_session() as sess:
boxes = array_ops.placeholder(boxes_np.dtype, shape=boxes_np.shape)
scores = array_ops.placeholder(scores_np.dtype, shape=scores_np.shape)
iou_threshold = array_ops.placeholder(iou_threshold_np.dtype,
iou_threshold_np.shape)
score_threshold = array_ops.placeholder(score_threshold_np.dtype,
score_threshold_np.shape)
with self.test_scope():
selected_indices = image_ops.non_max_suppression_padded(
boxes=boxes,
scores=scores,
max_output_size=max_output_size,
iou_threshold=iou_threshold,
score_threshold=score_threshold,
pad_to_max_output_size=True)
inputs_feed = {
boxes: boxes_np,
scores: scores_np,
score_threshold: score_threshold_np,
iou_threshold: iou_threshold_np
}
(indices_tf, _) = sess.run(selected_indices, feed_dict=inputs_feed)
self.assertEqual(indices_tf.size, max_output_size)
def test_placeholder(self):
def check(equation, *input_and_placeholder_shapes):
r = np.random.RandomState(0)
inputs = []
input_placeholders = []
for actual_shape, placeholder_shape in input_and_placeholder_shapes:
input_np = np.array(r.randn(*actual_shape))
inputs.append(input_np)
input_placeholders.append(
array_ops.placeholder_with_default(input_np,
placeholder_shape))
a = np.einsum(equation, *inputs)
b = self.evaluate(
special_math_ops.einsum(equation, *input_placeholders))
self.assertAllClose(a, b, atol=1e-4, rtol=1e-4)
check('bijl,bjkm->bik', ((9, 2, 3, 5), (None, None, None, 5)),
((9, 3, 4, 7), (None, None, 4, None)))
check('...ij,...->...i', ((4, 3, 1, 2), (None, 3, None, 2)),
((4, 3), (None, 3)))
# Ellipsis with unknown rank.
with compat.forward_compatibility_horizon(2019, 10, 19):
check('bijl,bjkm->bik', ((9, 2, 3, 5), None), ((9, 3, 4, 7), None))
check('...ij,...jk->...ik', ((3, 1, 2, 3), None),
((1, 7, 3, 4), None))
def testBatchNormGradShape5(self):
with compat.forward_compatibility_horizon(2019, 6, 7):
for is_training in [True, False]:
x_shape = [0, 7, 11, 4]
for dtype in [np.float16, np.float32]:
if test.is_gpu_available(cuda_only=True):
self._test_gradient(
x_shape,
dtype, [7],
np.float32,
use_gpu=True,
data_format='NCHW',
is_training=is_training)
self._test_gradient(
x_shape,
dtype, [4],
np.float32,
use_gpu=True,
data_format='NHWC',
is_training=is_training)
self._test_gradient(
x_shape,
dtype, [4],
np.float32,
use_gpu=False,
data_format='NHWC',
is_training=is_training)
def test_conv_bn_dropout(self):
"""Test dropout precision of convolution batch norm graph."""
with compat.forward_compatibility_horizon(2019, 6, 7):
if test.is_gpu_available(cuda_only=True):
random_seed.set_random_seed(0)
x = _input([2, 8, 8, 1])
y = _conv_bn(x)
y = nn.dropout(y, rate=0.5)
y = _conv_bn(y)
y = array_ops.identity(y)
optimizer = gradient_descent.GradientDescentOptimizer(
learning_rate=0.01)
g = optimizer.compute_gradients(y, [x])
output = (y, g)
output_val_ref, output_val, cost_graph = self._run(output)
node_map = _build_node_map(cost_graph.node)
self._assert_output_fp16(node_map, 'Conv2D')
self._assert_output_fp16(node_map, 'FusedBatchNormV3')
self._assert_output_fp16(node_map, 'dropout/mul')
self._assert_output_fp16(node_map, 'Conv2D_1')
output_val_ref, output_val, cost_graph = self._run(output)
self.assertAllClose(output_val_ref,
output_val,
atol=1e-3,
rtol=1e-3)
def benchmarkBatchMatMulBroadcast(self):
for (a_shape, b_shape) in self.shape_pairs:
with compat.forward_compatibility_horizon(2019, 4, 26):
with ops.Graph().as_default(), \
session.Session(config=benchmark.benchmark_config()) as sess, \
ops.device("/cpu:0"):
matrix_a = variables.Variable(
GetRandomNormalInput(a_shape, np.float32))
matrix_b = variables.Variable(
GetRandomNormalInput(b_shape, np.float32))
variables.global_variables_initializer().run()
# Use batch matmul op's internal broadcasting.
self.run_op_benchmark(
sess,
math_ops.matmul(matrix_a, matrix_b),
min_iters=50,
name="batch_matmul_cpu_{}_{}".format(a_shape, b_shape))
# Manually broadcast the input matrices using the broadcast_to op.
broadcasted_batch_shape = array_ops.broadcast_static_shape(
matrix_a.shape[:-2], matrix_b.shape[:-2])
broadcasted_a_shape = broadcasted_batch_shape.concatenate(
matrix_a.shape[-2:])
broadcasted_b_shape = broadcasted_batch_shape.concatenate(
matrix_b.shape[-2:])
self.run_op_benchmark(
sess,
math_ops.matmul(
array_ops.broadcast_to(matrix_a, broadcasted_a_shape),
array_ops.broadcast_to(matrix_b, broadcasted_b_shape)),
min_iters=50,
name="batch_matmul_manual_broadcast_cpu_{}_{}".format(
a_shape, b_shape))
def testExternalStatePolicyFail(self):
with compat.forward_compatibility_horizon(2019, 11, 30):
with ops.device(self._device0):
dataset0 = dataset_ops.Dataset.range(100).map(
lambda _: random_ops.random_uniform( # pylint:disable=g-long-lambda
[],
minval=1,
maxval=10,
dtype=dtypes.float32))
opt = dataset_ops.Options()
opt.experimental_external_state_policy = (
dataset_ops.ExternalStatePolicy.FAIL)
dataset0 = dataset0.with_options(opt)
with self.assertRaises(errors.FailedPreconditionError):
replicated_ds = distribute.replicate(dataset0,
[self._device1, self._device2])
dataset1 = replicated_ds[self._device1]
dataset2 = replicated_ds[self._device2]
with ops.device(self._device0):
get_next0 = self.getNext(dataset0)
with ops.device(self._device1):
get_next1 = self.getNext(dataset1)
with ops.device(self._device2):
get_next2 = self.getNext(dataset2)
for _ in range(100):
self.evaluate(get_next0())
self.evaluate(get_next1())
self.evaluate(get_next2())
def testAllowStatefulOp(self):
with compat.forward_compatibility_horizon(2019, 9, 12):
with ops.device(self._device0):
dataset0 = dataset_ops.Dataset.range(100).map(
lambda _: random_ops.random_uniform( # pylint:disable=g-long-lambda
[],
minval=1,
maxval=10,
dtype=dtypes.float32))
opt = dataset_ops.Options()
opt.experimental_allow_stateful = True
dataset0 = dataset0.with_options(opt)
replicated_ds = distribute.replicate(dataset0,
[self._device1, self._device2])
dataset1 = replicated_ds[self._device1]
dataset2 = replicated_ds[self._device2]
with ops.device(self._device0):
get_next0 = self.getNext(dataset0)
with ops.device(self._device1):
get_next1 = self.getNext(dataset1)
with ops.device(self._device2):
get_next2 = self.getNext(dataset2)
for _ in range(100):
self.evaluate(get_next0())
self.evaluate(get_next1())
self.evaluate(get_next2())
def testCopyToDevicePingPongCPUGPU(self):
if not test_util.is_gpu_available():
self.skipTest("No GPU available")
with compat.forward_compatibility_horizon(2018, 8, 4):
host_dataset = dataset_ops.Dataset.range(10)
device_dataset = host_dataset.apply(
prefetching_ops.copy_to_device("/gpu:0",
source_device="/cpu:0"))
back_to_cpu_dataset = device_dataset.apply(
prefetching_ops.copy_to_device("/cpu:0",
source_device="/gpu:0"))
with ops.device("/cpu:0"):
iterator = dataset_ops.make_initializable_iterator(
back_to_cpu_dataset)
next_element = iterator.get_next()
with self.cached_session(config=config_pb2.ConfigProto(
allow_soft_placement=False)):
self.evaluate(iterator.initializer)
for i in range(10):
self.assertEqual(i, self.evaluate(next_element))
with self.assertRaises(errors.OutOfRangeError):
self.evaluate(next_element)
def testSwitchingConditionalAccumulatorForV1(self):
# Test into the future.
with compat.forward_compatibility_horizon(2019, 8, 9):
train_input_fn = _make_train_input_fn_dataset(
is_classification=True)
predict_input_fn = numpy_io.numpy_input_fn(x=FEATURES_DICT,
y=None,
batch_size=1,
num_epochs=1,
shuffle=False)
est = boosted_trees.boosted_trees_classifier_train_in_memory(
train_input_fn=train_input_fn,
feature_columns=self._numeric_feature_columns,
n_trees=1,
max_depth=5,
quantile_sketch_epsilon=0.33)
# It will stop after 5 steps because of the max depth and num trees.
self._assert_checkpoint(est.model_dir,
global_step=5,
finalized_trees=1,
attempted_layers=5,
bucket_boundaries=[[-2.001, -1.999, 12.5],
[-3., 0.4995, 2.],
[-100., 20., 102.75]])
eval_res = est.evaluate(input_fn=train_input_fn, steps=1)
self.assertAllClose(eval_res['accuracy'], 1.0)
predictions = list(est.predict(input_fn=predict_input_fn))
self.assertAllClose([[0], [1], [1], [0], [0]],
[pred['class_ids'] for pred in predictions])
def test_empty(self):
def check(equation, input_shapes, output_shape):
# All these cases result in an output filled with zeros, so we don't call
# np.einsum. Also np.einsum doesn't support generalized diagonals which
# are needed for EinsumOp gradients.
r = np.random.RandomState(0)
inputs = [np.array(r.randn(*shape)) for shape in input_shapes]
input_tensors = [
constant_op.constant(x, shape=x.shape) for x in inputs
]
output = self.evaluate(
special_math_ops.einsum(equation, *input_tensors))
self.assertAllClose(output,
np.zeros(output_shape),
atol=1e-4,
rtol=1e-4)
# Contractions along zero-sized dimensons.
check('ab,bc->ac', [(0, 10), (10, 10)], (0, 10))
# From transformer xl.
check('ibnd,ijbn->jnd', [(1, 0, 5, 10), (1, 1, 0, 5)], (1, 5, 10))
with compat.forward_compatibility_horizon(2019, 10, 19):
# Generalized traces with zero-sized dimensions.
check('aab,bc->ac', [(0, 0, 10), (10, 10)], (0, 10))
check('aaab,bc->c', [(0, 0, 0, 3), (3, 4)], (4, ))
def test_scope_disable(self):
"""Test graph with convolution followed by batch norm."""
with compat.forward_compatibility_horizon(2019, 11, 11):
if test.is_gpu_available(cuda_only=True):
random_seed.set_random_seed(0)
y = _input([2, 8, 8, 1])
with auto_mixed_precision_scope(False):
x = _conv_bn(y)
with auto_mixed_precision_scope(True):
x = _conv_bn(x)
output = gradients.gradients(x, [y])
output_val_ref, output_val, cost_graph, partition_graphs = self._run(
output)
node_map = _build_node_map(cost_graph.node)
num_to_fp16, num_to_fp32 = _count_casts(partition_graphs[0])
self._assert_output_fp32(node_map, 'Conv2D')
self._assert_output_fp32(node_map, 'FusedBatchNormV3')
self._assert_output_fp16(node_map, 'Conv2D_1')
self._assert_output_fp32(node_map, 'FusedBatchNormV3_1')
self._assert_output_fp32(
node_map, 'gradients/Conv2D_grad/Conv2DBackpropInput')
self._assert_output_fp16(
node_map, 'gradients/Conv2D_1_grad/Conv2DBackpropInput')
self.assertEqual(num_to_fp16,
2) # Before Conv2D_1:0, Conv2D_1:1
self.assertEqual(num_to_fp32,
2) # After Conv2D_1 and Conv2D_1_grad
self.assertAllClose(output_val_ref,
output_val,
atol=1e-3,
rtol=1e-3)
def test_conv3d_bn(self):
"""Test graph with convolution followed by batch norm."""
# TODO(nluehr) re-enable once nvbug 3098132 is fixed.
self.skipTest('Test case should be skipped when cuDNN < 8.0.4')
with compat.forward_compatibility_horizon(2019, 6, 7):
if test.is_gpu_available(cuda_only=True):
random_seed.set_random_seed(0)
x = _input([2, 8, 8, 8, 1])
x = _conv3d_bn(x)
output = _conv3d_bn(x)
output_val_ref, output_val, cost_graph, partition_graphs = self._run(
output)
node_map = _build_node_map(cost_graph.node)
num_to_fp16, num_to_fp32 = _count_casts(partition_graphs[0])
self._assert_output_fp16(node_map, 'Conv3D')
self._assert_output_fp16(node_map, 'FusedBatchNormV3')
self._assert_output_fp16(node_map, 'Conv3D_1')
self.assertEqual(num_to_fp16,
3) # Before Conv3D:0, Conv3D:1, Conv3D_1:1
self.assertEqual(num_to_fp32, 1) # After FusedBatchNormV3:0
self.assertAllClose(output_val_ref,
output_val,
atol=1e-2,
rtol=1e-2)
def test_gradients_numerical(self, signal_length, frame_length, frame_step,
fft_length, np_rtype, forward_tol,
backward_tol):
# Enable float64 support for RFFTs.
with compat.forward_compatibility_horizon(2019, 10, 13):
# TODO(rjryan): Investigate why STFT gradient error is so high.
signal = np.random.rand(signal_length).astype(np_rtype) * 2 - 1
def forward(signal):
return spectral_ops.stft(signal,
frame_length,
frame_step,
fft_length,
pad_end=False)
((f_jacob_t, ),
(f_jacob_n, )) = gradient_checker_v2.compute_gradient(
forward, [signal])
self.assertAllClose(f_jacob_t,
f_jacob_n,
rtol=forward_tol,
atol=forward_tol)
def backward(stft):
return spectral_ops.inverse_stft(stft, frame_length,
frame_step, fft_length)
stft = forward(signal)
((b_jacob_t, ),
(b_jacob_n, )) = gradient_checker_v2.compute_gradient(
backward, [stft])
self.assertAllClose(b_jacob_t,
b_jacob_n,
rtol=backward_tol,
atol=backward_tol)
def testStaticRegexReplaceDelegation(self):
with compat.forward_compatibility_horizon(2018, 10, 11):
with self.test_session():
input_vector = constant_op.constant("foo", dtypes.string)
pattern = "[a-z]"
replace = "."
op = string_ops.regex_replace(input_vector, pattern, replace)
self.assertTrue(op.name.startswith("StaticRegexReplace"))
def test_repeated_indices(self):
with compat.forward_compatibility_horizon(2019, 10, 19):
# Repeated indices.
self._check('ijj,k->ik', (2, 3, 3), (4, ))
self._check('aba,a->b', (3, 4, 3), (3, ))
# From https://github.com/dask/dask/pull/3412#discussion_r182413444
self._check('aab,bc->ac', (2, 2, 3), (3, 4))
self._check('aab,bcc->ac', (2, 2, 3), (3, 4, 4))
def test3DTensorAsInputNoReshape(self):
with compat.forward_compatibility_horizon(2018, 8, 27):
self._testSoftmax(
np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]],
[[2., 3., 4., 5.], [6., 7., 8., 9.]],
[[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32),
use_gpu=False)
self._testOverflow(use_gpu=False)
def testRegexReplaceDelegation(self, pattern_fn, rewrite_fn):
with compat.forward_compatibility_horizon(2018, 10, 11):
with self.test_session():
input_vector = constant_op.constant("foo", dtypes.string)
pattern = pattern_fn("[a-z]")
replace = rewrite_fn(".")
op = string_ops.regex_replace(input_vector, pattern, replace)
self.assertTrue(op.name.startswith("RegexReplace"))
def _forward_compat_context(np_dtype):
@contextlib.contextmanager
def null_context():
yield
if np_dtype in (np.float64, np.complex128):
return compat.forward_compatibility_horizon(2019, 10, 13)
else:
return null_context()
def test_decorator(self):
compatibility_date = self._compatibility_date()
one_day_after = self._n_days_after(1)
with compat.forward_compatibility_horizon(*one_day_after):
self.assertTrue(compat.forward_compatible(*compatibility_date))
self.assertFalse(compat.forward_compatible(*one_day_after))
# After exiting context manager, value should be reset.
self.assertFalse(compat.forward_compatible(*compatibility_date))
def test_decorator(self):
compatibility_date = self._compatibility_date()
one_day_after = self._n_days_after(1)
with compat.forward_compatibility_horizon(*one_day_after):
self.assertTrue(compat.forward_compatible(*compatibility_date))
self.assertFalse(compat.forward_compatible(*one_day_after))
# After exiting context manager, value should be reset.
self.assertFalse(compat.forward_compatible(*compatibility_date))
def testGatherNdResourceVariable(self):
with compat.forward_compatibility_horizon(2019, 4, 30):
with self.cached_session():
v = resource_variable_ops.ResourceVariable(
constant_op.constant([[1, 2], [3, 4], [5, 6]]))
self.evaluate(variables.global_variables_initializer())
gather = array_ops.gather_nd(v, [[0, 1], [2, 0]])
if not context.executing_eagerly(): # .op doesn't make sense in Eager
self.assertEqual("ResourceGatherNd", gather.op.inputs[0].op.type)
self.assertAllEqual([2, 5], gather)
def testGatherNdResourceVariable(self):
with compat.forward_compatibility_horizon(2019, 4, 30):
with self.cached_session():
v = resource_variable_ops.ResourceVariable(
constant_op.constant([[1, 2], [3, 4], [5, 6]]))
self.evaluate(variables.global_variables_initializer())
gather = array_ops.gather_nd(v, [[0, 1], [2, 0]])
if not context.executing_eagerly(): # .op doesn't make sense in Eager
self.assertEqual("ResourceGatherNd", gather.op.inputs[0].op.type)
self.assertAllEqual([2, 5], gather)
def testBatchDims(self, params, indices, batch_dims, expected=None,
axis=None):
result = array_ops.gather(params, indices, axis=axis, batch_dims=batch_dims)
self.assertAllEqual(expected, result)
with compat.forward_compatibility_horizon(2019, 6, 11):
result = array_ops.gather(
params, indices, axis=axis, batch_dims=batch_dims)
self.assertAllEqual(expected, result)
def testStaticRegexFullMatchDelegation(self):
with compat.forward_compatibility_horizon(2018, 11, 20):
with self.cached_session():
input_tensor = constant_op.constant("foo", dtypes.string)
pattern = "[a-z]*"
op = string_ops.regex_full_match(input_tensor, pattern)
self.assertTrue(op.name.startswith("StaticRegexFullMatch"), op.name)
pattern_tensor = constant_op.constant("[a-z]*", dtypes.string)
op_vec = string_ops.regex_full_match(input_tensor, pattern_tensor)
self.assertTrue(op_vec.name.startswith("RegexFullMatch"), op.name)
def testStaticRegexFullMatchDelegation(self):
with compat.forward_compatibility_horizon(2018, 11, 20):
with self.test_session():
input_tensor = constant_op.constant("foo", dtypes.string)
pattern = "[a-z]*"
op = string_ops.regex_full_match(input_tensor, pattern)
self.assertTrue(op.name.startswith("StaticRegexFullMatch"), op.name)
pattern_tensor = constant_op.constant("[a-z]*", dtypes.string)
op_vec = string_ops.regex_full_match(input_tensor, pattern_tensor)
self.assertTrue(op_vec.name.startswith("RegexFullMatch"), op.name)
def _compute_stft_gradient(signal,
frame_length=32,
frame_step=16,
fft_length=32):
"""Computes the gradient of the STFT with respect to `signal`."""
# Enable float64 support for RFFTs.
with compat.forward_compatibility_horizon(2019, 10, 13):
stft = spectral_ops.stft(signal, frame_length, frame_step,
fft_length)
magnitude_stft = math_ops.abs(stft)
loss = math_ops.reduce_sum(magnitude_stft)
return gradients_impl.gradients([loss], [signal])[0]
def test_broadcasting(self):
with compat.forward_compatibility_horizon(2019, 10, 19):
self._check_gradient('...ij,...jk->...ik', (3, 2), (2, 4))
self._check_gradient('ij...,jk...->ik...', (3, 2, 1), (2, 4))
self._check_gradient('...ij,...jk->...ik', (3, 1, 3, 2),
(1, 5, 2, 4))
self._check_gradient('ij,jk...k->i...', (3, 2), (2, 4, 1, 4))
self._check_gradient('aab,b...c->a...c', (1, 1, 3), (3, 1, 1, 4))
# Tests from dask.
self._check_gradient('...i,...j,...k->...ijk', (1, 4, 1, 2),
(5, 1, 1, 3), (1, 1, 1, 1, 9))
self._check_gradient('...i,...j,...k->...ijk', (1, ), (1, ), (1, ))
def Test(self):
def CheckGradients(self, a_shape, b_shape):
self._compare(a_shape, b_shape, dtype, adjoint_a, adjoint_b)
with compat.forward_compatibility_horizon(2019, 4, 19):
CheckGradients(self, [1, 5, 2, 3], [7, 1, 3, 2])
CheckGradients(self, [2, 3], [1, 3, 5])
CheckGradients(self, [2, 3], [5, 3, 5])
CheckGradients(self, [5, 2, 5], [5, 3])
CheckGradients(self, [5, 2, 2, 3], [3, 5])
CheckGradients(self, [4, 5, 1, 2, 3], [1, 1, 3, 5])
CheckGradients(self, [1, 2, 1, 4, 2, 1, 3, 4], [3, 2, 1, 1, 1, 2, 4, 2])
def testUnary(self):
for dtype in self.float_types:
self._testUnary(
lambda x: special_math_ops.einsum('ijk->kji', x),
np.array([[[1, 3], [2, 5], [6, 8]]], dtype=dtype),
expected=np.array([[[1], [2], [6]], [[3], [5], [8]]], dtype=dtype))
with compat.forward_compatibility_horizon(2019, 10, 19):
self._testUnary(
lambda x: special_math_ops.einsum('ijk->kji', x),
np.array([[[1, 3], [2, 5], [6, 8]]], dtype=dtype),
expected=np.array([[[1], [2], [6]], [[3], [5], [8]]], dtype=dtype))
def test_numpy_input(self):
with compat.forward_compatibility_horizon(2019, 10, 19):
# In addition to Tensors, we also support raw numpy arrays as inputs.
r = np.random.RandomState(0)
s = 'ijk,ijl,ikl->i'
x = r.randn(1, 2, 3)
y = r.randn(1, 2, 4)
z = r.randn(1, 3, 4)
a = np.einsum(s, x, y, z)
b = self.evaluate(special_math_ops.einsum(s, x, y, z))
self.assertAllClose(a, b, atol=1e-4, rtol=1e-4)
def test_environment_override(self):
var_name = 'TF_FORWARD_COMPATIBILITY_DELTA_DAYS'
def remove_os_environment_var():
try:
del os.environ[var_name]
except KeyError:
pass
self.addCleanup(remove_os_environment_var)
compatibility_date = self._compatibility_date()
one_day_before = self._n_days_after(-1)
one_day_after = self._n_days_after(1)
ten_days_after = self._n_days_after(10)
nine_days_after = self._n_days_after(9)
self.assertTrue(compat.forward_compatible(*one_day_before))
self.assertFalse(compat.forward_compatible(*compatibility_date))
self.assertFalse(compat.forward_compatible(*one_day_after))
self.assertFalse(compat.forward_compatible(*nine_days_after))
self.assertFalse(compat.forward_compatible(*ten_days_after))
os.environ[var_name] = '10'
compat._update_forward_compatibility_date_number()
self.assertTrue(compat.forward_compatible(*one_day_before))
self.assertTrue(compat.forward_compatible(*compatibility_date))
self.assertTrue(compat.forward_compatible(*one_day_after))
self.assertTrue(compat.forward_compatible(*nine_days_after))
self.assertFalse(compat.forward_compatible(*ten_days_after))
del os.environ[var_name]
compat._update_forward_compatibility_date_number()
self.assertTrue(compat.forward_compatible(*one_day_before))
self.assertFalse(compat.forward_compatible(*compatibility_date))
self.assertFalse(compat.forward_compatible(*one_day_after))
self.assertFalse(compat.forward_compatible(*nine_days_after))
self.assertFalse(compat.forward_compatible(*ten_days_after))
# Now test interaction between environment variable and context func.
os.environ[var_name] = '10'
compat._update_forward_compatibility_date_number()
self.assertTrue(compat.forward_compatible(*one_day_after))
with compat.forward_compatibility_horizon(*one_day_after):
self.assertTrue(compat.forward_compatible(*one_day_before))
self.assertTrue(compat.forward_compatible(*compatibility_date))
self.assertFalse(compat.forward_compatible(*one_day_after))
self.assertFalse(compat.forward_compatible(*nine_days_after))
self.assertFalse(compat.forward_compatible(*ten_days_after))
self.assertTrue(compat.forward_compatible(*one_day_after))
def test_fused_batch_norm(self):
with compat.forward_compatibility_horizon(2019, 6, 7):
data_formats = ["NHWC"]
if test.is_gpu_available():
data_formats.append("NCHW")
for is_training in (True, False):
for data_format in data_formats:
with backprop.GradientTape(persistent=True) as g:
if data_format == "NCHW":
x = random_ops.random_uniform([3, 1, 2, 5, 5])
else:
x = random_ops.random_uniform([3, 1, 5, 5, 2])
g.watch(x)
scale = random_ops.random_uniform([2])
g.watch(scale)
offset = random_ops.random_uniform([2])
g.watch(offset)
mean = None if is_training else random_ops.random_uniform([2])
variance = None if is_training else random_ops.random_uniform([2])
# pylint: disable=cell-var-from-loop
def loop_fn(i):
with g:
x1 = array_ops.gather(x, i)
outputs = nn.fused_batch_norm(
x1,
scale,
offset,
mean=mean,
variance=variance,
epsilon=0.01,
data_format=data_format,
is_training=is_training)
outputs = list(outputs)
# We only test the first value of outputs when is_training is
# False. It looks like CPU and GPU have different outputs for
# batch_mean and batch_variance for this case.
if not is_training:
outputs[1] = constant_op.constant(0.)
outputs[2] = constant_op.constant(0.)
loss = nn.l2_loss(outputs[0])
if is_training:
gradients = g.gradient(loss, [x1, scale, offset])
else:
gradients = [constant_op.constant(0.)] * 3
return outputs + gradients
# pylint: enable=cell-var-from-loop
self._test_loop_fn(loop_fn, 3)
def test_decorator_with_failure(self):
compatibility_date = self._compatibility_date()
one_day_after = self._n_days_after(1)
class DummyError(Exception):
pass
try:
with compat.forward_compatibility_horizon(*one_day_after):
raise DummyError()
except DummyError:
pass # silence DummyError
# After exiting context manager, value should be reset.
self.assertFalse(compat.forward_compatible(*compatibility_date))
def testNMS3Then2WithScoreThresh(self):
# Three boxes are selected based on IOU.
# One is filtered out by score threshold.
# TODO(b/26783907): The Sort HLO is not implemented on CPU or GPU.
if self.device in ["XLA_CPU", "XLA_GPU"]:
return
with compat.forward_compatibility_horizon(2018, 8, 8):
boxes_data = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9],
[0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]]
boxes_np = np.array(boxes_data, dtype=np.float32)
scores_data = [0.9, 0.75, 0.6, 0.95, 0.5, 0.3]
scores_np = np.array(scores_data, dtype=np.float32)
max_output_size = 3
iou_threshold_np = np.array(0.5, dtype=np.float32)
score_threshold_np = np.array(0.4, dtype=np.float32)
with self.cached_session() as sess:
boxes = array_ops.placeholder(boxes_np.dtype, shape=boxes_np.shape)
scores = array_ops.placeholder(scores_np.dtype, shape=scores_np.shape)
iou_threshold = array_ops.placeholder(iou_threshold_np.dtype,
iou_threshold_np.shape)
score_threshold = array_ops.placeholder(score_threshold_np.dtype,
score_threshold_np.shape)
with self.test_scope():
selected_indices = image_ops.non_max_suppression_padded(
boxes=boxes,
scores=scores,
max_output_size=max_output_size,
iou_threshold=iou_threshold,
score_threshold=score_threshold,
pad_to_max_output_size=True)
inputs_feed = {
boxes: boxes_np,
scores: scores_np,
iou_threshold: iou_threshold_np,
score_threshold: score_threshold_np
}
(indices_tf, num_valid) = sess.run(
selected_indices, feed_dict=inputs_feed)
self.assertEqual(indices_tf.size, max_output_size)
self.assertEqual(num_valid, 2)
self.assertAllClose(indices_tf[:num_valid], [3, 0])
def testGradGradFloat32(self):
with compat.forward_compatibility_horizon(2018, 11, 2):
with self.test_session():
x = constant_op.constant(
[-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
shape=[2, 5],
name="x")
y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu")
z = gradients_impl.gradients(y, x)
x_init = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float32,
order="F")
err = gradient_checker.compute_gradient_error(
x, [2, 5], z[0], [2, 5], x_init_value=x_init)
print("leaky_relu (float32) gradient of gradient err = ", err)
self.assertLess(err, 1e-4)
def testBasicGpu(self):
if not test_util.is_gpu_available():
self.skipTest("No GPU available")
with compat.forward_compatibility_horizon(2018, 8, 4):
dataset = dataset_ops.Dataset.range(10)
multi_device_iterator = prefetching_ops.MultiDeviceIterator(
dataset, ["/cpu:1", "/gpu:0"])
elem_on_1, elem_on_2 = multi_device_iterator.get_next()
config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
with self.test_session(config=config) as sess:
sess.run(multi_device_iterator.initializer)
for i in range(0, 10, 2):
self.assertEqual(i, sess.run(elem_on_1))
self.assertEqual(i + 1, sess.run(elem_on_2))
with self.assertRaises(errors.OutOfRangeError):
sess.run(elem_on_1)
sess.run(elem_on_2)
def testGradGradFloat64(self):
with compat.forward_compatibility_horizon(2018, 11, 2):
with self.cached_session():
def f(x):
assert x.dtype == dtypes.float64
with backprop.GradientTape() as tape:
tape.watch(x)
y = nn_ops.leaky_relu(x)
return tape.gradient(y, x)
x = np.asarray(
[[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
dtype=np.float64,
order="F")
err = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(f, [x]))
print("leaky_relu (float64) gradient of gradient err = ", err)
self.assertLess(err, 1e-10)
def testNMS3From6Boxes(self):
with compat.forward_compatibility_horizon(2018, 8, 8):
# Three boxes are selected based on IOU.
boxes_data = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9],
[0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]]
boxes_np = np.array(boxes_data, dtype=np.float32)
scores_data = [0.9, 0.75, 0.6, 0.95, 0.5, 0.3]
scores_np = np.array(scores_data, dtype=np.float32)
max_output_size = 3
iou_threshold_np = np.array(0.5, dtype=np.float32)
score_threshold_np = np.array(0.0, dtype=np.float32)
with self.cached_session() as sess:
boxes = array_ops.placeholder(boxes_np.dtype, shape=boxes_np.shape)
scores = array_ops.placeholder(scores_np.dtype, shape=scores_np.shape)
iou_threshold = array_ops.placeholder(iou_threshold_np.dtype,
iou_threshold_np.shape)
score_threshold = array_ops.placeholder(score_threshold_np.dtype,
score_threshold_np.shape)
with self.test_scope():
selected_indices = image_ops.non_max_suppression_padded(
boxes=boxes,
scores=scores,
max_output_size=max_output_size,
iou_threshold=iou_threshold,
score_threshold=score_threshold,
pad_to_max_output_size=True)
inputs_feed = {
boxes: boxes_np,
scores: scores_np,
score_threshold: score_threshold_np,
iou_threshold: iou_threshold_np
}
(indices_tf, num_valid) = sess.run(
selected_indices, feed_dict=inputs_feed)
self.assertEqual(indices_tf.size, max_output_size)
self.assertEqual(num_valid, 3)
self.assertAllClose(indices_tf[:num_valid], [3, 0, 5])
def testCopyToDevicePingPongCPUGPU(self):
if not test_util.is_gpu_available():
self.skipTest("No GPU available")
with compat.forward_compatibility_horizon(2018, 8, 4):
host_dataset = dataset_ops.Dataset.range(10)
device_dataset = host_dataset.apply(
prefetching_ops.copy_to_device("/gpu:0", source_device="/cpu:0"))
back_to_cpu_dataset = device_dataset.apply(
prefetching_ops.copy_to_device("/cpu:0", source_device="/gpu:0"))
with ops.device("/cpu:0"):
iterator = back_to_cpu_dataset.make_initializable_iterator()
next_element = iterator.get_next()
with self.cached_session() as sess:
sess.run(iterator.initializer)
for i in range(10):
self.assertEqual(i, sess.run(next_element))
with self.assertRaises(errors.OutOfRangeError):
sess.run(next_element)
def test1DTensorAsInputNoReshape(self):
with compat.forward_compatibility_horizon(2018, 8, 27):
self._testSoftmax(
np.array([3., 2., 3., 9.]).astype(np.float64), use_gpu=False)
self._testOverflow(use_gpu=False)
def Test(self):
with compat.forward_compatibility_horizon(2019, 4, 19):
np.random.seed(42)
self._testBroadcasting(dtype, adjoint_a, adjoint_b, use_static_shape)
def testIteratorStringHandleFuture(self):
with forward_compat.forward_compatibility_horizon(2018, 8, 4):
dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
dataset_4 = dataset_ops.Dataset.from_tensor_slices([10, 20, 30, 40])
iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
iterator_4 = dataset_ops.make_one_shot_iterator(dataset_4)
handle_placeholder = array_ops.placeholder(dtypes.string, shape=[])
feedable_iterator = iterator_ops.Iterator.from_string_handle(
handle_placeholder, dataset_ops.get_legacy_output_types(dataset_3),
dataset_ops.get_legacy_output_shapes(dataset_3))
next_element = feedable_iterator.get_next()
self.assertTrue(dataset_ops.get_structure(dataset_3).is_compatible_with(
dataset_ops.get_structure(feedable_iterator)))
self.assertTrue(dataset_ops.get_structure(dataset_4).is_compatible_with(
dataset_ops.get_structure(feedable_iterator)))
with self.cached_session() as sess:
iterator_3_handle = sess.run(iterator_3.string_handle())
iterator_4_handle = sess.run(iterator_4.string_handle())
self.assertEqual(
10,
sess.run(
next_element,
feed_dict={handle_placeholder: iterator_4_handle}))
self.assertEqual(
1,
sess.run(
next_element,
feed_dict={handle_placeholder: iterator_3_handle}))
self.assertEqual(
20,
sess.run(
next_element,
feed_dict={handle_placeholder: iterator_4_handle}))
self.assertEqual(
2,
sess.run(
next_element,
feed_dict={handle_placeholder: iterator_3_handle}))
self.assertEqual(
30,
sess.run(
next_element,
feed_dict={handle_placeholder: iterator_4_handle}))
self.assertEqual(
3,
sess.run(
next_element,
feed_dict={handle_placeholder: iterator_3_handle}))
self.assertEqual(
40,
sess.run(
next_element,
feed_dict={handle_placeholder: iterator_4_handle}))
with self.assertRaises(errors.OutOfRangeError):
sess.run(
next_element, feed_dict={handle_placeholder: iterator_3_handle})
with self.assertRaises(errors.OutOfRangeError):
sess.run(
next_element, feed_dict={handle_placeholder: iterator_4_handle})
