def test_bincount_determinism_error(self):
arr = np.random.randint(0, 1000, size=1000)
with test_util.deterministic_ops(), self.assertRaisesRegex(
errors_impl.UnimplementedError,
"Determinism is not yet supported in GPU implementation of Bincount."):
self.evaluate(bincount_ops.bincount(arr, None, axis=None))
arr = np.random.randint(0, 1000, size=(100, 100))
with test_util.deterministic_ops(), self.assertRaisesRegex(
errors_impl.UnimplementedError,
"Determinism is not yet supported in GPU implementation of "
"DenseBincount."):
self.evaluate(bincount_ops.bincount(arr, None, axis=-1))
def test_deterministic_attribute(self, local_determinism,
global_determinism):
self._set_seed()
with test_util.deterministic_ops():
def sleep(x):
time.sleep(0.1)
return x
def map_function(x):
if math_ops.equal(x, 0):
return script_ops.py_func(sleep, [x],
x.dtype,
stateful=False)
else:
return x
dataset = dataset_ops.Dataset.range(100)
dataset = dataset.map(map_function,
num_parallel_calls=2,
deterministic=local_determinism)
opts = options_lib.Options()
opts.deterministic = global_determinism
dataset = dataset.with_options(opts)
self.assertDatasetProduces(dataset, expected_output=range(100))
def test_stateful_ops_map_and_batch(self, use_function,
use_legacy_map_and_batch):
with test_util.deterministic_ops():
v = variables.Variable(0.)
def map_fn(x):
v.assign_add(1.)
return (x, v.read_value())
if use_function:
map_fn = def_function.function(map_fn)
dataset = dataset_ops.Dataset.range(5)
if use_legacy_map_and_batch:
dataset = dataset.apply(
batching.map_and_batch(map_fn, 2, num_parallel_calls=5))
else:
dataset = dataset.map(map_fn, num_parallel_calls=5)
dataset = dataset.batch(2)
self.evaluate(variables.global_variables_initializer())
expected_output = [
(np.array([0, 1]), np.array([1, 2])),
(np.array([2, 3]), np.array([3, 4])),
(np.array([4]), np.array([5])),
]
self.assertDatasetProduces(dataset,
expected_output=expected_output,
requires_initialization=True)
def test_no_stateful_ops_interleave(self, use_function,
use_legacy_interleave):
self._set_seed()
with test_util.deterministic_ops():
def interleave_fn(x):
del x
return dataset_ops.Dataset.range(2)
if use_function:
interleave_fn = def_function.function(interleave_fn)
dataset = dataset_ops.Dataset.range(5)
if use_legacy_interleave:
dataset = dataset.apply(
testing.assert_next(["LegacyParallelInterleaveV2"]))
dataset = dataset.apply(
interleave_ops.parallel_interleave(interleave_fn,
cycle_length=5))
else:
dataset = dataset.apply(
testing.assert_next(["ParallelInterleave"]))
dataset = dataset.interleave(interleave_fn,
cycle_length=5,
num_parallel_calls=3)
options = options_lib.Options()
options.experimental_optimization.apply_default_optimizations = False
dataset = dataset.with_options(options)
self.evaluate(variables.global_variables_initializer())
self.assertDatasetProduces(dataset,
expected_output=[0] * 5 + [1] * 5)
def test_text_line_dataset(self, use_function):
self._set_seed()
with test_util.deterministic_ops():
def write_nums_to_file(filename, numbers):
path = os.path.join(self.get_temp_dir(), filename)
with open(path, "w") as f:
f.write("\n".join(str(n) for n in numbers))
return path
f1 = write_nums_to_file("f1", (1, 2, 3))
f2 = write_nums_to_file("f2", (4, 5, 6))
f3 = write_nums_to_file("f3", (7, 8, 9))
def interleave_fn(filename):
return reader_ops.TextLineDataset(filename)
if use_function:
interleave_fn = def_function.function(interleave_fn)
dataset = dataset_ops.Dataset.from_tensor_slices([f1, f2, f3])
dataset = dataset.apply(testing.assert_next(["ParallelInterleave"
]))
dataset = dataset.interleave(interleave_fn,
cycle_length=3,
num_parallel_calls=3)
options = options_lib.Options()
options.experimental_optimization.apply_default_optimizations = False
dataset = dataset.with_options(options)
self.assertDatasetProduces(
dataset,
expected_output=["1", "4", "7", "2", "5", "8", "3", "6", "9"])
def test_stateful_ops_interleave(self, use_function,
use_legacy_interleave):
with test_util.deterministic_ops():
v = variables.Variable(0.)
def map_fn(x):
v.assign_add(1.)
return (x, v.read_value())
def interleave_fn(x):
del x
return dataset_ops.Dataset.range(2).map(map_fn)
if use_function:
map_fn = def_function.function(map_fn)
interleave_fn = def_function.function(interleave_fn)
dataset = dataset_ops.Dataset.range(5)
if use_legacy_interleave:
dataset = dataset.apply(
interleave_ops.parallel_interleave(interleave_fn,
cycle_length=5))
else:
dataset = dataset.interleave(interleave_fn,
cycle_length=5,
num_parallel_calls=3)
options = options_lib.Options()
options.experimental_optimization.apply_default_optimizations = False
dataset = dataset.with_options(options)
self.evaluate(variables.global_variables_initializer())
expected_output = list(zip([0] * 5 + [1] * 5, range(1, 11)))
self.assertDatasetProduces(dataset,
expected_output=expected_output,
requires_initialization=True)
def testThrowDeterminismError(self):
shape = [6, 5]
seed = [42, 24]
matrix1 = stateless_random_ops.stateless_random_normal(shape, seed)
with test_util.deterministic_ops():
if test_util.is_gpu_available(cuda_only=True):
with self.assertRaisesRegex(
errors_impl.UnimplementedError, "Determinism is not yet supported "
"for Svd."):
self.evaluate(linalg_ops.svd(matrix1))
def testForwardAndBackward(self, sparse_labels, logits_time_major):
with test_util.deterministic_ops():
for seed in range(2):
loss_a, gradient_a = self._forwardAndBackward(
sparse_labels, logits_time_major, seed)
loss_b, gradient_b = self._forwardAndBackward(
sparse_labels, logits_time_major, seed)
loss_a, loss_b, gradient_a, gradient_b = self.evaluate(
(loss_a, loss_b, gradient_a, gradient_b))
self.assertAllEqual(loss_a, loss_b, "Loss mismatch")
self.assertAllEqual(gradient_a, gradient_b,
"Gradient mismatch")
def test_stateful_ops_map_with_random_ops(self):
with test_util.deterministic_ops():
def map_fn(x):
return x + random_ops.random_uniform(
(), 0, 2, dtype=dtypes.int64, seed=1)
dataset = dataset_ops.Dataset.range(5)
dataset = dataset.apply(testing.assert_next(["Map",
"ParallelMap"]))
dataset = dataset.map(map_fn, num_parallel_calls=5)
get_next = self.getNext(dataset, requires_initialization=True)
for i in range(5):
self.assertIn(self.evaluate(get_next()), [i, i + 1])
def _testForwardCase(self,
use_cudnn=False,
data_format="NHWC",
dtype=dtypes.float32):
for seed in range(5):
p = self._genParams(use_cudnn, data_format, dtype, seed=seed)
input_data, filter_data, strides, padding, _ = p
with test_util.deterministic_ops():
result_a = nn_impl.depthwise_conv2d_v2(input_data, filter_data, strides,
padding, data_format)
result_b = nn_impl.depthwise_conv2d_v2(input_data, filter_data, strides,
padding, data_format)
self.assertAllEqual(result_a, result_b)
def test_no_stateful_ops_map(self, use_function):
self._set_seed()
with test_util.deterministic_ops():
def map_fn(x):
return x + 1
if use_function:
map_fn = def_function.function(map_fn)
dataset = dataset_ops.Dataset.range(5)
dataset = dataset.apply(testing.assert_next(["ParallelMap"]))
dataset = dataset.map(map_fn, num_parallel_calls=5)
self.evaluate(variables.global_variables_initializer())
expected_output = range(1, 6)
self.assertDatasetProduces(dataset,
expected_output=expected_output)
def testDeterminismExceptionThrowing(self):
tensor_shape = (5, 20, 20, 3)
rand_mat = self._PRNG.random_sample(tensor_shape) * 1000 - 500
with test_util.deterministic_ops():
with self.assertRaisesRegex(
ValueError,
"requires a non-zero seed to be passed in when "
"determinism is enabled"):
nn_ops.fractional_max_pool_v2(rand_mat, [1, 1.5, 1.5, 1])
nn_ops.fractional_max_pool_v2(rand_mat, [1, 1.5, 1.5, 1], seed=1)
with self.assertRaisesRegex(
ValueError, 'requires "seed" and "seed2" to be non-zero'):
nn_ops.fractional_max_pool(rand_mat, [1, 1.5, 1.5, 1])
nn_ops.fractional_max_pool(rand_mat, [1, 1.5, 1.5, 1],
seed=1,
seed2=1,
deterministic=True)
def test_stateful_ops_map(self, use_function):
with test_util.deterministic_ops():
v = variables.Variable(0.)
def map_fn(x):
v.assign_add(1.)
return (x, v.read_value())
if use_function:
map_fn = def_function.function(map_fn)
dataset = dataset_ops.Dataset.range(5)
dataset = dataset.map(map_fn, num_parallel_calls=5)
self.evaluate(variables.global_variables_initializer())
expected_output = list(zip(range(0, 5), range(1, 6)))
self.assertDatasetProduces(dataset,
expected_output=expected_output,
requires_initialization=True)
def test_rewrite_prefetch(self):
with test_util.deterministic_ops():
v = variables.Variable(-1, dtype=dtypes.int64)
def map_fn(x):
v.assign(x)
return x
dataset = dataset_ops.Dataset.range(5)
dataset = dataset.map(map_fn)
dataset = dataset.prefetch(5)
self.evaluate(variables.global_variables_initializer())
get_next = self.getNext(dataset, requires_initialization=True)
self.assertEqual(self.evaluate(v), -1)
self.assertEqual(self.evaluate(get_next()), 0)
time.sleep(0.01)
self.assertEqual(self.evaluate(v), 0)
self.assertEqual(self.evaluate(get_next()), 1)
time.sleep(0.01)
self.assertEqual(self.evaluate(v), 1)
def test_stateful_ops_map_ignore_input(self, use_function):
with test_util.deterministic_ops():
v = variables.Variable(0.)
def map_fn(x):
del x
v.assign_add(1.)
return math_ops.constant_op.constant(1.)
if use_function:
map_fn = def_function.function(map_fn)
dataset = dataset_ops.Dataset.range(5)
dataset = dataset.map(map_fn, num_parallel_calls=5)
self.evaluate(variables.global_variables_initializer())
expected_output = [1.] * 5
self.assertDatasetProduces(dataset,
expected_output=expected_output,
requires_initialization=True)
def test_text_line_dataset(self, use_function, use_control_flow):
self._set_seed()
with test_util.deterministic_ops():
def write_nums_to_file(filename, numbers):
path = os.path.join(self.get_temp_dir(), filename)
with open(path, "w") as f:
f.write("\n".join(str(n) for n in numbers))
return path
f1 = write_nums_to_file("f1", (1, 2, 3))
f2 = write_nums_to_file("f2", (4, 5, 6))
f3 = write_nums_to_file("f3", (7, 8, 9))
if use_control_flow:
def interleave_fn(filename):
# Test function that uses control flow. The True branch is never taken
concat = string_ops.string_join([filename, "abc"])
return control_flow_ops.cond(
math_ops.equal(filename, "abc"),
lambda: reader_ops.TextLineDataset(concat),
lambda: reader_ops.TextLineDataset(filename))
else:
def interleave_fn(filename):
return reader_ops.TextLineDataset(filename)
if use_function:
interleave_fn = def_function.function(interleave_fn)
dataset = dataset_ops.Dataset.from_tensor_slices([f1, f2, f3])
dataset = dataset.apply(testing.assert_next(["ParallelInterleave"
]))
dataset = dataset.interleave(interleave_fn,
cycle_length=3,
num_parallel_calls=3)
self.assertDatasetProduces(
dataset,
expected_output=["1", "4", "7", "2", "5", "8", "3", "6", "9"])
def testDebugNumericSummaryV2OpDeterminism(self):
x = np.zeros([100, 100, 50], dtype=np.float64)
x = constant_op.constant(x)
modes = (
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
debug_event_pb2.TensorDebugMode.FULL_HEALTH,
)
for mode in modes:
debug_mode = debug_event_pb2.TensorDebugMode.Name(mode)
with test_util.deterministic_ops():
if test_util.is_gpu_available(cuda_only=True):
with self.assertRaisesRegex(
errors_impl.UnimplementedError, "Determinism is not yet "
"supported for DebugNumericSummaryV2 when tensor_debug_mode is "
+ debug_mode + "."):
self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=mode,
tensor_id=x._id,
output_dtype=dtypes.float64))
def _testBackwardCase(self,
using_gpu=False,
use_cudnn=False,
data_format="NHWC",
dtype=dtypes.float32):
p = self._genParams(use_cudnn, data_format, dtype, seed=123)
input_data, filter_data, strides, padding, output_shape = p
with test_util.deterministic_ops():
def Gradients(upstream_gradients):
with backprop.GradientTape() as tape:
tape.watch(input_data)
tape.watch(filter_data)
op_output = nn_impl.depthwise_conv2d_v2(input_data, filter_data,
strides, padding, data_format)
gradient_injector_output = op_output * upstream_gradients
return tape.gradient(gradient_injector_output,
[input_data, filter_data])
if using_gpu and not use_cudnn:
# This tests depends on other tests, tests which do not enable
# op-determinism, to ensure that determinism-unimplemented exceptions
# are not erroneously thrown when op-determinism is not enabled.
ctx_mgr = self.assertRaisesRegex(
errors.UnimplementedError, "A deterministic GPU implementation of" +
" DepthwiseConvBackpropFilter is not currently available.")
else:
ctx_mgr = contextlib.suppress()
with ctx_mgr:
# Test only two seeds, since testing takes a long time
for seed in (987, 988):
upstream_gradients = random_ops.random_normal(output_shape,
dtype=dtype, seed=seed)
input_gradients_a, filter_gradients_a = Gradients(upstream_gradients)
input_gradients_b, filter_gradients_b = Gradients(upstream_gradients)
self.assertAllEqual(input_gradients_a, input_gradients_b)
self.assertAllEqual(filter_gradients_a, filter_gradients_b)
def test_stateful_ops_batch(self, use_function):
with test_util.deterministic_ops():
v = variables.Variable(0.)
def map_fn(x):
return (x, v.read_value())
if use_function:
map_fn = def_function.function(map_fn)
dataset = dataset_ops.Dataset.range(5)
dataset = dataset.map(map_fn)
dataset = dataset.apply(testing.assert_next(["Batch"]))
dataset = dataset.batch(2, num_parallel_calls=2)
self.evaluate(variables.global_variables_initializer())
expected_output = [
(np.array([0, 1]), np.array([0, 0])),
(np.array([2, 3]), np.array([0, 0])),
(np.array([4]), np.array([0])),
]
self.assertDatasetProduces(dataset,
expected_output=expected_output,
requires_initialization=True)
