def testDeterminism(self):
# This does not test any ops are deterministic, because that is tested by
# many kernel tests.
try:
config.disable_op_determinism()
self.assertFalse(config.is_op_determinism_enabled())
config.enable_op_determinism()
self.assertTrue(config.is_op_determinism_enabled())
finally:
config.disable_op_determinism()
def tensorflow_random_state(seed: int) -> Generator[None, None, None]:
# Save values
origin_gpu_det = os.environ.get("TF_DETERMINISTIC_OPS", None)
orig_random_state = random.getstate()
orig_np_random_state = np.random.get_state()
if context.executing_eagerly():
tf_random_seed = context.global_seed()
else:
tf_random_seed = ops.get_default_graph().seed
determism_enabled = config.is_op_determinism_enabled()
config.enable_op_determinism()
# Set values
os.environ["TF_DETERMINISTIC_OPS"] = "1"
random.seed(seed)
np.random.seed(seed)
tf.random.set_seed(seed)
yield
# Reset values
if origin_gpu_det is not None:
os.environ["TF_DETERMINISTIC_OPS"] = origin_gpu_det
else:
os.environ.pop("TF_DETERMINISTIC_OPS")
random.setstate(orig_random_state)
np.random.set_state(orig_np_random_state)
tf.random.set_seed(tf_random_seed)
if not determism_enabled:
config.disable_op_determinism()
def testSecondGradient(self):
with self.session() as sess:
labels = constant_op.constant([3, 0, 1], name="labels")
logits = constant_op.constant(
[0.3, 0.4, 0.1, 1.2, 0.1, 1.9, 0.1, 0.7, 0.8, 0.2, 1.3, 1.3],
shape=[3, 4],
dtype=dtypes.float64,
name="logits")
def xent_grad(logits):
with backprop_lib.GradientTape() as tape:
tape.watch(logits)
return tape.gradient(
nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
labels=labels, logits=logits, name="xent"),
[logits])[0]
analytical, numerical = gradient_checker_v2.compute_gradient(
xent_grad, [logits])
if (not context.executing_eagerly()
and not config.is_op_determinism_enabled()):
# Check that second derivative is calculated.
# (it is equivalent to being `BatchMatMul` op in the graph because of
# implementation of xentropy grad)
op_names = [
op.op_def.name for op in sess.graph.get_operations()
if op.op_def
]
self.assertIn("BatchMatMulV2", op_names)
tol = 5e-8
self.assertAllClose(analytical, numerical, atol=tol, rtol=tol)
def testSecondGradient(self):
with self.cached_session() as sess:
labels = constant_op.constant([
0.0, 0.0, 1.0 / 3, 0.0, 1.0 / 3, 0.0, 0.0, 0.0, 0.0, 0.5 / 3,
0.0, 0.5 / 3
],
shape=[12],
dtype=dtypes.float64,
name="labels")
logits = constant_op.constant(
[0.1, 0.2, 0.3, 0.4, 0.1, 0.4, 0.9, 1.6, 0.1, 0.8, 2.7, 6.4],
shape=[12],
dtype=dtypes.float64,
name="logits")
x = nn_ops.softmax_cross_entropy_with_logits(labels=labels,
logits=logits,
name="xent")
loss = math_ops.reduce_sum(x)
gradients = gradients_impl.gradients(loss, [logits])[0]
err = gradient_checker.compute_gradient_error(
logits, [12], gradients, [12])
if not config.is_op_determinism_enabled():
# Check how second derivative is calculated.
# (it is equivalent to a `BatchMatMul` op being in the graph because of
# the implementation in SoftmaxCrossEntropyWithLogitsGrad)
op_names = [
op.op_def.name for op in sess.graph.get_operations()
if op.op_def
]
self.assertIn("BatchMatMulV2", op_names)
self.assertLess(err, 5e-8)
def get_seed(op_seed):
"""Returns the local seeds an operation should use given an op-specific seed.
Given operation-specific seed, `op_seed`, this helper function returns two
seeds derived from graph-level and op-level seeds. Many random operations
internally use the two seeds to allow user to change the seed globally for a
graph, or for only specific operations.
For details on how the graph-level seed interacts with op seeds, see
`tf.compat.v1.random.set_random_seed`.
Args:
op_seed: integer.
Returns:
A tuple of two integers that should be used for the local seed of this
operation.
"""
eager = context.executing_eagerly()
if eager:
global_seed = context.global_seed()
else:
global_seed = ops.get_default_graph().seed
if global_seed is not None:
if op_seed is None:
# pylint: disable=protected-access
if hasattr(ops.get_default_graph(), '_seed_used'):
ops.get_default_graph()._seed_used = True
if eager:
op_seed = context.internal_operation_seed()
else:
op_seed = _graph_to_seed_dict.setdefault(
ops.get_default_graph(), 0)
_graph_to_seed_dict[ops.get_default_graph()] += 1
seeds = _truncate_seed(global_seed), _truncate_seed(op_seed)
else:
if op_seed is not None:
seeds = DEFAULT_GRAPH_SEED, _truncate_seed(op_seed)
else:
seeds = None, None
if seeds == (None, None) and config.is_op_determinism_enabled():
raise RuntimeError( # pylint: disable=g-doc-exception
'Random ops require a seed to be set when determinism is enabled. '
'Please set a seed before running the op, e.g. by calling '
'tf.random.set_seed(1).')
# Avoid (0, 0) as the C++ ops interpret it as nondeterminism, which would
# be unexpected since Python docs say nondeterminism is (None, None).
if seeds == (0, 0):
return (0, _MAXINT32)
return seeds
def get_global_generator():
"""Retrieves the global generator.
This function will create the global generator the first time it is called,
and the generator will be placed at the default device at that time, so one
needs to be careful when this function is first called. Using a generator
placed on a less-ideal device will incur performance regression.
Returns:
The global `tf.random.Generator` object.
"""
global global_generator
if global_generator is None:
if config.is_op_determinism_enabled():
raise RuntimeError('"get_global_generator" cannot be called if ' # pylint: disable=g-doc-exception
"determinism is enabled, unless "
'"set_global_generator" has already been called. '
'Please call "set_global_generator" first.')
with ops.init_scope():
global_generator = Generator.from_non_deterministic_state()
return global_generator
def from_non_deterministic_state(cls, alg=None):
"""Creates a generator by non-deterministically initializing its state.
The source of the non-determinism will be platform- and time-dependent.
Args:
alg: (optional) the RNG algorithm. If None, it will be auto-selected. See
`__init__` for its possible values.
Returns:
The new generator.
"""
if config.is_op_determinism_enabled():
raise RuntimeError('"from_non_deterministic_state" cannot be called when ' # pylint: disable=g-doc-exception
"determinism is enabled.")
if alg is None:
# TODO(b/170668986): more sophisticated algorithm selection
alg = DEFAULT_ALGORITHM
alg = stateless_random_ops.convert_alg_to_int(alg)
state = non_deterministic_ints(shape=[_get_state_size(alg)],
dtype=SEED_TYPE)
return cls(state=state, alg=alg)
def testGradRandomBoxes(self):
"""Test that the gradient is correct for randomly generated boxes.
The mapping is piecewise differentiable with respect to the box coordinates.
The points where the function is not differentiable are those which are
mapped to image pixels, i.e., the normalized y coordinates in
np.linspace(0, 1, image_height) and normalized x coordinates in
np.linspace(0, 1, image_width). Make sure that the box coordinates are
sufficiently far away from those rectangular grid centers that are points of
discontinuity, so that the finite difference Jacobian is close to the
computed one.
"""
np.random.seed(1) # Make it reproducible.
delta = 1e-3
radius = 2 * delta
low, high = -0.5, 1.5 # Also covers the case of extrapolation.
image_height = 4
for image_width in range(1, 3):
for crop_height in range(1, 3):
for crop_width in range(2, 4):
for depth in range(1, 3):
for num_boxes in range(1, 3):
batch = num_boxes
image_shape = [batch, image_height, image_width, depth]
crop_size = [crop_height, crop_width]
image = np.arange(0, batch * image_height * image_width *
depth).reshape(image_shape).astype(np.float32)
boxes = []
for _ in range(num_boxes):
# pylint: disable=unbalanced-tuple-unpacking
y1, y2 = self._randomUniformAvoidAnchors(
low, high, np.linspace(0, 1, image_height), radius, 2)
x1, x2 = self._randomUniformAvoidAnchors(
low, high, np.linspace(0, 1, image_width), radius, 2)
# pylint: enable=unbalanced-tuple-unpacking
boxes.append([y1, x1, y2, x2])
boxes = np.array(boxes, dtype=np.float32)
box_ind = np.arange(batch, dtype=np.int32)
image_tensor = constant_op.constant(image, shape=image_shape)
boxes_tensor = constant_op.constant(boxes, shape=[num_boxes, 4])
box_ind_tensor = constant_op.constant(box_ind, shape=[num_boxes])
def crop_resize(image_tensor, boxes_tensor):
# pylint: disable=cell-var-from-loop
return image_ops.crop_and_resize(
image_tensor, boxes_tensor, box_ind_tensor,
constant_op.constant(crop_size, shape=[2]))
with test_util.device(use_gpu=True):
with self.cached_session():
# pylint: disable=cell-var-from-loop
if (config.is_op_determinism_enabled() and
test_util.is_gpu_available()):
with self.assertRaises(errors_impl.UnimplementedError):
gradient_checker_v2.compute_gradient(
lambda x: crop_resize(x, boxes_tensor),
[image_tensor])
with self.assertRaises(errors_impl.UnimplementedError):
gradient_checker_v2.compute_gradient(
lambda x: crop_resize(image_tensor, x),
[boxes_tensor])
else:
err1 = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(
lambda x: crop_resize(x, boxes_tensor),
[image_tensor]))
err2 = gradient_checker_v2.max_error(
*gradient_checker_v2.compute_gradient(
lambda x: crop_resize(image_tensor, x),
[boxes_tensor]))
err = max(err1, err2)
self.assertLess(err, 2e-3)
