def test_noise_shape_correctness(self): x = np.random.normal(size=(4, 16)).astype(np.float32) dropped_rows = self.evaluate( stateless_dropout_lib.stateless_dropout(x, 0.8, seed=[2, 4], noise_shape=[4, 1])) dropped_columns = self.evaluate( stateless_dropout_lib.stateless_dropout(x, 0.8, seed=[2, 4], noise_shape=[1, 16])) self.assertBetween(np.sum(np.sum(dropped_rows, axis=-1) == 0), 1, 3) self.assertBetween(np.sum(np.sum(dropped_columns, axis=0) == 0), 5, 15)
def f(x, seed=np.array(1, dtype=np.int32)): if recompute_grad_lib.get_recompute_context() is not None: seed = recompute_grad_lib.get_recompute_context().seed return stateless_dropout_lib.stateless_dropout(model(x), rate=0.5, seed=tf.stack( [seed, 0]))
def dropped_inputs(): """Randomly drops elements of `inputs` when `training=True`.""" recompute_context = recompute_grad_lib.get_recompute_context() if recompute_context is None: if self.force_recomputation: raise ValueError( 'RecomputeContext is required when force_recomputation=True.') return tf.nn.dropout( inputs, noise_shape=self._get_noise_shape(inputs), seed=self.seed, rate=self.rate) seed = tf.stack([recompute_context.seed, self._recompute_seed]) return stateless_dropout_lib.stateless_dropout( inputs, rate=self.rate, seed=seed, noise_shape=self._get_noise_shape(inputs))
def f(x): if recompute_grad_lib.get_recompute_context() is None: generator = tf.random.experimental.get_global_generator() recompute_grad_seed = tf.stack( (generator.uniform_full_int([], tf.int32, name='seed'), 0)) else: recompute_grad_seed = tf.stack( (recompute_grad_lib.get_recompute_context().seed, 0)) seeds = tf.random.stateless_uniform([len(self._units), 2], recompute_grad_seed, minval=-2**31, maxval=2**31 - 1, dtype=tf.int32, name='dropout_seeds') for i, (kernel, bias) in enumerate(zip(self._kernels, self._biases)): x = tf.nn.tanh(tf.matmul(x, kernel) + bias) x = stateless_dropout_lib.stateless_dropout( x, self._rate, seeds[i]) return x
def test_deterministic(self): x = np.random.normal(size=(4, 8)).astype(np.float32) y = self.evaluate( stateless_dropout_lib.stateless_dropout(x, 0.5, seed=[2, 4])) self.assertAllEqual( stateless_dropout_lib.stateless_dropout(x, 0.5, seed=[2, 4]), y)
def test_rate_correctness(self): x = np.random.normal(size=(4, 8)).astype(np.float32) y = self.evaluate( stateless_dropout_lib.stateless_dropout(x, 0.8, seed=[2, 4])) self.assertBetween(np.sum(y == 0), 10, 31)
def test_rate_zero(self): x = np.random.normal(size=(4, 8)).astype(np.float32) self.assertAllEqual( stateless_dropout_lib.stateless_dropout(x, 0, seed=[1, 0]), x)