def testDeterministicDropoutTest(self): x = tf.ones([4, 6], dtype=tf.float32) x = py_utils.DeterministicDropout(x, keep_prob=0.7, seeds=[1234, 5678]) with self.session() as sess: x_val = sess.run(x) # pyformat: disable self.assertAllClose( [[1.0 / 0.7, 0.0000000, 0.0000000, 0.0000000, 1.0 / 0.7, 1.0 / 0.7], [1.0 / 0.7, 1.0 / 0.7, 1.0 / 0.7, 1.0 / 0.7, 1.0 / 0.7, 1.0 / 0.7], [1.0 / 0.7, 0.0000000, 0.0000000, 1.0 / 0.7, 1.0 / 0.7, 0.0000000], [1.0 / 0.7, 0.0000000, 0.0000000, 1.0 / 0.7, 1.0 / 0.7, 1.0 / 0.7]], x_val) # pyformat: enable self.assertAllClose(22.85714, np.sum(x_val)) self.assertEqual(x_val.dtype, np.float32)
def _GetWeight(self, theta): p = self.params filter_w = theta.w # First normalize filter_w over the temporal dimension here. filter_w = tf.nn.softmax(filter_w / p.temperature, axis=0) # Add dropconnect on the weights for regularization. if p.dropconnect_prob > 0.0 and not self.do_eval: if p.deterministic_dropout: filter_w = py_utils.DeterministicDropout( filter_w, 1.0 - p.dropconnect_prob, py_utils.GenerateStepSeedPair(p)) else: filter_w = tf.nn.dropout( filter_w, rate=p.dropconnect_prob, seed=p.random_seed) # Tie the parameters of every subsequent number of weight_tiling_factor # channels. filter_w = tf.tile(filter_w, [1, 1, p.weight_tiling_factor, 1]) return filter_w