def test_unary_cwise_real_ops_2(self): real_ops = [ math_ops.neg, math_ops.negative, math_ops.reciprocal, math_ops.rint, math_ops.round, math_ops.rsqrt, math_ops.sigmoid, math_ops.sign, math_ops.sin, math_ops.sinh, math_ops.sqrt, math_ops.square, math_ops.tan, math_ops.tanh, nn.elu, nn.relu, nn.relu6, lambda t: nn.leaky_relu(t, alpha=0.1), nn.selu, nn.softplus, nn.softsign, ] self._test_unary_cwise_ops(real_ops, False)
def dropped_inputs_training(): with ops.name_scope("random_relu_training"): x = ops.convert_to_tensor(inputs, name="x") if not x.dtype.is_floating: raise ValueError( "x has to be a floating point tensor since it's going to" " be scaled. Got a %s tensor instead." % x.dtype) if isinstance(self.a, numbers.Real) and not 0 < self.a: raise ValueError( "a must be a scalar tensor or a float positive " ", got %g" % self.a) if isinstance(self.b, numbers.Real) and not 0 < self.b: raise ValueError( "b must be a scalar tensor or a float positive " ", got %g" % self.b) if isinstance(self.b, numbers.Real) and \ isinstance(self.a, numbers.Real) and \ not self.a < self.b: raise ValueError( "a and b must be a scalar tensor or a float such that" " a < b, got {} and {}".format(self.a, self.b)) else: a = ops.convert_to_tensor(self.a, dtype=x.dtype, name="a") b = ops.convert_to_tensor(self.b, dtype=x.dtype, name="b") a.get_shape().assert_is_compatible_with( tensor_shape.scalar()) b.get_shape().assert_is_compatible_with( tensor_shape.scalar()) # TODO : confirm this noise shape !! noise_shape = array_ops.shape(x) random_tensor = math_ops.divide(1, b) random_tensor += math_ops.divide( b - a, a * b) * random_ops.random_uniform( noise_shape, seed=self.seed, dtype=x.dtype) ret = nn.leaky_relu(x, alpha=random_tensor) if not context.executing_eagerly(): ret.set_shape(x.get_shape()) return ret
def test_two_conv2d_fusions(self): """Test two Conv2D patterns and only the second is fusable.""" if not test_util.is_gpu_available(cuda_only=True, min_cuda_compute_capability=(8, 0)): self.skipTest('No GPU with compute compatibility >= 8.0 available') N, H, W, C = (5, 3, 3, 8) # pylint: disable=invalid-name ops.reset_default_graph() x_shape = [N, C, H, W] x_format, b_format = ('NCHW', 'NC..') x = _input(x_shape) w = _weight([2, 2, C, C]) b = _bias([C]) y = nn_ops.conv2d(x, w, strides=(1, 1), padding='SAME', data_format=x_format) y = nn.bias_add(y, b, data_format=b_format) y = nn.leaky_relu(y) y = nn_ops.conv2d(y, w, strides=(1, 1), padding='SAME', data_format=x_format) y = nn.bias_add(y, b, data_format=b_format) y = nn.relu(y) out = array_ops.identity(y) # The first Conv-BiasAdd-LeakyRelu is not fusable because cuDNN requires # fp16 for this pattern. The second Conv-BiasAdd-Relu is fusable. epilog_ops = [b'BiasAdd', b'Relu'] fused_op = ['_FusedConv2D'] self._VerifyValues(out, False, fused_op, epilog_ops)
def dropped_inputs_testing(): with ops.name_scope("drop_activation_testing"): # in testing phase, deterministic activation function # leaky ReLU with slope = 1. - p return nn.leaky_relu(inputs, alpha=1 - self.p)
def dropped_inputs_testing(): with ops.name_scope("random_relu_testing"): # in testing phase, deterministic activation function # leaky ReLU with slope = 1. - p return nn.leaky_relu(inputs, alpha=1 / (0.5 * (self.a + self.b)))
def leaky_relu(x, alpha=0.1): return nn.leaky_relu(x, alpha=alpha)