def test_elu(): x = K.placeholder(ndim=2) f = K.function([x], [activations.elu(x, 0.5)]) test_values = get_standard_values() result = f([test_values])[0] assert_allclose(result, test_values, rtol=1e-05) negative_values = np.array([[-1, -2]], dtype=K.floatx()) # cntk can't rebind the input shape, so create the model again to test different batch size if (K.backend() == 'cntk'): x2 = K.placeholder(ndim=2) f = K.function([x2], [activations.elu(x2, 0.5)]) result = f([negative_values])[0] true_result = (np.exp(negative_values) - 1) / 2 assert_allclose(result, true_result)
def test_elu(): x = K.placeholder(ndim=2) f = K.function([x], [activations.elu(x, 0.5)]) test_values = get_standard_values() result = f([test_values])[0] assert_allclose(result, test_values, rtol=1e-05) negative_values = np.array([[-1, -2]], dtype=K.floatx()) # cntk can't rebind the input shape, so create the model again to test different batch size if (K.backend() == 'cntk'): x2 = K.placeholder(ndim=2) f = K.function([x2], [activations.elu(x2, 0.5)]) result = f([negative_values])[0] true_result = (np.exp(negative_values) - 1) / 2 assert_allclose(result, true_result)
def selu(x): """Scaled Exponential Linear Unit. (Klambauer et al., 2017) # Arguments x: A tensor or variable to compute the activation function for. # References - [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515) """ alpha = 1.6732632423543772848170429916717 scale = 1.0507009873554804934193349852946 return scale * elu(x, alpha)
def test_elu(self): x = backend.placeholder(ndim=2) f = backend.function([x], [activations.elu(x, 0.5)]) test_values = np.random.random((2, 5)) result = f([test_values])[0] self.assertAllClose(result, test_values, rtol=1e-05) negative_values = np.array([[-1, -2]], dtype=backend.floatx()) result = f([negative_values])[0] true_result = (np.exp(negative_values) - 1) / 2 self.assertAllClose(result, true_result)
def test_elu(): x = K.placeholder(ndim=2) f = K.function([x], [activations.elu(x, 0.5)]) test_values = get_standard_values() result = f([test_values])[0] assert_allclose(result, test_values, rtol=1e-05) negative_values = np.array([[-1, -2]], dtype=K.floatx()) result = f([negative_values])[0] true_result = (np.exp(negative_values) - 1) / 2 assert_allclose(result, true_result)
def test_elu(): x = K.placeholder(ndim=2) f = K.function([x], [activations.elu(x, 0.5)]) test_values = get_standard_values() result = f([test_values])[0] assert_allclose(result, test_values, rtol=1e-05) negative_values = np.array([[-1, -2]], dtype=K.floatx()) result = f([negative_values])[0] true_result = (np.exp(negative_values) - 1) / 2 assert_allclose(result, true_result)
def selu(x): """ Scaled Exponential Linear Unit. (Klambauer et al., 2017) # References - [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515) :param x: tensor or variable to compute the activation function :return: """ from keras.activations import elu alpha = 1.6732632423543772848170429916717 scale_ = 1.0507009873554804934193349852946 return scale_ * elu(x, alpha)
def eluu(x): return elu(x, alpha=2)
def selu(x): alpha = 1.6732632423543772848170429916717 scale = 1.0507009873554804934193349852946 return scale * elu(x, alpha)
import numpy as np import matplotlib.pyplot as plt from keras.activations import elu ''' def elu(x, a=1): y_list = [] for x in x: if x > 0 : y = x if x<= 0 : y = a*(np.exp(x)-1) y_list.append(y) return y_list ''' def elu(x, a=1): return list(map(lambda x: x if x > 0 else a * (np.exp(x) - 1), x)) x = np.arange(-5, 5, 0.1) y = elu(x) plt.plot(x, y) plt.grid() plt.show()
}) ## nonlinearities: add magic constants def gelu(x): cdf = 0.5 * (1.0 + tf.erf(x / tf.sqrt(2.0))) return x * cdf nonlinearities = { 'identity': Lambda(lambda x: x), 'celu': Lambda(lambda x: tf.nn.crelu(x) * 1.270926833152771), 'elu': Lambda(lambda x: elu(x) * 1.2716004848480225), 'gelu': Lambda(lambda x: gelu(x) * 1.7015043497085571), # 'glu': lambda x: jax.nn.glu(x) * 1.8484294414520264, 'leaky_relu': Lambda(lambda x: tf.nn.leaky_relu(x) * 1.70590341091156), 'log_sigmoid': Lambda(lambda x: tf.math.log(tf.nn.sigmoid(x)) * 1.9193484783172607), 'log_softmax': Lambda(lambda x: tf.math.log(tf.nn.softmax(x)) * 1.0002083778381348), 'relu': Lambda(lambda x: relu(x) * 1.7139588594436646), 'relu6': Lambda(lambda x: tf.nn.relu6(x) * 1.7131484746932983), 'selu': Lambda(lambda x: selu(x) * 1.0008515119552612),