def build(self, input_shape): #The initialisation parameters self.mean = 0.0 self.stddev = 1.0 dtype = 'float32' self.seed = 1 #The output and the inut dimension k = self.output_dim d = self.input_dim #Initialise the variables to be trained. The variables are according to the #function defined. self.W = K.random_normal_variable((k, d, d), self.mean, self.stddev, dtype=dtype) self.V = K.random_normal_variable((2 * d, k), self.mean, self.stddev, dtype=dtype) self.b = K.zeros((self.input_dim, )) #Set the variables to be trained. self.trainable_weights = [self.W, self.V, self.b]
def sampling(args): z_mean, z_log_sigma = args epsilon = K.random_normal_variable(shape=(self.config.batch_size, self.config.latent_dim), mean=0., scale=1.) return z_mean + K.exp(z_log_sigma / 2.) * epsilon
def build(self, input_shape): self.patch_layer.build(input_shape) self.input_shape_t = self.patch_layer.compute_output_shape(input_shape) self.dim = self.input_shape_t[-1] self.filters = self.dict_size self.strides = (1, self.dim) self.kernel_shape = (1, self.dim, self.dict_size) self.D0 = K.random_normal_variable((self.dim, self.dict_size), mean=0, scale=1) self.D = tf.matmul(tf.diag(1 / tf.norm(self.D0, axis=1)), self.D0) self.D_ols = tf.matmul(tf.linalg.inv( tf.matmul(self.D, self.D, transpose_a=True) + self.alpha * tf.eye(self.dict_size)), self.D, transpose_b=True) self.kernel = K.reshape(self.D_ols, self.kernel_shape) #self.add_weight(shape=self.kernel_shape, # initializer='glorot_uniform', # name='kernel') self.D_kernel = K.reshape(tf.matmul(self.D, self.D_ols), (1, self.dim, self.dim)) self.trainable_weights = [self.D0]
def test_specify_initial_state(layer_class): num_states = 2 if layer_class is recurrent.LSTM else 1 # Test with Keras tensor inputs = Input((timesteps, embedding_dim)) initial_state = [Input((units, )) for _ in range(num_states)] layer = layer_class(units) output = layer(inputs, initial_state=initial_state) model = Model([inputs] + initial_state, output) model.compile(loss='categorical_crossentropy', optimizer='adam') inputs = np.random.random((num_samples, timesteps, embedding_dim)) initial_states = [ np.random.random((num_samples, units)) for _ in range(num_states) ] targets = np.random.random((num_samples, units)) model.fit([inputs] + initial_states, targets) # Test with non-Keras tensor inputs = Input((timesteps, embedding_dim)) initial_state = [ K.random_normal_variable((units, ), 0, 1) for _ in range(num_states) ] layer = layer_class(units) output = layer(inputs, initial_state=initial_state) model = Model([inputs], output) model.compile(loss='categorical_crossentropy', optimizer='adam') inputs = np.random.random((num_samples, timesteps, embedding_dim)) targets = np.random.random((num_samples, units)) model.fit(inputs, targets)
def test_specify_initial_state(layer_class): num_states = 2 if layer_class is recurrent.LSTM else 1 # Test with Keras tensor inputs = Input((timesteps, embedding_dim)) initial_state = [Input((units,)) for _ in range(num_states)] layer = layer_class(units) output = layer(inputs, initial_state=initial_state) model = Model([inputs] + initial_state, output) model.compile(loss='categorical_crossentropy', optimizer='adam') inputs = np.random.random((num_samples, timesteps, embedding_dim)) initial_states = [np.random.random((num_samples, units)) for _ in range(num_states)] targets = np.random.random((num_samples, units)) model.fit([inputs] + initial_states, targets) # Test with non-Keras tensor inputs = Input((timesteps, embedding_dim)) initial_state = [K.random_normal_variable((units,), 0, 1) for _ in range(num_states)] layer = layer_class(units) output = layer(inputs, initial_state=initial_state) model = Model([inputs], output) model.compile(loss='categorical_crossentropy', optimizer='adam') inputs = np.random.random((num_samples, timesteps, embedding_dim)) targets = np.random.random((num_samples, units)) model.fit(inputs, targets)
def __init__(self, val_shape, state_shape, output_shape, dropout): #inputs x = K.placeholder(shape=val_shape) h_1 = K.placeholder(shape=val_shape) C_1 = K.placeholder(shape=state_shape) #Forget gate comb = K.concatenate([h_1, x]) Wf = K.random_normal_variable(shape=K.int_shape(comb), mean=0, scale=1, name='forget_W') bf = K.random_normal_variable(shape=K.int_shape(comb), mean=0.3, scale=1, name='forget_b') ft = self._sigmoid_node(x, h_1, Wf, bf) #input gate layer Wi = K.random_normal_variable(shape=K.int_shape(comb), mean=0, scale=1, name='i_W') bi = K.random_normal_variable(shape=K.int_shape(comb), mean=0, scale=1, name='i_b') it = self._sigmoid_node(x, h_1, Wi, b) #new state inclusion layer Wcl = K.random_normal_variable(shape=K.int_shape(comb), mean=0, scale=1, name='cl_W') bcl = K.random_normal_variable(shape=K.int_shape(comb), mean=0, scale=1, name='cl_b') CLt = K.tanh((Wc * comb) + bc) #New state self.Ct = (ft * C_1) + (it * CLt) #Output layer Wo = K.random_normal_variable(shape=K.int_shape(comb), mean=0, scale=1, name='o_W') bo = K.random_normal_variable(shape=K.int_shape(comb), mean=0, scale=1, name='o_b') ot = self._sigmoid_node(x, h_1, Wo, bo) self.ht = ot * K.tanh(Ct)
def sampling(args): z_mean, z_log_var = args epsilon = K.random_normal_variable(shape=(params['batch_size'], params['hidden_dim']), mean=0., scale=1.) # insert kl loss here z_rand = z_mean + K.exp(z_log_var / 2) * kl_loss_var * epsilon return K.in_train_phase(z_rand, z_mean)
def __call__(self, w): if self.sd_lambda == 0: return 0 else: w = K.reshape(w, [-1, w.shape.as_list()[-1]]) x = K.random_normal_variable(shape=(int(w.shape[1]), 1), mean=0, scale=1) for i in range(0, self.iterations): x_p = K.dot(w, x) x = K.dot(K.transpose(w), x_p) return self.sd_lambda * K.sum(K.pow(K.dot(w, x), 2.0)) / K.sum(K.pow(x, 2.0))
def __call__(self, w): norm = self.max_k if len(w.shape) == 4: x = K.random_normal_variable(shape=(1,) + self.in_shape[1:3] + (self.in_shape[0],), mean=0, scale=1) for i in range(0, self.iterations): x_p = K.conv2d(x, w, strides=self.stride, padding=self.padding) x = K.conv2d_transpose(x_p, w, x.shape, strides=self.stride, padding=self.padding) Wx = K.conv2d(x, w, strides=self.stride, padding=self.padding) norm = K.sqrt(K.sum(K.pow(Wx, 2.0)) / K.sum(K.pow(x, 2.0))) else: x = K.random_normal_variable(shape=(int(w.shape[1]), 1), mean=0, scale=1) for i in range(0, self.iterations): x_p = K.dot(w, x) x = K.dot(K.transpose(w), x_p) norm = K.sqrt(K.sum(K.pow(K.dot(w, x), 2.0)) / K.sum(K.pow(x, 2.0))) return w * (1.0 / K.maximum(1.0, norm / self.max_k))
def sampling(args): ## could not save model(pickle issue), should do some modify. z_mean, z_log_var = args epsilon = K.random_normal_variable(shape=(params['batch_size'], params['hidden_dim']), mean=0., scale=1.) # insert kl loss here z_rand = z_mean + K.exp(z_log_var / 2) * kl_loss_var * epsilon return K.in_train_phase(z_rand, z_mean)
def cross_block(x, y): ''' x_(l+1) = x_0 * x^T_l * w_l + b_l + x_l x : input feature y : the l-th feature after cross feature layer ''' y_shape = K.int_shape(y) weight = K.random_normal_variable(shape=(y_shape[1], 1), mean=0, scale=0.01) tmp = K.dot(x, weight) x_shape = K.int_shape(x) tmp = Lambda(lambda z: K.repeat_elements(z, x_shape[1], axis=1))(tmp) # tmp = Lambda(lambda z: K.dot(K.permute_dimensions(z[0], (0, 2, 1)), z[1]))([y, weight]) return multiply([x, tmp])
def func(x): o, u = x[0], x[1] A = [] for i in range(5): A.append(x[i + 2]) alpha = K.random_normal_variable(shape=(1, ), mean=0, scale=1, name='alpha') dot_prod3 = Lambda(name='dot_prod3_' + pfx, function=lambda x: K.sum( multiply([x[0], x[1]]), keepdims=True, axis=1)) scores = [] for i in range(5): scores.append(dot_prod3([(alpha * o + (1 - alpha) * u), A[i]])) # shape = (None, 1) z = Activation('softmax')(concatenate(scores)) # shape = (None, 5) return z
def build(self, input_shape): if self.multiple_inputs: n_in = input_shape[1][-1] else: n_in = input_shape[-1] print(n_in) #sda #n_in=n_in+1 n_out = self.attention_dim lim = np.sqrt(6. / (n_in + n_out)) W = K.random_uniform_variable((n_in, n_out), -lim, lim, name='{}_W'.format(self.name)) b = K.zeros((n_out, ), name='{}_b'.format(self.name)) self.W = W self.b = b self.v = K.random_normal_variable(shape=(n_out, 1), mean=0, scale=0.1, name='{}_v'.format(self.name)) self.trainable_weights = [self.W, self.v, self.b]
def init_normal(shape, name=None): return K.random_normal_variable(shape, mean = 0.0, scale=0.01, name=name)
def norm_weight(shape, scale=0.01, name=None): return K.random_normal_variable(shape, 0.0, scale, name=name)
def normal(shape, scale=0.05, name=None, dim_ordering='th'): return K.random_normal_variable(shape, 0.0, scale, name=name)
def init_normal(shape, name=None): return K.random_normal_variable(shape, 0.0, 0.01, name=name)
def sampling(self, args): z_mean, z_log_sigma = args epsilon = K.random_normal_variable(shape=(BATCH_SIZE, NOISE_DIM), mean=0.0, scale=1.0) return z_mean + K.exp(z_log_sigma) * epsilon
def init_w_k(shape, dtype=None): return K.random_normal_variable(kernel_shape, mean=0, scale=s, dtype=tf.float32, seed=self.seed)
def small_init(shape, dtype=None): return K.random_normal_variable(shape=shape, mean=.0, scale=.01)
# Set up the decoder, using `encoder_states` as initial state. decoder_inputs = Input(shape=(max_decoder_seq_length, num_decoder_tokens)) # We set up our decoder to return full output sequences, # and to return internal states as well. We don't use the # return states in the training model, but we will use them in inference. decoder_lstm = LSTM(latent_dim, return_state=True, return_sequences =True) decoder_outputs, decoder_hidden, _ = decoder_lstm(decoder_inputs, initial_state=encoder_states) print ("decoder_outputs") print (decoder_outputs) #decoder_dense = Dense(num_decoder_tokens, activation='softmax') #decoder_outputs = decoder_dense(decoder_outputs) vt = K.random_normal_variable(shape= (1,latent_dim), mean=0, scale=1) # Gaussian distribution (input_seq_lenth,1)) print ("vt") print (vt) print ("decoder_hidden") print (decoder_hidden) #en_seq = Reshape((-1,1,latent_dim))(encoder_outputs) #?,latent_dim #en_seq =K.squeeze(en_seq,0) en_seq = encoder_outputs #en_seq = K.repeat(en_seq, max_encoder_seq_length) print ("en_seq") print (en_seq) #dec_seq = Reshape((-1,1,latent_dim))(decoder_hidden) dec_seq = K.repeat(decoder_hidden, max_encoder_seq_length)
def sampling(args): z_mean, z_log_var = args epsilon = K.random_normal_variable(shape=(z_mean.get_shape()), mean=0., scale=epsilon_std) return z_mean + K.exp(z_log_var / 2) * epsilon
def build(self, input_shape): self.input_spec = [InputSpec(shape=input_shape)] self.input_dim = input_shape[2] if self.stateful: self.reset_states() else: # initial states: 2 all-zero tensors of shape (output_dim) self.states = [None, None] if self.consume_less == 'gpu': self.W = self.init((self.input_dim, 4 * self.output_dim), name='{}_W'.format(self.name)) self.U = self.inner_init((self.output_dim, 4 * self.output_dim), name='{}_U'.format(self.name)) self.b = K.variable(np.hstack( (np.zeros(self.output_dim), K.get_value(self.forget_bias_init((self.output_dim, ))), np.zeros(self.output_dim), np.zeros(self.output_dim))), name='{}_b'.format(self.name)) self.trainable_weights = [self.W, self.U, self.b] else: self.W_i = self.init((self.input_dim, self.output_dim), name='{}_W_i'.format(self.name)) self.U_i = self.inner_init((self.output_dim, self.output_dim), name='{}_U_i'.format(self.name)) self.b_i = K.zeros((self.output_dim, ), name='{}_b_i'.format(self.name)) self.W_f = self.init((self.input_dim, self.output_dim), name='{}_W_f'.format(self.name)) self.U_f = self.inner_init((self.output_dim, self.output_dim), name='{}_U_f'.format(self.name)) self.b_f = self.forget_bias_init((self.output_dim, ), name='{}_b_f'.format(self.name)) self.W_c = self.init((self.input_dim, self.output_dim), name='{}_W_c'.format(self.name)) self.U_c = self.inner_init((self.output_dim, self.output_dim), name='{}_U_c'.format(self.name)) self.b_c = K.zeros((self.output_dim, ), name='{}_b_c'.format(self.name)) self.W_o = self.init((self.input_dim, self.output_dim), name='{}_W_o'.format(self.name)) self.U_o = self.inner_init((self.output_dim, self.output_dim), name='{}_U_o'.format(self.name)) self.b_o = K.zeros((self.output_dim, ), name='{}_b_o'.format(self.name)) # added by Di Kai self.W_h = self.init((self.output_dim, self.output_dim), name='{}_W_h'.format(self.name)) self.u_h = K.random_normal_variable( (self.output_dim), 1.0, 0.1, name='{}_u_h'.format(self.name)) self.trainable_weights = [ self.W_i, self.U_i, self.b_i, self.W_c, self.U_c, self.b_c, self.W_f, self.U_f, self.b_f, self.W_o, self.U_o, self.b_o, self.W_h, self.u_h ] self.W = K.concatenate([self.W_i, self.W_f, self.W_c, self.W_o]) self.U = K.concatenate([self.U_i, self.U_f, self.U_c, self.U_o]) self.b = K.concatenate([self.b_i, self.b_f, self.b_c, self.b_o]) self.regularizers = [] if self.W_regularizer: self.W_regularizer.set_param(self.W) self.regularizers.append(self.W_regularizer) if self.U_regularizer: self.U_regularizer.set_param(self.U) self.regularizers.append(self.U_regularizer) if self.b_regularizer: self.b_regularizer.set_param(self.b) self.regularizers.append(self.b_regularizer) if self.initial_weights is not None: self.set_weights(self.initial_weights) del self.initial_weights
def sampling(args): encoder_mean, encoder_log_sigma = args epsilon = K.random_normal_variable(shape=(batch_size, 2048), mean=0., scale=0.1) return encoder_mean + K.exp(encoder_log_sigma) * epsilon
import tensorflow as tf import matplotlib.pyplot as plt # Fixed batch = 4000 alpha = np.random.normal(0, 0.5, 100) X_tr = np.random.normal(0, 0.5, (batch, )) Z_tr = np.mean(np.exp(-X_tr.reshape(-1, 1)**2 * alpha), axis=1) # sort idx = np.argsort(X_tr) X_tr = X_tr[idx] Z_tr = Z_tr[idx] # Learned coefs = K.random_normal_variable((100, ), 0, 0.1) pows = K.arange(100, dtype='float32') X = K.placeholder((None, 1)) Y = K.mean(K.exp(-K.square(X) * coefs), axis=1) Z = K.placeholder(ndim=1) loss = K.mean(K.square(Y - Z)) optimizer = tf.train.AdamOptimizer() opt = optimizer.minimize(loss, var_list=[coefs]) grad = K.Function([X, Z], [opt]) get_loss = K.Function([X, Z], [loss]) get_Y = K.Function([X], [Y]) for i in range(1000):
from keras.losses import categorical_crossentropy from keras.activations import softmax from keras import optimizers import numpy as np dataset_size = 200000 X = np.random.rand(dataset_size, 2) labels = np.zeros((dataset_size, 3)) labels[X[:, 0] > X[:,1]] = [0,0,1] labels[X[:, 0] <= X[:,1]] = [1,0,0] labels[X[:,1] + X[:, 0] > 1] = [0, 1, 0] x = K.placeholder(shape=(None, 2)) t = K.placeholder(shape=(None, 3)) theta1 = K.random_normal_variable(shape=(2, 12), mean=0, scale=0.01) bias1 = K.random_normal_variable(shape=(1, 12), mean=0, scale=0.01) theta2 = K.random_normal_variable(shape=(12, 3), mean=0, scale=0.01) bias2 = K.random_normal_variable(shape=(1, 3), mean=0, scale=0.01) def forward(x): y = K.dot(x, theta1) + bias1 y = K.maximum(y, 0.) return K.dot(y, theta2) + bias2 loss = categorical_crossentropy(softmax(forward(x)),t) params= [theta1, bias1, theta2, bias2] # sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
def init_normal(shape, dtype=None): return K.random_normal_variable(shape, 0.0, scale=0.01, dtype=dtype)
# K.tensorflow_backend._get_available_gpus() from keras import metrics from keras import backend as K import numpy as np def squared_error(a, b): return K.square(a - b) a = K.random_normal_variable(shape=(3, 4), mean=0, scale=1) b = K.random_normal_variable(shape=(3, 4), mean=0, scale=1) c = squared_error(a, b) print(c)
# -*- coding: utf-8 -*- """ Created on Fri Sep 14 13:10:30 2018 @author: Youssef """ from keras import backend as K inputs = K.placeholder(shape=(2, 4, 5)) # also works: inputs = K.placeholder(ndim=3) import numpy as np x = np.random.random((3, 4, 5)) y = K.variable(value=x) # Initializing Tensors with Random Numbers b = K.random_uniform_variable(shape=(3, 4), low=0, high=1) # Uniform distribution c = K.random_normal_variable(shape=(3, 4), mean=0, scale=1) # Gaussian distribution d = K.random_normal_variable(shape=(3, 4), mean=0, scale=1) a = b + c * K.abs(d) c = K.dot(a, K.transpose(b)) a = K.sum(b, axis=1) a = K.softmax(b) a = K.concatenate([b, c], axis=-1)
def _subNM(x): return K.map_fn( lambda i: K.random_normal_variable( shape=(1, ), mean=m[i], scale=s[i]), K.arange(K.constant(len_u)))
def sampling(args): z_mean, z_log_var = args epsilon = K.random_normal_variable(shape=(batch_size, dim_latent), mean=0., scale=epsilon_std) return z_mean + K.exp(z_log_var / 2) * epsilon
start.append(mean) mean = np.mean([model.dict_to_array(vals) for vals in start], axis=0) var = np.ones_like(mean) potential = quadpotential.QuadPotentialDiagAdapt( model.ndim, mean, var, 10) return pm.step_methods.HamiltonianMC(step_scale=stepsize, potential=potential, path_length=1) else: raise NotImplementedError() ENERGY_FUNCTIONS = OrderedDict(( ("banana", (Banana(), lambda: [ K.random_normal_variable(mean=0., scale=1., shape=(1, )), K.random_normal_variable(mean=0., scale=1., shape=(1, )) ])), ("gmm1", ( Gmm1(), lambda: [K.variable(K.random_normal((1, )), name="x", dtype=K.floatx())], )), ("gmm2", ( Gmm2(), lambda: [K.variable(K.random_normal((1, )), name="x", dtype=K.floatx())], )), ("gmm3", ( Gmm3(), lambda: