def build(self, input_shape): """ This method must be defined for any custom layer, here you define the training parameters. input_shape: a tensor that automatically captures the dimensions of the input by tensorflow. """ # get the size of the input self.N = input_shape.as_list()[1] # construct the DFT matrix self.F = dft(self.N) / self.fs # construct the trapezoidal rule matrix self.D = np.eye(self.N // 2) self.D[0, 0] = 0.5 self.D[-1, -1] = 0.5 # define the trainable parameters representing the double side band PSD of noise self.S = self.add_weight(name="S", shape=tf.TensorShape((self.N // 2, 1)), initializer=initializers.Ones(), constraint=constraints.NonNeg(), trainable=True) # this has to be called for any tensorflow custom layer super(coherence_Layer, self).build(input_shape)
def bias_initializer(_, *args, **kwargs): return tf.concat([ self.bias_initializer((self.units, ), *args, **kwargs), initializers.Ones()((self.units, ), *args, **kwargs), self.bias_initializer( (self.units * 2, ), *args, **kwargs), ], -1)
def bias_initializer(shape, *args, **kwargs): return K.concatenate([ self.bias_initializer((self.units, ), *args, **kwargs), initializers.Ones()((self.units, ), *args, **kwargs), self.bias_initializer((self.units * 2, ), *args, **kwargs), ])
def get_ann( n_hidden=4, n_neurons=20, kernel_initializer="he_normal", bias_initializer=initializers.Ones() ): model = Sequential() model.add( Dense( units=n_neurons, input_dim=14, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, ) ) model.add(keras.layers.LeakyReLU(alpha=0.2)) model.add(Dropout(rate=0.1)) for _ in range(n_hidden): model.add( Dense( units=n_neurons, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, ) ) model.add(keras.layers.LeakyReLU(alpha=0.2)) model.add(Dropout(rate=0.1)) model.add(Dense(units=1, activation="linear")) optimizer = optimizers.RMSprop() model.compile(loss="mse", optimizer=optimizer, metrics=["mse", "mae"]) return model
def bias_initializer(_, *args, **kwargs): return kb.concatenate( [ self.bias_initializer((self.n_hidden,), *args, **kwargs), initializers.Ones()((self.n_hidden,), *args, **kwargs), self.bias_initializer((self.n_hidden * 2,), *args, **kwargs), ] )
def build(self, input_shape): self.input_spec = [InputSpec(shape=input_shape)] output_shape = (int(input_shape[self.axis]),) gamma_initializer = initializers.Ones() beta_initializer = initializers.Zeros() self.gamma = K.variable(gamma_initializer(output_shape)) self.beta = K.variable(beta_initializer(output_shape)) self.trainable_weights = [self.gamma, self.beta]
def construct_actor_network(bandits): """Construct the actor network with mu and sigma as output""" inputs = layers.Input(shape=(1,)) #input dimension hidden1 = layers.Dense(5, activation="relu",kernel_initializer=initializers.he_normal())(inputs) hidden2 = layers.Dense(5, activation="relu",kernel_initializer=initializers.he_normal())(hidden1) probabilities = layers.Dense(len(bandits), kernel_initializer=initializers.Ones(),activation="softmax")(hidden2) actor_network = keras.Model(inputs=inputs, outputs=[probabilities]) return actor_network
def build(self, input_shape): self.gain = self.add_weight(name='gain', shape=input_shape[-1:], initializer=initializers.Ones(), trainable=True) self.bias = self.add_weight(name='bias', shape=input_shape[-1:], initializer=initializers.Zeros(), trainable=True) super().build(input_shape)
def create_model(opt, nFeatures=1): model = ks.models.Sequential() #lys = Dense(2, input_shape=(1,)) #print(lys.get_weights()) #print(lys.get_config()) #print('nFeatures=', nFeatures) k_initializer = initializers.Ones() b_initializer = initializers.Ones() if 1: # 1 layer model.add( Dense(5, input_shape=(nFeatures, ), kernel_initializer=k_initializer, bias_initializer=b_initializer)) model.add(lys.ReLU()) model.add( Dense(5, kernel_initializer=k_initializer, bias_initializer=b_initializer)) model.add(lys.ReLU()) model.add( Dense(1, kernel_initializer=k_initializer, bias_initializer=b_initializer)) else: # multify layers model.add(Dense(10, input_shape=(nFeatures, ))) model.add(Dense(8, input_shape=(10, ))) model.add(Dense(2, input_shape=(8, ))) model.add(Dense(1, input_shape=(2, ))) model.compile(optimizer=opt, loss='mean_squared_error') #metrics=['accuracy'] #model.summary() return model
def get_lstm(kernel_initializer="he_uniform", bias_initializer=initializers.Ones()): model = Sequential() if lstm_params["n_hidden"] == 1: model.add( LSTM( units=lstm_params["units"], input_shape=(lstm_params["steps"], lstm_params["features_num"]), kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, )) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.2)) if lstm_params["n_hidden"] == 2: model.add( LSTM( units=lstm_params["units"], input_shape=(lstm_params["steps"], lstm_params["features_num"]), return_sequences=True, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, )) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.2)) model.add( LSTM( units=lstm_params["units"], input_shape=(lstm_params["steps"], lstm_params["features_num"]), kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, )) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.2)) model.add(Dense(1, activation="linear")) optimizer = optimizers.RMSprop() model.compile(loss="mse", metrics=["mse", "mae"], optimizer=optimizer) return model
def _build_model(self): model = tf.keras.Sequential([ layers.Dense( 100, input_shape=(4,), kernel_initializer=initializers.RandomNormal(stddev=5.0), bias_initializer=initializers.Ones(), # kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4), activation='sigmoid', name='state' ), layers.Dense( 2, #input_shape=(4,), activation='relu' ), layers.Dense(1, name='action', activation='tanh'), ]) model.summary() model.compile( loss='hinge', optimizer=optimizers.RMSprop(lr=self.lr) ) return model
lr = 0.1 m = 0.6 weight_decay = [0.1, 0.5, 0.9] exp_metrics = np.zeros([len(weight_decay), 3]) i = 0 for r in weight_decay: RMSEs = [] MAEs = [] for j in range(0, 5): # Model model = Sequential() # Input layer model.add( Dense(N, kernel_initializer=initializers.Ones(), bias_initializer=initializers.Zeros(), input_dim=N)) # Leaky ReLU activation function def lrelu(x): return relu(x, alpha=0.01) # Hidden layer model.add( Dense(H, kernel_initializer=initializers.glorot_uniform(), bias_initializer=initializers.glorot_uniform(), activation=lrelu, kernel_regularizer=l1(r),
def make_network(self, amino_acid_encoding, peptide_max_length, n_flank_length, c_flank_length, flanking_averages, convolutional_filters, convolutional_kernel_size, convolutional_activation, convolutional_kernel_l1_l2, dropout_rate, post_convolutional_dense_layer_sizes): """ Helper function to make a keras network given hyperparameters. """ # We import keras here to avoid tensorflow debug output, etc. unless we # are actually about to use Keras. configure_tensorflow() from tensorflow.keras.layers import (Input, Dense, Dropout, Concatenate, Conv1D, Lambda) from tensorflow.keras.models import Model from tensorflow.keras import regularizers, initializers model_inputs = {} empty_x_dict = self.network_input(FlankingEncoding([], [], [])) sequence_dims = empty_x_dict['sequence'].shape[1:] numpy.testing.assert_equal( sequence_dims[0], peptide_max_length + n_flank_length + c_flank_length) model_inputs['sequence'] = Input(shape=sequence_dims, dtype='float32', name='sequence') model_inputs['peptide_length'] = Input(shape=(1, ), dtype='int32', name='peptide_length') current_layer = model_inputs['sequence'] current_layer = Conv1D( filters=convolutional_filters, kernel_size=convolutional_kernel_size, kernel_regularizer=regularizers.l1_l2(*convolutional_kernel_l1_l2), padding="same", activation=convolutional_activation, name="conv1")(current_layer) if dropout_rate > 0: current_layer = Dropout( name="conv1_dropout", rate=dropout_rate, noise_shape=(None, 1, int( current_layer.get_shape()[-1])))(current_layer) convolutional_result = current_layer outputs_for_final_dense = [] for flank in ["n_flank", "c_flank"]: current_layer = convolutional_result for (i, size) in enumerate( list(post_convolutional_dense_layer_sizes) + [1]): current_layer = Conv1D( name="%s_post_%d" % (flank, i), filters=size, kernel_size=1, kernel_regularizer=regularizers.l1_l2( *convolutional_kernel_l1_l2), activation=("tanh" if size == 1 else convolutional_activation))(current_layer) single_output_result = current_layer dense_flank = None if flank == "n_flank": def cleavage_extractor(x): return x[:, n_flank_length] single_output_at_cleavage_position = Lambda( cleavage_extractor, name="%s_cleaved" % flank)(single_output_result) def max_pool_over_peptide_extractor(lst): import tensorflow as tf (x, peptide_length) = lst # We generate a per-sample mask that is 1 for all peptide # positions except the first position, and 0 for all other # positions (i.e. n flank, c flank, and the first peptide # position). starts = n_flank_length + 1 limits = n_flank_length + peptide_length row = tf.expand_dims(tf.range(0, x.shape[1]), axis=0) mask = tf.logical_and(tf.greater_equal(row, starts), tf.less(row, limits)) # We are assuming that x >= -1. The final activation in the # previous layer should be a function that satisfies this # (e.g. sigmoid, tanh, relu). max_value = tf.reduce_max( (x + 1) * tf.expand_dims(tf.cast(mask, tf.float32), axis=-1), axis=1) - 1 # We flip the sign so that initializing the final dense # layer weights to 1s is reasonable. return -1 * max_value max_over_peptide = Lambda(max_pool_over_peptide_extractor, name="%s_internal_cleaved" % flank)([ single_output_result, model_inputs['peptide_length'] ]) def flanking_extractor(lst): import tensorflow as tf (x, peptide_length) = lst # mask is 1 for n_flank positions and 0 elsewhere. starts = 0 limits = n_flank_length row = tf.expand_dims(tf.range(0, x.shape[1]), axis=0) mask = tf.logical_and(tf.greater_equal(row, starts), tf.less(row, limits)) # We are assuming that x >= -1. The final activation in the # previous layer should be a function that satisfies this # (e.g. sigmoid, tanh, relu). average_value = tf.reduce_mean( (x + 1) * tf.expand_dims(tf.cast(mask, tf.float32), axis=-1), axis=1) - 1 return average_value if flanking_averages and n_flank_length > 0: # Also include average pooled of flanking sequences pooled_flank = Lambda(flanking_extractor, name="%s_extracted" % flank)([ convolutional_result, model_inputs['peptide_length'] ]) dense_flank = Dense(1, activation="tanh", name="%s_avg_dense" % flank)(pooled_flank) else: assert flank == "c_flank" def cleavage_extractor(lst): import tensorflow as tf (x, peptide_length) = lst indexer = peptide_length + n_flank_length - 1 result = tf.squeeze( tf.gather(x, indexer, batch_dims=1, axis=1), -1) return result single_output_at_cleavage_position = Lambda( cleavage_extractor, name="%s_cleaved" % flank)( [single_output_result, model_inputs['peptide_length']]) def max_pool_over_peptide_extractor(lst): import tensorflow as tf (x, peptide_length) = lst # We generate a per-sample mask that is 1 for all peptide # positions except the last position, and 0 for all other # positions (i.e. n flank, c flank, and the last peptide # position). starts = n_flank_length limits = n_flank_length + peptide_length - 1 row = tf.expand_dims(tf.range(0, x.shape[1]), axis=0) mask = tf.logical_and(tf.greater_equal(row, starts), tf.less(row, limits)) # We are assuming that x >= -1. The final activation in the # previous layer should be a function that satisfies this # (e.g. sigmoid, tanh, relu). max_value = tf.reduce_max( (x + 1) * tf.expand_dims(tf.cast(mask, tf.float32), axis=-1), axis=1) - 1 # We flip the sign so that initializing the final dense # layer weights to 1s is reasonable. return -1 * max_value max_over_peptide = Lambda(max_pool_over_peptide_extractor, name="%s_internal_cleaved" % flank)([ single_output_result, model_inputs['peptide_length'] ]) def flanking_extractor(lst): import tensorflow as tf (x, peptide_length) = lst # mask is 1 for c_flank positions and 0 elsewhere. starts = n_flank_length + peptide_length limits = n_flank_length + peptide_length + c_flank_length row = tf.expand_dims(tf.range(0, x.shape[1]), axis=0) mask = tf.logical_and(tf.greater_equal(row, starts), tf.less(row, limits)) # We are assuming that x >= -1. The final activation in the # previous layer should be a function that satisfies this # (e.g. sigmoid, tanh, relu). average_value = tf.reduce_mean( (x + 1) * tf.expand_dims(tf.cast(mask, tf.float32), axis=-1), axis=1) - 1 return average_value if flanking_averages and c_flank_length > 0: # Also include average pooled of flanking sequences pooled_flank = Lambda(flanking_extractor, name="%s_extracted" % flank)([ convolutional_result, model_inputs['peptide_length'] ]) dense_flank = Dense(1, activation="tanh", name="%s_avg_dense" % flank)(pooled_flank) outputs_for_final_dense.append(single_output_at_cleavage_position) outputs_for_final_dense.append(max_over_peptide) if dense_flank is not None: outputs_for_final_dense.append(dense_flank) if len(outputs_for_final_dense) == 1: (current_layer, ) = outputs_for_final_dense else: current_layer = Concatenate(name="final")(outputs_for_final_dense) output = Dense( 1, activation="sigmoid", name="output", kernel_initializer=initializers.Ones(), )(current_layer) model = Model( inputs=[model_inputs[name] for name in sorted(model_inputs)], outputs=[output], name="predictor") return model
def build(self, input_shape): input_dim = input_shape[-1] input_dim -= 1 # We add time and event afterwards if type(self.recurrent_initializer).__name__ == 'Identity': def recurrent_identity(shape, gain=1., dtype=None): del dtype return gain * np.concatenate( [np.identity(shape[0])] * (shape[1] // shape[0]), axis=1) self.recurrent_initializer = recurrent_identity self.input_embedding = self.add_weight( shape=(input_dim, self.embedding_size), name='embedding_matrix', initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.kernel = self.add_weight(shape=(self.embedding_size, self.units * 4), name='kernel', initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) self.recurrent_kernel = self.add_weight( shape=(self.units, self.units * 4), name='recurrent_kernel', initializer=self.recurrent_initializer, regularizer=self.recurrent_regularizer, constraint=self.recurrent_constraint) self.time_kernel = self.add_weight(shape=(1, self.projection_size), name='time_kernel', initializer=self.time_initializer, regularizer=self.time_regularizer, constraint=self.time_constraint) self.embedding_projector = self.add_weight( shape=(self.projection_size, self.embedding_size), name='time_projection_onto_embedding_space', initializer=self.time_initializer, regularizer=self.time_regularizer, constraint=self.time_constraint) if self.use_bias: if self.unit_forget_bias: def bias_initializer(_, *args, **kwargs): return K.concatenate([ self.bias_initializer((self.units, ), *args, **kwargs), initializers.Ones()((self.units, ), *args, **kwargs), self.bias_initializer((self.units * 2, ), *args, **kwargs), ]) else: bias_initializer = self.bias_initializer self.bias = self.add_weight(shape=(self.units * 4, ), name='bias', initializer=bias_initializer, regularizer=self.bias_regularizer, constraint=self.bias_constraint) time_bias_initializer = initializers.Ones()( (self.projection_size, )) self.time_bias = self.add_weight(shape=(self.projection_size, ), name='time_bias', initializer=None, regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.time_bias = None self.kernel_i = self.kernel[:, :self.units] self.kernel_f = self.kernel[:, self.units:self.units * 2] self.kernel_c = self.kernel[:, self.units * 2:self.units * 3] self.kernel_o = self.kernel[:, self.units * 3:self.units * 4] self.recurrent_kernel_i = self.recurrent_kernel[:, :self.units] self.recurrent_kernel_f = ( self.recurrent_kernel[:, self.units:self.units * 2]) self.recurrent_kernel_c = (self.recurrent_kernel[:, self.units * 2:self.units * 3]) self.recurrent_kernel_o = self.recurrent_kernel[:, self.units * 3:self.units * 4] if self.use_bias: self.bias_i = self.bias[:self.units] self.bias_f = self.bias[self.units:self.units * 2] self.bias_c = self.bias[self.units * 2:self.units * 3] self.bias_o = self.bias[self.units * 3:self.units * 4] self.bias_t = self.time_bias else: self.bias_i = None self.bias_f = None self.bias_c = None self.bias_o = None self.bias_t = None self.built = True
def __init__(self, alpha_initializer=initializers.Ones(), **kwargs): self.alpha_initializer = alpha_initializer super().__init__(**kwargs)
test_ind = np.load('test_ind.npy', allow_pickle=True) H = 15 learning_rate = [0.001, 0.001, 0.05, 0.1] momentum = [0.2, 0.6, 0.6, 0.6] exp_metrics = np.zeros([4, 4]) i = 0 for lr, m in zip(learning_rate, momentum): RMSEs = [] MAEs = [] for j in range(0, 5): # Model model = Sequential() # Input layer model.add(Dense(N, kernel_initializer=initializers.Ones(), bias_initializer=initializers.Zeros(), input_dim=N)) # Leaky ReLU activation function def lrelu(x): return relu(x, alpha=0.01) # Hidden layer model.add(Dense(H, kernel_initializer=initializers.glorot_uniform(), bias_initializer=initializers.glorot_uniform(), activation=lrelu)) # Output layer model.add(Dense(M, kernel_initializer=initializers.glorot_uniform(), bias_initializer=initializers.glorot_uniform(), activation='sigmoid')) # SGD optimiser sgd = SGD(learning_rate=lr, momentum=m, nesterov=False) def rmse(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true)))