def inner_layer_fn(): return Dense(conv_size, activation=conv_activation, bias=conv_bias, W_regularizer=l1_l2(conv_l1, conv_l2), b_regularizer=l1_l2(conv_l1, conv_l2), **conv_kwargs)
def basic_model_3(x_size, y_size): t_model = Sequential() t_model.add( Dense(80, activation="tanh", kernel_initializer='normal', input_shape=(x_size, ))) t_model.add(Dropout(0.2)) t_model.add( Dense(120, activation="relu", kernel_initializer='normal', kernel_regularizer=regularizers.l1(0.01), bias_regularizer=regularizers.l1(0.01))) t_model.add(Dropout(0.1)) t_model.add( Dense(20, activation="relu", kernel_initializer='normal', kernel_regularizer=regularizers.l1_l2(0.01), bias_regularizer=regularizers.l1_l2(0.01))) t_model.add(Dropout(0.1)) t_model.add(Dense(10, activation="relu", kernel_initializer='normal')) t_model.add(Dropout(0.0)) t_model.add(Dense(y_size)) t_model.compile( loss='mean_squared_error', # optimizer=tf.keras.optimizers.Nadam(lr=0.0005), optimizer=tf.keras.optimizers.RMSprop(0.001), metrics=[metrics.mae]) return (t_model)
def create_model_meta(NUM_CLASS, shape, isPreTrained=False, pathToMetaModelWeights=None, isTrainable=True): initializer = GlorotNormal() inputs = Input(shape=(shape)) x = Dense(60, activation='relu', kernel_regularizer=l1_l2(l1=1e-5, l2=1e-4), kernel_initializer=initializer)(inputs) x = Dense(30, activation='relu', kernel_regularizer=l1_l2(l1=1e-5, l2=1e-4), kernel_initializer=initializer)(x) x = Dense(6, activation='relu', kernel_regularizer=l1_l2(l1=1e-5, l2=1e-4), kernel_initializer=initializer, name='final_output')(x) output = Dense(2, activation='softmax')(x) model = Model(inputs, output) if not isPreTrained: return model else: model.load_weights(pathToMetaModelWeights) if not isTrainable: for layer in model.layers: layer.trainable = False return model, 1
def rnn_conv_block(self, n_filter, kernel_size, strides, x, conv_type=Conv2D, drop=True, flatten=False, batchnorm=True, **kwargs): x = TD( conv_type( n_filter, kernel_size=kernel_size, strides=strides, use_bias=self.use_bias, padding=self.padding, kernel_regularizer=l1_l2(self.regularizer[0], self.regularizer[1]), bias_regularizer=l1_l2(self.regularizer[0], self.regularizer[1]), ), **kwargs)(x) if batchnorm: x = TD(BatchNormalization())(x) x = TD(Activation(self.prev_act))(x) if drop: x = TD(Dropout(self.drop_rate))(x) if flatten: x = TD(Flatten())(x) return x
def trainCNN(train, test, nrows = 200, ncols = 2500, size_batch = 32): '''Train CNN on aligned haplotypes. Setup to train on nrows (haplotypes) X ncols (base pair) alignments''' import tensorflow as tf from tensorflow.keras import datasets, layers, models, regularizers import numpy as np tf.random.set_seed(123456) # Load data train_x, train_y = train test_x, test_y = test # Some more preprocessing train_x = train_x.reshape(train_x.shape + (1,)) test_x = test_x.reshape(test_x.shape + (1,)) train_y = np.array([1 if i == 'positive' else 0 for i in train_y]) test_y = np.array([1 if i == 'positive' else 0 for i in test_y]) # Early stopping callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3) # Initialize model model = models.Sequential() model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(nrows, ncols, 1), kernel_regularizer=regularizers.l1_l2(l1=0.008, l2=0.008))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation='relu', kernel_regularizer=regularizers.l1_l2(l1=0.006, l2=0.006))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation='relu', kernel_regularizer=regularizers.l1_l2(l1=0.002, l2=0.002))) model.add(layers.Flatten()) model.add(layers.Dense(64, activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) METRICS = [tf.keras.metrics.BinaryAccuracy(name='accuracy'),tf.keras.metrics.Precision(name='precision'),tf.keras.metrics.Recall(name='recall'),tf.keras.metrics.AUC(name='auc')] model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=METRICS) if size_batch > 0: history = model.fit(train_x, train_y, batch_size = size_batch, epochs=30, validation_data=(test_x, test_y), callbacks = [callback]) else: history = model.fit(train_x, train_y, epochs=25, validation_data=(test_x, test_y), callbacks = [callback]) return (model, history)
def attention_lstm(self): input_x = Input(shape = self.input_shape, name = 'input') X = input_x for i in range(self.lstm_blocks): query = Dense(10, name='query_' + str(i))(X) key = Dense(10, name='key_' + str(i))(X) attention_weights = AdditiveAttention(use_scale = False, name='attention_'+str(i))([query, X, key]) attention_weights = Dense(1, activation='softmax', name='attention_weights_'+str(i))(attention_weights) context = Multiply(name='context_'+str(i))([attention_weights,X]) X = LSTM(self.n_units, return_sequences = True, recurrent_dropout=self.recurrent_dropout, kernel_regularizer=l1_l2(self.lstm_l1, self.lstm_l2), activity_regularizer=l1_l2(self.lstm_l1, self.lstm_l2), name = 'lstm_' + str(i))(context) if self.dropout_rate > 0: X = Dropout(self.dropout_rate, name='dropout_'+str(i))(X) X = LSTM(self.n_units, return_sequences = False, recurrent_dropout=self.recurrent_dropout, kernel_regularizer=l1_l2(self.lstm_l1, self.lstm_l2), activity_regularizer=l1_l2(self.lstm_l1, self.lstm_l2), name = 'lstm_last')(X) if self.dropout_rate > 0: X = Dropout(self.dropout_rate, name='dropout_last')(X) X = Dense(self.n_outputs, activation=self.activation, name = 'output')(X) return Model(inputs=input_x, outputs=X, name='attention_lstm')
def conv_block(self, n_filter, kernel_size, strides, x, drop=False, conv_type=Conv2D, flatten=False, batchnorm=False, maxpool=False, **kwargs): x = conv_type(n_filter, kernel_size=kernel_size, strides=strides, use_bias=self.use_bias, kernel_regularizer=l1_l2(self.regularizer[0], self.regularizer[1]), bias_regularizer=l1_l2(self.regularizer[0], self.regularizer[1]), **kwargs)(x) if batchnorm: x = BatchNormalization()(x) x = Activation(self.prev_act)(x) if drop: x = Dropout(self.drop_rate)(x) if maxpool: x = MaxPooling2D()(x) if flatten: x = Flatten()(x) return x
def resnet_v1_eembc_tiny(input_shape=[32, 32, 3], num_classes=10, num_filters=[8], kernel_sizes=[3, 1], strides=[1, 2], l1p=1e-4, l2p=0): # Input layer, change kernel size to 7x7 and strides to 2 for an official resnet inputs = Input(shape=input_shape) x = Conv2D(num_filters[0], kernel_size=kernel_sizes[0], strides=strides[0], padding='same', kernel_initializer='he_normal', kernel_regularizer=l1_l2(l1=l1p, l2=l2p))(inputs) x = BatchNormalization()(x) x = Activation('relu')(x) # First stack # Weight layers y = Conv2D(num_filters[0], kernel_size=kernel_sizes[0], strides=strides[1], padding='same', kernel_initializer='he_normal', kernel_regularizer=l1_l2(l1=l1p, l2=l2p))(x) y = BatchNormalization()(y) y = Activation('relu')(y) y = Conv2D(num_filters[0], kernel_size=kernel_sizes[0], strides=strides[0], padding='same', kernel_initializer='he_normal', kernel_regularizer=l1_l2(l1=l1p, l2=l2p))(y) y = BatchNormalization()(y) # Adjust for change in dimension due to stride in identity x = Conv2D(num_filters[0], kernel_size=kernel_sizes[1], strides=strides[1], padding='same', kernel_initializer='he_normal', kernel_regularizer=l1_l2(l1=l1p, l2=l2p))(x) # Overall residual, connect weight layer and identity paths x = Add()([x, y]) x = Activation('relu')(x) # Final classification layer. pool_size = int(np.amin(x.shape[1:3])) x = AveragePooling2D(pool_size=pool_size)(x) y = Flatten()(x) outputs = Dense(num_classes, activation='softmax', kernel_initializer='he_normal')(y) # Instantiate model. model = Model(inputs=inputs, outputs=outputs) return model
def get_model(): model = Sequential() model.add(Conv3D(32, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform', input_shape=(10, 240, 320, 3),padding ="same", kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4))) model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) model.add(MaxPooling3D(pool_size=(2, 2, 2))) model.add(BatchNormalization(center=True, scale=True)) model.add(Dropout(0.7)) model.add(Conv3D(32, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform', kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4))) model.add(MaxPooling3D(pool_size=(2, 2, 2))) model.add(BatchNormalization(center=True, scale=True)) model.add(Dropout(0.7)) model.add(Flatten()) model.add(Dense(128, activation='relu', kernel_initializer='he_uniform', kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4))) model.add(Dropout(0.2)) model.add(Dense(50, activation='softmax')) # Compile the model model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.001), metrics=['accuracy']) model.summary() # plot_model(model, to_file='images/baseline.png', show_shapes=True, show_layer_names=True) return model
def trainwithfit(lr, reg_coef): dl_model = keras.Sequential([ layers.Conv1D(32, context_size, input_shape=(1500, vec_length), kernel_regularizer=regularizers.l1_l2(l1=reg_coef, l2=reg_coef)), layers.GlobalMaxPooling1D(data_format="channels_first"), layers.Dense(32, kernel_regularizer=regularizers.l1_l2(l1=reg_coef, l2=reg_coef)), layers.Dense(2, activation='softmax', kernel_regularizer=regularizers.l1_l2(l1=reg_coef, l2=reg_coef)) ]) dl_model.compile( optimizer=tf.optimizers.Adam(learning_rate=lr), loss='binary_crossentropy', metrics=['binary_accuracy'], ) generatorTrain = MyDataset("trainData") history = dl_model.fit(x=generatorTrain, verbose=2, epochs=2, callbacks=[tensorboard_callback]) generatorTest = MyDataset("testData") res = dl_model.evaluate(x=generatorTest, ) return (dl_model, res)
def __init__(self, embedding_dim, units, vocab_size, p_dropout = 0, l1_reg = 0, l2_reg = 0): super(RNN_Decoder, self).__init__() self.units = units self.embedding = Embedding(vocab_size, embedding_dim) self.gru = GRU(self.units, return_sequences = True, return_state = True, recurrent_initializer = 'glorot_uniform', dropout = p_dropout, # recurrent_dropout = p_dropout, kernel_regularizer = l1_l2(l1_reg, l2_reg)) self.fc1 = Dense(self.units, kernel_regularizer=l1_l2(l1_reg, l2_reg)) self.fc2 = Dense(vocab_size, kernel_regularizer=l1_l2(l1_reg, l2_reg)) self.attention = BahdanauAttention(units=units, p_dropout = p_dropout, l1_reg = l1_reg, l2_reg = l2_reg) self.dropout = Dropout(p_dropout)
def fitnessLearnModel(max_len, features, lr, cells=32, regularization_base=2e-6): inp = Input(shape=(max_len, features), name='fitnessModel_inputs1') inp2 = Input(shape=(max_len, features), name='fitnessModel_inputs2') mult = Multiply()([inp, inp2]) mask = Masking(0.0)(mult) lstm_Layer = LSTM(cells, activation='relu', return_sequences=False, kernel_initializer=he_normal(24353), name='fitnessModel_lstm1', return_state=False, recurrent_regularizer=l1_l2(regularization_base / 20, regularization_base / 20), kernel_regularizer=l1_l2(regularization_base, regularization_base), bias_regularizer=l1_l2(regularization_base * 2, regularization_base * 2)) lstm_out = lstm_Layer(mask) out = Dense(1, activation='linear', kernel_initializer=he_normal(53436), name='fitnessModel_denseOut')(lstm_out) fitnessModel = Model(inputs=[inp, inp2], outputs=out) fitnessModel.compile(optimizer=Adam(lr, clipnorm=1.0, clipvalue=0.5), loss='mse') return fitnessModel
def convolutional_block(X, f, filters, stage, block, s=2, l1=0.0, l2=0.01): # Defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2, F3 = filters # Save the input value X_shortcut = X X = Conv2D(filters=F1, kernel_size=(1, 1), strides=(s, s), data_format='channels_first', padding='same', name=conv_name_base + '2a', kernel_initializer=glorot_uniform(seed=0), activity_regularizer=l1_l2(l1, l2))(X) X = BatchNormalization(axis=1, name=bn_name_base + '2a')(X) X = Activation('relu')(X) X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), data_format='channels_first', padding='same', name=conv_name_base + '2b', kernel_initializer=glorot_uniform(seed=0), activity_regularizer=l1_l2(l1, l2))(X) X = BatchNormalization(axis=1, name=bn_name_base + '2b')(X) X = Activation('relu')(X) X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), data_format='channels_first', padding='same', name=conv_name_base + '2c', kernel_initializer=glorot_uniform(seed=0), activity_regularizer=l1_l2(l1, l2))(X) X = BatchNormalization(axis=1, name=bn_name_base + '2c')(X) X_shortcut = Conv2D(filters=F3, kernel_size=(1, 1), strides=(s, s), data_format='channels_first', padding='same', name=conv_name_base + '1', kernel_initializer=glorot_uniform(seed=0), activity_regularizer=l1_l2(l1, l2))(X_shortcut) X_shortcut = BatchNormalization(axis=1, name=bn_name_base + '1')(X_shortcut) X = Add()([X, X_shortcut]) X = Activation('relu')(X) return X
def get_compiled_model(): # shape=(batch_size, time_steps, channels, row, col) input_flir = Input(shape=( 6, 3, 480, 640, )) input_bottom = Input(shape=( 6, 3, 480, 640, )) input_top = Input(shape=( 6, 3, 480, 640, )) x_concat = Concatenate(axis=-1)([input_flir, input_bottom, input_top]) x_concat = TimeDistributed( SeparableConv2D(8, (4, 4), activation="relu", padding="same"))(x_concat) x_ConvLSTM2D = ConvLSTM2D(32, (6, 6), padding="same", kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4), dropout=0.1, recurrent_dropout=0.1, return_sequences=True)(x_concat) x_ConvLSTM2D = BatchNormalization()(x_ConvLSTM2D) x_ConvLSTM2D = ConvLSTM2D(64, (4, 4), padding="same", kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4), dropout=0.1, recurrent_dropout=0.1, return_sequences=False)(x_ConvLSTM2D) x_ConvLSTM2D = BatchNormalization()(x_ConvLSTM2D) x_flat = GlobalAveragePooling2D()(x_ConvLSTM2D) x_flat = Dropout(.2)(x_flat) yh = Dense(3, activation="softmax", kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4))(x_flat) model = Model([input_flir, input_bottom, input_top], yh) opt = SGD(lr=1e-4, momentum=0.9, decay=1e-4) model.compile(loss=categorical_crossentropy, optimizer=opt, metrics=["accuracy"]) return model
def generate_model_2D(_input_shape): model = Sequential() model.add(Conv2D(140, (3, 3), input_shape=_input_shape)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(70, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(20, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Activation('relu')) model.add( Dense(256, kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4), bias_regularizer=regularizers.l2(1e-4), activity_regularizer=regularizers.l2(1e-5))) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Activation('relu')) model.add( Dense(64, kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4), bias_regularizer=regularizers.l2(1e-4), activity_regularizer=regularizers.l2(1e-5))) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Activation('relu')) model.add( Dense(20, kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4), bias_regularizer=regularizers.l2(1e-4), activity_regularizer=regularizers.l2(1e-5))) model.add(Dense(2)) model.add(Activation('softmax')) model.compile( loss='binary_crossentropy', optimizer='adam', #metrics=['accuracy', metrics.auc]) metrics=['accuracy']) return model
def EEGNet_old(nb_classes, Chans = 64, Samples = 128, regRate = 0.0001, dropoutRate = 0.25, kernels = [(2, 32), (8, 4)], strides = (2, 4)): """ Keras Implementation of EEGNet_v1 (https://arxiv.org/abs/1611.08024v2) This model is the original EEGNet model proposed on arxiv https://arxiv.org/abs/1611.08024v2 with a few modifications: we use striding instead of max-pooling as this helped slightly in classification performance while also providing a computational speed-up. Note that we no longer recommend the use of this architecture, as the new version of EEGNet performs much better overall and has nicer properties. Inputs: nb_classes : total number of final categories Chans, Samples : number of EEG channels and samples, respectively regRate : regularization rate for L1 and L2 regularizations dropoutRate : dropout fraction kernels : the 2nd and 3rd layer kernel dimensions (default is the [2, 32] x [8, 4] configuration) strides : the stride size (note that this replaces the max-pool used in the original paper) """ # start the model input_main = Input((1, Chans, Samples)) layer1 = Conv2D(16, (Chans, 1), input_shape=(1, Chans, Samples), kernel_regularizer = l1_l2(l1=regRate, l2=regRate))(input_main) layer1 = BatchNormalization(axis=1)(layer1) layer1 = Activation('elu')(layer1) layer1 = Dropout(dropoutRate)(layer1) permute_dims = 2, 1, 3 permute1 = Permute(permute_dims)(layer1) layer2 = Conv2D(4, kernels[0], padding = 'same', kernel_regularizer=l1_l2(l1=0.0, l2=regRate), strides = strides)(permute1) layer2 = BatchNormalization(axis=1)(layer2) layer2 = Activation('elu')(layer2) layer2 = Dropout(dropoutRate)(layer2) layer3 = Conv2D(4, kernels[1], padding = 'same', kernel_regularizer=l1_l2(l1=0.0, l2=regRate), strides = strides)(layer2) layer3 = BatchNormalization(axis=1)(layer3) layer3 = Activation('elu')(layer3) layer3 = Dropout(dropoutRate)(layer3) flatten = Flatten(name = 'flatten')(layer3) dense = Dense(nb_classes, name = 'dense')(flatten) softmax = Activation('softmax', name = 'softmax')(dense) return Model(inputs=input_main, outputs=softmax)
def ResNet(chans=64, samples=121, l1=0.0, l2=0.01): input_shape = (1, chans, samples) input1 = Input(shape=input_shape) X = Conv2D(128, (1, 64), padding='same', input_shape=input_shape, data_format='channels_first', kernel_initializer=glorot_uniform(seed=0), activity_regularizer=l1_l2(l1, l2))(input1) X = BatchNormalization(axis=1)(X) X = DepthwiseConv2D((64, 1), kernel_initializer=glorot_uniform(seed=0), depth_multiplier=2, data_format='channels_first', activity_regularizer=l1_l2(l1, l2))(X) X = BatchNormalization(axis=1)(X) X = Activation('elu')(X) X = AveragePooling2D((1, 2), data_format='channels_first')(X) X = convolutional_block(X, f=3, filters=[64, 64, 256], stage=2, block='a', s=1, l1=l1, l2=l2) X = identity_block(X, 3, [64, 64, 256], stage=2, block='b', l1=l1, l2=l2) X = identity_block(X, 3, [64, 64, 256], stage=2, block='c', l1=l1, l2=l2) X = convolutional_block(X, f=3, filters=[128, 128, 512], stage=3, block='a', s=1, l1=l1, l2=l2) X = identity_block(X, 3, [128, 128, 512], stage=3, block='b', l1=l1, l2=l2) X = identity_block(X, 3, [128, 128, 512], stage=3, block='c', l1=l1, l2=l2) X = identity_block(X, 3, [128, 128, 512], stage=3, block='d', l1=l1, l2=l2) X = AveragePooling2D(pool_size=(2, 2), padding='same')(X) X = Flatten()(X) X = Dense(64, activation='relu')(X) X = Dense(32, activation='relu')(X) X = Dense(1, activation='sigmoid', name='fc1', kernel_initializer=glorot_uniform(seed=0))(X) model = Model(inputs=input1, outputs=X, name='ResNet50') return model
def encoder_decoder2( latent_dim1, n_outputs, n_timesteps, n_features, l1, l2, ): model = tf.keras.Sequential() model.add( GRU( latent_dim1, activation="relu", return_sequences=True, recurrent_regularizer=l1_l2(0, 0), kernel_regularizer=l1_l2(l1=l1, l2=l2), dropout=0.0, recurrent_dropout=0.0, input_shape=(n_timesteps, n_features), ), ) model.add(Reshape((n_timesteps * latent_dim1 ,))) model.add(RepeatVector(n_outputs)) model.add(Reshape((n_outputs, n_timesteps, latent_dim1 ))) model.add(Dropout(0.1)) model.add( TimeDistributed( GRU( latent_dim1, activation="relu", # return_sequences=True, kernel_regularizer=l1_l2(l1=l1, l2=l2), dropout=0.0, recurrent_dropout=0.0, ) ) ) model.add(TimeDistributed(Dense(10, activation="relu", kernel_regularizer=l1_l2(l1=0, l2=0)))) # model.add( # Dense( # 10, # activation="relu", # kernel_regularizer=l1_l2(l1=l1, l2=l2), # ) # ) # model.add( # Dense( # 10, # activation="relu", # kernel_regularizer=l1_l2(l1=l1, l2=l2), # ) # ) model.add(TimeDistributed(Dense(1,))) return model
def get_attention(units, lstm_units, n_layers_att, p_dropout=0, l1_reg=0, l2_reg=0): # W1 = Dense(units, kernel_regularizer=l1_l2(l1_reg, l2_reg), name = 'W_feats') W1 = [ Dense(units, activation='tanh', name='W_feats_{}'.format(i)) for i in range(n_layers_att) ] W2 = Dense(units, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='W_hidden') V = Dense(1, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='V') f_beta = Dense(1, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='f_beta') dropout = Dropout(p_dropout, name='dropout') # shape = (batch, attn_features, features_shape) encoder_output = Input(feature_vector_shape, name='image_features') # shape = (batch, lstm_units) hidden_last = Input(lstm_units, name='last_hidden_state') projected_features = dropout(W1[0](encoder_output)) for i in range(1, n_layers_att): projected_features = dropout(W1[i](projected_features)) # shape = (batch, attn_features, 1) score = V(tanh( projected_features + \ dropout(W2(tf.expand_dims(hidden_last, axis = 1))) )) # shape = (batch, attn_features) score = dropout(tf.reduce_sum(score, axis=2)) attention_weights = Activation('softmax', dtype='float32')(score) # beta = f_beta(dropout(tf.expand_dims(hidden_last, axis = 1))) # shape = (batch, 1) beta = f_beta(hidden_last) beta = Activation('sigmoid', dtype='float32')(beta) # shape = (batch, attn_features, features_shape) context_vector = tf.expand_dims(attention_weights, axis=2) * encoder_output # shape = (batch, features_shape) context_vector = beta * tf.reduce_sum(context_vector, axis=1) context_vector = Activation('linear', dtype='float32')(context_vector) return Model(inputs=[encoder_output, hidden_last], outputs=[context_vector, attention_weights], name='attention')
def identity_block(X, f, filters, stage, block, l1=0.0, l2=0.01): # Defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2, F3 = filters # Save the input value X_shortcut = X # First component of main path X = Conv2D(filters=F1, kernel_size=(1, 1), strides=(1, 1), data_format='channels_first', padding='valid', name=conv_name_base + '2a', kernel_initializer=glorot_uniform(seed=0), activity_regularizer=l1_l2(l1, l2))(X) X = BatchNormalization(axis=1, name=bn_name_base + '2a')(X) X = Activation('relu')(X) # Second component of main path X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), data_format='channels_first', padding='same', name=conv_name_base + '2b', kernel_initializer=glorot_uniform(seed=0), activity_regularizer=l1_l2(l1, l2))(X) X = BatchNormalization(axis=1, name=bn_name_base + '2b')(X) X = Activation('relu')(X) # Third component of main path X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), data_format='channels_first', padding='valid', name=conv_name_base + '2c', kernel_initializer=glorot_uniform(seed=0), activity_regularizer=l1_l2(l1, l2))(X) X = BatchNormalization(axis=1, name=bn_name_base + '2c')(X) # Final step: Add shortcut value to main path, and pass it through a RELU activation X = Add()([X, X_shortcut]) X = Activation('relu')(X) return X
def residual_unit(x, activation, n_weights): res = x out = BatchNormalization()(x) out = activation(out) out = Dense(n_weights, activation=None, kernel_regularizer=l1_l2(1e-4, 1e-4))(out) out = BatchNormalization()(x) out = activation(out) out = Dense(n_weights, activation=None, kernel_regularizer=l1_l2(1e-4, 1e-4))(out) out = add([res, out]) return out
def test_constructor(self): ConditionalRNN(5, cell='GRU', kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4)) ConditionalRNN(5, cell='LSTM', kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4)) ConditionalRNN(5, cell='RNN', kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4)) ConditionalRNN(5, cell='RNN', return_sequences=True) ConditionalRNN(5, cell='LSTM', return_sequences=True) ConditionalRNN(5, cell='GRU', return_sequences=True)
def build_model(self, n_features, n_labels): """ The method builds a new member of the ensemble and returns it. :type n_features: int :param n_features: The number of features. :type n_labels: int :param n_labels: The number of labels. """ # initialize optimizer and early stopping self.optimizer = Adam(lr=self.hyperparameters['lr'], beta_1=0.9, beta_2=0.999, epsilon=None, decay=0., amsgrad=False) self.es = EarlyStopping(monitor=f'val_mean_squared_error', min_delta=0.0, patience=self.patience, verbose=1, mode='min', restore_best_weights=True) inputs = Input(shape=(n_features,)) h = GaussianNoise(self.hyperparameters['noise'])(inputs) # the encoder part for i in range(self.bottelneck_layer): h = Dense(self.hyperparameters['neurons'][i], activation='relu', kernel_regularizer=regularizers.l1_l2(self.hyperparameters['l1_hidden'], self.hyperparameters['l2_hidden']))(h) h = Dropout(self.hyperparameters['dropout'])(h) latent = Dense(self.hyperparameters['neurons'][self.bottelneck_layer], activation='linear', kernel_regularizer=regularizers.l1_l2(self.hyperparameters['l1_hidden'], self.hyperparameters['l2_hidden']))(h) encoder = Model(inputs=inputs, outputs=latent, name='encoder') # the decoder part latent_inputs = Input(shape=(self.hyperparameters['neurons'][self.bottelneck_layer],)) h = GaussianNoise(0.0)(latent_inputs) for i in range(self.bottelneck_layer + 1, self.n_hidden_layers - 1): h = Dense(self.hyperparameters['neurons'][i], activation='relu', kernel_regularizer=regularizers.l1_l2(self.hyperparameters['l1_hidden'], self.hyperparameters['l2_hidden']))(h) h = Dropout(self.hyperparameters['dropout'])(h) decoded = Dense(n_labels, activation='linear', kernel_regularizer=regularizers.l1_l2(self.hyperparameters['l1_out'], self.hyperparameters['l2_out']))(h) decoder = Model(inputs=latent_inputs, outputs=decoded, name='decoder') # endocder-decoder model encoder_decoder = Model(inputs, decoder(encoder(inputs)), name='encoder_decoder') return encoder_decoder, encoder, decoder
def Build_model(news_input_shape, num_classes): model = Sequential() #model.add(AveragePooling1D(pool_size=3, strides=1, input_shape=(news_input_shape.shape[1], 1)) model.add( Conv1D(filters=1, kernel_size=1, name='gaa', input_shape=(news_input_shape.shape[1], 1))) model.add(Conv1D(filters=4, kernel_size=4, name='0_conv_conc1')) model.add(Conv1D(filters=5, kernel_size=3, name='0_conv_conc2')) model.add(Conv1D(filters=1, kernel_size=1, name='0_conv_conc3')) model.add(Flatten()) model.add(Dense(units=320, activation='relu', name='dense_layer_200')) model.add(Reshape(target_shape=(320, 1))) model.add(Conv1D(filters=3, kernel_size=3, name='1_conv_conc')) model.add(Conv1D(filters=3, kernel_size=3, name='2_conv_conc')) model.add(MaxPooling1D(pool_size=3, strides=1, name='first_avgPool')) model.add(Conv1D(filters=3, kernel_size=3, name='2_conv_sconc')) model.add(MaxPooling1D(pool_size=3, strides=1, name='first_avgsssPsool')) model.add(Flatten()) model.add(Dropout(rate=0.3)) model.add(Dense(units=180, activation='relu', name="1_dense_layer")) model.add(Reshape(target_shape=(180, 1))) model.add(MaxPooling1D(pool_size=3, strides=1, name='first_avgssPosol')) model.add(Conv1D(filters=3, kernel_size=2)) model.add(Conv1D(filters=5, kernel_size=5)) model.add(MaxPooling1D(pool_size=3, strides=1, name='first_avgPosol1')) model.add(Flatten()) model.add( Dense(units=120, activation='relu', kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4))) model.add( Dense(units=num_classes, activation='relu', name="2_dense_layer", kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4), bias_regularizer=regularizers.l2(1e-4), activity_regularizer=regularizers.l2(1e-5))) #model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) #model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.compile(optimizer='rmsprop', loss='mean_squared_error', metrics=['accuracy']) model.summary() return model
def get_encoder(node_num, d, K, n_units, nu1, nu2, activation_fn): # Input x = Input(shape=(node_num,)) # Encoder layers y = [None] * (K + 1) y[0] = x # y[0] is assigned the input for i in range(K - 1): y[i + 1] = Dense(n_units[i], activation=activation_fn, kernel_regularizer=Reg.l1_l2(l1=nu1, l2=nu2))(y[i]) y[K] = Dense(d, activation=activation_fn, kernel_regularizer =Reg.l1_l2(l1=nu1, l2=nu2))(y[K - 1]) # Encoder model encoder = Model(inputs=x, outputs=y[K]) return encoder
def build_model(self, n_features): """ The method builds a new member of the ensemble and returns it. """ # derived parameters self.hyperparameters['n_members'] = self.hyperparameters[ 'n_segments'] * self.hyperparameters['n_members_segment'] # initialize optimizer and early stopping self.optimizer = Adam(lr=self.hyperparameters['lr'], beta_1=0.9, beta_2=0.999, epsilon=None, decay=0., amsgrad=False) self.es = EarlyStopping(monitor=f'val_{self.loss_name}', min_delta=0.0, patience=self.hyperparameters['patience'], verbose=1, mode='min', restore_best_weights=True) inputs = Input(shape=(n_features, )) h = GaussianNoise(self.hyperparameters['noise_in'], name='noise_input')(inputs) for i in range(self.hyperparameters['layers']): h = Dense(self.hyperparameters['neurons'], activation='tanh', kernel_regularizer=regularizers.l1_l2( self.hyperparameters['l1_hidden'], self.hyperparameters['l2_hidden']), kernel_initializer='random_uniform', bias_initializer='random_uniform', name=f'hidden_{i}')(h) h = Dropout(self.hyperparameters['dropout'], name=f'hidden_dropout_{i}')(h) out = Dense(self.n_outputs, activation='softmax', kernel_regularizer=regularizers.l1_l2( self.hyperparameters['l1_out'], self.hyperparameters['l2_out']), kernel_initializer='random_uniform', bias_initializer='random_uniform', name='output')(h) model = Model(inputs=inputs, outputs=out) return model
def build(self, input_shape) -> None: self.embedding: Embedding = Embedding(input_dim=self.vocabulary_size, output_dim=self.embedding_size, input_length=self.sentence_len, trainable=True) self.conv_1: Conv1D = Conv1D(filters=self.conv_filter, kernel_size=3, activation="relu", name="conv_1") self.conv_2: Conv1D = Conv1D(filters=self.conv_filter, kernel_size=4, activation="relu", name="conv_2") self.conv_3: Conv1D = Conv1D(filters=self.conv_filter, kernel_size=5, activation="relu", name="conv_3") if not self.global_max_pool: self.pool_1: MaxPool1D = MaxPool1D(pool_size=self.pool_size, strides=1, name="pool_1") self.pool_2: MaxPool1D = MaxPool1D(pool_size=self.pool_size, strides=1, name="pool_2") self.pool_3: MaxPool1D = MaxPool1D(pool_size=self.pool_size, strides=1, name="pool_3") else: self.pool_1: GlobalMaxPool1D = GlobalMaxPool1D(name="pool_1") self.pool_2: GlobalMaxPool1D = GlobalMaxPool1D(name="pool_2") self.pool_3: GlobalMaxPool1D = GlobalMaxPool1D(name="pool_3") self.concatenate: Concatenate = Concatenate(axis=1) self.flatten: Flatten = Flatten() self.dropout_1: Dropout = Dropout(self.drop_rate, name="dropout_1") self.dense1 = Dense(self.dense_size, activation="sigmoid", kernel_regularizer=regularizers.l1_l2( self.l1_regularization, self.l2_regularization)) self.dropout_2: Dropout = Dropout(self.drop_rate, name="dropout_2") self.dense: Dense = Dense(self.class_num, activation="softmax", kernel_regularizer=regularizers.l1_l2( self.l1_regularization, self.l2_regularization)) super(TextCNN, self).build(input_shape)
def Model(learn_rate=0.01, L1=0, L2=0, fc=4096, mm=0.9): # K.clear_session() model = Sequential() model.add( Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(new_height, new_width, ch))) model.add(Conv2D(64, (3, 3), activation='relu', padding='same')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(Conv2D(128, (3, 3), activation='relu', padding='same')) model.add(Conv2D(128, (3, 3), activation='relu', padding='same')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(Conv2D(256, (3, 3), activation='relu', padding='same')) model.add(Conv2D(256, (3, 3), activation='relu', padding='same')) model.add(Conv2D(256, (3, 3), activation='relu', padding='same')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(Flatten()) print(model.output_shape) model.add( Dense(fc, activation='relu', kernel_regularizer=regularizers.l1_l2(l1=L1, l2=L2))) model.add( Dense(fc, activation='relu', kernel_regularizer=regularizers.l1_l2(l1=L1, l2=L2))) model.add(Dense(2, activation='softmax')) model.compile(optimizer=SGD(lr=learn_rate, momentum=mm), loss='binary_crossentropy', metrics=['accuracy']) return model
def introduce_metadata(path, i): if dynamic_input_shapes: metadata_input_ = metadata_path = Input( shape=(None, None, None, metadata_sizes[i][-1] if isinstance( metadata_sizes[i], tuple) else metadata_sizes[i])) else: metadata_input_ = metadata_path = Input( shape=metadata_sizes[i] if isinstance(metadata_sizes[i], tuple) else (1, 1, 1, metadata_sizes[i])) metadata_inputs[i] = metadata_input_ for j, (m_n_f, m_d) in enumerate( zip(metadata_number_features[i], metadata_dropout[i])): metadata_path = Dropout(m_d)(metadata_path) metadata_path = Conv3D(filters=m_n_f, kernel_size=(1, 1, 1), padding=padding, kernel_initializer=kernel_initializer, kernel_regularizer=regularizers.l1_l2( l1_reg, l2_reg))(metadata_path) metadata_path = activation_function("m{}_activation{}".format( i, j))(metadata_path) if not isinstance(metadata_sizes[i], tuple): broadcast_shape = K.concatenate([ K.constant([1], dtype="int32"), K.shape(path)[1:-1], K.constant([1], dtype="int32") ]) metadata_path = K.tile(metadata_path, broadcast_shape) return Concatenate(axis=-1)([path, metadata_path])
def retrieve_extra_output(path, i): extra_output_path = path for j, (e_o_k_s, e_o_n_f, e_o_d) in enumerate( zip(extra_output_kernel_sizes[i], extra_output_number_features[i], extra_output_dropout[i])): extra_output_path = Dropout(e_o_d)(extra_output_path) extra_output_path = Conv3D( filters=e_o_n_f, kernel_size=e_o_k_s, activation=extra_output_activation_final_layer[i] if j + 1 == len(extra_output_number_features[i]) else None, padding=padding, kernel_initializer=kernel_initializer, kernel_regularizer=regularizers.l1_l2(l1_reg, l2_reg), name=f"s{i + 1}" if j + 1 == len(extra_output_number_features[i]) else None)(extra_output_path) if j + 1 < len(extra_output_number_features[i]): if (batch_normalization or instance_normalization ) and not relaxed_normalization_scheme: extra_output_path = normalization_function()( extra_output_path) extra_output_path = activation_function( "eo{}_activation{}".format(i, j))(extra_output_path) return extra_output_path