def test_find_activation_layer(): conv1_filters = 1 conv2_filters = 1 dense_units = 1 model = Sequential() model.add( Conv2D(conv1_filters, [3, 3], input_shape=(28, 28, 1), data_format="channels_last", name='conv_1')) model.add(Activation('relu', name='act_1')) model.add(MaxPool2D((2, 2), name='pool_1')) model.add( Conv2D(conv2_filters, [3, 3], data_format="channels_last", name='conv_2')) model.add(Activation('relu', name='act_2')) model.add(MaxPool2D((2, 2), name='pool_2')) model.add(Flatten(name='flat_1')) model.add(Dense(dense_units, name='dense_1')) model.add(Activation('relu', name='act_3')) model.add(Dense(10, name='dense_2')) model.add(Activation('softmax', name='act_4')) assert find_activation_layer(model.get_layer('conv_1'), 0) == (model.get_layer('act_1'), 0) assert find_activation_layer(model.get_layer('conv_2'), 0) == (model.get_layer('act_2'), 0) assert find_activation_layer(model.get_layer('dense_1'), 0) == (model.get_layer('act_3'), 0) assert find_activation_layer(model.get_layer('dense_2'), 0) == (model.get_layer('act_4'), 0)
def CAE(input_shape=(28, 28, 1), filters=[32, 64, 128, 10]): model = Sequential() if input_shape[0] % 8 == 0: pad3 = 'same' else: pad3 = 'valid' model.add(InputLayer(input_shape)) model.add(Conv2D(filters[0], 5, strides=2, padding='same', activation='relu', name='conv1')) model.add(Conv2D(filters[1], 5, strides=2, padding='same', activation='relu', name='conv2')) model.add(Conv2D(filters[2], 3, strides=2, padding=pad3, activation='relu', name='conv3')) model.add(Flatten()) model.add(Dense(units=filters[3], name='embedding')) model.add(Dense(units=filters[2]*int(input_shape[0]/8)*int(input_shape[0]/8), activation='relu')) model.add(Reshape((int(input_shape[0]/8), int(input_shape[0]/8), filters[2]))) model.add(Conv2DTranspose(filters[1], 3, strides=2, padding=pad3, activation='relu', name='deconv3')) model.add(Conv2DTranspose(filters[0], 5, strides=2, padding='same', activation='relu', name='deconv2')) model.add(Conv2DTranspose(input_shape[2], 5, strides=2, padding='same', name='deconv1')) if model.layers[0].input_shape != model.layers[-1].output_shape: crop = abs(model.layers[0].input_shape[1] - model.layers[-1].output_shape[1])//2 model.add(Cropping2D(crop)) encoder = Model(inputs=model.input, outputs=model.get_layer('embedding').output) return model, encoder
def universal_approximation(f, x): [train_x, test_x] = split_data(x, ratio=0.75, random=True) train_y = np.sin(train_x) test_x = np.sort(test_x, axis=0) test_y = f(test_x) # build simple FNN model = Sequential() model.add(Dense(50, input_shape=(1, ), activation='relu')) model.add(Dense(1)) model.compile(loss='mse', optimizer='adam') # training process model.fit(train_x, train_y, batch_size=100, epochs=1000) layer = model.get_layer(index=0) plt.plot(model.history.history['loss']) plt.show() # predict y_hat = model.predict(test_x) plt.plot(test_x, test_y, 'b-', label='original') plt.plot(test_x, y_hat, 'r-', label='predicted') plt.legend() plt.show()
def export_flatten(filename, input_shape): model = Sequential() model.add(Flatten(input_shape=input_shape[1:])) model.predict(np.random.uniform(size=input_shape)) sess = K.get_session() output = model.get_layer('flatten').output return export(output, filename, sess=sess)
def train_2nd_level( self, X_train: np.ndarray, X_valid: np.ndarray ) -> Tuple[np.ndarray, np.ndarray]: """train 2nd level DAE Note that input must to be scaled betweeen [0, 1] because output layer is governed by sigmoid function. Args X_train (np.ndarray): has shape (num_samples, output dim of 1st level) X_valid (np.ndarray): has shape (num_samples, output dim of 1st level) """ # Build base model model = Sequential() model.add(Dropout(0.4, seed=20, name="2nd_level_dropout")) model.add( Dense(30, input_dim=X_train.shape[1], activation="sigmoid", name="2nd_level_fc") ) # encoder model.add(Dense(X_train.shape[1], activation="sigmoid", name="2nd_level_output")) # decoder model.compile( loss="mean_squared_error", optimizer=optimizers.Adam(lr=self.pretrain_lr), metrics=["mse"], ) # Create callback callbacks = create_callback( model=model, path_chpt=f"{self.LOG_DIR}/trained_model_2nd_level_fold{self.fold_id}.h5", metric="mse", verbose=50, epochs=self.pretrain_epochs, ) fit = model.fit( x=X_train, y=X_train, batch_size=self.pretrain_batch_size, epochs=self.pretrain_epochs, verbose=self.verbose, validation_data=(X_valid, X_valid), callbacks=callbacks, ) plot_learning_history( fit=fit, metric="mse", path=f"{self.LOG_DIR}/history_2nd_level_fold{self.fold_id}.png" ) plot_model(model, path=f"{self.LOG_DIR}/model_2nd_level.png") # Load best model model = keras.models.load_model( f"{self.LOG_DIR}/trained_model_2nd_level_fold{self.fold_id}.h5" ) self.model_2nd_level = model encoder = Model(inputs=model.input, outputs=model.get_layer("2nd_level_fc").output) pred_train = encoder.predict(X_train) # predict with clean signal pred_valid = encoder.predict(X_valid) # predict with clean signal return pred_train, pred_valid
def vgg_layers(layer_names): red_model = Sequential() vgg = tf.keras.applications.vgg19.VGG19(include_top=False, weights='imagenet') vgg.trainable = False for layer in vgg.layers: if layer.__class__ == MaxPooling2D: red_model.add(AveragePooling2D()) else: red_model.add(layer) outputs = [red_model.get_layer(name).output for name in layer_names] model = Model([red_model.input], outputs) return model
def build_vae_encoder(self): encoder = Sequential() encoder.add( Conv2D(32, (4, 4), strides=(2, 2), activation='relu', name='enc_conv1', input_shape=self.state_size)) encoder.add( Conv2D(64, (4, 4), strides=(2, 2), activation='relu', name='enc_conv2')) encoder.add( Conv2D(128, (4, 4), strides=(2, 2), activation='relu', name='enc_conv3')) encoder.add( Conv2D(256, (4, 4), strides=(2, 2), activation='relu', name='enc_conv4')) encoder.add(Flatten()) # shape: [-1, 3 * 8 * 256] encoder.add(Dense(self.z_size, name='enc_fc_mu')) if self.vae_weights: encoder.get_layer('enc_conv1').set_weights(self.vae_weights[0:2]) encoder.get_layer('enc_conv2').set_weights(self.vae_weights[2:4]) encoder.get_layer('enc_conv3').set_weights(self.vae_weights[4:6]) encoder.get_layer('enc_conv4').set_weights(self.vae_weights[6:8]) encoder.get_layer('enc_fc_mu').set_weights(self.vae_weights[8:10]) encoder.trainable = False return encoder
def __init__(self, input_shape): model = Sequential() model.add(Conv2D(16,(5,5),padding='valid',input_shape = input_shape)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2,2),strides=2,padding = 'valid')) model.add(Dropout(0.4)) model.add(Conv2D(32,(5,5),padding='valid')) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2,2),strides=2,padding = 'valid')) model.add(Dropout(0.6)) model.add(Conv2D(64,(5,5),padding='valid')) model.add(Activation('relu')) model.add(Dropout(0.8)) model.add(Flatten()) model.add(Dense(2)) model.add(Activation('softmax')) self.transformer = Model(inputs=model.input,outputs=model.get_layer('dense_1').output)
def autoencoder(n_input, loss='mean_squared_error'): ''' model.fit(x_train, x_train, batch_size = 32, epochs = 500) bottleneck_representation = encoder.predict(x_train) ''' model = Sequential() # encoder model.add(Dense(512, activation='relu', input_shape=(n_input, ))) model.add(Dense(256, activation='relu')) model.add(Dense(64, activation='relu')) # bottleneck code model.add(Dense(20, activation='linear', name="bottleneck")) # decoder model.add(Dense(64, activation='relu')) model.add(Dense(256, activation='relu')) model.add(Dense(512, activation='relu')) model.add(Dense(n_input, activation='sigmoid')) model.compile(loss=loss, optimizer=Adam()) encoder = Model(model.input, model.get_layer('bottleneck').output) return model, encoder
def featureExtraction(x,y): x = x.to_numpy() y = y.to_numpy() x = x[:, :, newaxis] verbose, epochs, batch_size = 1, 50, 1 n_timesteps, n_features = x.shape[1], x.shape[2] model = Sequential() #model.build(1344) model.add(Conv1D(filters=100, kernel_size=6, activation='relu', input_shape=(n_timesteps,n_features))) model.add(Dropout(0.5)) model.add(MaxPooling1D(pool_size=1)) model.add(BatchNormalization()) model.add(Conv1D(filters=75, kernel_size=4, activation='relu')) model.add(Dropout(0.1)) model.add(MaxPooling1D(pool_size=1)) model.add(BatchNormalization()) model.add(Conv1D(filters=50, kernel_size=2, activation='relu')) model.add(Conv1D(filters=12, kernel_size=2, activation='relu')) model.add(Flatten()) model.add(Dense(15, activation='relu', name ='feature_dense')) model.add(Dense(1, activation='softmax')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) #fit network model.fit(x, y, validation_split=0.02, epochs=epochs, batch_size=batch_size, verbose=verbose) model.summary() feature_layer = Model(inputs=model.input, outputs=model.get_layer('feature_dense').output) feature_layer.summary() new_data = feature_layer.predict(x) new_data = pd.DataFrame(new_data ) return new_data
def cnn(train_dir, test_dir): # 定数定義 NUM_CLASSES = 4 # 識別クラス数 IMAGE_SIZE = 256 # 学習時の画像サイズ[px] IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE * 3 # 画像の次元数 LABEL_ANNOTATION_PATH = './label_annotation.txt' LOG_TRAINING_ACCURACY_GRAPH_PATH = './log/cnn/training_accuracy.png' LOG_TRAINING_LOSS_GRAPH_PATH = './log/cnn/training_loss.png' LOG_TRAINING_MODEL_PATH = './log/cnn/model.png' TRAINING_OPTIMIZER = "SGD(確率的勾配降下法)" ACTIVATION_FUNC = "relu" #活性化関数 # 学習データセットのインポート train_image, train_label, train_path = import_dataset( train_dir, IMAGE_SIZE, IMAGE_SIZE) #Kerasの学習モデルの構築 model = Sequential() # 畳み込み層 model.add( Conv2D(3, kernel_size=3, activation=ACTIVATION_FUNC, input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(3, kernel_size=3, activation=ACTIVATION_FUNC, input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(3, kernel_size=3, activation=ACTIVATION_FUNC, input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Activation(ACTIVATION_FUNC)) model.add(Dropout(0.2)) # 全結合層 model.add(Dense(200)) model.add(Activation(ACTIVATION_FUNC)) model.add(Dropout(0.2)) model.add(Dense(200)) model.add(Activation(ACTIVATION_FUNC)) model.add(Dropout(0.2)) model.add(Dense(200)) model.add(Activation(ACTIVATION_FUNC)) model.add(Dropout(0.2)) model.add(Dense(200)) model.add(Activation(ACTIVATION_FUNC)) model.add(Dropout(0.2)) model.add(Dense(NUM_CLASSES)) model.add(Activation("softmax")) # オプティマイザにAdamを使用 opt = Adam(lr=0.001) # モデルをコンパイル model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) # 学習を実行 Y = to_categorical(train_label, NUM_CLASSES) history = model.fit(train_image, Y, nb_epoch=40, batch_size=100, validation_split=0.1) export.plot(history) # テスト用データセットのインポート test_image, test_label, test_path = import_dataset(test_dir, IMAGE_SIZE, IMAGE_SIZE) result = model.predict_classes(test_image) result_prob = model.predict_proba(test_image) sum_accuracy = 0.0 for i in range(test_image.shape[0]): print("label:", test_label[i], "result:", result[i], "prob: ", result_prob[i]) if test_label[i] == result[i]: sum_accuracy += 1 sum_accuracy /= test_image.shape[0] print("accuracy: ", sum_accuracy) plot_model(model, show_shapes=True, to_file=LOG_TRAINING_MODEL_PATH) #中間層の出力 imm_layer = ['conv2d', 'conv2d_1', 'conv2d_2'] for layer_name in imm_layer: #中間層のmodelを作成 intermediate_layer_model = Model( inputs=model.input, outputs=model.get_layer(layer_name).output) #出力をmodel.predictで見る intermediate_output = intermediate_layer_model.predict(test_image) path = os.getcwd() + "/log/cnn/" + layer_name if os.path.exists(path) == False: # 出力先ディレクトリが存在しなければ新規作成する os.mkdir(path) for i in range(intermediate_output.shape[0]): cv2.imwrite(path + '/immidiate_' + str(i) + '.png', intermediate_output[i] * 255) #結果をhtmlファイル出力 result_dict = {'acc': 0, 'n_img': 0, 'opt': "", 'act_func': ""} result_dict['acc'] = sum_accuracy result_dict['n_img'] = train_image.shape[0] result_dict['opt'] = TRAINING_OPTIMIZER result_dict['act_func'] = ACTIVATION_FUNC export.cnn_html(result_dict, test_image, test_label, test_path, result, result_prob, imm_layer)
class CNN(object): def __init__(self): """ Initialize multi-layer neural network """ self.model = Sequential() def add_input_layer(self, shape=(2, ), name=""): """ This function adds an input layer to the neural network. If an input layer exist, then this function should replcae it with the new input layer. :param shape: input shape (tuple) :param name: Layer name (string) :return: None """ # self.model.add(Dense(input_shape=shape,name=name)) self.model.add(InputLayer(input_shape=shape, name=name)) def append_dense_layer(self, num_nodes, activation="relu", name="", trainable=True): """ This function adds a dense layer to the neural network :param num_nodes: Number of nodes :param activation: Activation function for the layer. Possible values are "Linear", "Relu", "Sigmoid", "Softmax" :param name: Layer name (string) :param trainable: Boolean :return: None """ self.model.add( Dense(units=num_nodes, activation=activation, name=name, trainable=trainable)) def append_conv2d_layer(self, num_of_filters, kernel_size=3, padding='same', strides=1, activation="Relu", name="", trainable=True): """ This function adds a conv2d layer to the neural network :param num_of_filters: Number of nodes :param num_nodes: Number of nodes :param kernel_size: Kernel size (assume that the kernel has the same horizontal and vertical size) :param padding: "same", "Valid" :param strides: strides :param activation: Activation function for the layer. Possible values are "Linear", "Relu", "Sigmoid" :param name: Layer name (string) :param trainable: Boolean :return: Layer object """ layer = Conv2D(num_of_filters, kernel_size=kernel_size, padding=padding, strides=strides, activation=activation, name=name, trainable=trainable) self.model.add( Conv2D(num_of_filters, kernel_size=kernel_size, padding=padding, strides=strides, activation=activation, name=name, trainable=trainable)) return layer def append_maxpooling2d_layer(self, pool_size=2, padding="same", strides=2, name=""): """ This function adds a maxpool2d layer to the neural network :param pool_size: Pool size (assume that the pool has the same horizontal and vertical size) :param padding: "same", "valid" :param strides: strides :param name: Layer name (string) :return: Layer object """ layer = MaxPooling2D(pool_size=pool_size, strides=strides, padding=padding, name=name) self.model.add( MaxPooling2D(pool_size=pool_size, strides=strides, padding=padding, name=name)) return layer def append_flatten_layer(self, name=""): """ This function adds a flattening layer to the neural network :param name: Layer name (string) :return: Layer object """ layer = Flatten(name=name) self.model.add(Flatten(name=name)) return layer def set_training_flag(self, layer_numbers=[], layer_names="", trainable_flag=True): """ This function sets the trainable flag for a given layer :param layer_number: an integer or a list of numbers.Layer numbers start from layer 0. :param layer_names: a string or a list of strings (if both layer_number and layer_name are specified, layer number takes precedence). :param trainable_flag: Set trainable flag :return: None """ def get_weights_without_biases(self, layer_number=None, layer_name=""): """ This function should return the weight matrix (without biases) for layer layer_number. layer numbers start from zero. This means that the first layer with activation function is layer zero :param layer_number: Layer number starting from layer 0. :param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence). :return: Weight matrix for the given layer (not including the biases). If the given layer does not have weights then None should be returned. """ if (not (layer_number is None)): if (layer_number >= 0): if (layer_number == 0): return None elif (self.model.get_layer( index=layer_number - 1, name=layer_name).count_params() == 0): return None else: return self.model.get_layer( index=layer_number - 1, name=layer_name).get_weights()[0] else: if (self.model.get_layer( index=layer_number).count_params() == 0): return None else: return self.model.get_layer( index=layer_number).get_weights()[0] else: if (self.model.get_layer(name=layer_name).count_params() == 0): return None else: return self.model.get_layer(name=layer_name).get_weights()[0] def get_biases(self, layer_number=None, layer_name=""): """ This function should return the biases for layer layer_number. layer numbers start from zero. This means that the first layer with activation function is layer zero :param layer_number: Layer number starting from layer 0 :param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence). :return: biases for the given layer (If the given layer does not have bias then None should be returned) """ if (not layer_number is None): if (layer_number == 0): return None elif (self.model.get_layer(index=layer_number - 1, name=layer_name).count_params() == 0): return None else: return self.model.get_layer(index=layer_number - 1, name=layer_name).get_weights()[1] else: if (self.model.get_layer(name=layer_name).count_params() == 0): return None else: return self.model.get_layer(name=layer_name).get_weights()[1] def set_weights_without_biases(self, weights, layer_number=None, layer_name=""): """ This function sets the weight matrix for layer layer_number. layer numbers start from zero. This means that the first layer with activation function is layer zero :param weights: weight matrix (without biases). Note that the shape of the weight matrix should be [input_dimensions][number of nodes] :param layer_number: Layer number starting from layer 0 :param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence). :return: None """ if (not layer_number is None): temp = self.get_biases(layer_number, layer_name) W = [weights, temp] self.model.get_layer(index=layer_number - 1, name=layer_name).set_weights(W) else: temp = self.get_biases(layer_number, layer_name) W = [weights, temp] self.model.get_layer(name=layer_name).set_weights(W) def set_biases(self, biases, layer_number=None, layer_name=""): """ This function sets the biases for layer layer_number. layer numbers start from zero. This means that the first layer with activation function is layer zero :param biases: biases. Note that the biases shape should be [1][number_of_nodes] :param layer_number: Layer number starting from layer 0 :param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence). :return: none """ if (not layer_number is None): temp = self.get_weights_without_biases(layer_number, layer_name) W = [temp, biases] self.model.get_layer(index=layer_number - 1, name=layer_name).set_weights(W) else: temp = self.get_weights_without_biases(layer_number, layer_name) W = [temp, biases] self.model.get_layer(name=layer_name).set_weights(W) def remove_last_layer(self): """ This function removes a layer from the model. :return: removed layer """ layer = self.model.get_layer(index=-1) self.model = Sequential(self.model.layers[:-1]) return layer # self.model.outputs = [self.model.layers[-1].output] def load_a_model(self, model_name="", model_file_name=""): """ This function loads a model architecture and weights. :param model_name: Name of the model to load. model_name should be one of the following: "VGG16", "VGG19" :param model_file_name: Name of the file to load the model (if both madel_name and model_file_name are specified, model_name takes precedence). :return: model """ if (model_name == "VGG16"): self.model = Sequential(VGG16().layers) elif (model_name == "VGG19"): self.model = Sequential(VGG19().layers) else: self.model = load_model(model_file_name) def save_model(self, model_file_name=""): """ This function saves the current model architecture and weights together in a HDF5 file. :param file_name: Name of file to save the model. :return: model """ self.model.save(model_file_name) def set_loss_function(self, loss="SparseCategoricalCrossentropy"): """ This function sets the loss function. :param loss: loss is a string with the following choices: "SparseCategoricalCrossentropy", "MeanSquaredError", "hinge". :return: none """ def set_metric(self, metric): """ This function sets the metric. :param metric: metric should be one of the following strings: "accuracy", "mse". :return: none """ def set_optimizer(self, optimizer="SGD", learning_rate=0.01, momentum=0.0): """ This function sets the optimizer. :param optimizer: Should be one of the following: "SGD" , "RMSprop" , "Adagrad" , :param learning_rate: Learning rate :param momentum: Momentum :return: none """ def predict(self, X): """ Given array of inputs, this function calculates the output of the multi-layer network. :param X: Input tensor. :return: Output tensor. """ return self.model.predict(X) def evaluate(self, X, y): """ Given array of inputs and desired ouputs, this function returns the loss value and metrics of the model. :param X: Array of input :param y: Array of desired (target) outputs :return: loss value and metric value """ def train(self, X_train, y_train, batch_size, num_epochs): """
dec.add(UpSampling2D((2, 2))) dec.add(Conv2D(16, (3, 3), activation='relu')) dec.add(UpSampling2D((2, 2))) dec.add(Conv2D(1, (3, 3), activation='sigmoid', padding='same')) dec.summary() encoder = Sequential([enc,dec]) encoder.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(lr=0.001)) encoder.fit(X_train,X_train,epochs=10,validation_data=(X_test,X_test)) !pip install bokeh from tensorflow.keras import Model layer_name = 'MUSAFA' intermediate_layer_model = Model(inputs=enc.input,outputs=enc.get_layer(layer_name).output) intermediate_output = intermediate_layer_model.predict(X_test) # BEFORE SHAPE WAS (10000,7,7,8) # 7*7*8 --> 392 intermediate_output = intermediate_output.reshape(10000,392) from sklearn.manifold import TSNE tsne_model = TSNE(n_components=2, verbose=1, random_state=0) tsne_img_label = tsne_model.fit_transform(intermediate_output) import pandas as pd import numpy as np tsne_df = pd.DataFrame(tsne_img_label, columns=['x', 'y']) tsne_df['image_label'] = y_test
plt.figure(figsize=(8,8)) for i in range(3): delta_1=W_1[i]-W_0[i] print(f"difference between biases is {b_1[i]-b_0[i]}") axis=plt.subplot(1,3,i+1) plt.imshow(delta_1) plt.title(f"layer {i+1}") plt.axis("off") plt.colorbar() plt.show()''' n_trainable_variables = model.trainable_variables print(n_trainable_variables) model.get_layer("dense").trainable = False model.compile(optimizer="adam", loss="mse", metrics=["accuracy"]) his = model.fit(Xtrain, ytrain, epochs=50) W_1 = [e.weights[0].numpy() for e in model.layers] b_1 = [b.bias.numpy() for b in model.layers] plt.figure(figsize=(8, 8)) for i in range(3): delta_1 = W_1[i] - W_0[i] print(f"difference between biases is {b_1[i]-b_0[i]}") axis = plt.subplot(1, 3, i + 1) plt.imshow(delta_1) plt.title(f"layer {i+1}") plt.axis("off")
class CNN(object): def __init__(self): """ Initialize multi-layer neural network """ self.model = Sequential() def add_input_layer(self, shape=(2,),name="" ): """ This function adds an input layer to the neural network. If an input layer exist, then this function should replcae it with the new input layer. :param shape: input shape (tuple) :param name: Layer name (string) :return: None """ self.builtIn = False if shape.__class__ != tuple and shape.__class__ == int: shape = (shape,) if self.model is None: self.model = Sequential() self.model.add(InputLayer(input_shape=shape, name=name)) def append_dense_layer(self, num_nodes,activation="relu",name="",trainable=True): """ This function adds a dense layer to the neural network :param num_nodes: Number of nodes :param activation: Activation function for the layer. Possible values are "Linear", "Relu", "Sigmoid", "Softmax" :param name: Layer name (string) :param trainable: Boolean :return: None """ self.model.add(Dense(units = num_nodes, activation=activation, name= name, trainable= trainable)) def append_conv2d_layer(self, num_of_filters, kernel_size=3, padding='same', strides=1, activation="Relu",name="",trainable=True, input_shape = None): """ This function adds a conv2d layer to the neural network :param num_of_filters: Number of nodes :param num_nodes: Number of nodes :param kernel_size: Kernel size (assume that the kernel has the same horizontal and vertical size) :param padding: "same", "Valid" :param strides: strides :param activation: Activation function for the layer. Possible values are "Linear", "Relu", "Sigmoid" :param name: Layer name (string) :param trainable: Boolean :return: Layer object """ if input_shape != None: conv2D_layer = Conv2D(num_of_filters, kernel_size=kernel_size, padding=padding, name=name, strides = strides, activation=activation, trainable=trainable, input_shape = input_shape) else: conv2D_layer = Conv2D(num_of_filters, kernel_size=kernel_size, padding=padding, name=name, strides = strides, activation=activation, trainable=trainable) self.model.add(conv2D_layer) return conv2D_layer def append_maxpooling2d_layer(self, pool_size=2, padding="same", strides=2,name=""): """ This function adds a maxpool2d layer to the neural network :param pool_size: Pool size (assume that the pool has the same horizontal and vertical size) :param padding: "same", "valid" :param strides: strides :param name: Layer name (string) :return: Layer object """ maxpooling2d_layer = MaxPooling2D(pool_size = pool_size, strides=strides, padding=padding, name=name) self.model.add(maxpooling2d_layer) return maxpooling2d_layer def append_flatten_layer(self,name=""): """ This function adds a flattening layer to the neural network :param name: Layer name (string) :return: Layer object """ flatten_layer = Flatten(name=name) self.model.add(flatten_layer) return flatten_layer def set_training_flag(self,layer_numbers=[],layer_names="",trainable_flag=True): """ This function sets the trainable flag for a given layer :param layer_number: an integer or a list of numbers.Layer numbers start from layer 0. :param layer_names: a string or a list of strings (if both layer_number and layer_name are specified, layer number takes precedence). :param trainable_flag: Set trainable flag :return: None """ for num in layer_numbers: self.model.layers[num].trainable = trainable_flag def get_weights_without_biases(self,layer_number=None,layer_name=""): """ This function should return the weight matrix (without biases) for layer layer_number. layer numbers start from zero. This means that the first layer with activation function is layer zero :param layer_number: Layer number starting from layer 0. :param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence). :return: Weight matrix for the given layer (not including the biases). If the given layer does not have weights then None should be returned. """ if(layer_number is not None): if(layer_number > 0): weights = self.model.get_layer(index = layer_number-1).get_weights() if len(weights) > 0: return weights[0] elif(layer_number < 0): return self.model.get_layer(index = layer_number).get_weights()[0] elif(layer_name != ""): try: if(self.model.get_layer(name=layer_name) != None and self.model.get_layer(name=layer_name).count_params() != 0): return self.model.get_layer(name = layer_name).get_weights()[0] except: return None return None def get_biases(self,layer_number=None,layer_name=""): """ This function should return the biases for layer layer_number. layer numbers start from zero. This means that the first layer with activation function is layer zero :param layer_number: Layer number starting from layer 0 :param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence). :return: biases for the given layer (If the given layer does not have bias then None should be returned) """ if(not layer_number is None): if(layer_number > 0): biases = self.model.get_layer(index=layer_number-1,name=layer_name).get_weights() if len(biases) > 0: return biases[1] elif layer_name != "": biases = self.model.get_layer(name=layer_name).get_weights() if len(biases) > 0: return biases[1] return None def set_weights_without_biases(self,weights,layer_number=None,layer_name=""): """ This function sets the weight matrix for layer layer_number. layer numbers start from zero. This means that the first layer with activation function is layer zero :param weights: weight matrix (without biases). Note that the shape of the weight matrix should be [input_dimensions][number of nodes] :param layer_number: Layer number starting from layer 0 :param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence). :return: None """ if(layer_number is not None): nweights = [weights,self.get_biases(layer_number, "")] self.model.get_layer(index=layer_number-1).set_weights(nweights) elif(layer_name != ""): nweights = [weights,self.get_biases(None,layer_name)] self.model.get_layer(name=layer_name).set_weights(nweights) def set_biases(self,biases,layer_number=None,layer_name=""): """ This function sets the biases for layer layer_number. layer numbers start from zero. This means that the first layer with activation function is layer zero :param biases: biases. Note that the biases shape should be [1][number_of_nodes] :param layer_number: Layer number starting from layer 0 :param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence). :return: none """ if(layer_number is not None): nbiases = [temp,self.get_weights_without_biases(layer_number,layer_name)] self.model.get_layer(index=layer_number-1,name=layer_name).set_weights(nbiases) else: nbiases = [self.get_weights_without_biases(layer_number,layer_name),biases] self.model.get_layer(name=layer_name).set_weights(nbiases) def remove_last_layer(self): """ This function removes a layer from the model. :return: removed layer """ layer=self.model._layers.pop() self.model= Model(self.model.input,self.model.layers[-1].output) return layer def load_a_model(self,model_name="",model_file_name=""): """ This function loads a model architecture and weights. :param model_name: Name of the model to load. model_name should be one of the following: "VGG16", "VGG19" :param model_file_name: Name of the file to load the model (if both madel_name and model_file_name are specified, model_name takes precedence). :return: model """ if(model_name=="VGG16"): self. model= Sequential(VGG16().layers) elif(model_name=="VGG19"): self. model= Sequential(VGG19().layers) else: self.model = Sequential() self.model = load_model(model_file_name) def save_model(self,model_file_name=""): """ This function saves the current model architecture and weights together in a HDF5 file. :param file_name: Name of file to save the model. :return: model """ self.model.save(model_file_name) def set_loss_function(self, loss="SparseCategoricalCrossentropy"): """ This function sets the loss function. :param loss: loss is a string with the following choices: "SparseCategoricalCrossentropy", "MeanSquaredError", "hinge". :return: none """ self.loss_function = None if "SparseCategoricalCrossentropy": self.loss_function = loss elif "MeanSquaredError": self.loss_function = loss elif "hinge": self.loss_function = loss else: self.loss_function = loss def set_metric(self,metric): """ This function sets the metric. :param metric: metric should be one of the following strings: "accuracy", "mse". :return: none """ self.metric = None if "accuracy": self.metric = ['acc'] elif "mse": self.metric = metric def set_optimizer(self,optimizer="SGD",learning_rate=0.01,momentum=0.0): """ This function sets the optimizer. :param optimizer: Should be one of the following: "SGD" , "RMSprop" , "Adagrad" , :param learning_rate: Learning rate :param momentum: Momentum :return: none """ self.optimizer = None if "RMSprop": self.optimizer = optimizers.RMSprop(lr=learning_rate) elif "SGD": self.optimizer = optimizers.SGD(lr=learning_rate, momentum=momentum) elif "Adagrad": self.optimizer = 'Adagrad' print("Optimizer : {0}".format(self.optimizer)) def predict(self, X): """ Given array of inputs, this function calculates the output of the multi-layer network. :param X: Input tensor. :return: Output tensor. """ output = self.model.predict(X) return output def evaluate(self,X,y): """ Given array of inputs and desired ouputs, this function returns the loss value and metrics of the model. :param X: Array of input :param y: Array of desired (target) outputs :return: loss value and metric value """ test_loss, test_acc = self.model.evaluate(X, y, verbose=2) return (test_loss, test_acc) def train(self, X_train, y_train, batch_size, num_epochs): """ Given a batch of data, and the necessary hyperparameters, this function trains the neural network by adjusting the weights and biases of all the layers. :param X_train: Array of input :param y_train: Array of desired (target) outputs :param X_validation: Array of input validation data :param y: Array of desired (target) validation outputs :param batch_size: number of samples in a batch :param num_epochs: Number of times training should be repeated over all input data :return: list of loss values. Each element of the list should be the value of loss after each epoch. """ self.model.compile(optimizer= self.optimizer, loss = self.loss_function, metrics = self.metric) seqModel = self.model.fit(x=X_train, y=y_train, batch_size=batch_size, epochs = num_epochs) return seqModel.history['loss']
Conv2D(filters=1, kernel_size=(3, 3), activation='sigmoid', padding='same')) autoencoder.summary() autoencoder.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) autoencoder.fit(previsoresTreinamento, previsoresTreinamento, epochs=100, batch_size=256, validation_data=(previsoresTeste, previsoresTeste)) # Flatten 128 encoder = Model(inputs=autoencoder.input, outputs=autoencoder.get_layer('flatten_6').output) encoder.summary() imagensCodificadas = encoder.predict(previsoresTeste) imagensDecodificadas = autoencoder.predict(previsoresTeste) numeroImagens = 10 imagensTeste = np.random.randint(previsoresTeste.shape[0], size=numeroImagens) plt.figure(figsize=(18, 18)) for i, indiceImagem in enumerate(imagensTeste): # imagem original eixo = plt.subplot(10, 10, i + 1) plt.imshow(previsoresTeste[indiceImagem].reshape(28, 28)) plt.xticks(()) plt.yticks(())
class FreezeUnfreeze: """ Implements a bottleneck neural network with Keras that can use pre-trained weights and first freezes certain layers before training all layers. """ def __init__(self, l1, l2, lr, act, input_dim, output_dim, unfreeze, pre_trained_weights = False, pre_trained_weights_h5 = None, \ nodes_list=[512, 128, 2, 128, 512]): """ Constructor. :param l1: int, lasso penalty :param l2: int, ridge penalty :param lr: int, learning rate for Adam :param act: string, activation function :param input_dim: int, input layer dimensionality :param output_dim: int, output layer dimensionality :param unfreeze: list of bools, if element in list is True that index corresponds to a layer that can be trained :param pre_trained_weights: bool, if True we have pre-trained weights we could use for intialisation (optional, default=False) :param pre_trained_weights_h5: .h5 file, pre_trained weights to use as intial weights for training deep regression (optional, default=None) :param nodes_list: list, integers denoting # nodes in each layer (optional, default=[512, 128, 2, 128, 512]) """ self.l1=l1 self.l2=l2 self.lr=lr self.act=act self.input_dim=input_dim self.output_dim=output_dim self.unfreeze = unfreeze self.pre_trained_weights = pre_trained_weights self.nodes_list=nodes_list # Architecture keras.backend.clear_session() self.m = Sequential() self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(self.input_dim, ), \ kernel_regularizer=ElasticNet(l1=self.l1, l2=self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ trainable = self.unfreeze[0], name = 'W1')) self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ trainable = self.unfreeze[1], name = 'W2')) # linear for latent space representation self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), trainable = self.unfreeze[2], name='bottleneck')) self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ trainable = self.unfreeze[3], name = 'W4')) self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ trainable = self.unfreeze[4], name = 'W5')) self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ trainable = self.unfreeze[5], \ #bias_regularizer=regularizers.l2(l2_parameter), \ name = 'W6_regr')) # Load weights from training a previous network on classification if self.pre_trained_weights: self.m.load_weights(pre_trained_weights_h5, by_name=True) self.m.compile(loss='mean_squared_error', \ optimizer=keras.optimizers.Adam(learning_rate=lr), \ metrics=[r2_score, 'mse']) def train(self, x_train, y_train, x_test, y_test, epochs, bs, patience, cvfold_id=0, l1_id=0, l2_id=0, verbose=1, \ prune=False, geneNames=None, citeseq=False, report_individual_ephys_feature_test_R2=False): """ Train the bottleneck. If you don't prune the training lasts for 2*epochs iterations (freezing+unfreezing). If you prune the training lasts for 4*epochs (freezing (epochs) + unfreezing (epochs) + pruning (2*epochs)) iterations in total. Parameters ---------- :param x_train, y_train: numpy 2D matrices, input and output training set :param x_test, y_test: numpy 2D matrices, input and output validation set :param epochs: int, # of training iterations :param bs: int, batch size :param patience: int, early stopping :param cvfold_id: int, cross validation set id (optional, default=0), for saving :param l1_id: int, lasso penalty id (optional, default=0), for saving :param l2_id: int, ridge penalty id (optional, default=0), for saving :param verbose: int, print info or not (optional, default=1, i.e. print info -- verbose=0 means not printing info) :param prune: bool, if True we additionally prune the network (new input layer with lower dimensionality) (optional, default=False) :param geneNames: numpy 1D array, contains name of the corresponding gene of every input neuron (optional, default=None) :param citeseq: bool, if True we're performing the training procedure on a CITE-seq dataset and will use the best epoch during unfreezing for use afterwards when we prune (optional, default=False) :param report_individual_ephys_feature_test_R2: bool, if True save test R^2 scores for every individual ephys feature Returns ------- Training and validation loss R^2 score for all epochs """ # Settings for early stopping and saving best model es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=patience) if not self.pre_trained_weights: mc = ModelCheckpoint('KerasSavedModels/FreezeUnfreeze_weights_before_unfreezing_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) else: mc = ModelCheckpoint('KerasSavedModels/PreTrFreezeUnfreeze_weights_before_unfreezing_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) # train the network print("[INFO] training network...") H = self.m.fit(x_train, y_train, batch_size=bs, validation_data=(x_test, y_test), epochs=epochs, verbose=verbose, callbacks = [es, mc]) train_loss_freeze_unfreeze = np.array(H.history["r2_score"]) val_loss_freeze_unfreeze = np.array(H.history["val_r2_score"]) # Now UNFREEZE all layers for layer in self.m.layers: layer.trainable = True if verbose!=0: for layer in self.m.layers: print(layer, 'trainable?', layer.trainable) # Since we’ve unfrozen additional layers, we must re-compile the model and let us decrease the learning rate by a half self.m.compile(loss='mean_squared_error', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse']) # Settings for early stopping and saving best model es = EarlyStopping(monitor='val_mse', mode='min', verbose=1, patience=patience) if not self.pre_trained_weights: mc = ModelCheckpoint('KerasSavedModels/FreezeUnfreeze_weights_after_unfreezing_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) else: mc = ModelCheckpoint('KerasSavedModels/PreTrFreezeUnfreeze_weights_after_unfreezing_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) # train the network again print("[INFO] training network...") H = self.m.fit(x_train, y_train, batch_size=bs, validation_data=(x_test, y_test), epochs=epochs, verbose=verbose, callbacks = [es, mc]) train_loss_freeze_unfreeze = np.concatenate([train_loss_freeze_unfreeze, np.array(H.history["r2_score"])]) val_loss_freeze_unfreeze = np.concatenate([val_loss_freeze_unfreeze, np.array(H.history["val_r2_score"])]) # Retrieve activations and ephys prediction if not self.pre_trained_weights: saved_model = load_model('KerasSavedModels/FreezeUnfreeze_weights_before_unfreezing_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) saved_model_2 = load_model('KerasSavedModels/FreezeUnfreeze_weights_after_unfreezing_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) else: saved_model = load_model('KerasSavedModels/PreTrFreezeUnfreeze_weights_before_unfreezing_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) saved_model_2 = load_model('KerasSavedModels/PreTrFreezeUnfreeze_weights_after_unfreezing_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) r2_before_unfreezing_train = 1-np.sum((y_train - saved_model.predict(x_train))**2) / np.sum(y_train**2) r2_before_unfreezing_test = 1-np.sum((y_test - saved_model.predict(x_test))**2) / np.sum(y_test**2) r2_after_unfreezing_train = 1-np.sum((y_train - saved_model_2.predict(x_train))**2) / np.sum(y_train**2) r2_after_unfreezing_test = 1-np.sum((y_test - saved_model_2.predict(x_test))**2) / np.sum(y_test**2) self.m.save('KerasSavedModels/FreezeUnfreeze_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id)) if prune: saved_model_ = load_model('KerasSavedModels/FreezeUnfreeze_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) weight = saved_model_.get_weights()[0] bias = saved_model_.get_weights()[1] ind_genes = np.argsort(np.linalg.norm(weight, ord=2, axis=1))[-25:] print('The 25 genes that make it: ', geneNames[ind_genes]) # Architecture keras.backend.clear_session() self.m = Sequential() self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(25, ), \ kernel_regularizer=ElasticNet(l1=0, l2=self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W1_25')) self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W2')) # linear for latent space representation self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), name='bottleneck')) self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W4')) self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W5')) self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ #bias_regularizer=regularizers.l2(l2_parameter), \ name = 'W6_regr')) # Load weights from training a previous network on regression self.m.load_weights('KerasSavedModels/FreezeUnfreeze_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), by_name=True) # We transfer the weights of those 25 genes now manually too self.m.get_layer('W1_25').set_weights([weight[ind_genes, :], bias]) self.m.compile(loss='mse', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse']) es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=2*patience) if not self.pre_trained_weights: mc = ModelCheckpoint('KerasSavedModels/FreezeUnfreeze_weights_after_unfreezing_and_pruning_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) else: mc = ModelCheckpoint('KerasSavedModels/PreTrFreezeUnfreeze_weights_after_unfreezing_and_pruning_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) # train the network print("[INFO] training network...") H = self.m.fit(x_train[:, ind_genes], y_train, batch_size=bs, validation_data=(x_test[:, ind_genes], y_test), epochs=2*epochs, verbose=verbose, callbacks = [es, mc]) train_loss_freeze_unfreeze = np.concatenate([train_loss_freeze_unfreeze, np.array(H.history["r2_score"])]) val_loss_freeze_unfreeze = np.concatenate([val_loss_freeze_unfreeze, np.array(H.history["val_r2_score"])]) if prune: if not self.pre_trained_weights: saved_model_3 = load_model('KerasSavedModels/FreezeUnfreeze_weights_after_unfreezing_and_pruning_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) else: saved_model_3 = load_model('KerasSavedModels/PreTrFreezeUnfreeze_weights_after_unfreezing_and_pruning_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) r2_after_unfreezing_and_pruning_train = 1-np.sum((y_train - saved_model_3.predict(x_train[:, ind_genes]))**2) \ / np.sum(y_train**2) r2_after_unfreezing_and_pruning_test = 1-np.sum((y_test - saved_model_3.predict(x_test[:, ind_genes]))**2) \ / np.sum(y_test**2) if report_individual_ephys_feature_test_R2: if not citeseq: if self.nodes_list[2]==2: np.savez('KerasSavedModels/individual_ephys_feature_test_R2_{}_{}_{}.npz'.format(cvfold_id, l1_id, l2_id), \ R2=np.array([1-np.sum((y_test[:,i] - saved_model_3.predict(x_test[:, ind_genes])[:,i])**2) \ / np.sum(y_test[:,i]**2) for i in range(y_test.shape[1])]) ) else: np.savez('KerasSavedModels/individual_ephys_feature_test_R2_{}_{}_{}_nb.npz'.format(cvfold_id, l1_id, l2_id), \ R2=np.array([1-np.sum((y_test[:,i] - saved_model_3.predict(x_test[:, ind_genes])[:,i])**2) \ / np.sum(y_test[:,i]**2) for i in range(y_test.shape[1])]) ) else: if self.nodes_list[2]==2: np.savez('KerasSavedModels/individual_ephys_feature_test_R2_{}_{}_{}_citeseq.npz'.format(cvfold_id, l1_id, l2_id), \ R2=np.array([1-np.sum((y_test[:,i] - saved_model_3.predict(x_test[:, ind_genes])[:,i])**2) \ / np.sum(y_test[:,i]**2) for i in range(y_test.shape[1])]) ) else: np.savez('KerasSavedModels/individual_ephys_feature_test_R2_{}_{}_{}_citeseq_nb.npz'.format(cvfold_id, l1_id, \ l2_id), \ R2=np.array([1-np.sum((y_test[:,i] - saved_model_3.predict(x_test[:, ind_genes])[:,i])**2) \ / np.sum(y_test[:,i]**2) for i in range(y_test.shape[1])]) ) print('Train R^2 before unfreezing: ', r2_before_unfreezing_train) print('Test R^2 before unfreezing: ', r2_before_unfreezing_test) print('Train R^2 after unfreezing: ', r2_after_unfreezing_train) print('Test R^2 after unfreezing: ', r2_after_unfreezing_test) if prune: print('Train R^2 after unfreezing and pruning: ', r2_after_unfreezing_and_pruning_train) print('Test R^2 after unfreezing and pruning: ', r2_after_unfreezing_and_pruning_test) if not prune: return r2_before_unfreezing_train, r2_before_unfreezing_test, \ r2_after_unfreezing_train, r2_after_unfreezing_test, \ train_loss_freeze_unfreeze, val_loss_freeze_unfreeze else: return r2_before_unfreezing_train, r2_before_unfreezing_test, \ r2_after_unfreezing_train, r2_after_unfreezing_test, \ r2_after_unfreezing_and_pruning_train, r2_after_unfreezing_and_pruning_test, \ train_loss_freeze_unfreeze, val_loss_freeze_unfreeze def train_full_dataset(self, x_train, y_train, epochs, bs, patience, cvfold_id=0, l1_id=0, l2_id=0, verbose=1, \ prune=False, geneNames=None, add_autoencoder=False, output_name=None): """ Train the bottleneck for the full dataset (no validation). If you don't prune the training lasts for 2*epochs iterations (freezing+unfreezing). If you prune the training lasts for 4*epochs (freezing (epochs) + unfreezing (epochs) + pruning (2*epochs)) iterations in total. Parameters ---------- :param x_train, y_train: numpy 2D matrices, input and output training set :param epochs: int, # of training iterations :param bs: int, batch size :param patience: int, early stopping :param cvfold_id: int, cross validation set id (optional, default=0) :param l1_id: int, lasso penalty id (optional, default=0) :param l2_id: int, ridge penalty id (optional, default=0) :param verbose: int, print info or not (optional, default=1, i.e. print info -- verbose=0 means not printing info) :param prune: bool, if True we additionally prune the network (new input layer with lower dimensionality) (optional, default=False) :param geneNames: numpy 1D array, contains name of the corresponding gene of every input neuron (optional, default=None) :param add_autoencoder: bool, if True we add an autoencoder to selected genes from the bottleneck layer :param output_name: string, if provided we save file with the string in it (optional, default = None) Returns ------- Training loss R^2 score for all epochs """ # train the network print("[INFO] training network...") H = self.m.fit(x_train, y_train, batch_size=bs, epochs=epochs, verbose=verbose) train_loss_freeze_unfreeze = np.array(H.history["r2_score"]) # Now UNFREEZE all layers for layer in self.m.layers: layer.trainable = True if verbose!=0: for layer in self.m.layers: print(layer, 'trainable?', layer.trainable) # Since we’ve unfrozen additional layers, we must re-compile the model and let us decrease the learning rate by a half self.m.compile(loss='mean_squared_error', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse']) if output_name is None: self.m.save('KerasSavedModels/FreezeUnfreeze_before_unfreezing_full_dataset_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id)) else: self.m.save('KerasSavedModels/{}/FreezeUnfreeze_before_unfreezing_full_dataset_{}_{}_{}.h5'.format\ (output_name, cvfold_id, l1_id, l2_id)) # train the network again print("[INFO] training ephys prediction network...") H = self.m.fit(x_train, y_train, batch_size=bs, epochs=epochs, verbose=verbose) train_loss_freeze_unfreeze = np.concatenate([train_loss_freeze_unfreeze, np.array(H.history["r2_score"])]) if output_name is None: self.m.save('KerasSavedModels/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id)) else: self.m.save('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.format\ (output_name, cvfold_id, l1_id, l2_id)) if prune: if output_name is None: saved_model_ = load_model('KerasSavedModels/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) else: saved_model_ = load_model('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.\ format(output_name, cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) weight = saved_model_.get_weights()[0] bias = saved_model_.get_weights()[1] ind_genes = np.argsort(np.linalg.norm(weight, ord=2, axis=1))[-25:] print('The 25 genes that make it: ', geneNames[ind_genes]) # Architecture keras.backend.clear_session() self.m = Sequential() self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(25, ), \ kernel_regularizer=ElasticNet(l1=0, l2=self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W1_25')) self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W2')) # linear for latent space representation self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), name='bottleneck')) self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W4')) self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W5')) self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ #bias_regularizer=regularizers.l2(l2_parameter), \ name = 'W6_regr')) # Load weights from training a previous network on regression if output_name is None: self.m.load_weights('KerasSavedModels/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), by_name=True) else: self.m.load_weights('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.\ format(output_name, cvfold_id, l1_id, l2_id), by_name=True) # We transfer the weights of those 25 genes now manually too self.m.get_layer('W1_25').set_weights([weight[ind_genes, :], bias]) self.m.compile(loss='mse', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse']) # train the network print("[INFO] training pruning network...") H=self.m.fit(x_train[:, ind_genes], y_train, batch_size=bs, epochs=2*epochs, verbose=verbose) if output_name is None: self.m.save('KerasSavedModels/FreezeUnfreeze_after_unfreezing_ap_full_dataset_{}_{}_{}.h5'.format\ (cvfold_id, l1_id, l2_id)) else: self.m.save('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_ap_full_dataset_{}_{}_{}.h5'.format\ (output_name, cvfold_id, l1_id, l2_id)) train_loss_freeze_unfreeze=np.concatenate([train_loss_freeze_unfreeze, np.array(H.history["r2_score"])]) if add_autoencoder: # The latent space is the same, but instead of predicting ephys we want to predict selected genes. This leads us to # latent space visualisations that can be overlayed with gene model predictions. # Retrieve bottleneck activations if output_name is None: saved_model_AE = load_model('KerasSavedModels/FreezeUnfreeze_after_unfreezing_ap_full_dataset_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) else: saved_model_AE = load_model('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_ap_full_dataset_{}_{}_{}.h5'.\ format(output_name, cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) encoder = Model(saved_model_AE.input, saved_model_AE.get_layer('bottleneck').output) latent = encoder.predict(x_train[:, ind_genes]) # Retrieve weights W4=saved_model_AE.get_weights()[6] W4_bias=saved_model_AE.get_weights()[7] W5=saved_model_AE.get_weights()[8] W5_bias=saved_model_AE.get_weights()[9] # Architecture keras.backend.clear_session() self.m = Sequential() self.m.add(Dense(self.nodes_list[3], activation=self.act, input_shape=(self.nodes_list[2], ), \ kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name='W4')) self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name='W5')) self.m.add(Dense(25, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ #bias_regularizer=regularizers.l2(self.l2), \ name='W6_regr_AE')) # Set initial weights self.m.get_layer('W4').set_weights([W4, W4_bias]) self.m.get_layer('W5').set_weights([W5, W5_bias]) self.m.compile(loss='mse', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse']) # train the network print("[INFO] training autoencoder network") H=self.m.fit(latent, x_train[:,ind_genes], batch_size=bs, epochs=2*epochs, verbose=verbose) if output_name is None: self.m.save('KerasSavedModels/FreezeUnfreeze_after_unfreezing_ap_full_dataset_AE_{}_{}_{}.h5'\ .format(cvfold_id, l1_id, l2_id)) else: self.m.save('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_ap_full_dataset_AE_{}_{}_{}.h5'\ .format(output_name, cvfold_id, l1_id, l2_id)) train_loss_freeze_unfreeze_AE=np.array(H.history["r2_score"]) if not add_autoencoder: return train_loss_freeze_unfreeze else: return train_loss_freeze_unfreeze, train_loss_freeze_unfreeze_AE
def createModel(self): self.model_instance += 1 clear_session() features, label = self.getDataset() X_train, y_train = self.createLag(features, label) X_train = X_train[:, self.lags] learning_rate = float(self.hyperparameters["Learning_Rate"].get()) momentum = float(self.hyperparameters["Momentum"].get()) optimizers = { "Adam": Adam(learning_rate=learning_rate), "SGD": SGD(learning_rate=learning_rate, momentum=momentum), "RMSprop": RMSprop(learning_rate=learning_rate, momentum=momentum) } shape = (X_train.shape[1], X_train.shape[2]) model_choice = self.model_var.get() if not self.do_optimization: model = Sequential() model.add(Input(shape=shape)) if model_choice == 0: model.add(Flatten()) layers = self.no_optimization_choice_var.get() for i in range(layers): neuron_number = self.neuron_numbers_var[i].get() activation_function = self.activation_var[i].get() if model_choice == 0: model.add(Dense(neuron_number, activation=activation_function, kernel_initializer=GlorotUniform(seed=0))) elif model_choice == 1: model.add(Conv1D(filters=neuron_number, kernel_size=2, activation=activation_function, kernel_initializer=GlorotUniform(seed=0))) model.add(MaxPooling1D(pool_size=2)) elif model_choice == 2: if i == layers-1: model.add(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) else: model.add(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) elif model_choice == 3: if i == layers-1: model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))) model.add(Dropout(0.2)) else: model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))) model.add(Dropout(0.2)) elif model_choice == 4: if i == layers-1: model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) else: model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) elif model_choice == 5: if i == layers-1: model.add(GRU(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) else: model.add(GRU(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))) model.add(Dropout(0.2)) if model_choice == 1: model.add(Flatten()) model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0))) model.add(Dense(1, activation=self.output_activation.get(), kernel_initializer=GlorotUniform(seed=0))) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1, shuffle=False) loss = history.history["loss"][-1] self.train_loss.set(loss) elif self.do_optimization: layer = self.optimization_choice_var.get() if model_choice == 0: def build_model(hp): model = Sequential() model.add(Input(shape=shape)) model.add(Flatten()) for i in range(layer): n_min = self.neuron_min_number_var[i].get() n_max = self.neuron_max_number_var[i].get() step = int((n_max - n_min)/4) model.add(Dense(units=hp.Int('MLP_'+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu')) model.add(Dense(1)) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) return model name = str(self.model_instance) + ". MLP" elif model_choice == 1: def build_model(hp): model = Sequential() model.add(Input(shape=shape)) for i in range(layer): n_min = self.neuron_min_number_var[i].get() n_max = self.neuron_max_number_var[i].get() step = int((n_max-n_min)/4) model.add(Conv1D(filters=hp.Int("CNN_"+str(i), min_value=n_min, max_value=n_max, step=step), kernel_size=2, activation="relu", kernel_initializer=GlorotUniform(seed=0))) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0))) model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0))) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) return model name = str(self.model_instance) + ". CNN" elif model_choice == 2: def build_model(hp): model = Sequential() model.add(Input(shape=shape)) for i in range(layer): n_min = self.neuron_min_number_var[i].get() n_max = self.neuron_max_number_var[i].get() step = int((n_max - n_min)/4) model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0))) if i == layer-1: model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0))) model.add(Dense(1)) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) return model name = str(self.model_instance) + ". LSTM" elif model_choice == 3: def build_model(hp): model = Sequential() model.add(Input(shape=shape)) for i in range(layer): n_min = self.neuron_min_number_var[i].get() n_max = self.neuron_max_number_var[i].get() step = int((n_max - n_min)/4) model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0)))) if i == layer-1: model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0)))) model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0))) model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get()) return model name = str(self.model_instance) + ". Bi-LSTM" tuner = RandomSearch(build_model, objective='loss', max_trials=25, executions_per_trial=2, directory=self.runtime, project_name=name) tuner.search(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get()) hps = tuner.get_best_hyperparameters(num_trials = 1)[0] model = tuner.hypermodel.build(hps) history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1) loss = history.history["loss"][-1] self.train_loss.set(loss) for i in range(layer): if model_choice == 0: self.best_model_neurons[i].set(model.get_layer(index=i+1).get_config()["units"]) elif model_choice == 1: self.best_model_neurons[i].set(model.get_layer(index=(2*i)).get_config()["filters"]) elif model_choice == 2: self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["units"]) elif model_choice == 3: self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["layer"]["config"]["units"]) model.summary() self.model = model
BACKENDS = ['tf', 'pt'] FILE_FORMATS = ['hdf5', 'npy', 'mat', 'txt'] DISTANCES = ['correlation', 'cosine', 'euclidean', 'gaussian'] BATCH_SIZE = 16 NUM_OBJECTS = 1854 # we want to iterate over two batches to exhaustively test mini-batching NUM_SAMPLES = int(BATCH_SIZE * 2) DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' tf_model = Sequential() tf_model.add( Dense(2, input_dim=1, activation='relu', use_bias=False, name='relu')) weights = np.array([[[1, 1]]]) tf_model.get_layer('relu').set_weights(weights) class NN(nn.Module): def __init__(self, in_size: int, out_size: int): super(NN, self).__init__() self.linear = nn.Linear(in_size, out_size, bias=False) self.relu = nn.ReLU() # exchange weight value with 1. self.linear.weight = nn.Parameter(torch.tensor([[1.], [1.]])) def forward(self, x): x = self.linear(x) act = self.relu(x) return act
autoencoder = Sequential() autoencoder.add(Dense(512, activation='elu', input_shape=(784, ))) autoencoder.add(Dense(128, activation='elu')) autoencoder.add(Dense(10, activation='linear', name="bottleneck")) autoencoder.add(Dense(128, activation='elu')) autoencoder.add(Dense(512, activation='elu')) autoencoder.add(Dense(784, activation='sigmoid')) autoencoder.compile(loss='mean_squared_error', optimizer=Adam()) trained_model = autoencoder.fit(X_train, X_train, batch_size=1024, epochs=10, verbose=1, validation_data=(val_x, val_x)) encoder = Model(autoencoder.input, autoencoder.get_layer('bottleneck').output) encoded_data = encoder.predict(X_train) # bottleneck representation decoded_output = autoencoder.predict(X_train) # reconstruction encoding_dim = 10 print("Encoded Image: {}".format( encoder.predict(X_train[0].reshape(1, -1))[0])) encoded_image = encoder.predict(X_train[0].reshape(1, -1))[0] # return the decoder encoded_input = Input(shape=(encoding_dim, )) decoder = autoencoder.layers[-3](encoded_input) decoder = autoencoder.layers[-2](decoder) decoder = autoencoder.layers[-1](decoder) decoder = Model(encoded_input, decoder)
VGGFace16.add(Flatten(name='flatten')) VGGFace16.add(layers.Dense(units=512, name='fc6')) VGGFace16.add(Activation(activation='relu', name='fc6/relu')) VGGFace16.add(layers.Dense(units=512, name='fc7')) VGGFace16.add(Activation(activation='relu', name='fc7/relu')) VGGFace16.add(layers.Dense(8, name='fc8')) VGGFace16.add(Activation(activation='softmax', name='fc8/softmax')) # load the pretrained weights VGGFace16.load_weights( '/home/ubuntu/Notebooks/Models/AffectNet_VGG16_v3-1.h5', by_name=True) # construct new classification layers nb_class = 8 hidden_dim = 512 last_layer = VGGFace16.get_layer('pool5').output x = Flatten(name='flatten')(last_layer) x = layers.Dense(hidden_dim, activation='relu', name='fc6', kernel_regularizer=l2(5 * 1e-4))(x) x = layers.Dense(hidden_dim, activation='relu', name='fc7', kernel_regularizer=l2(5 * 1e-4))(x) out = layers.Dense(nb_class, activation='softmax', name='fc8')(x) FER_VGGFace16 = tensorflow.keras.Model(VGGFace16.input, out) # compile the model # initial_learning_rate = 1e-3
import matplotlib.pyplot as plt # 指定亂數種子 seed = 7 np.random.seed(seed) # 載入資料集 (X_train, Y_train), (_, _) = mnist.load_data() # 將圖片轉換成 4D 張量 X_train = X_train.reshape(X_train.shape[0], 28, 28, 1).astype("float32") # 因為是固定範圍, 所以執行正規化, 從 0-255 至 0-1 X_train = X_train / 255 # 建立Keras的Sequential模型 model = Sequential() model = load_model("mnist.h5") model.summary() # 顯示模型摘要資訊 # 編譯模型 model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) # 使用 Model 建立 Conv2D 層 from tensorflow.keras.models import Model layer_name = "conv2d_1" model_test = Model(inputs=model.input, outputs=model.get_layer(layer_name).output) output = model_test.predict(X_train[0].reshape(1,28,28,1)) # 繪出第1個 Conv2D 層的輸出 plt.figure(figsize=(10,8)) for i in range(0,16): plt.subplot(4,4,i+1) plt.imshow(output[0,:,:,i], cmap="gray") plt.axis("off")
ae.add(Dense(2, activation='linear', name="bottleneck")) ae.add(Dense(128, activation='relu')) ae.add(Dense(512, activation='relu')) ae.add(Dense(784, activation='sigmoid')) ae.compile(loss='mean_squared_error', optimizer=Adam()) def explained_variance(projections): var = np.array([(projections[:,i]**2).sum()/(n_sample-1) for i in range(k)]).round(2) return var def fit(): history = ae.fit(x_train, x_train, batch_size=128, epochs=10, verbose=1, validation_data=(x_test, x_test)) fit() encoder = Model(ae.input, ae.get_layer('bottleneck').output) Zenc = encoder.predict(x_train) print("Pca explained variance:", explained_variance(Zpca)) print("ae explained variance:", explained_variance(Zenc)) # PLOT plt.figure(figsize=(8, 4)) plt.subplot(121) plt.title('PCA') plt.scatter(Zpca[:5000, 0], Zpca[:5000, 1], c=y_train[:5000], s=8, cmap='tab10')
class Gan: def __init__(self,folder,loadFile): self.folder=folder self.modelFolder='models' self.loadModelNumb=-1 self.modelName='model' self.loadModel=False self.noiseNo=100 self.loadFile=loadFile self.picFolder='pics' self.picName='pic%d.png' self.x=pickle.load(open(loadFile+'.pickle','rb')) self.x=self.x/255.0#normalise self.imgDim=self.x.shape[1]#width and height self.imgDepth=self.x.shape[3]#depth 1 or black and white, 3 for colour self.D = Sequential() self.G = Sequential() self.AM = Sequential() def descriminator(self,depth,dropout,alpha,window,stridedLayers,unstridedLayers): #dropout ignore units during training prevent over fitting #alpha leaked negative gradient size strides=2 input_shape = (self.imgDim, self.imgDim, self.imgDepth) self.D.add(Conv2D(depth*1, window, strides=strides, input_shape=input_shape,padding='same')) self.D.add(LeakyReLU(alpha=alpha)) self.D.add(Dropout(dropout)) for i in range(stridedLayers-1+unstridedLayers): depth=depth*2 self.D.add(Conv2D(depth, window, strides=strides, padding='same')) self.D.add(LeakyReLU(alpha=alpha)) self.D.add(Dropout(dropout)) if(i+2==stridedLayers): strides-=1 self.D.add(Flatten()) self.D.add(Dense(1))#real or not self.D.add(Activation('sigmoid')) def gen(self,depth,dim,momentum,dropout,window,upSampledLayers,additionalLayers): self.G.add(Dense(dim*dim*depth, input_dim=self.noiseNo)) self.G.add(BatchNormalization(momentum=momentum))#stablise learing reduce weight shifting in hidden by reducing by mean self.G.add(Activation('relu')) self.G.add(Reshape((dim, dim, depth)))#7x7x256 self.G.add(Dropout(dropout)) for i in range(upSampledLayers+additionalLayers): depth=int(depth/2) if(i<upSampledLayers): self.G.add(UpSampling2D())#repeats rows and columns e.g 32x32 becomes 64x64 self.G.add(Conv2DTranspose(depth, window, padding='same'))#reduce depth to 128 self.G.add(BatchNormalization(momentum=momentum)) self.G.add(Activation('relu')) self.G.add(Conv2DTranspose(self.imgDepth, window, padding='same')) self.G.add(Activation('sigmoid')) def combine(self,dlr,dcv,ddcy,alr,acv,adcy): optimizer = RMSprop(lr=dlr, clipvalue=dcv, decay=ddcy) self.D.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) self.D.trainable=False#freeze weights in descriminator so gen weights update instead optimizer = RMSprop(lr=alr, clipvalue=acv, decay=adcy) self.AM.add(self.G)#gen then input to discrim self.AM.add(self.D) self.AM.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) self.createDir() self.writeParams() ws=self.AM.get_layer("",1).get_weights()[0][:,:,0,:] for i in range(1,26): plt.subplot(5,5,i) plt.imshow(ws[:,:,i],interpolation="nearest") plt.show() def saveImage(self, numb): filename = self.folder+'/'+self.picFolder+'/'+self.picName % numb images = self.G.predict(noise) image=images[0] plt.figure(figsize=(10,10)) if(self.imgDepth==3): image = np.reshape(image, [self.imgDim , self.imgDim,self.imgDepth]) plt.imshow(image) else: image = np.reshape(image, [self.imgDim , self.imgDim]) plt.imshow(image, cmap='gray') plt.axis('off') plt.tight_layout() plt.savefig(filename) plt.close('all') def showConv(self): ws=self.AM.get_layer("",2).get_weights()[0][:,:,0,:] for i in range(1,26): plt.subplot(5,5,i) plt.imshow(ws[:,:,i],interpolation="nearest") plt.show() ws=self.AM.get_layer("",1).get_weights()[0][:,:,0,:] for i in range(1,26): plt.subplot(5,5,i) plt.imshow(ws[:,:,i],interpolation="nearest") plt.show() def loadModel(self,loadModelNumb): try: self.loadModel=True self.D=self.load_model(modelLoc(loadModelNumb,'descriminator')) self.G=self.load_model(modelLoc(loadModelNumb,'generator')) self.AM=self.load_model(modelLoc(loadModelNumb,'Adversarial')) except OSError as er: print('Error loading file ',er) self.createDir() self.writeParams() def train(self,batchSize,trainingSteps,saveImgInterval,saveModelInterval): self.batchSize=batchSize self.trainingSteps=trainingSteps self.saveImgInterval=saveImgInterval self.saveModelInterval=saveModelInterval noise = np.random.uniform(-1.0, 1.0, size=[self.batchSize, self.noiseNo])#random noise of 100 units for each batch for i in range(self.trainingSteps): if(self.loadModel): i+=self.loadModelNumb+1 images_train = self.x[np.random.randint(0,self.x.shape[0],size=self.batchSize),:,:,:]#index random images of batch size #noise = np.random.uniform(-1.0, 1.0, size=[self.batchSize, self.noiseNo])#random noise of 100 units for each batch images_fake = self.G.predict(noise)#gen fake image xs = np.concatenate((images_train, images_fake))#concat real then fake y = np.ones([2*self.batchSize, 1])#first half 1 for real second 0 for fake y[self.batchSize:, :] = 0 dLoss = self.D.train_on_batch(xs, y)#train on half real half fake images y = np.ones([self.batchSize, 1])#all 1 so mislabled for real #noise = np.random.uniform(-1.0, 1.0, size=[self.batchSize, self.noiseNo])#new noise as learned from old noise knows it's wrong, might help aLoss = self.AM.train_on_batch(noise, y) msg = '%d: [D loss: %f, acc: %f]' % (i, dLoss[0], dLoss[1]) msg = '%s [A loss: %f, acc: %f]' % (msg, aLoss[0], aLoss[1]) print(msg) if (i % self.saveImgInterval==0): self.showConv() self.saveImage(noise, i) if (i % self.saveModelInterval==0 and i !=0): self.D.save(self.modelLoc(i,'descriminator')) self.G.save(self.modelLoc(i,'generator')) self.AM.save(self.modelLoc(i,'Adversarial')) self.D.save(self.modelLoc(self.trainingSteps,'descriminator')) self.G.save(self.modelLoc(self.trainingSteps,'generator')) self.AM.save(self.modelLoc(self.trainingSteps,'Adversarial')) def modelLoc(self,num,netType): return self.folder+'/'+self.modelFolder+'/'+str(num)+self.modelName+'_'+netType+'.model' def writeParams(self): with open(self.folder+'/params.txt', 'w+') as f: size=str(self.imgDim)+'X'+str(self.imgDim)+'X'+str(self.imgDepth) #batchSteps='batch size: '+str(self.batchSize)+' '+'trainingSteps: '+str(self.trainingSteps) f.write(self.folder+'\n'+'\n'+size+'\n') with redirect_stdout(f): self.D.summary() with redirect_stdout(f): self.G.summary() def createDir(self): if not os.path.exists(self.folder): os.makedirs(self.folder) if not os.path.exists(self.folder+'/'+self.picFolder): os.makedirs(self.folder+'/'+self.picFolder) if not os.path.exists(self.folder+'/'+self.modelFolder): os.makedirs(self.folder+'/'+self.modelFolder)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy') model.add(MaxPool2D()) model.add(Flatten()) model.add(Dense(units=4096,activation='relu')) model.add(Dense(units=4096,activation='relu')) model.add(Dense(units=2,activation='softmax')) #train the fine -tuned vgg16 model model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy']) #fit our model model.fit(x=train_batches,validation_data=valid_batches,epochs=20,verbose=2) ##### extract the fc2 layer ("dense6) fc2_output = tf.keras.backend.function(model.input, model.get_layer('dense_6').output) # put the images from train set to extract the features ( pleon ta flatten, fc2, and predict) exoun ekpaideutei me tis fwto moy import cv2 images = [cv2.imread(file) for file in glob.glob("/content/drive/My Drive/trainn/train/Covid/*.png")] y=[] for i in range(1,200): gray=cv2.resize(images[i],(224,224)) y.append(gray) list_cov=tf.convert_to_tensor(y) list11=fc2_output(list_cov) pd_list_cov=pd.DataFrame(list11) import cv2 images = [cv2.imread(file) for file in glob.glob("/content/drive/My Drive/trainn/train/Non-Covid/*.png")] y1=[]
class ConvModel: """ Conv-2,4 and 6 model class with the functions needed for iterative pruning """ input_data_shape = 1 def __init__(self, input_data_shape, model_name='conv-6', use_dropout=False): # Hyper-params that can be changed self.K = 10 self.learning_rate = 0.0003 self.batch_size = 60 self.input_data_shape = input_data_shape self._use_dropout = use_dropout assert model_name in ['conv-2', 'conv-4', 'conv-6'], "Use Conv-2, Conv-4 or Conv-6 model" self._model_name = model_name # All the layers with relevant weights for the pruning self._weight_layers = [] # The percentages we should prune for each layer. Eg. { "conv-1": 0.2 ... } self._prune_percentages = {} # Initially build it with no pruning self._build() self._compile() # The initial weights are stored the first time we build the model only, # so we can reset it when we change the pruning self._initial_weights = self._get_trainable_weights() # Save last used masks so that it can be used when we reinitialize the layers self._initial_random_weights = None self._last_used_masks = None def _build(self): """ Builds the model from scratch. If we have set the pruning percentages we will build the model with pruning. """ self._weight_layers = [] self._model = Sequential() self._add_layer( Conv2D(64, (3, 3), name='conv-1', padding='same', activation='relu', input_shape=self.input_data_shape, kernel_initializer='glorot_normal')) self._add_layer( Conv2D(64, (3, 3), name='conv-2', padding='same', activation='relu', kernel_initializer='glorot_normal')) self._add_layer(MaxPooling2D((2, 2))) self._weight_layers.extend(['conv-1', 'conv-2']) if self._model_name in ['conv-2', 'conv-4']: self._add_layer( Conv2D(128, (3, 3), name='conv-3', padding='same', activation='relu', kernel_initializer='glorot_normal')) self._add_layer( Conv2D(128, (3, 3), name='conv-4', padding='same', activation='relu', kernel_initializer='glorot_normal')) self._add_layer(MaxPooling2D((2, 2))) self._weight_layers.extend(['conv-3', 'conv-4']) if self._model_name == 'conv-6': self._add_layer( Conv2D(256, (3, 3), name='conv-5', padding='same', activation='relu', kernel_initializer='glorot_normal')) self._add_layer( Conv2D(256, (3, 3), name='conv-6', padding='same', activation='relu', kernel_initializer='glorot_normal')) self._add_layer(MaxPooling2D((2, 2))) self._weight_layers.extend(['conv-5', 'conv-6']) self._add_layer(Flatten()) if self._use_dropout: self._add_layer(Dropout(0.5)) self._add_layer( Dense(256, name='dense-1', activation='relu', kernel_initializer='glorot_normal')) if self._use_dropout: self._add_layer(Dropout(0.5)) self._add_layer( Dense(256, name='dense-2', activation='relu', kernel_initializer='glorot_normal')) if self._use_dropout: self._add_layer(Dropout(0.5)) self._add_layer( Dense(self.K, name='dense-3', activation='softmax', kernel_initializer='glorot_normal')) self._weight_layers.extend(['dense-1', 'dense-2', 'dense-3']) def _add_layer(self, layer): """ Helper for adding layer to the current model, also sets the pruning if we have a pruning percentage for the layer :param layer: The layer we should add """ if self._prune_percentages is not None and layer.name in self._prune_percentages: return self._model.add( tfmot.sparsity.keras.prune_low_magnitude( layer, pruning_schedule=tfmot.sparsity.keras.ConstantSparsity( self._prune_percentages[layer.name], begin_step=0, end_step=1, frequency=1), name=layer.name)) self._model.add(layer) def _compile(self): adam = optimizers.Adam(self.learning_rate) self._model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) def predict(self, x): """ Make predictions :param x: input data :return: output """ return self._model.predict(x) def evaluate(self, x, y): """ Evaluate a model on test data :param x: input :param y: labels """ eval = self._model.evaluate(x, y) print("Accuracy: " + str(eval[1]) + ", Loss: " + str(eval[0])) return eval def train(self, x_train, y_train, iterations=30000, early_stopping=False): """ Train the network. If we want pruning the method set pruning should be called prior to this :param x_train: training inputs :param y_train: training labels :param iterations: number of iterations to run, will be rounded to nearest epoch count :param early_stopping: True if early stopping should be used """ callbacks = [tfmot.sparsity.keras.UpdatePruningStep()] val_split = 0.0 if early_stopping: early_stopping = tf.keras.callbacks.EarlyStopping( monitor="val_loss", patience=3) val_split = 0.2 callbacks.append(early_stopping) epochs = int(iterations * self.batch_size / (x_train.shape[0] * (1 - val_split))) return self._model.fit(x_train, y_train, self.batch_size, epochs, callbacks=callbacks, validation_split=val_split) def set_pruning(self, prune_percentages=None): """ This method resets the network, applies pruning that will be used on next training round and re-initializes the unpruned weights to their initial values. :param prune_percentages: Dictionary of pruning values to use. Eg. { 'conv-1': 0.2, ... } """ # Create a mask based on the current values in the network masks = {} # Prunes the additional smallest weights apart from the ones in earlier steps. Uses the weights # from the final step in the earlier training session for name, weights in self._get_trainable_weights().items(): # Calculate how many weights we need to prune prune_count = int(round(prune_percentages[name] * weights.size)) # Sort by the magnitude of the weights for the weights and extract the element value where # we have gone past 'prune_count' nodes. cutoff_value = np.sort(np.absolute(weights.ravel()))[prune_count] # Update mask by disabling the nodes with magnitude smaller than/equal to our cutoff value masks[name] = np.where(np.absolute(weights) > cutoff_value, 1, 0) # Store last used mask globally self._last_used_masks = masks # Set new prune percentages and rebuild self._prune_percentages = prune_percentages self._build() self._compile() # Set initial weights and mask already pruned values. # So when we start the training it will prune away the values that was pruned in the previous training round # plus (p - p_previous) % of the lowest values where p is the current pruning amount and p_previous was the # pruning percentage from the previous training session for name in self._weight_layers: layer = self._get_layer(name) weights = layer.get_weights() weights[0] = self._initial_weights[name] * masks[name] layer.set_weights(weights) def set_pruning_random_init(self, prune_percentages=None): """ This method resets the network, applies pruning that will be used on next training round and re-initializes the unpruned weights to their initial values. :param prune_percentages: Dictionary of pruning values to use. Eg. { 'conv-1': 0.2, ... } """ # Use the most recent masks masks = self._last_used_masks # Set new prune percentages and rebuild self._prune_percentages = prune_percentages self._build() self._compile() # Use the random weights if self._initial_random_weights is None: self._initial_random_weights = self._get_trainable_weights() return # Set weights and mask already pruned values. # So when we start the training it will prune away the values that was pruned in the previous training round # plus (p - p_previous) % of the lowest values where p is the current pruning amount and p_previous was the # pruning percentage from the previous training session for name in self._weight_layers: layer = self._get_layer(name) weights = layer.get_weights() weights[0] = self._initial_random_weights[name] * masks[name] layer.set_weights(weights) def checkpoint_weights(self): """Create a check point so that we do not lose trained weights if needed for pruning""" self.last_used_trainable = self._get_trainable_weights() def reset_to_old_weights(self): """Reset the model to the weights that are in our check point""" for name, weights_m in self.last_used_trainable.items(): layer = self._get_layer(name) weights = layer.get_weights() weights[0] = weights_m layer.set_weights(weights) pass def _get_trainable_weights(self): """ Returns the current trainable weights in a dictionary. Dict keys are layer names and dict values are the layers in numpy format. Could be named like this: {'conv-1': w, ... } """ weights = {} for l in self._weight_layers: layer = self._get_layer(l) weights[l] = layer.get_weights()[0] return weights def get_layer_names(self): """ Returns the relevant layer names in an array. This can be used to set pruning values :return: Array, eg. ['conv-1', 'conv-2', ... 'dense-1', ...] """ return self._weight_layers def _get_layer(self, name): """ Helper for getting a layer from the underlying model :param name: String :return: keras layer """ return self._model.get_layer(( "prune_low_magnitude_" + name) if self._prune_percentages is not None and name in self._prune_percentages else name)
good_prediction=lambda df: df.target.iloc[0], ).loc[lambda df: df.target].agg('mean'))).agg('mean'))] scores = pd.DataFrame(scores)[[ 'score', 'average_precision', 'good_prediction' ]] plt.clf() scores.boxplot() plt.savefig(output_folder / 'scores_boxplot.png') plt.clf() scores.good_prediction.hist() plt.savefig(output_folder / 'scores_good_predictions.png') scores.to_csv(output_folder / 'scores.csv', index=False) #%% Export classification model with SavedModel model.load_weights(str(output_folder / 'kernel_loss_best_loss_weights.h5')) classifier = Sequential([ siamese_nets.get_layer('branch_model'), Classification(kernel=siamese_nets.get_layer('head_model')), Activation('softmax'), ]) tf.saved_model.save(classifier, str(output_folder / 'saved_model/1/')) #%% Example of use as classifier classifier.get_layer('classification').set_support_set( support_tensors=tf.convert_to_tensor(support_set_embeddings, tf.float32), support_labels=tf.convert_to_tensor( pd.get_dummies(support_set.label.values).values, tf.float32), ) y = classifier.predict_generator(test_sequence, verbose=1)
class RNNModel: def __init__(self, nt = 784, nin = 1, nh = 64, nout = 10,\ lr=0.001, mod_type='rnn', batch_size=64,\ is_complex=False, contractive=False): """ A RNN network model nt: num time steps per sample nin: num inputs per time step nout: num outputs per time step nh: num hidden units lr: Learning rate mod_type: 'rnn', 'urnn' or 'lstm' batch_size: Batch size for the optimization is_complex: If complex is to be performed applies only to 'rnn' or 'urnn'. """ # Save dimensions self.nt = nt self.nin = nin self.nh = nh self.nout = nout # Save parameters self.batch_size = batch_size self.mod_type = mod_type self.lr = lr self.is_complex = is_complex self.contractive = contractive # Create the model self.create_model() def create_model(self): """ Creates the model """ self.mod = Sequential() # Add the RNN layer # Note 1: For RNN, we set unroll=True to enable fast GPU usage # Note 2: You need to set the CuDNN version of LSTM unroll = tf.test.is_gpu_available() if (self.mod_type == 'lstm'): # LSTM model if tf.test.is_gpu_available(): self.mod.add(CuDNNLSTM(self.nh, input_shape=(self.nt, self.nin),\ return_sequences=False, name='RNN')) else: self.mod.add(LSTM(self.nh, input_shape=(self.nt, self.nin),\ return_sequences=False, name='RNN',unroll=unroll)) elif self.is_complex: # Complex RNN cell = ComplexRNNCell(nh=self.nh) self.mod.add(RNN(cell, input_shape=(self.nt, self.nin),\ return_sequences=False, name='RNN',unroll=True)) else: # Real RNN model self.mod.add(SimpleRNN(self.nh, input_shape=(self.nt, self.nin),\ return_sequences=False, name='RNN',activation='relu',unroll=unroll)) self.mod.add(Dense(nout, activation='softmax', name='Output')) self.mod.summary() def fit(self, Xtr, Ytr, Xts, Yts, nepochs=10): """ Fits the model parameters """ # Compile the model #opt = Adam(lr=self.lr) opt = RMSprop(lr=self.lr) self.mod.compile(loss='sparse_categorical_crossentropy', optimizer=opt,\ metrics=['accuracy']) # For URNN, add a callback that projects the weight matrix to unitary if mod_type == 'urnn': rnn_layer = self.mod.get_layer('RNN') callbacks = [ ProjectCB(rnn_layer, unitary=True, is_complex=self.is_complex) ] elif mod_type == 'rnn': rnn_layer = self.mod.get_layer('RNN') callbacks = [ProjectCB(rnn_layer,unitary=False,is_complex=self.is_complex,\ contractive=contractive)] else: callbacks = [] # Print progress if self.mod_type == 'lstm': cstr = '' elif self.is_complex: cstr = ' (complex)' else: cstr = ' (real)' print('%s%s nh=%d' % (self.mod_type, cstr, self.nh)) # Fit the model hist = self.mod.fit(Xtr,Ytr,epochs=nepochs, batch_size=self.batch_size,\ callbacks=callbacks,validation_data=(Xts,Yts)) self.tr_acc = hist.history['acc'] self.val_acc = hist.history['val_acc']
print("Expected: {:<25} Pred: {:<25} Difference: {:<25}".format( test_stacked_df_eval[0, 0, i], yhat[0, 0, i], err)) totalError += err print("Total Error: ", totalError) # Testing results = model.evaluate( test_stacked_df, test_stacked_df, verbose=1, steps=(num_of_test_files * len(file_list) // b_size + 1)) print(model.metrics_names) print(results) encoder_layer = Model(inputs=model.input, outputs=model.get_layer("leaky_re_lu_2").output) encoder_layer.trainable = False predictions = encoder_layer.predict(stacked_df, verbose=0, steps=1) print(predictions) print(dy_train) np.save("predictions.npy", predictions) np.save("dy_train.npy", dy_train) print("Data saved - predictions and train") print(predictions.shape) print(len(dy_train)) predictions = encoder_layer.predict(val_stacked_df, verbose=0, steps=1) print(predictions) print(dy_val) np.save("val_predictions.npy", predictions)
class StraightRegression: """ Implements a bottleneck neural network for regression with keras that can use pre-trained weights. """ def __init__(self, l1, l2, lr, act, input_dim, output_dim, pre_trained_weights = False, pre_trained_weights_h5 = None, \ nodes_list=[512, 128, 2, 128, 512]): """ Constructor. :param l1: int, lasso penalty :param l2: int, ridge penalty :param lr: int, learning rate for Adam :param act: string, activation function :param input_dim: int, input layer dimensionality :param output_dim: int, output layer dimensionality :param pre_trained_weights: bool, if True we have pre-trained weights we could use for intialisation (optional, default=False) :param pre_trained_weights_h5: .h5 file, pre_trained weights to use as intial weights for training deep regression (optional, default=None) :param nodes_list: list, integers denoting # nodes in each layer (optional, default=[512, 128, 2, 128, 512]) """ self.l1=l1 self.l2=l2 self.lr=lr self.act=act self.input_dim=input_dim self.output_dim=output_dim self.pre_trained_weights = pre_trained_weights self.nodes_list=nodes_list # Architecture keras.backend.clear_session() self.m = Sequential() self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(self.input_dim, ), \ kernel_regularizer=ElasticNet(l1=self.l1, l2=self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W1')) self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W2')) # linear for latent space representation self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name='bottleneck')) self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W4')) self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W5')) self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ #bias_regularizer=regularizers.l2(self.l2), \ name = 'W6_regr')) if self.pre_trained_weights: self.m.load_weights(pre_trained_weights_h5, by_name=True) self.m.compile(loss='mean_squared_error', \ optimizer=keras.optimizers.Adam(learning_rate=lr), \ metrics=[r2_score, 'mse']) def train(self, x_train, y_train, x_test, y_test, epochs, bs, patience, cvfold_id=0, l1_id=0, l2_id=0, verbose=1, \ prune=False, geneNames=None): """ Train the bottleneck. If you don't prune the training lasts for epochs iterations. If you prune the training lasts for 4*epochs iterations in total (2*epochs normal training, 2*epochs pruning). Parameters ---------- :param x_train, y_train: numpy 2D matrices, input and output training set :param x_test, y_test: numpy 2D matrices, input and output validation set :param epochs: int, # of training iterations :param bs: int, bullshit, ah no, batch size :param patience: int, early stopping :param cvfold_id: int, cross validation set id (optional, default=0), for saving :param l1_id: int, lasso penalty id (optional, default=0), for saving :param l2_id: int, ridge penalty id (optional, default=0), for saving :param verbose: int, print info or not (optional, default=1, i.e. print info -- verbose=0 means not printing info) :param prune: bool, if True we additionally prune the network (new input layer with lower dimensionality) (optional, default=False) :param geneNames: numpy 1D array, contains name of the corresponding gene of every input neuron (optional, default=None) Returns ------- Training and validation loss R^2 score for all epochs """ # Settings for early stopping and saving best model if not prune: es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=patience) else: es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=2*patience) if not self.pre_trained_weights: mc = ModelCheckpoint('KerasSavedModels/StraightRegression_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) else: mc = ModelCheckpoint('KerasSavedModels/PreTrRegression_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) # train the network print("[INFO] training network...") if not prune: H = self.m.fit(x_train, y_train, batch_size=bs, validation_data=(x_test, y_test), epochs=epochs, verbose=verbose, callbacks = [es, mc]) else: H = self.m.fit(x_train, y_train, batch_size=bs, validation_data=(x_test, y_test), epochs=2*epochs, verbose=verbose, callbacks = [es, mc]) train_loss_straight_regr = np.array(H.history["r2_score"]) val_loss_straight_regr = np.array(H.history["val_r2_score"]) MSE_tr = np.array(H.history["loss"]) MSE_val = np.array(H.history["val_loss"]) # Retrieve activations and ephys prediction if not self.pre_trained_weights: saved_model = load_model('KerasSavedModels/StraightRegression_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) else: saved_model = load_model('KerasSavedModels/PreTrRegression_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) #print('predict: ', saved_model.predict(x_train)) #print('y_train: ', y_train) #print('MSE: ', np.sum((y_train - saved_model.predict(x_train))**2)) #print('denominator: ', np.sum(y_train**2)) r2_train = 1-np.sum((y_train - saved_model.predict(x_train))**2) / np.sum(y_train**2) r2_test = 1-np.sum((y_test - saved_model.predict(x_test))**2) / np.sum(y_test**2) self.m.save('KerasSavedModels/Regression_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id)) if prune: saved_model_ = load_model('KerasSavedModels/Regression_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) weight = saved_model_.get_weights()[0] bias = saved_model_.get_weights()[1] ind_genes = np.argsort(np.linalg.norm(weight, ord=2, axis=1))[-25:] print('The 25 genes that make it: ', geneNames[ind_genes]) # Architecture keras.backend.clear_session() self.m = Sequential() self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(25, ), \ kernel_regularizer=ElasticNet(l1=0, l2=self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W1_25')) self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W2')) # linear for latent space representation self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), name='bottleneck')) self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W4')) self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \ bias_regularizer=regularizers.l2(self.l2), \ name = 'W5')) self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \ #bias_regularizer=regularizers.l2(l2_parameter), \ name = 'W6_regr')) # Load weights from training a previous network on regression self.m.load_weights('KerasSavedModels/Regression_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), by_name=True) # We transfer the weights of those 25 genes now manually too self.m.get_layer('W1_25').set_weights([weight[ind_genes, :], bias]) self.m.compile(loss='mse', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse']) es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=2*patience) if not self.pre_trained_weights: mc = ModelCheckpoint('KerasSavedModels/StraightRegression_weights_after_pruning_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) else: mc = ModelCheckpoint('KerasSavedModels/PreTrRegression_weights_after_pruning_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ monitor='val_mse', mode='min', verbose=verbose, save_best_only=True) # train the network print("[INFO] training network...") H = self.m.fit(x_train[:, ind_genes], y_train, batch_size=bs, validation_data=(x_test[:, ind_genes], y_test), epochs=2*epochs, verbose=verbose, callbacks = [es, mc]) MSE_tr = np.concatenate([MSE_tr, np.array(H.history["loss"])]) MSE_val = np.concatenate([MSE_val, np.array(H.history["val_loss"])]) train_loss_straight_regr = np.concatenate([train_loss_straight_regr, np.array(H.history["r2_score"])]) val_loss_straight_regr = np.concatenate([val_loss_straight_regr, np.array(H.history["val_r2_score"])]) if prune: if not self.pre_trained_weights: saved_model_2 = load_model('KerasSavedModels/StraightRegression_weights_after_pruning_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet}) else: saved_model_2 = load_model('KerasSavedModels/PreTrRegression_weights_after_pruning_{}_{}_{}.h5'.\ format(cvfold_id, l1_id, l2_id), \ custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet, 'residual': residual}) r2_after_pruning_train = 1-np.sum((y_train - saved_model_2.predict(x_train[:, ind_genes]))**2) \ / np.sum(y_train**2) r2_after_pruning_test = 1-np.sum((y_test - saved_model_2.predict(x_test[:, ind_genes]))**2) \ / np.sum(y_test**2) if not prune: print('Train R^2: ', r2_train) print('Test R^2: ', r2_test) else: print('Train R^2 before pruning: ', r2_train) print('Test R^2 after pruning: ', r2_test) print('Train R^2 after pruning: ', r2_after_pruning_train) print('Test R^2 after pruning: ', r2_after_pruning_test) if not prune: return saved_model.predict(x_train), saved_model.predict(x_test), \ r2_train, r2_test, train_loss_straight_regr, val_loss_straight_regr, \ MSE_tr, MSE_val else: return r2_train, r2_test, \ r2_after_pruning_train, r2_after_pruning_test, \ train_loss_straight_regr, val_loss_straight_regr