def test_find_activation_layer():
conv1_filters = 1
conv2_filters = 1
dense_units = 1
model = Sequential()
model.add(
Conv2D(conv1_filters, [3, 3],
input_shape=(28, 28, 1),
data_format="channels_last",
name='conv_1'))
model.add(Activation('relu', name='act_1'))
model.add(MaxPool2D((2, 2), name='pool_1'))
model.add(
Conv2D(conv2_filters, [3, 3],
data_format="channels_last",
name='conv_2'))
model.add(Activation('relu', name='act_2'))
model.add(MaxPool2D((2, 2), name='pool_2'))
model.add(Flatten(name='flat_1'))
model.add(Dense(dense_units, name='dense_1'))
model.add(Activation('relu', name='act_3'))
model.add(Dense(10, name='dense_2'))
model.add(Activation('softmax', name='act_4'))
assert find_activation_layer(model.get_layer('conv_1'),
0) == (model.get_layer('act_1'), 0)
assert find_activation_layer(model.get_layer('conv_2'),
0) == (model.get_layer('act_2'), 0)
assert find_activation_layer(model.get_layer('dense_1'),
0) == (model.get_layer('act_3'), 0)
assert find_activation_layer(model.get_layer('dense_2'),
0) == (model.get_layer('act_4'), 0)
def CAE(input_shape=(28, 28, 1), filters=[32, 64, 128, 10]):
model = Sequential()
if input_shape[0] % 8 == 0:
pad3 = 'same'
else:
pad3 = 'valid'
model.add(InputLayer(input_shape))
model.add(Conv2D(filters[0], 5, strides=2, padding='same', activation='relu', name='conv1'))
model.add(Conv2D(filters[1], 5, strides=2, padding='same', activation='relu', name='conv2'))
model.add(Conv2D(filters[2], 3, strides=2, padding=pad3, activation='relu', name='conv3'))
model.add(Flatten())
model.add(Dense(units=filters[3], name='embedding'))
model.add(Dense(units=filters[2]*int(input_shape[0]/8)*int(input_shape[0]/8), activation='relu'))
model.add(Reshape((int(input_shape[0]/8), int(input_shape[0]/8), filters[2])))
model.add(Conv2DTranspose(filters[1], 3, strides=2, padding=pad3, activation='relu', name='deconv3'))
model.add(Conv2DTranspose(filters[0], 5, strides=2, padding='same', activation='relu', name='deconv2'))
model.add(Conv2DTranspose(input_shape[2], 5, strides=2, padding='same', name='deconv1'))
if model.layers[0].input_shape != model.layers[-1].output_shape:
crop = abs(model.layers[0].input_shape[1] - model.layers[-1].output_shape[1])//2
model.add(Cropping2D(crop))
encoder = Model(inputs=model.input, outputs=model.get_layer('embedding').output)
return model, encoder
def universal_approximation(f, x):
[train_x, test_x] = split_data(x, ratio=0.75, random=True)
train_y = np.sin(train_x)
test_x = np.sort(test_x, axis=0)
test_y = f(test_x)
# build simple FNN
model = Sequential()
model.add(Dense(50, input_shape=(1, ), activation='relu'))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
# training process
model.fit(train_x, train_y, batch_size=100, epochs=1000)
layer = model.get_layer(index=0)
plt.plot(model.history.history['loss'])
plt.show()
# predict
y_hat = model.predict(test_x)
plt.plot(test_x, test_y, 'b-', label='original')
plt.plot(test_x, y_hat, 'r-', label='predicted')
plt.legend()
plt.show()
def export_flatten(filename, input_shape):
model = Sequential()
model.add(Flatten(input_shape=input_shape[1:]))
model.predict(np.random.uniform(size=input_shape))
sess = K.get_session()
output = model.get_layer('flatten').output
return export(output, filename, sess=sess)
def train_2nd_level(
self, X_train: np.ndarray, X_valid: np.ndarray
) -> Tuple[np.ndarray, np.ndarray]:
"""train 2nd level DAE
Note that input must to be scaled betweeen [0, 1] because output layer is governed by sigmoid function.
Args
X_train (np.ndarray): has shape (num_samples, output dim of 1st level)
X_valid (np.ndarray): has shape (num_samples, output dim of 1st level)
"""
# Build base model
model = Sequential()
model.add(Dropout(0.4, seed=20, name="2nd_level_dropout"))
model.add(
Dense(30, input_dim=X_train.shape[1], activation="sigmoid", name="2nd_level_fc")
) # encoder
model.add(Dense(X_train.shape[1], activation="sigmoid", name="2nd_level_output")) # decoder
model.compile(
loss="mean_squared_error",
optimizer=optimizers.Adam(lr=self.pretrain_lr),
metrics=["mse"],
)
# Create callback
callbacks = create_callback(
model=model,
path_chpt=f"{self.LOG_DIR}/trained_model_2nd_level_fold{self.fold_id}.h5",
metric="mse",
verbose=50,
epochs=self.pretrain_epochs,
)
fit = model.fit(
x=X_train,
y=X_train,
batch_size=self.pretrain_batch_size,
epochs=self.pretrain_epochs,
verbose=self.verbose,
validation_data=(X_valid, X_valid),
callbacks=callbacks,
)
plot_learning_history(
fit=fit, metric="mse", path=f"{self.LOG_DIR}/history_2nd_level_fold{self.fold_id}.png"
)
plot_model(model, path=f"{self.LOG_DIR}/model_2nd_level.png")
# Load best model
model = keras.models.load_model(
f"{self.LOG_DIR}/trained_model_2nd_level_fold{self.fold_id}.h5"
)
self.model_2nd_level = model
encoder = Model(inputs=model.input, outputs=model.get_layer("2nd_level_fc").output)
pred_train = encoder.predict(X_train) # predict with clean signal
pred_valid = encoder.predict(X_valid) # predict with clean signal
return pred_train, pred_valid
def vgg_layers(layer_names):
red_model = Sequential()
vgg = tf.keras.applications.vgg19.VGG19(include_top=False, weights='imagenet')
vgg.trainable = False
for layer in vgg.layers:
if layer.__class__ == MaxPooling2D:
red_model.add(AveragePooling2D())
else:
red_model.add(layer)
outputs = [red_model.get_layer(name).output for name in layer_names]
model = Model([red_model.input], outputs)
return model
def build_vae_encoder(self):
encoder = Sequential()
encoder.add(
Conv2D(32, (4, 4),
strides=(2, 2),
activation='relu',
name='enc_conv1',
input_shape=self.state_size))
encoder.add(
Conv2D(64, (4, 4),
strides=(2, 2),
activation='relu',
name='enc_conv2'))
encoder.add(
Conv2D(128, (4, 4),
strides=(2, 2),
activation='relu',
name='enc_conv3'))
encoder.add(
Conv2D(256, (4, 4),
strides=(2, 2),
activation='relu',
name='enc_conv4'))
encoder.add(Flatten()) # shape: [-1, 3 * 8 * 256]
encoder.add(Dense(self.z_size, name='enc_fc_mu'))
if self.vae_weights:
encoder.get_layer('enc_conv1').set_weights(self.vae_weights[0:2])
encoder.get_layer('enc_conv2').set_weights(self.vae_weights[2:4])
encoder.get_layer('enc_conv3').set_weights(self.vae_weights[4:6])
encoder.get_layer('enc_conv4').set_weights(self.vae_weights[6:8])
encoder.get_layer('enc_fc_mu').set_weights(self.vae_weights[8:10])
encoder.trainable = False
return encoder
def __init__(self, input_shape):
model = Sequential()
model.add(Conv2D(16,(5,5),padding='valid',input_shape = input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2),strides=2,padding = 'valid'))
model.add(Dropout(0.4))
model.add(Conv2D(32,(5,5),padding='valid'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2),strides=2,padding = 'valid'))
model.add(Dropout(0.6))
model.add(Conv2D(64,(5,5),padding='valid'))
model.add(Activation('relu'))
model.add(Dropout(0.8))
model.add(Flatten())
model.add(Dense(2))
model.add(Activation('softmax'))
self.transformer = Model(inputs=model.input,outputs=model.get_layer('dense_1').output)
def autoencoder(n_input, loss='mean_squared_error'):
'''
model.fit(x_train, x_train, batch_size = 32, epochs = 500)
bottleneck_representation = encoder.predict(x_train)
'''
model = Sequential()
# encoder
model.add(Dense(512, activation='relu', input_shape=(n_input, )))
model.add(Dense(256, activation='relu'))
model.add(Dense(64, activation='relu'))
# bottleneck code
model.add(Dense(20, activation='linear', name="bottleneck"))
# decoder
model.add(Dense(64, activation='relu'))
model.add(Dense(256, activation='relu'))
model.add(Dense(512, activation='relu'))
model.add(Dense(n_input, activation='sigmoid'))
model.compile(loss=loss, optimizer=Adam())
encoder = Model(model.input, model.get_layer('bottleneck').output)
return model, encoder
def featureExtraction(x,y):
x = x.to_numpy()
y = y.to_numpy()
x = x[:, :, newaxis]
verbose, epochs, batch_size = 1, 50, 1
n_timesteps, n_features = x.shape[1], x.shape[2]
model = Sequential()
#model.build(1344)
model.add(Conv1D(filters=100, kernel_size=6, activation='relu', input_shape=(n_timesteps,n_features)))
model.add(Dropout(0.5))
model.add(MaxPooling1D(pool_size=1))
model.add(BatchNormalization())
model.add(Conv1D(filters=75, kernel_size=4, activation='relu'))
model.add(Dropout(0.1))
model.add(MaxPooling1D(pool_size=1))
model.add(BatchNormalization())
model.add(Conv1D(filters=50, kernel_size=2, activation='relu'))
model.add(Conv1D(filters=12, kernel_size=2, activation='relu'))
model.add(Flatten())
model.add(Dense(15, activation='relu', name ='feature_dense'))
model.add(Dense(1, activation='softmax'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
#fit network
model.fit(x, y, validation_split=0.02, epochs=epochs, batch_size=batch_size, verbose=verbose)
model.summary()
feature_layer = Model(inputs=model.input,
outputs=model.get_layer('feature_dense').output)
feature_layer.summary()
new_data = feature_layer.predict(x)
new_data = pd.DataFrame(new_data )
return new_data
def cnn(train_dir, test_dir):
# 定数定義
NUM_CLASSES = 4 # 識別クラス数
IMAGE_SIZE = 256 # 学習時の画像サイズ[px]
IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE * 3 # 画像の次元数
LABEL_ANNOTATION_PATH = './label_annotation.txt'
LOG_TRAINING_ACCURACY_GRAPH_PATH = './log/cnn/training_accuracy.png'
LOG_TRAINING_LOSS_GRAPH_PATH = './log/cnn/training_loss.png'
LOG_TRAINING_MODEL_PATH = './log/cnn/model.png'
TRAINING_OPTIMIZER = "SGD(確率的勾配降下法)"
ACTIVATION_FUNC = "relu" #活性化関数
# 学習データセットのインポート
train_image, train_label, train_path = import_dataset(
train_dir, IMAGE_SIZE, IMAGE_SIZE)
#Kerasの学習モデルの構築
model = Sequential()
# 畳み込み層
model.add(
Conv2D(3,
kernel_size=3,
activation=ACTIVATION_FUNC,
input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(3,
kernel_size=3,
activation=ACTIVATION_FUNC,
input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(3,
kernel_size=3,
activation=ACTIVATION_FUNC,
input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Activation(ACTIVATION_FUNC))
model.add(Dropout(0.2))
# 全結合層
model.add(Dense(200))
model.add(Activation(ACTIVATION_FUNC))
model.add(Dropout(0.2))
model.add(Dense(200))
model.add(Activation(ACTIVATION_FUNC))
model.add(Dropout(0.2))
model.add(Dense(200))
model.add(Activation(ACTIVATION_FUNC))
model.add(Dropout(0.2))
model.add(Dense(200))
model.add(Activation(ACTIVATION_FUNC))
model.add(Dropout(0.2))
model.add(Dense(NUM_CLASSES))
model.add(Activation("softmax"))
# オプティマイザにAdamを使用
opt = Adam(lr=0.001)
# モデルをコンパイル
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
# 学習を実行
Y = to_categorical(train_label, NUM_CLASSES)
history = model.fit(train_image,
Y,
nb_epoch=40,
batch_size=100,
validation_split=0.1)
export.plot(history)
# テスト用データセットのインポート
test_image, test_label, test_path = import_dataset(test_dir, IMAGE_SIZE,
IMAGE_SIZE)
result = model.predict_classes(test_image)
result_prob = model.predict_proba(test_image)
sum_accuracy = 0.0
for i in range(test_image.shape[0]):
print("label:", test_label[i], "result:", result[i], "prob: ",
result_prob[i])
if test_label[i] == result[i]:
sum_accuracy += 1
sum_accuracy /= test_image.shape[0]
print("accuracy: ", sum_accuracy)
plot_model(model, show_shapes=True, to_file=LOG_TRAINING_MODEL_PATH)
#中間層の出力
imm_layer = ['conv2d', 'conv2d_1', 'conv2d_2']
for layer_name in imm_layer:
#中間層のmodelを作成
intermediate_layer_model = Model(
inputs=model.input, outputs=model.get_layer(layer_name).output)
#出力をmodel.predictで見る
intermediate_output = intermediate_layer_model.predict(test_image)
path = os.getcwd() + "/log/cnn/" + layer_name
if os.path.exists(path) == False: # 出力先ディレクトリが存在しなければ新規作成する
os.mkdir(path)
for i in range(intermediate_output.shape[0]):
cv2.imwrite(path + '/immidiate_' + str(i) + '.png',
intermediate_output[i] * 255)
#結果をhtmlファイル出力
result_dict = {'acc': 0, 'n_img': 0, 'opt': "", 'act_func': ""}
result_dict['acc'] = sum_accuracy
result_dict['n_img'] = train_image.shape[0]
result_dict['opt'] = TRAINING_OPTIMIZER
result_dict['act_func'] = ACTIVATION_FUNC
export.cnn_html(result_dict, test_image, test_label, test_path, result,
result_prob, imm_layer)
class CNN(object):
def __init__(self):
"""
Initialize multi-layer neural network
"""
self.model = Sequential()
def add_input_layer(self, shape=(2, ), name=""):
"""
This function adds an input layer to the neural network. If an input layer exist, then this function
should replcae it with the new input layer.
:param shape: input shape (tuple)
:param name: Layer name (string)
:return: None
"""
# self.model.add(Dense(input_shape=shape,name=name))
self.model.add(InputLayer(input_shape=shape, name=name))
def append_dense_layer(self,
num_nodes,
activation="relu",
name="",
trainable=True):
"""
This function adds a dense layer to the neural network
:param num_nodes: Number of nodes
:param activation: Activation function for the layer. Possible values are "Linear", "Relu", "Sigmoid",
"Softmax"
:param name: Layer name (string)
:param trainable: Boolean
:return: None
"""
self.model.add(
Dense(units=num_nodes,
activation=activation,
name=name,
trainable=trainable))
def append_conv2d_layer(self,
num_of_filters,
kernel_size=3,
padding='same',
strides=1,
activation="Relu",
name="",
trainable=True):
"""
This function adds a conv2d layer to the neural network
:param num_of_filters: Number of nodes
:param num_nodes: Number of nodes
:param kernel_size: Kernel size (assume that the kernel has the same horizontal and vertical size)
:param padding: "same", "Valid"
:param strides: strides
:param activation: Activation function for the layer. Possible values are "Linear", "Relu", "Sigmoid"
:param name: Layer name (string)
:param trainable: Boolean
:return: Layer object
"""
layer = Conv2D(num_of_filters,
kernel_size=kernel_size,
padding=padding,
strides=strides,
activation=activation,
name=name,
trainable=trainable)
self.model.add(
Conv2D(num_of_filters,
kernel_size=kernel_size,
padding=padding,
strides=strides,
activation=activation,
name=name,
trainable=trainable))
return layer
def append_maxpooling2d_layer(self,
pool_size=2,
padding="same",
strides=2,
name=""):
"""
This function adds a maxpool2d layer to the neural network
:param pool_size: Pool size (assume that the pool has the same horizontal and vertical size)
:param padding: "same", "valid"
:param strides: strides
:param name: Layer name (string)
:return: Layer object
"""
layer = MaxPooling2D(pool_size=pool_size,
strides=strides,
padding=padding,
name=name)
self.model.add(
MaxPooling2D(pool_size=pool_size,
strides=strides,
padding=padding,
name=name))
return layer
def append_flatten_layer(self, name=""):
"""
This function adds a flattening layer to the neural network
:param name: Layer name (string)
:return: Layer object
"""
layer = Flatten(name=name)
self.model.add(Flatten(name=name))
return layer
def set_training_flag(self,
layer_numbers=[],
layer_names="",
trainable_flag=True):
"""
This function sets the trainable flag for a given layer
:param layer_number: an integer or a list of numbers.Layer numbers start from layer 0.
:param layer_names: a string or a list of strings (if both layer_number and layer_name are specified, layer number takes precedence).
:param trainable_flag: Set trainable flag
:return: None
"""
def get_weights_without_biases(self, layer_number=None, layer_name=""):
"""
This function should return the weight matrix (without biases) for layer layer_number.
layer numbers start from zero.
This means that the first layer with activation function is layer zero
:param layer_number: Layer number starting from layer 0.
:param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence).
:return: Weight matrix for the given layer (not including the biases). If the given layer does not have
weights then None should be returned.
"""
if (not (layer_number is None)):
if (layer_number >= 0):
if (layer_number == 0):
return None
elif (self.model.get_layer(
index=layer_number - 1,
name=layer_name).count_params() == 0):
return None
else:
return self.model.get_layer(
index=layer_number - 1,
name=layer_name).get_weights()[0]
else:
if (self.model.get_layer(
index=layer_number).count_params() == 0):
return None
else:
return self.model.get_layer(
index=layer_number).get_weights()[0]
else:
if (self.model.get_layer(name=layer_name).count_params() == 0):
return None
else:
return self.model.get_layer(name=layer_name).get_weights()[0]
def get_biases(self, layer_number=None, layer_name=""):
"""
This function should return the biases for layer layer_number.
layer numbers start from zero.
This means that the first layer with activation function is layer zero
:param layer_number: Layer number starting from layer 0
:param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence).
:return: biases for the given layer (If the given layer does not have bias then None should be returned)
"""
if (not layer_number is None):
if (layer_number == 0):
return None
elif (self.model.get_layer(index=layer_number - 1,
name=layer_name).count_params() == 0):
return None
else:
return self.model.get_layer(index=layer_number - 1,
name=layer_name).get_weights()[1]
else:
if (self.model.get_layer(name=layer_name).count_params() == 0):
return None
else:
return self.model.get_layer(name=layer_name).get_weights()[1]
def set_weights_without_biases(self,
weights,
layer_number=None,
layer_name=""):
"""
This function sets the weight matrix for layer layer_number.
layer numbers start from zero.
This means that the first layer with activation function is layer zero
:param weights: weight matrix (without biases). Note that the shape of the weight matrix should be
[input_dimensions][number of nodes]
:param layer_number: Layer number starting from layer 0
:param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence).
:return: None
"""
if (not layer_number is None):
temp = self.get_biases(layer_number, layer_name)
W = [weights, temp]
self.model.get_layer(index=layer_number - 1,
name=layer_name).set_weights(W)
else:
temp = self.get_biases(layer_number, layer_name)
W = [weights, temp]
self.model.get_layer(name=layer_name).set_weights(W)
def set_biases(self, biases, layer_number=None, layer_name=""):
"""
This function sets the biases for layer layer_number.
layer numbers start from zero.
This means that the first layer with activation function is layer zero
:param biases: biases. Note that the biases shape should be [1][number_of_nodes]
:param layer_number: Layer number starting from layer 0
:param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence).
:return: none
"""
if (not layer_number is None):
temp = self.get_weights_without_biases(layer_number, layer_name)
W = [temp, biases]
self.model.get_layer(index=layer_number - 1,
name=layer_name).set_weights(W)
else:
temp = self.get_weights_without_biases(layer_number, layer_name)
W = [temp, biases]
self.model.get_layer(name=layer_name).set_weights(W)
def remove_last_layer(self):
"""
This function removes a layer from the model.
:return: removed layer
"""
layer = self.model.get_layer(index=-1)
self.model = Sequential(self.model.layers[:-1])
return layer
# self.model.outputs = [self.model.layers[-1].output]
def load_a_model(self, model_name="", model_file_name=""):
"""
This function loads a model architecture and weights.
:param model_name: Name of the model to load. model_name should be one of the following:
"VGG16", "VGG19"
:param model_file_name: Name of the file to load the model (if both madel_name and
model_file_name are specified, model_name takes precedence).
:return: model
"""
if (model_name == "VGG16"):
self.model = Sequential(VGG16().layers)
elif (model_name == "VGG19"):
self.model = Sequential(VGG19().layers)
else:
self.model = load_model(model_file_name)
def save_model(self, model_file_name=""):
"""
This function saves the current model architecture and weights together in a HDF5 file.
:param file_name: Name of file to save the model.
:return: model
"""
self.model.save(model_file_name)
def set_loss_function(self, loss="SparseCategoricalCrossentropy"):
"""
This function sets the loss function.
:param loss: loss is a string with the following choices:
"SparseCategoricalCrossentropy", "MeanSquaredError", "hinge".
:return: none
"""
def set_metric(self, metric):
"""
This function sets the metric.
:param metric: metric should be one of the following strings:
"accuracy", "mse".
:return: none
"""
def set_optimizer(self, optimizer="SGD", learning_rate=0.01, momentum=0.0):
"""
This function sets the optimizer.
:param optimizer: Should be one of the following:
"SGD" , "RMSprop" , "Adagrad" ,
:param learning_rate: Learning rate
:param momentum: Momentum
:return: none
"""
def predict(self, X):
"""
Given array of inputs, this function calculates the output of the multi-layer network.
:param X: Input tensor.
:return: Output tensor.
"""
return self.model.predict(X)
def evaluate(self, X, y):
"""
Given array of inputs and desired ouputs, this function returns the loss value and metrics of the model.
:param X: Array of input
:param y: Array of desired (target) outputs
:return: loss value and metric value
"""
def train(self, X_train, y_train, batch_size, num_epochs):
"""
dec.add(UpSampling2D((2, 2))) dec.add(Conv2D(16, (3, 3), activation='relu')) dec.add(UpSampling2D((2, 2))) dec.add(Conv2D(1, (3, 3), activation='sigmoid', padding='same')) dec.summary() encoder = Sequential([enc,dec]) encoder.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(lr=0.001)) encoder.fit(X_train,X_train,epochs=10,validation_data=(X_test,X_test)) !pip install bokeh from tensorflow.keras import Model layer_name = 'MUSAFA' intermediate_layer_model = Model(inputs=enc.input,outputs=enc.get_layer(layer_name).output) intermediate_output = intermediate_layer_model.predict(X_test) # BEFORE SHAPE WAS (10000,7,7,8) # 7*7*8 --> 392 intermediate_output = intermediate_output.reshape(10000,392) from sklearn.manifold import TSNE tsne_model = TSNE(n_components=2, verbose=1, random_state=0) tsne_img_label = tsne_model.fit_transform(intermediate_output) import pandas as pd import numpy as np tsne_df = pd.DataFrame(tsne_img_label, columns=['x', 'y']) tsne_df['image_label'] = y_test
plt.figure(figsize=(8,8))
for i in range(3):
delta_1=W_1[i]-W_0[i]
print(f"difference between biases is {b_1[i]-b_0[i]}")
axis=plt.subplot(1,3,i+1)
plt.imshow(delta_1)
plt.title(f"layer {i+1}")
plt.axis("off")
plt.colorbar()
plt.show()'''
n_trainable_variables = model.trainable_variables
print(n_trainable_variables)
model.get_layer("dense").trainable = False
model.compile(optimizer="adam", loss="mse", metrics=["accuracy"])
his = model.fit(Xtrain, ytrain, epochs=50)
W_1 = [e.weights[0].numpy() for e in model.layers]
b_1 = [b.bias.numpy() for b in model.layers]
plt.figure(figsize=(8, 8))
for i in range(3):
delta_1 = W_1[i] - W_0[i]
print(f"difference between biases is {b_1[i]-b_0[i]}")
axis = plt.subplot(1, 3, i + 1)
plt.imshow(delta_1)
plt.title(f"layer {i+1}")
plt.axis("off")
class CNN(object):
def __init__(self):
"""
Initialize multi-layer neural network
"""
self.model = Sequential()
def add_input_layer(self, shape=(2,),name="" ):
"""
This function adds an input layer to the neural network. If an input layer exist, then this function
should replcae it with the new input layer.
:param shape: input shape (tuple)
:param name: Layer name (string)
:return: None
"""
self.builtIn = False
if shape.__class__ != tuple and shape.__class__ == int:
shape = (shape,)
if self.model is None:
self.model = Sequential()
self.model.add(InputLayer(input_shape=shape, name=name))
def append_dense_layer(self, num_nodes,activation="relu",name="",trainable=True):
"""
This function adds a dense layer to the neural network
:param num_nodes: Number of nodes
:param activation: Activation function for the layer. Possible values are "Linear", "Relu", "Sigmoid",
"Softmax"
:param name: Layer name (string)
:param trainable: Boolean
:return: None
"""
self.model.add(Dense(units = num_nodes, activation=activation, name= name, trainable= trainable))
def append_conv2d_layer(self, num_of_filters, kernel_size=3, padding='same', strides=1,
activation="Relu",name="",trainable=True, input_shape = None):
"""
This function adds a conv2d layer to the neural network
:param num_of_filters: Number of nodes
:param num_nodes: Number of nodes
:param kernel_size: Kernel size (assume that the kernel has the same horizontal and vertical size)
:param padding: "same", "Valid"
:param strides: strides
:param activation: Activation function for the layer. Possible values are "Linear", "Relu", "Sigmoid"
:param name: Layer name (string)
:param trainable: Boolean
:return: Layer object
"""
if input_shape != None:
conv2D_layer = Conv2D(num_of_filters, kernel_size=kernel_size, padding=padding, name=name, strides = strides, activation=activation, trainable=trainable, input_shape = input_shape)
else:
conv2D_layer = Conv2D(num_of_filters, kernel_size=kernel_size, padding=padding, name=name, strides = strides, activation=activation, trainable=trainable)
self.model.add(conv2D_layer)
return conv2D_layer
def append_maxpooling2d_layer(self, pool_size=2, padding="same", strides=2,name=""):
"""
This function adds a maxpool2d layer to the neural network
:param pool_size: Pool size (assume that the pool has the same horizontal and vertical size)
:param padding: "same", "valid"
:param strides: strides
:param name: Layer name (string)
:return: Layer object
"""
maxpooling2d_layer = MaxPooling2D(pool_size = pool_size, strides=strides, padding=padding, name=name)
self.model.add(maxpooling2d_layer)
return maxpooling2d_layer
def append_flatten_layer(self,name=""):
"""
This function adds a flattening layer to the neural network
:param name: Layer name (string)
:return: Layer object
"""
flatten_layer = Flatten(name=name)
self.model.add(flatten_layer)
return flatten_layer
def set_training_flag(self,layer_numbers=[],layer_names="",trainable_flag=True):
"""
This function sets the trainable flag for a given layer
:param layer_number: an integer or a list of numbers.Layer numbers start from layer 0.
:param layer_names: a string or a list of strings (if both layer_number and layer_name are specified, layer number takes precedence).
:param trainable_flag: Set trainable flag
:return: None
"""
for num in layer_numbers:
self.model.layers[num].trainable = trainable_flag
def get_weights_without_biases(self,layer_number=None,layer_name=""):
"""
This function should return the weight matrix (without biases) for layer layer_number.
layer numbers start from zero.
This means that the first layer with activation function is layer zero
:param layer_number: Layer number starting from layer 0.
:param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence).
:return: Weight matrix for the given layer (not including the biases). If the given layer does not have
weights then None should be returned.
"""
if(layer_number is not None):
if(layer_number > 0):
weights = self.model.get_layer(index = layer_number-1).get_weights()
if len(weights) > 0:
return weights[0]
elif(layer_number < 0):
return self.model.get_layer(index = layer_number).get_weights()[0]
elif(layer_name != ""):
try:
if(self.model.get_layer(name=layer_name) != None and self.model.get_layer(name=layer_name).count_params() != 0):
return self.model.get_layer(name = layer_name).get_weights()[0]
except:
return None
return None
def get_biases(self,layer_number=None,layer_name=""):
"""
This function should return the biases for layer layer_number.
layer numbers start from zero.
This means that the first layer with activation function is layer zero
:param layer_number: Layer number starting from layer 0
:param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence).
:return: biases for the given layer (If the given layer does not have bias then None should be returned)
"""
if(not layer_number is None):
if(layer_number > 0):
biases = self.model.get_layer(index=layer_number-1,name=layer_name).get_weights()
if len(biases) > 0:
return biases[1]
elif layer_name != "":
biases = self.model.get_layer(name=layer_name).get_weights()
if len(biases) > 0:
return biases[1]
return None
def set_weights_without_biases(self,weights,layer_number=None,layer_name=""):
"""
This function sets the weight matrix for layer layer_number.
layer numbers start from zero.
This means that the first layer with activation function is layer zero
:param weights: weight matrix (without biases). Note that the shape of the weight matrix should be
[input_dimensions][number of nodes]
:param layer_number: Layer number starting from layer 0
:param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence).
:return: None
"""
if(layer_number is not None):
nweights = [weights,self.get_biases(layer_number, "")]
self.model.get_layer(index=layer_number-1).set_weights(nweights)
elif(layer_name != ""):
nweights = [weights,self.get_biases(None,layer_name)]
self.model.get_layer(name=layer_name).set_weights(nweights)
def set_biases(self,biases,layer_number=None,layer_name=""):
"""
This function sets the biases for layer layer_number.
layer numbers start from zero.
This means that the first layer with activation function is layer zero
:param biases: biases. Note that the biases shape should be [1][number_of_nodes]
:param layer_number: Layer number starting from layer 0
:param layer_name: Layer name (if both layer_number and layer_name are specified, layer number takes precedence).
:return: none
"""
if(layer_number is not None):
nbiases = [temp,self.get_weights_without_biases(layer_number,layer_name)]
self.model.get_layer(index=layer_number-1,name=layer_name).set_weights(nbiases)
else:
nbiases = [self.get_weights_without_biases(layer_number,layer_name),biases]
self.model.get_layer(name=layer_name).set_weights(nbiases)
def remove_last_layer(self):
"""
This function removes a layer from the model.
:return: removed layer
"""
layer=self.model._layers.pop()
self.model= Model(self.model.input,self.model.layers[-1].output)
return layer
def load_a_model(self,model_name="",model_file_name=""):
"""
This function loads a model architecture and weights.
:param model_name: Name of the model to load. model_name should be one of the following:
"VGG16", "VGG19"
:param model_file_name: Name of the file to load the model (if both madel_name and
model_file_name are specified, model_name takes precedence).
:return: model
"""
if(model_name=="VGG16"):
self. model= Sequential(VGG16().layers)
elif(model_name=="VGG19"):
self. model= Sequential(VGG19().layers)
else:
self.model = Sequential()
self.model = load_model(model_file_name)
def save_model(self,model_file_name=""):
"""
This function saves the current model architecture and weights together in a HDF5 file.
:param file_name: Name of file to save the model.
:return: model
"""
self.model.save(model_file_name)
def set_loss_function(self, loss="SparseCategoricalCrossentropy"):
"""
This function sets the loss function.
:param loss: loss is a string with the following choices:
"SparseCategoricalCrossentropy", "MeanSquaredError", "hinge".
:return: none
"""
self.loss_function = None
if "SparseCategoricalCrossentropy":
self.loss_function = loss
elif "MeanSquaredError":
self.loss_function = loss
elif "hinge":
self.loss_function = loss
else:
self.loss_function = loss
def set_metric(self,metric):
"""
This function sets the metric.
:param metric: metric should be one of the following strings:
"accuracy", "mse".
:return: none
"""
self.metric = None
if "accuracy":
self.metric = ['acc']
elif "mse":
self.metric = metric
def set_optimizer(self,optimizer="SGD",learning_rate=0.01,momentum=0.0):
"""
This function sets the optimizer.
:param optimizer: Should be one of the following:
"SGD" , "RMSprop" , "Adagrad" ,
:param learning_rate: Learning rate
:param momentum: Momentum
:return: none
"""
self.optimizer = None
if "RMSprop":
self.optimizer = optimizers.RMSprop(lr=learning_rate)
elif "SGD":
self.optimizer = optimizers.SGD(lr=learning_rate, momentum=momentum)
elif "Adagrad":
self.optimizer = 'Adagrad'
print("Optimizer : {0}".format(self.optimizer))
def predict(self, X):
"""
Given array of inputs, this function calculates the output of the multi-layer network.
:param X: Input tensor.
:return: Output tensor.
"""
output = self.model.predict(X)
return output
def evaluate(self,X,y):
"""
Given array of inputs and desired ouputs, this function returns the loss value and metrics of the model.
:param X: Array of input
:param y: Array of desired (target) outputs
:return: loss value and metric value
"""
test_loss, test_acc = self.model.evaluate(X, y, verbose=2)
return (test_loss, test_acc)
def train(self, X_train, y_train, batch_size, num_epochs):
"""
Given a batch of data, and the necessary hyperparameters,
this function trains the neural network by adjusting the weights and biases of all the layers.
:param X_train: Array of input
:param y_train: Array of desired (target) outputs
:param X_validation: Array of input validation data
:param y: Array of desired (target) validation outputs
:param batch_size: number of samples in a batch
:param num_epochs: Number of times training should be repeated over all input data
:return: list of loss values. Each element of the list should be the value of loss after each epoch.
"""
self.model.compile(optimizer= self.optimizer, loss = self.loss_function, metrics = self.metric)
seqModel = self.model.fit(x=X_train, y=y_train, batch_size=batch_size, epochs = num_epochs)
return seqModel.history['loss']
Conv2D(filters=1, kernel_size=(3, 3), activation='sigmoid',
padding='same'))
autoencoder.summary()
autoencoder.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
autoencoder.fit(previsoresTreinamento,
previsoresTreinamento,
epochs=100,
batch_size=256,
validation_data=(previsoresTeste, previsoresTeste))
# Flatten 128
encoder = Model(inputs=autoencoder.input,
outputs=autoencoder.get_layer('flatten_6').output)
encoder.summary()
imagensCodificadas = encoder.predict(previsoresTeste)
imagensDecodificadas = autoencoder.predict(previsoresTeste)
numeroImagens = 10
imagensTeste = np.random.randint(previsoresTeste.shape[0], size=numeroImagens)
plt.figure(figsize=(18, 18))
for i, indiceImagem in enumerate(imagensTeste):
# imagem original
eixo = plt.subplot(10, 10, i + 1)
plt.imshow(previsoresTeste[indiceImagem].reshape(28, 28))
plt.xticks(())
plt.yticks(())
class FreezeUnfreeze:
"""
Implements a bottleneck neural network with Keras that can use pre-trained weights and first freezes certain layers before training
all layers.
"""
def __init__(self, l1, l2, lr, act, input_dim, output_dim, unfreeze, pre_trained_weights = False, pre_trained_weights_h5 = None, \
nodes_list=[512, 128, 2, 128, 512]):
"""
Constructor.
:param l1: int, lasso penalty
:param l2: int, ridge penalty
:param lr: int, learning rate for Adam
:param act: string, activation function
:param input_dim: int, input layer dimensionality
:param output_dim: int, output layer dimensionality
:param unfreeze: list of bools, if element in list is True that index corresponds to a layer that can be trained
:param pre_trained_weights: bool, if True we have pre-trained weights we could use for intialisation
(optional, default=False)
:param pre_trained_weights_h5: .h5 file, pre_trained weights to use as intial weights for training deep regression
(optional, default=None)
:param nodes_list: list, integers denoting # nodes in each layer (optional, default=[512, 128, 2, 128, 512])
"""
self.l1=l1
self.l2=l2
self.lr=lr
self.act=act
self.input_dim=input_dim
self.output_dim=output_dim
self.unfreeze = unfreeze
self.pre_trained_weights = pre_trained_weights
self.nodes_list=nodes_list
# Architecture
keras.backend.clear_session()
self.m = Sequential()
self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(self.input_dim, ), \
kernel_regularizer=ElasticNet(l1=self.l1, l2=self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
trainable = self.unfreeze[0], name = 'W1'))
self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
trainable = self.unfreeze[1], name = 'W2'))
# linear for latent space representation
self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2),
trainable = self.unfreeze[2], name='bottleneck'))
self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
trainable = self.unfreeze[3], name = 'W4'))
self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
trainable = self.unfreeze[4], name = 'W5'))
self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
trainable = self.unfreeze[5], \
#bias_regularizer=regularizers.l2(l2_parameter), \
name = 'W6_regr'))
# Load weights from training a previous network on classification
if self.pre_trained_weights:
self.m.load_weights(pre_trained_weights_h5, by_name=True)
self.m.compile(loss='mean_squared_error', \
optimizer=keras.optimizers.Adam(learning_rate=lr), \
metrics=[r2_score, 'mse'])
def train(self, x_train, y_train, x_test, y_test, epochs, bs, patience, cvfold_id=0, l1_id=0, l2_id=0, verbose=1, \
prune=False, geneNames=None, citeseq=False, report_individual_ephys_feature_test_R2=False):
"""
Train the bottleneck. If you don't prune the training lasts for 2*epochs iterations (freezing+unfreezing). If you prune the
training lasts for 4*epochs (freezing (epochs) + unfreezing (epochs) + pruning (2*epochs)) iterations in total.
Parameters
----------
:param x_train, y_train: numpy 2D matrices, input and output training set
:param x_test, y_test: numpy 2D matrices, input and output validation set
:param epochs: int, # of training iterations
:param bs: int, batch size
:param patience: int, early stopping
:param cvfold_id: int, cross validation set id (optional, default=0), for saving
:param l1_id: int, lasso penalty id (optional, default=0), for saving
:param l2_id: int, ridge penalty id (optional, default=0), for saving
:param verbose: int, print info or not (optional, default=1, i.e. print info -- verbose=0 means not printing info)
:param prune: bool, if True we additionally prune the network (new input layer with lower dimensionality)
(optional, default=False)
:param geneNames: numpy 1D array, contains name of the corresponding gene of every input neuron (optional, default=None)
:param citeseq: bool, if True we're performing the training procedure on a CITE-seq dataset and will use the best epoch
during unfreezing for use afterwards when we prune (optional, default=False)
:param report_individual_ephys_feature_test_R2: bool, if True save test R^2 scores for every individual ephys feature
Returns
-------
Training and validation loss R^2 score for all epochs
"""
# Settings for early stopping and saving best model
es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=patience)
if not self.pre_trained_weights:
mc = ModelCheckpoint('KerasSavedModels/FreezeUnfreeze_weights_before_unfreezing_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
else:
mc = ModelCheckpoint('KerasSavedModels/PreTrFreezeUnfreeze_weights_before_unfreezing_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
# train the network
print("[INFO] training network...")
H = self.m.fit(x_train, y_train, batch_size=bs,
validation_data=(x_test, y_test),
epochs=epochs, verbose=verbose, callbacks = [es, mc])
train_loss_freeze_unfreeze = np.array(H.history["r2_score"])
val_loss_freeze_unfreeze = np.array(H.history["val_r2_score"])
# Now UNFREEZE all layers
for layer in self.m.layers:
layer.trainable = True
if verbose!=0:
for layer in self.m.layers:
print(layer, 'trainable?', layer.trainable)
# Since we’ve unfrozen additional layers, we must re-compile the model and let us decrease the learning rate by a half
self.m.compile(loss='mean_squared_error', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse'])
# Settings for early stopping and saving best model
es = EarlyStopping(monitor='val_mse', mode='min', verbose=1, patience=patience)
if not self.pre_trained_weights:
mc = ModelCheckpoint('KerasSavedModels/FreezeUnfreeze_weights_after_unfreezing_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
else:
mc = ModelCheckpoint('KerasSavedModels/PreTrFreezeUnfreeze_weights_after_unfreezing_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
# train the network again
print("[INFO] training network...")
H = self.m.fit(x_train, y_train, batch_size=bs,
validation_data=(x_test, y_test),
epochs=epochs, verbose=verbose, callbacks = [es, mc])
train_loss_freeze_unfreeze = np.concatenate([train_loss_freeze_unfreeze, np.array(H.history["r2_score"])])
val_loss_freeze_unfreeze = np.concatenate([val_loss_freeze_unfreeze, np.array(H.history["val_r2_score"])])
# Retrieve activations and ephys prediction
if not self.pre_trained_weights:
saved_model = load_model('KerasSavedModels/FreezeUnfreeze_weights_before_unfreezing_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
saved_model_2 = load_model('KerasSavedModels/FreezeUnfreeze_weights_after_unfreezing_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
else:
saved_model = load_model('KerasSavedModels/PreTrFreezeUnfreeze_weights_before_unfreezing_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
saved_model_2 = load_model('KerasSavedModels/PreTrFreezeUnfreeze_weights_after_unfreezing_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
r2_before_unfreezing_train = 1-np.sum((y_train - saved_model.predict(x_train))**2) / np.sum(y_train**2)
r2_before_unfreezing_test = 1-np.sum((y_test - saved_model.predict(x_test))**2) / np.sum(y_test**2)
r2_after_unfreezing_train = 1-np.sum((y_train - saved_model_2.predict(x_train))**2) / np.sum(y_train**2)
r2_after_unfreezing_test = 1-np.sum((y_test - saved_model_2.predict(x_test))**2) / np.sum(y_test**2)
self.m.save('KerasSavedModels/FreezeUnfreeze_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id))
if prune:
saved_model_ = load_model('KerasSavedModels/FreezeUnfreeze_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
weight = saved_model_.get_weights()[0]
bias = saved_model_.get_weights()[1]
ind_genes = np.argsort(np.linalg.norm(weight, ord=2, axis=1))[-25:]
print('The 25 genes that make it: ', geneNames[ind_genes])
# Architecture
keras.backend.clear_session()
self.m = Sequential()
self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(25, ), \
kernel_regularizer=ElasticNet(l1=0, l2=self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W1_25'))
self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W2'))
# linear for latent space representation
self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2),
name='bottleneck'))
self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W4'))
self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W5'))
self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
#bias_regularizer=regularizers.l2(l2_parameter), \
name = 'W6_regr'))
# Load weights from training a previous network on regression
self.m.load_weights('KerasSavedModels/FreezeUnfreeze_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), by_name=True)
# We transfer the weights of those 25 genes now manually too
self.m.get_layer('W1_25').set_weights([weight[ind_genes, :], bias])
self.m.compile(loss='mse', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse'])
es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=2*patience)
if not self.pre_trained_weights:
mc = ModelCheckpoint('KerasSavedModels/FreezeUnfreeze_weights_after_unfreezing_and_pruning_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
else:
mc = ModelCheckpoint('KerasSavedModels/PreTrFreezeUnfreeze_weights_after_unfreezing_and_pruning_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
# train the network
print("[INFO] training network...")
H = self.m.fit(x_train[:, ind_genes], y_train, batch_size=bs,
validation_data=(x_test[:, ind_genes], y_test),
epochs=2*epochs, verbose=verbose, callbacks = [es, mc])
train_loss_freeze_unfreeze = np.concatenate([train_loss_freeze_unfreeze, np.array(H.history["r2_score"])])
val_loss_freeze_unfreeze = np.concatenate([val_loss_freeze_unfreeze, np.array(H.history["val_r2_score"])])
if prune:
if not self.pre_trained_weights:
saved_model_3 = load_model('KerasSavedModels/FreezeUnfreeze_weights_after_unfreezing_and_pruning_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
else:
saved_model_3 = load_model('KerasSavedModels/PreTrFreezeUnfreeze_weights_after_unfreezing_and_pruning_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
r2_after_unfreezing_and_pruning_train = 1-np.sum((y_train - saved_model_3.predict(x_train[:, ind_genes]))**2) \
/ np.sum(y_train**2)
r2_after_unfreezing_and_pruning_test = 1-np.sum((y_test - saved_model_3.predict(x_test[:, ind_genes]))**2) \
/ np.sum(y_test**2)
if report_individual_ephys_feature_test_R2:
if not citeseq:
if self.nodes_list[2]==2:
np.savez('KerasSavedModels/individual_ephys_feature_test_R2_{}_{}_{}.npz'.format(cvfold_id, l1_id, l2_id), \
R2=np.array([1-np.sum((y_test[:,i] - saved_model_3.predict(x_test[:, ind_genes])[:,i])**2) \
/ np.sum(y_test[:,i]**2) for i in range(y_test.shape[1])])
)
else:
np.savez('KerasSavedModels/individual_ephys_feature_test_R2_{}_{}_{}_nb.npz'.format(cvfold_id, l1_id, l2_id), \
R2=np.array([1-np.sum((y_test[:,i] - saved_model_3.predict(x_test[:, ind_genes])[:,i])**2) \
/ np.sum(y_test[:,i]**2) for i in range(y_test.shape[1])])
)
else:
if self.nodes_list[2]==2:
np.savez('KerasSavedModels/individual_ephys_feature_test_R2_{}_{}_{}_citeseq.npz'.format(cvfold_id, l1_id, l2_id), \
R2=np.array([1-np.sum((y_test[:,i] - saved_model_3.predict(x_test[:, ind_genes])[:,i])**2) \
/ np.sum(y_test[:,i]**2) for i in range(y_test.shape[1])])
)
else:
np.savez('KerasSavedModels/individual_ephys_feature_test_R2_{}_{}_{}_citeseq_nb.npz'.format(cvfold_id, l1_id, \
l2_id), \
R2=np.array([1-np.sum((y_test[:,i] - saved_model_3.predict(x_test[:, ind_genes])[:,i])**2) \
/ np.sum(y_test[:,i]**2) for i in range(y_test.shape[1])])
)
print('Train R^2 before unfreezing: ', r2_before_unfreezing_train)
print('Test R^2 before unfreezing: ', r2_before_unfreezing_test)
print('Train R^2 after unfreezing: ', r2_after_unfreezing_train)
print('Test R^2 after unfreezing: ', r2_after_unfreezing_test)
if prune:
print('Train R^2 after unfreezing and pruning: ', r2_after_unfreezing_and_pruning_train)
print('Test R^2 after unfreezing and pruning: ', r2_after_unfreezing_and_pruning_test)
if not prune:
return r2_before_unfreezing_train, r2_before_unfreezing_test, \
r2_after_unfreezing_train, r2_after_unfreezing_test, \
train_loss_freeze_unfreeze, val_loss_freeze_unfreeze
else:
return r2_before_unfreezing_train, r2_before_unfreezing_test, \
r2_after_unfreezing_train, r2_after_unfreezing_test, \
r2_after_unfreezing_and_pruning_train, r2_after_unfreezing_and_pruning_test, \
train_loss_freeze_unfreeze, val_loss_freeze_unfreeze
def train_full_dataset(self, x_train, y_train, epochs, bs, patience, cvfold_id=0, l1_id=0, l2_id=0, verbose=1, \
prune=False, geneNames=None, add_autoencoder=False, output_name=None):
"""
Train the bottleneck for the full dataset (no validation). If you don't prune the training lasts for 2*epochs iterations
(freezing+unfreezing). If you prune the training lasts for 4*epochs (freezing (epochs) + unfreezing (epochs) + pruning (2*epochs))
iterations in total.
Parameters
----------
:param x_train, y_train: numpy 2D matrices, input and output training set
:param epochs: int, # of training iterations
:param bs: int, batch size
:param patience: int, early stopping
:param cvfold_id: int, cross validation set id (optional, default=0)
:param l1_id: int, lasso penalty id (optional, default=0)
:param l2_id: int, ridge penalty id (optional, default=0)
:param verbose: int, print info or not (optional, default=1, i.e. print info -- verbose=0 means not printing info)
:param prune: bool, if True we additionally prune the network (new input layer with lower dimensionality)
(optional, default=False)
:param geneNames: numpy 1D array, contains name of the corresponding gene of every input neuron (optional, default=None)
:param add_autoencoder: bool, if True we add an autoencoder to selected genes from the bottleneck layer
:param output_name: string, if provided we save file with the string in it (optional, default = None)
Returns
-------
Training loss R^2 score for all epochs
"""
# train the network
print("[INFO] training network...")
H = self.m.fit(x_train, y_train, batch_size=bs,
epochs=epochs, verbose=verbose)
train_loss_freeze_unfreeze = np.array(H.history["r2_score"])
# Now UNFREEZE all layers
for layer in self.m.layers:
layer.trainable = True
if verbose!=0:
for layer in self.m.layers:
print(layer, 'trainable?', layer.trainable)
# Since we’ve unfrozen additional layers, we must re-compile the model and let us decrease the learning rate by a half
self.m.compile(loss='mean_squared_error', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse'])
if output_name is None:
self.m.save('KerasSavedModels/FreezeUnfreeze_before_unfreezing_full_dataset_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id))
else:
self.m.save('KerasSavedModels/{}/FreezeUnfreeze_before_unfreezing_full_dataset_{}_{}_{}.h5'.format\
(output_name, cvfold_id, l1_id, l2_id))
# train the network again
print("[INFO] training ephys prediction network...")
H = self.m.fit(x_train, y_train, batch_size=bs,
epochs=epochs, verbose=verbose)
train_loss_freeze_unfreeze = np.concatenate([train_loss_freeze_unfreeze, np.array(H.history["r2_score"])])
if output_name is None:
self.m.save('KerasSavedModels/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id))
else:
self.m.save('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.format\
(output_name, cvfold_id, l1_id, l2_id))
if prune:
if output_name is None:
saved_model_ = load_model('KerasSavedModels/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
else:
saved_model_ = load_model('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.\
format(output_name, cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
weight = saved_model_.get_weights()[0]
bias = saved_model_.get_weights()[1]
ind_genes = np.argsort(np.linalg.norm(weight, ord=2, axis=1))[-25:]
print('The 25 genes that make it: ', geneNames[ind_genes])
# Architecture
keras.backend.clear_session()
self.m = Sequential()
self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(25, ), \
kernel_regularizer=ElasticNet(l1=0, l2=self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W1_25'))
self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W2'))
# linear for latent space representation
self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2),
name='bottleneck'))
self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W4'))
self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W5'))
self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
#bias_regularizer=regularizers.l2(l2_parameter), \
name = 'W6_regr'))
# Load weights from training a previous network on regression
if output_name is None:
self.m.load_weights('KerasSavedModels/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), by_name=True)
else:
self.m.load_weights('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_full_dataset_{}_{}_{}.h5'.\
format(output_name, cvfold_id, l1_id, l2_id), by_name=True)
# We transfer the weights of those 25 genes now manually too
self.m.get_layer('W1_25').set_weights([weight[ind_genes, :], bias])
self.m.compile(loss='mse', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse'])
# train the network
print("[INFO] training pruning network...")
H=self.m.fit(x_train[:, ind_genes], y_train, batch_size=bs,
epochs=2*epochs, verbose=verbose)
if output_name is None:
self.m.save('KerasSavedModels/FreezeUnfreeze_after_unfreezing_ap_full_dataset_{}_{}_{}.h5'.format\
(cvfold_id, l1_id, l2_id))
else:
self.m.save('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_ap_full_dataset_{}_{}_{}.h5'.format\
(output_name, cvfold_id, l1_id, l2_id))
train_loss_freeze_unfreeze=np.concatenate([train_loss_freeze_unfreeze, np.array(H.history["r2_score"])])
if add_autoencoder:
# The latent space is the same, but instead of predicting ephys we want to predict selected genes. This leads us to
# latent space visualisations that can be overlayed with gene model predictions.
# Retrieve bottleneck activations
if output_name is None:
saved_model_AE = load_model('KerasSavedModels/FreezeUnfreeze_after_unfreezing_ap_full_dataset_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
else:
saved_model_AE = load_model('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_ap_full_dataset_{}_{}_{}.h5'.\
format(output_name, cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
encoder = Model(saved_model_AE.input, saved_model_AE.get_layer('bottleneck').output)
latent = encoder.predict(x_train[:, ind_genes])
# Retrieve weights
W4=saved_model_AE.get_weights()[6]
W4_bias=saved_model_AE.get_weights()[7]
W5=saved_model_AE.get_weights()[8]
W5_bias=saved_model_AE.get_weights()[9]
# Architecture
keras.backend.clear_session()
self.m = Sequential()
self.m.add(Dense(self.nodes_list[3], activation=self.act, input_shape=(self.nodes_list[2], ), \
kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name='W4'))
self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name='W5'))
self.m.add(Dense(25, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
#bias_regularizer=regularizers.l2(self.l2), \
name='W6_regr_AE'))
# Set initial weights
self.m.get_layer('W4').set_weights([W4, W4_bias])
self.m.get_layer('W5').set_weights([W5, W5_bias])
self.m.compile(loss='mse', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse'])
# train the network
print("[INFO] training autoencoder network")
H=self.m.fit(latent, x_train[:,ind_genes], batch_size=bs,
epochs=2*epochs, verbose=verbose)
if output_name is None:
self.m.save('KerasSavedModels/FreezeUnfreeze_after_unfreezing_ap_full_dataset_AE_{}_{}_{}.h5'\
.format(cvfold_id, l1_id, l2_id))
else:
self.m.save('KerasSavedModels/{}/FreezeUnfreeze_after_unfreezing_ap_full_dataset_AE_{}_{}_{}.h5'\
.format(output_name, cvfold_id, l1_id, l2_id))
train_loss_freeze_unfreeze_AE=np.array(H.history["r2_score"])
if not add_autoencoder:
return train_loss_freeze_unfreeze
else: return train_loss_freeze_unfreeze, train_loss_freeze_unfreeze_AE
def createModel(self):
self.model_instance += 1
clear_session()
features, label = self.getDataset()
X_train, y_train = self.createLag(features, label)
X_train = X_train[:, self.lags]
learning_rate = float(self.hyperparameters["Learning_Rate"].get())
momentum = float(self.hyperparameters["Momentum"].get())
optimizers = {
"Adam": Adam(learning_rate=learning_rate),
"SGD": SGD(learning_rate=learning_rate, momentum=momentum),
"RMSprop": RMSprop(learning_rate=learning_rate, momentum=momentum)
}
shape = (X_train.shape[1], X_train.shape[2])
model_choice = self.model_var.get()
if not self.do_optimization:
model = Sequential()
model.add(Input(shape=shape))
if model_choice == 0:
model.add(Flatten())
layers = self.no_optimization_choice_var.get()
for i in range(layers):
neuron_number = self.neuron_numbers_var[i].get()
activation_function = self.activation_var[i].get()
if model_choice == 0:
model.add(Dense(neuron_number, activation=activation_function, kernel_initializer=GlorotUniform(seed=0)))
elif model_choice == 1:
model.add(Conv1D(filters=neuron_number, kernel_size=2, activation=activation_function, kernel_initializer=GlorotUniform(seed=0)))
model.add(MaxPooling1D(pool_size=2))
elif model_choice == 2:
if i == layers-1:
model.add(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
elif model_choice == 3:
if i == layers-1:
model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))))
model.add(Dropout(0.2))
else:
model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))))
model.add(Dropout(0.2))
elif model_choice == 4:
if i == layers-1:
model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
elif model_choice == 5:
if i == layers-1:
model.add(GRU(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(GRU(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
if model_choice == 1:
model.add(Flatten())
model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1, activation=self.output_activation.get(), kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1, shuffle=False)
loss = history.history["loss"][-1]
self.train_loss.set(loss)
elif self.do_optimization:
layer = self.optimization_choice_var.get()
if model_choice == 0:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
model.add(Flatten())
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(Dense(units=hp.Int('MLP_'+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu'))
model.add(Dense(1))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". MLP"
elif model_choice == 1:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max-n_min)/4)
model.add(Conv1D(filters=hp.Int("CNN_"+str(i), min_value=n_min, max_value=n_max, step=step), kernel_size=2, activation="relu", kernel_initializer=GlorotUniform(seed=0)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". CNN"
elif model_choice == 2:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0)))
if i == layer-1:
model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". LSTM"
elif model_choice == 3:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0))))
if i == layer-1:
model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0))))
model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". Bi-LSTM"
tuner = RandomSearch(build_model, objective='loss', max_trials=25, executions_per_trial=2, directory=self.runtime, project_name=name)
tuner.search(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get())
hps = tuner.get_best_hyperparameters(num_trials = 1)[0]
model = tuner.hypermodel.build(hps)
history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1)
loss = history.history["loss"][-1]
self.train_loss.set(loss)
for i in range(layer):
if model_choice == 0:
self.best_model_neurons[i].set(model.get_layer(index=i+1).get_config()["units"])
elif model_choice == 1:
self.best_model_neurons[i].set(model.get_layer(index=(2*i)).get_config()["filters"])
elif model_choice == 2:
self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["units"])
elif model_choice == 3:
self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["layer"]["config"]["units"])
model.summary()
self.model = model
BACKENDS = ['tf', 'pt']
FILE_FORMATS = ['hdf5', 'npy', 'mat', 'txt']
DISTANCES = ['correlation', 'cosine', 'euclidean', 'gaussian']
BATCH_SIZE = 16
NUM_OBJECTS = 1854
# we want to iterate over two batches to exhaustively test mini-batching
NUM_SAMPLES = int(BATCH_SIZE * 2)
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
tf_model = Sequential()
tf_model.add(
Dense(2, input_dim=1, activation='relu', use_bias=False, name='relu'))
weights = np.array([[[1, 1]]])
tf_model.get_layer('relu').set_weights(weights)
class NN(nn.Module):
def __init__(self, in_size: int, out_size: int):
super(NN, self).__init__()
self.linear = nn.Linear(in_size, out_size, bias=False)
self.relu = nn.ReLU()
# exchange weight value with 1.
self.linear.weight = nn.Parameter(torch.tensor([[1.], [1.]]))
def forward(self, x):
x = self.linear(x)
act = self.relu(x)
return act
autoencoder = Sequential()
autoencoder.add(Dense(512, activation='elu', input_shape=(784, )))
autoencoder.add(Dense(128, activation='elu'))
autoencoder.add(Dense(10, activation='linear', name="bottleneck"))
autoencoder.add(Dense(128, activation='elu'))
autoencoder.add(Dense(512, activation='elu'))
autoencoder.add(Dense(784, activation='sigmoid'))
autoencoder.compile(loss='mean_squared_error', optimizer=Adam())
trained_model = autoencoder.fit(X_train,
X_train,
batch_size=1024,
epochs=10,
verbose=1,
validation_data=(val_x, val_x))
encoder = Model(autoencoder.input, autoencoder.get_layer('bottleneck').output)
encoded_data = encoder.predict(X_train) # bottleneck representation
decoded_output = autoencoder.predict(X_train) # reconstruction
encoding_dim = 10
print("Encoded Image: {}".format(
encoder.predict(X_train[0].reshape(1, -1))[0]))
encoded_image = encoder.predict(X_train[0].reshape(1, -1))[0]
# return the decoder
encoded_input = Input(shape=(encoding_dim, ))
decoder = autoencoder.layers[-3](encoded_input)
decoder = autoencoder.layers[-2](decoder)
decoder = autoencoder.layers[-1](decoder)
decoder = Model(encoded_input, decoder)
VGGFace16.add(Flatten(name='flatten'))
VGGFace16.add(layers.Dense(units=512, name='fc6'))
VGGFace16.add(Activation(activation='relu', name='fc6/relu'))
VGGFace16.add(layers.Dense(units=512, name='fc7'))
VGGFace16.add(Activation(activation='relu', name='fc7/relu'))
VGGFace16.add(layers.Dense(8, name='fc8'))
VGGFace16.add(Activation(activation='softmax', name='fc8/softmax'))
# load the pretrained weights
VGGFace16.load_weights(
'/home/ubuntu/Notebooks/Models/AffectNet_VGG16_v3-1.h5', by_name=True)
# construct new classification layers
nb_class = 8
hidden_dim = 512
last_layer = VGGFace16.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = layers.Dense(hidden_dim,
activation='relu',
name='fc6',
kernel_regularizer=l2(5 * 1e-4))(x)
x = layers.Dense(hidden_dim,
activation='relu',
name='fc7',
kernel_regularizer=l2(5 * 1e-4))(x)
out = layers.Dense(nb_class, activation='softmax', name='fc8')(x)
FER_VGGFace16 = tensorflow.keras.Model(VGGFace16.input, out)
# compile the model
# initial_learning_rate = 1e-3
import matplotlib.pyplot as plt
# 指定亂數種子
seed = 7
np.random.seed(seed)
# 載入資料集
(X_train, Y_train), (_, _) = mnist.load_data()
# 將圖片轉換成 4D 張量
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1).astype("float32")
# 因為是固定範圍, 所以執行正規化, 從 0-255 至 0-1
X_train = X_train / 255
# 建立Keras的Sequential模型
model = Sequential()
model = load_model("mnist.h5")
model.summary() # 顯示模型摘要資訊
# 編譯模型
model.compile(loss="categorical_crossentropy", optimizer="adam",
metrics=["accuracy"])
# 使用 Model 建立 Conv2D 層
from tensorflow.keras.models import Model
layer_name = "conv2d_1"
model_test = Model(inputs=model.input,
outputs=model.get_layer(layer_name).output)
output = model_test.predict(X_train[0].reshape(1,28,28,1))
# 繪出第1個 Conv2D 層的輸出
plt.figure(figsize=(10,8))
for i in range(0,16):
plt.subplot(4,4,i+1)
plt.imshow(output[0,:,:,i], cmap="gray")
plt.axis("off")
ae.add(Dense(2, activation='linear', name="bottleneck"))
ae.add(Dense(128, activation='relu'))
ae.add(Dense(512, activation='relu'))
ae.add(Dense(784, activation='sigmoid'))
ae.compile(loss='mean_squared_error', optimizer=Adam())
def explained_variance(projections):
var = np.array([(projections[:,i]**2).sum()/(n_sample-1) for i in range(k)]).round(2)
return var
def fit():
history = ae.fit(x_train, x_train, batch_size=128, epochs=10,
verbose=1, validation_data=(x_test, x_test))
fit()
encoder = Model(ae.input, ae.get_layer('bottleneck').output)
Zenc = encoder.predict(x_train)
print("Pca explained variance:", explained_variance(Zpca))
print("ae explained variance:", explained_variance(Zenc))
# PLOT
plt.figure(figsize=(8, 4))
plt.subplot(121)
plt.title('PCA')
plt.scatter(Zpca[:5000, 0], Zpca[:5000, 1],
c=y_train[:5000], s=8, cmap='tab10')
class Gan:
def __init__(self,folder,loadFile):
self.folder=folder
self.modelFolder='models'
self.loadModelNumb=-1
self.modelName='model'
self.loadModel=False
self.noiseNo=100
self.loadFile=loadFile
self.picFolder='pics'
self.picName='pic%d.png'
self.x=pickle.load(open(loadFile+'.pickle','rb'))
self.x=self.x/255.0#normalise
self.imgDim=self.x.shape[1]#width and height
self.imgDepth=self.x.shape[3]#depth 1 or black and white, 3 for colour
self.D = Sequential()
self.G = Sequential()
self.AM = Sequential()
def descriminator(self,depth,dropout,alpha,window,stridedLayers,unstridedLayers):
#dropout ignore units during training prevent over fitting
#alpha leaked negative gradient size
strides=2
input_shape = (self.imgDim, self.imgDim, self.imgDepth)
self.D.add(Conv2D(depth*1, window, strides=strides, input_shape=input_shape,padding='same'))
self.D.add(LeakyReLU(alpha=alpha))
self.D.add(Dropout(dropout))
for i in range(stridedLayers-1+unstridedLayers):
depth=depth*2
self.D.add(Conv2D(depth, window, strides=strides, padding='same'))
self.D.add(LeakyReLU(alpha=alpha))
self.D.add(Dropout(dropout))
if(i+2==stridedLayers):
strides-=1
self.D.add(Flatten())
self.D.add(Dense(1))#real or not
self.D.add(Activation('sigmoid'))
def gen(self,depth,dim,momentum,dropout,window,upSampledLayers,additionalLayers):
self.G.add(Dense(dim*dim*depth, input_dim=self.noiseNo))
self.G.add(BatchNormalization(momentum=momentum))#stablise learing reduce weight shifting in hidden by reducing by mean
self.G.add(Activation('relu'))
self.G.add(Reshape((dim, dim, depth)))#7x7x256
self.G.add(Dropout(dropout))
for i in range(upSampledLayers+additionalLayers):
depth=int(depth/2)
if(i<upSampledLayers):
self.G.add(UpSampling2D())#repeats rows and columns e.g 32x32 becomes 64x64
self.G.add(Conv2DTranspose(depth, window, padding='same'))#reduce depth to 128
self.G.add(BatchNormalization(momentum=momentum))
self.G.add(Activation('relu'))
self.G.add(Conv2DTranspose(self.imgDepth, window, padding='same'))
self.G.add(Activation('sigmoid'))
def combine(self,dlr,dcv,ddcy,alr,acv,adcy):
optimizer = RMSprop(lr=dlr, clipvalue=dcv, decay=ddcy)
self.D.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
self.D.trainable=False#freeze weights in descriminator so gen weights update instead
optimizer = RMSprop(lr=alr, clipvalue=acv, decay=adcy)
self.AM.add(self.G)#gen then input to discrim
self.AM.add(self.D)
self.AM.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
self.createDir()
self.writeParams()
ws=self.AM.get_layer("",1).get_weights()[0][:,:,0,:]
for i in range(1,26):
plt.subplot(5,5,i)
plt.imshow(ws[:,:,i],interpolation="nearest")
plt.show()
def saveImage(self, numb):
filename = self.folder+'/'+self.picFolder+'/'+self.picName % numb
images = self.G.predict(noise)
image=images[0]
plt.figure(figsize=(10,10))
if(self.imgDepth==3):
image = np.reshape(image, [self.imgDim , self.imgDim,self.imgDepth])
plt.imshow(image)
else:
image = np.reshape(image, [self.imgDim , self.imgDim])
plt.imshow(image, cmap='gray')
plt.axis('off')
plt.tight_layout()
plt.savefig(filename)
plt.close('all')
def showConv(self):
ws=self.AM.get_layer("",2).get_weights()[0][:,:,0,:]
for i in range(1,26):
plt.subplot(5,5,i)
plt.imshow(ws[:,:,i],interpolation="nearest")
plt.show()
ws=self.AM.get_layer("",1).get_weights()[0][:,:,0,:]
for i in range(1,26):
plt.subplot(5,5,i)
plt.imshow(ws[:,:,i],interpolation="nearest")
plt.show()
def loadModel(self,loadModelNumb):
try:
self.loadModel=True
self.D=self.load_model(modelLoc(loadModelNumb,'descriminator'))
self.G=self.load_model(modelLoc(loadModelNumb,'generator'))
self.AM=self.load_model(modelLoc(loadModelNumb,'Adversarial'))
except OSError as er:
print('Error loading file ',er)
self.createDir()
self.writeParams()
def train(self,batchSize,trainingSteps,saveImgInterval,saveModelInterval):
self.batchSize=batchSize
self.trainingSteps=trainingSteps
self.saveImgInterval=saveImgInterval
self.saveModelInterval=saveModelInterval
noise = np.random.uniform(-1.0, 1.0, size=[self.batchSize, self.noiseNo])#random noise of 100 units for each batch
for i in range(self.trainingSteps):
if(self.loadModel):
i+=self.loadModelNumb+1
images_train = self.x[np.random.randint(0,self.x.shape[0],size=self.batchSize),:,:,:]#index random images of batch size
#noise = np.random.uniform(-1.0, 1.0, size=[self.batchSize, self.noiseNo])#random noise of 100 units for each batch
images_fake = self.G.predict(noise)#gen fake image
xs = np.concatenate((images_train, images_fake))#concat real then fake
y = np.ones([2*self.batchSize, 1])#first half 1 for real second 0 for fake
y[self.batchSize:, :] = 0
dLoss = self.D.train_on_batch(xs, y)#train on half real half fake images
y = np.ones([self.batchSize, 1])#all 1 so mislabled for real
#noise = np.random.uniform(-1.0, 1.0, size=[self.batchSize, self.noiseNo])#new noise as learned from old noise knows it's wrong, might help
aLoss = self.AM.train_on_batch(noise, y)
msg = '%d: [D loss: %f, acc: %f]' % (i, dLoss[0], dLoss[1])
msg = '%s [A loss: %f, acc: %f]' % (msg, aLoss[0], aLoss[1])
print(msg)
if (i % self.saveImgInterval==0):
self.showConv()
self.saveImage(noise, i)
if (i % self.saveModelInterval==0 and i !=0):
self.D.save(self.modelLoc(i,'descriminator'))
self.G.save(self.modelLoc(i,'generator'))
self.AM.save(self.modelLoc(i,'Adversarial'))
self.D.save(self.modelLoc(self.trainingSteps,'descriminator'))
self.G.save(self.modelLoc(self.trainingSteps,'generator'))
self.AM.save(self.modelLoc(self.trainingSteps,'Adversarial'))
def modelLoc(self,num,netType):
return self.folder+'/'+self.modelFolder+'/'+str(num)+self.modelName+'_'+netType+'.model'
def writeParams(self):
with open(self.folder+'/params.txt', 'w+') as f:
size=str(self.imgDim)+'X'+str(self.imgDim)+'X'+str(self.imgDepth)
#batchSteps='batch size: '+str(self.batchSize)+' '+'trainingSteps: '+str(self.trainingSteps)
f.write(self.folder+'\n'+'\n'+size+'\n')
with redirect_stdout(f):
self.D.summary()
with redirect_stdout(f):
self.G.summary()
def createDir(self):
if not os.path.exists(self.folder):
os.makedirs(self.folder)
if not os.path.exists(self.folder+'/'+self.picFolder):
os.makedirs(self.folder+'/'+self.picFolder)
if not os.path.exists(self.folder+'/'+self.modelFolder):
os.makedirs(self.folder+'/'+self.modelFolder)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
model.add(MaxPool2D())
model.add(Flatten())
model.add(Dense(units=4096,activation='relu'))
model.add(Dense(units=4096,activation='relu'))
model.add(Dense(units=2,activation='softmax'))
#train the fine -tuned vgg16 model
model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])
#fit our model
model.fit(x=train_batches,validation_data=valid_batches,epochs=20,verbose=2)
##### extract the fc2 layer ("dense6)
fc2_output = tf.keras.backend.function(model.input, model.get_layer('dense_6').output)
# put the images from train set to extract the features ( pleon ta flatten, fc2, and predict) exoun ekpaideutei me tis fwto moy
import cv2
images = [cv2.imread(file) for file in glob.glob("/content/drive/My Drive/trainn/train/Covid/*.png")]
y=[]
for i in range(1,200):
gray=cv2.resize(images[i],(224,224))
y.append(gray)
list_cov=tf.convert_to_tensor(y)
list11=fc2_output(list_cov)
pd_list_cov=pd.DataFrame(list11)
import cv2
images = [cv2.imread(file) for file in glob.glob("/content/drive/My Drive/trainn/train/Non-Covid/*.png")]
y1=[]
class ConvModel:
"""
Conv-2,4 and 6 model class with the functions needed for iterative pruning
"""
input_data_shape = 1
def __init__(self,
input_data_shape,
model_name='conv-6',
use_dropout=False):
# Hyper-params that can be changed
self.K = 10
self.learning_rate = 0.0003
self.batch_size = 60
self.input_data_shape = input_data_shape
self._use_dropout = use_dropout
assert model_name in ['conv-2', 'conv-4',
'conv-6'], "Use Conv-2, Conv-4 or Conv-6 model"
self._model_name = model_name
# All the layers with relevant weights for the pruning
self._weight_layers = []
# The percentages we should prune for each layer. Eg. { "conv-1": 0.2 ... }
self._prune_percentages = {}
# Initially build it with no pruning
self._build()
self._compile()
# The initial weights are stored the first time we build the model only,
# so we can reset it when we change the pruning
self._initial_weights = self._get_trainable_weights()
# Save last used masks so that it can be used when we reinitialize the layers
self._initial_random_weights = None
self._last_used_masks = None
def _build(self):
"""
Builds the model from scratch. If we have set the pruning percentages we will build the model with pruning.
"""
self._weight_layers = []
self._model = Sequential()
self._add_layer(
Conv2D(64, (3, 3),
name='conv-1',
padding='same',
activation='relu',
input_shape=self.input_data_shape,
kernel_initializer='glorot_normal'))
self._add_layer(
Conv2D(64, (3, 3),
name='conv-2',
padding='same',
activation='relu',
kernel_initializer='glorot_normal'))
self._add_layer(MaxPooling2D((2, 2)))
self._weight_layers.extend(['conv-1', 'conv-2'])
if self._model_name in ['conv-2', 'conv-4']:
self._add_layer(
Conv2D(128, (3, 3),
name='conv-3',
padding='same',
activation='relu',
kernel_initializer='glorot_normal'))
self._add_layer(
Conv2D(128, (3, 3),
name='conv-4',
padding='same',
activation='relu',
kernel_initializer='glorot_normal'))
self._add_layer(MaxPooling2D((2, 2)))
self._weight_layers.extend(['conv-3', 'conv-4'])
if self._model_name == 'conv-6':
self._add_layer(
Conv2D(256, (3, 3),
name='conv-5',
padding='same',
activation='relu',
kernel_initializer='glorot_normal'))
self._add_layer(
Conv2D(256, (3, 3),
name='conv-6',
padding='same',
activation='relu',
kernel_initializer='glorot_normal'))
self._add_layer(MaxPooling2D((2, 2)))
self._weight_layers.extend(['conv-5', 'conv-6'])
self._add_layer(Flatten())
if self._use_dropout: self._add_layer(Dropout(0.5))
self._add_layer(
Dense(256,
name='dense-1',
activation='relu',
kernel_initializer='glorot_normal'))
if self._use_dropout: self._add_layer(Dropout(0.5))
self._add_layer(
Dense(256,
name='dense-2',
activation='relu',
kernel_initializer='glorot_normal'))
if self._use_dropout: self._add_layer(Dropout(0.5))
self._add_layer(
Dense(self.K,
name='dense-3',
activation='softmax',
kernel_initializer='glorot_normal'))
self._weight_layers.extend(['dense-1', 'dense-2', 'dense-3'])
def _add_layer(self, layer):
"""
Helper for adding layer to the current model, also sets the pruning if we have a pruning percentage for the layer
:param layer: The layer we should add
"""
if self._prune_percentages is not None and layer.name in self._prune_percentages:
return self._model.add(
tfmot.sparsity.keras.prune_low_magnitude(
layer,
pruning_schedule=tfmot.sparsity.keras.ConstantSparsity(
self._prune_percentages[layer.name],
begin_step=0,
end_step=1,
frequency=1),
name=layer.name))
self._model.add(layer)
def _compile(self):
adam = optimizers.Adam(self.learning_rate)
self._model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
def predict(self, x):
"""
Make predictions
:param x: input data
:return: output
"""
return self._model.predict(x)
def evaluate(self, x, y):
"""
Evaluate a model on test data
:param x: input
:param y: labels
"""
eval = self._model.evaluate(x, y)
print("Accuracy: " + str(eval[1]) + ", Loss: " + str(eval[0]))
return eval
def train(self, x_train, y_train, iterations=30000, early_stopping=False):
"""
Train the network. If we want pruning the method set pruning should be called prior to this
:param x_train: training inputs
:param y_train: training labels
:param iterations: number of iterations to run, will be rounded to nearest epoch count
:param early_stopping: True if early stopping should be used
"""
callbacks = [tfmot.sparsity.keras.UpdatePruningStep()]
val_split = 0.0
if early_stopping:
early_stopping = tf.keras.callbacks.EarlyStopping(
monitor="val_loss", patience=3)
val_split = 0.2
callbacks.append(early_stopping)
epochs = int(iterations * self.batch_size / (x_train.shape[0] *
(1 - val_split)))
return self._model.fit(x_train,
y_train,
self.batch_size,
epochs,
callbacks=callbacks,
validation_split=val_split)
def set_pruning(self, prune_percentages=None):
"""
This method resets the network, applies pruning that will be used on next training round and re-initializes
the unpruned weights to their initial values.
:param prune_percentages: Dictionary of pruning values to use. Eg. { 'conv-1': 0.2, ... }
"""
# Create a mask based on the current values in the network
masks = {}
# Prunes the additional smallest weights apart from the ones in earlier steps. Uses the weights
# from the final step in the earlier training session
for name, weights in self._get_trainable_weights().items():
# Calculate how many weights we need to prune
prune_count = int(round(prune_percentages[name] * weights.size))
# Sort by the magnitude of the weights for the weights and extract the element value where
# we have gone past 'prune_count' nodes.
cutoff_value = np.sort(np.absolute(weights.ravel()))[prune_count]
# Update mask by disabling the nodes with magnitude smaller than/equal to our cutoff value
masks[name] = np.where(np.absolute(weights) > cutoff_value, 1, 0)
# Store last used mask globally
self._last_used_masks = masks
# Set new prune percentages and rebuild
self._prune_percentages = prune_percentages
self._build()
self._compile()
# Set initial weights and mask already pruned values.
# So when we start the training it will prune away the values that was pruned in the previous training round
# plus (p - p_previous) % of the lowest values where p is the current pruning amount and p_previous was the
# pruning percentage from the previous training session
for name in self._weight_layers:
layer = self._get_layer(name)
weights = layer.get_weights()
weights[0] = self._initial_weights[name] * masks[name]
layer.set_weights(weights)
def set_pruning_random_init(self, prune_percentages=None):
"""
This method resets the network, applies pruning that will be used on next training round and re-initializes
the unpruned weights to their initial values.
:param prune_percentages: Dictionary of pruning values to use. Eg. { 'conv-1': 0.2, ... }
"""
# Use the most recent masks
masks = self._last_used_masks
# Set new prune percentages and rebuild
self._prune_percentages = prune_percentages
self._build()
self._compile()
# Use the random weights
if self._initial_random_weights is None:
self._initial_random_weights = self._get_trainable_weights()
return
# Set weights and mask already pruned values.
# So when we start the training it will prune away the values that was pruned in the previous training round
# plus (p - p_previous) % of the lowest values where p is the current pruning amount and p_previous was the
# pruning percentage from the previous training session
for name in self._weight_layers:
layer = self._get_layer(name)
weights = layer.get_weights()
weights[0] = self._initial_random_weights[name] * masks[name]
layer.set_weights(weights)
def checkpoint_weights(self):
"""Create a check point so that we do not lose trained weights if needed for pruning"""
self.last_used_trainable = self._get_trainable_weights()
def reset_to_old_weights(self):
"""Reset the model to the weights that are in our check point"""
for name, weights_m in self.last_used_trainable.items():
layer = self._get_layer(name)
weights = layer.get_weights()
weights[0] = weights_m
layer.set_weights(weights)
pass
def _get_trainable_weights(self):
"""
Returns the current trainable weights in a dictionary.
Dict keys are layer names and dict values are the layers in numpy format.
Could be named like this: {'conv-1': w, ... }
"""
weights = {}
for l in self._weight_layers:
layer = self._get_layer(l)
weights[l] = layer.get_weights()[0]
return weights
def get_layer_names(self):
"""
Returns the relevant layer names in an array. This can be used to set pruning values
:return: Array, eg. ['conv-1', 'conv-2', ... 'dense-1', ...]
"""
return self._weight_layers
def _get_layer(self, name):
"""
Helper for getting a layer from the underlying model
:param name: String
:return: keras layer
"""
return self._model.get_layer((
"prune_low_magnitude_" +
name) if self._prune_percentages is not None and name in
self._prune_percentages else name)
good_prediction=lambda df: df.target.iloc[0],
).loc[lambda df: df.target].agg('mean'))).agg('mean'))]
scores = pd.DataFrame(scores)[[
'score', 'average_precision', 'good_prediction'
]]
plt.clf()
scores.boxplot()
plt.savefig(output_folder / 'scores_boxplot.png')
plt.clf()
scores.good_prediction.hist()
plt.savefig(output_folder / 'scores_good_predictions.png')
scores.to_csv(output_folder / 'scores.csv', index=False)
#%% Export classification model with SavedModel
model.load_weights(str(output_folder / 'kernel_loss_best_loss_weights.h5'))
classifier = Sequential([
siamese_nets.get_layer('branch_model'),
Classification(kernel=siamese_nets.get_layer('head_model')),
Activation('softmax'),
])
tf.saved_model.save(classifier, str(output_folder / 'saved_model/1/'))
#%% Example of use as classifier
classifier.get_layer('classification').set_support_set(
support_tensors=tf.convert_to_tensor(support_set_embeddings, tf.float32),
support_labels=tf.convert_to_tensor(
pd.get_dummies(support_set.label.values).values, tf.float32),
)
y = classifier.predict_generator(test_sequence, verbose=1)
class RNNModel:
def __init__(self, nt = 784, nin = 1, nh = 64, nout = 10,\
lr=0.001, mod_type='rnn', batch_size=64,\
is_complex=False, contractive=False):
"""
A RNN network model
nt: num time steps per sample
nin: num inputs per time step
nout: num outputs per time step
nh: num hidden units
lr: Learning rate
mod_type: 'rnn', 'urnn' or 'lstm'
batch_size: Batch size for the optimization
is_complex: If complex is to be performed
applies only to 'rnn' or 'urnn'.
"""
# Save dimensions
self.nt = nt
self.nin = nin
self.nh = nh
self.nout = nout
# Save parameters
self.batch_size = batch_size
self.mod_type = mod_type
self.lr = lr
self.is_complex = is_complex
self.contractive = contractive
# Create the model
self.create_model()
def create_model(self):
"""
Creates the model
"""
self.mod = Sequential()
# Add the RNN layer
# Note 1: For RNN, we set unroll=True to enable fast GPU usage
# Note 2: You need to set the CuDNN version of LSTM
unroll = tf.test.is_gpu_available()
if (self.mod_type == 'lstm'):
# LSTM model
if tf.test.is_gpu_available():
self.mod.add(CuDNNLSTM(self.nh, input_shape=(self.nt, self.nin),\
return_sequences=False, name='RNN'))
else:
self.mod.add(LSTM(self.nh, input_shape=(self.nt, self.nin),\
return_sequences=False, name='RNN',unroll=unroll))
elif self.is_complex:
# Complex RNN
cell = ComplexRNNCell(nh=self.nh)
self.mod.add(RNN(cell, input_shape=(self.nt, self.nin),\
return_sequences=False, name='RNN',unroll=True))
else:
# Real RNN model
self.mod.add(SimpleRNN(self.nh, input_shape=(self.nt, self.nin),\
return_sequences=False, name='RNN',activation='relu',unroll=unroll))
self.mod.add(Dense(nout, activation='softmax', name='Output'))
self.mod.summary()
def fit(self, Xtr, Ytr, Xts, Yts, nepochs=10):
"""
Fits the model parameters
"""
# Compile the model
#opt = Adam(lr=self.lr)
opt = RMSprop(lr=self.lr)
self.mod.compile(loss='sparse_categorical_crossentropy', optimizer=opt,\
metrics=['accuracy'])
# For URNN, add a callback that projects the weight matrix to unitary
if mod_type == 'urnn':
rnn_layer = self.mod.get_layer('RNN')
callbacks = [
ProjectCB(rnn_layer, unitary=True, is_complex=self.is_complex)
]
elif mod_type == 'rnn':
rnn_layer = self.mod.get_layer('RNN')
callbacks = [ProjectCB(rnn_layer,unitary=False,is_complex=self.is_complex,\
contractive=contractive)]
else:
callbacks = []
# Print progress
if self.mod_type == 'lstm':
cstr = ''
elif self.is_complex:
cstr = ' (complex)'
else:
cstr = ' (real)'
print('%s%s nh=%d' % (self.mod_type, cstr, self.nh))
# Fit the model
hist = self.mod.fit(Xtr,Ytr,epochs=nepochs, batch_size=self.batch_size,\
callbacks=callbacks,validation_data=(Xts,Yts))
self.tr_acc = hist.history['acc']
self.val_acc = hist.history['val_acc']
print("Expected: {:<25} Pred: {:<25} Difference: {:<25}".format(
test_stacked_df_eval[0, 0, i], yhat[0, 0, i], err))
totalError += err
print("Total Error: ", totalError)
# Testing
results = model.evaluate(
test_stacked_df,
test_stacked_df,
verbose=1,
steps=(num_of_test_files * len(file_list) // b_size + 1))
print(model.metrics_names)
print(results)
encoder_layer = Model(inputs=model.input,
outputs=model.get_layer("leaky_re_lu_2").output)
encoder_layer.trainable = False
predictions = encoder_layer.predict(stacked_df, verbose=0, steps=1)
print(predictions)
print(dy_train)
np.save("predictions.npy", predictions)
np.save("dy_train.npy", dy_train)
print("Data saved - predictions and train")
print(predictions.shape)
print(len(dy_train))
predictions = encoder_layer.predict(val_stacked_df, verbose=0, steps=1)
print(predictions)
print(dy_val)
np.save("val_predictions.npy", predictions)
class StraightRegression:
"""
Implements a bottleneck neural network for regression with keras that can use pre-trained weights.
"""
def __init__(self, l1, l2, lr, act, input_dim, output_dim, pre_trained_weights = False, pre_trained_weights_h5 = None, \
nodes_list=[512, 128, 2, 128, 512]):
"""
Constructor.
:param l1: int, lasso penalty
:param l2: int, ridge penalty
:param lr: int, learning rate for Adam
:param act: string, activation function
:param input_dim: int, input layer dimensionality
:param output_dim: int, output layer dimensionality
:param pre_trained_weights: bool, if True we have pre-trained weights we could use for intialisation
(optional, default=False)
:param pre_trained_weights_h5: .h5 file, pre_trained weights to use as intial weights for training deep regression
(optional, default=None)
:param nodes_list: list, integers denoting # nodes in each layer (optional, default=[512, 128, 2, 128, 512])
"""
self.l1=l1
self.l2=l2
self.lr=lr
self.act=act
self.input_dim=input_dim
self.output_dim=output_dim
self.pre_trained_weights = pre_trained_weights
self.nodes_list=nodes_list
# Architecture
keras.backend.clear_session()
self.m = Sequential()
self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(self.input_dim, ), \
kernel_regularizer=ElasticNet(l1=self.l1, l2=self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W1'))
self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W2'))
# linear for latent space representation
self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name='bottleneck'))
self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W4'))
self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W5'))
self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
#bias_regularizer=regularizers.l2(self.l2), \
name = 'W6_regr'))
if self.pre_trained_weights:
self.m.load_weights(pre_trained_weights_h5, by_name=True)
self.m.compile(loss='mean_squared_error', \
optimizer=keras.optimizers.Adam(learning_rate=lr), \
metrics=[r2_score, 'mse'])
def train(self, x_train, y_train, x_test, y_test, epochs, bs, patience, cvfold_id=0, l1_id=0, l2_id=0, verbose=1, \
prune=False, geneNames=None):
"""
Train the bottleneck. If you don't prune the training lasts for epochs iterations. If you prune the training lasts for 4*epochs
iterations in total (2*epochs normal training, 2*epochs pruning).
Parameters
----------
:param x_train, y_train: numpy 2D matrices, input and output training set
:param x_test, y_test: numpy 2D matrices, input and output validation set
:param epochs: int, # of training iterations
:param bs: int, bullshit, ah no, batch size
:param patience: int, early stopping
:param cvfold_id: int, cross validation set id (optional, default=0), for saving
:param l1_id: int, lasso penalty id (optional, default=0), for saving
:param l2_id: int, ridge penalty id (optional, default=0), for saving
:param verbose: int, print info or not (optional, default=1, i.e. print info -- verbose=0 means not printing info)
:param prune: bool, if True we additionally prune the network (new input layer with lower dimensionality)
(optional, default=False)
:param geneNames: numpy 1D array, contains name of the corresponding gene of every input neuron (optional, default=None)
Returns
-------
Training and validation loss R^2 score for all epochs
"""
# Settings for early stopping and saving best model
if not prune:
es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=patience)
else:
es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=2*patience)
if not self.pre_trained_weights:
mc = ModelCheckpoint('KerasSavedModels/StraightRegression_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
else:
mc = ModelCheckpoint('KerasSavedModels/PreTrRegression_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
# train the network
print("[INFO] training network...")
if not prune:
H = self.m.fit(x_train, y_train, batch_size=bs,
validation_data=(x_test, y_test),
epochs=epochs, verbose=verbose, callbacks = [es, mc])
else:
H = self.m.fit(x_train, y_train, batch_size=bs,
validation_data=(x_test, y_test),
epochs=2*epochs, verbose=verbose, callbacks = [es, mc])
train_loss_straight_regr = np.array(H.history["r2_score"])
val_loss_straight_regr = np.array(H.history["val_r2_score"])
MSE_tr = np.array(H.history["loss"])
MSE_val = np.array(H.history["val_loss"])
# Retrieve activations and ephys prediction
if not self.pre_trained_weights:
saved_model = load_model('KerasSavedModels/StraightRegression_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
else:
saved_model = load_model('KerasSavedModels/PreTrRegression_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
#print('predict: ', saved_model.predict(x_train))
#print('y_train: ', y_train)
#print('MSE: ', np.sum((y_train - saved_model.predict(x_train))**2))
#print('denominator: ', np.sum(y_train**2))
r2_train = 1-np.sum((y_train - saved_model.predict(x_train))**2) / np.sum(y_train**2)
r2_test = 1-np.sum((y_test - saved_model.predict(x_test))**2) / np.sum(y_test**2)
self.m.save('KerasSavedModels/Regression_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id))
if prune:
saved_model_ = load_model('KerasSavedModels/Regression_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
weight = saved_model_.get_weights()[0]
bias = saved_model_.get_weights()[1]
ind_genes = np.argsort(np.linalg.norm(weight, ord=2, axis=1))[-25:]
print('The 25 genes that make it: ', geneNames[ind_genes])
# Architecture
keras.backend.clear_session()
self.m = Sequential()
self.m.add(Dense(self.nodes_list[0], activation=self.act, input_shape=(25, ), \
kernel_regularizer=ElasticNet(l1=0, l2=self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W1_25'))
self.m.add(Dense(self.nodes_list[1], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W2'))
# linear for latent space representation
self.m.add(Dense(self.nodes_list[2], activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2),
name='bottleneck'))
self.m.add(Dense(self.nodes_list[3], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W4'))
self.m.add(Dense(self.nodes_list[4], activation=self.act, kernel_regularizer=regularizers.l2(self.l2), \
bias_regularizer=regularizers.l2(self.l2), \
name = 'W5'))
self.m.add(Dense(self.output_dim, activation='linear', kernel_regularizer=regularizers.l2(self.l2), \
#bias_regularizer=regularizers.l2(l2_parameter), \
name = 'W6_regr'))
# Load weights from training a previous network on regression
self.m.load_weights('KerasSavedModels/Regression_last_weights_{}_{}_{}.h5'.format(cvfold_id, l1_id, l2_id), by_name=True)
# We transfer the weights of those 25 genes now manually too
self.m.get_layer('W1_25').set_weights([weight[ind_genes, :], bias])
self.m.compile(loss='mse', optimizer=keras.optimizers.Adam(learning_rate=self.lr/2), metrics=[r2_score, 'mse'])
es = EarlyStopping(monitor='val_mse', mode='min', verbose=verbose, patience=2*patience)
if not self.pre_trained_weights:
mc = ModelCheckpoint('KerasSavedModels/StraightRegression_weights_after_pruning_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
else:
mc = ModelCheckpoint('KerasSavedModels/PreTrRegression_weights_after_pruning_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
monitor='val_mse', mode='min', verbose=verbose, save_best_only=True)
# train the network
print("[INFO] training network...")
H = self.m.fit(x_train[:, ind_genes], y_train, batch_size=bs,
validation_data=(x_test[:, ind_genes], y_test),
epochs=2*epochs, verbose=verbose, callbacks = [es, mc])
MSE_tr = np.concatenate([MSE_tr, np.array(H.history["loss"])])
MSE_val = np.concatenate([MSE_val, np.array(H.history["val_loss"])])
train_loss_straight_regr = np.concatenate([train_loss_straight_regr, np.array(H.history["r2_score"])])
val_loss_straight_regr = np.concatenate([val_loss_straight_regr, np.array(H.history["val_r2_score"])])
if prune:
if not self.pre_trained_weights:
saved_model_2 = load_model('KerasSavedModels/StraightRegression_weights_after_pruning_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet})
else:
saved_model_2 = load_model('KerasSavedModels/PreTrRegression_weights_after_pruning_{}_{}_{}.h5'.\
format(cvfold_id, l1_id, l2_id), \
custom_objects={'r2_score': r2_score, 'ElasticNet': ElasticNet, 'residual': residual})
r2_after_pruning_train = 1-np.sum((y_train - saved_model_2.predict(x_train[:, ind_genes]))**2) \
/ np.sum(y_train**2)
r2_after_pruning_test = 1-np.sum((y_test - saved_model_2.predict(x_test[:, ind_genes]))**2) \
/ np.sum(y_test**2)
if not prune:
print('Train R^2: ', r2_train)
print('Test R^2: ', r2_test)
else:
print('Train R^2 before pruning: ', r2_train)
print('Test R^2 after pruning: ', r2_test)
print('Train R^2 after pruning: ', r2_after_pruning_train)
print('Test R^2 after pruning: ', r2_after_pruning_test)
if not prune:
return saved_model.predict(x_train), saved_model.predict(x_test), \
r2_train, r2_test, train_loss_straight_regr, val_loss_straight_regr, \
MSE_tr, MSE_val
else:
return r2_train, r2_test, \
r2_after_pruning_train, r2_after_pruning_test, \
train_loss_straight_regr, val_loss_straight_regr
