class model:
"""
Model to recognised prompt (label) based on the input MFCC feature
CNN-model
Lstm-model
End2end-model
Attention-model
"""
def __init__(self, train_mfcc, train_label, test_mfcc, test_label, validate_mfcc, validate_label):
self.train_mfcc = train_mfcc
self.train_label = train_label
self.test_mfcc = test_mfcc
self.test_label = test_label
self.validate_mfcc = validate_mfcc
self.validate_label = validate_label
self.history = None
def CNN_model(self, learning_rate, epoch, batchsize, whether_Adam, Momentum_gamma, weight_decay, whether_load, cnn_type):
"""
Resnet model
:param learning_rate
:param epoch
:param batchsize
:param whether_Adam: whether to perform Adam optimiser, if not perform Momentum
:param Momentum gamma: a variable of Momentum
:param weight_decay: weight decay for Momentum
:param whether_load: whether to load trained Resnet model in if it exists (or cover it)
"""
test_cnn_mfcc = self.train_mfcc
test_cnn_label = self.train_label
if(isfile("model/resnet_label.hdf5") and whether_load):
self.cnn_model = load_model("model/resnet_label.hdf5")
else:
train_cnn_mfcc = self.test_mfcc
train_cnn_label = self.test_label
val_cnn_mfcc = self.validate_mfcc
val_cnn_label = self.validate_label
# input
input = Input(shape=(self.test_mfcc.shape[1], self.test_mfcc.shape[2], 1))
# Concatenate -1 dimension to be three channels, to fit the input need in ResNet50
input_concate = Concatenate()([input,input,input])
# CNN series network (VGG+Resnet)
# reference: https://keras.io/api/applications/
if(cnn_type == 'ResNet50'):
from tensorflow.keras.applications import ResNet50
cnn_output = ResNet50(pooling = 'avg')(input_concate)
elif(cnn_type == 'ResNet101'):
from tensorflow.keras.applications import ResNet101
cnn_output = ResNet101(pooling = 'avg')(input_concate)
elif(cnn_type == 'ResNet152'):
from tensorflow.keras.applications import ResNet152
cnn_output = ResNet152(pooling = 'avg')(input_concate)
elif(cnn_type == 'ResNet50V2'):
from tensorflow.keras.applications import ResNet50V2
cnn_output = ResNet50V2(pooling = 'avg')(input_concate)
elif(cnn_type == 'ResNet101V2'):
from tensorflow.keras.applications import ResNet101V2
cnn_output = ResNet101V2(pooling = 'avg')(input_concate)
elif(cnn_type == 'ResNet152V2'):
from tensorflow.keras.applications import ResNet152V2
cnn_output = ResNet152V2(pooling = 'avg')(input_concate)
elif(cnn_type == 'VGG16'):
# width and height should not smaller than 32
from tensorflow.keras.applications import VGG16
cnn_output = VGG16(include_top = False, pooling = 'avg')(input_concate)
cnn_output = Flatten()(cnn_output)
elif(cnn_type == 'VGG19'):
# width and height should not smaller than 32
from tensorflow.keras.applications import VGG19
cnn_output = VGG19(include_top = False, pooling = 'avg')(input_concate)
cnn_output = Flatten()(cnn_output)
else:
# CNN layers we design
print("No recognised CNN network. The CNN layers we designed are performed")
# convolution layers
conv_output1 = Conv2D(filters=32, strides=(1, 1), kernel_size=5, activation='relu')(input)
# pool_output1 = MaxPool2D(pool_size=(2, 2))(conv_output1)
conv_output2 = Conv2D(filters=8, strides=(2, 2), kernel_size=4, activation='relu')(conv_output1)
conv_output2 = Dropout(0.2)(conv_output2)
conv_output2_batch = BatchNormalization()(conv_output2)
cnn_output = Flatten()(conv_output2_batch)
cnn_output = Flatten()(cnn_output)
# dense with sigmoid
Dense_sigmoid = Dense(24, activation='sigmoid')(cnn_output)
Dense_sigmoid = Dropout(0.2)(Dense_sigmoid)
# dense output
output = Dense(self.test_label.shape[1], activation='softmax')(Dense_sigmoid)
# cnn model for labels recognision
self.cnn_model = Model(input, output)
# optimizer
if whether_Adam:
optimizer = optimizers.Adam(lr=learning_rate, beta_1 = Momentum_gamma, decay=weight_decay)
else:
optimizer = optimizers.SGD(lr=learning_rate, momentum=Momentum_gamma, nesterov=True, decay=weight_decay)
self.cnn_model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['mse', 'accuracy'])
start = time.time()
self.history = self.cnn_model.fit(train_cnn_mfcc, train_cnn_label, epochs=epoch, batch_size=batchsize, validation_data=[val_cnn_mfcc,val_cnn_label])
self.training_time = time.time() - start
self.cnn_model.save("model/resnet_label.hdf5")
# model evaluation
self.cnn_model.predict(test_cnn_mfcc)
self.score = self.cnn_model.evaluate(test_cnn_mfcc, test_cnn_label)
print("test loss: ", self.score[0], ", mse: ", self.score[1], ", accuracy", self.score[2])
def LSTM_model(self, learning_rate, epoch, batchsize, Lstm_hidden_num, whether_Adam,
Momentum_gamma, weight_decay, whether_load, bidirectional = True):
"""
LSTM with concated hidden states connected to output layer
:param learning_rate
:param epoch
:param batchsize
:param Lstm_hidden_num: number of hidden units of LSTM layer
:param whether_Adam: whether to perform Adam optimiser, if not perform Momentum
:param Momentum gamma: a variable of Momentum
:param weight_decay: weight decay for Momentum
:param whether_load: whether to load trained LSTM model in if it exists (or cover it)
:param bidirectional: whether to use bidirectional LSTM or a normal one
"""
test_lstm_mfcc = self.train_mfcc.reshape(self.train_mfcc.shape[0],self.train_mfcc.shape[1],self.train_mfcc.shape[2])
test_lstm_label = self.train_label
if (isfile("model/lstm_label.hdf5") and whether_load):
self.lstm_model = load_model("model/lstm_label.hdf5")
else:
train_lstm_mfcc = self.test_mfcc.reshape(self.test_mfcc.shape[0], self.test_mfcc.shape[1],
self.test_mfcc.shape[2])
train_lstm_label = self.test_label
val_lstm_mfcc = self.validate_mfcc.reshape(self.validate_mfcc.shape[0], self.validate_mfcc.shape[1],
self.validate_mfcc.shape[2])
val_lstm_label = self.validate_label
# input
input = Input(shape=(None, self.test_mfcc.shape[2]))
lstm_layer = LSTM(Lstm_hidden_num, return_state=True)
if(bidirectional):
bi_lstm_layer = Bidirectional(lstm_layer)
_, forword_h, forword_c, backword_h, backword_c = bi_lstm_layer(input)
Concate = Concatenate()([forword_h, forword_c, backword_h, backword_c])
else:
_, h, c = lstm_layer(input)
Concate = Concatenate()([h, c])
Concate = Dropout(0.2)(Concate)
Concate_batchnorm = BatchNormalization()(Concate)
# dense output
output = Dense(self.test_label.shape[1], activation='softmax')(Concate_batchnorm)
# cnn model for labels recognision
self.lstm_model = Model(input, output)
# optimizer
if whether_Adam:
optimizer = optimizers.Adam(lr=learning_rate, beta_1 = Momentum_gamma, decay=weight_decay)
else:
optimizer = optimizers.SGD(lr=learning_rate, momentum=Momentum_gamma, nesterov=True, decay=weight_decay)
self.lstm_model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['mse', 'accuracy'])
start = time.time()
self.history = self.lstm_model.fit(train_lstm_mfcc, train_lstm_label, epochs=epoch, batch_size=batchsize, validation_data=[val_lstm_mfcc, val_lstm_label])
self.training_time = time.time() - start
self.lstm_model.save("model/lstm_label.hdf5")
# model evaluation
self.lstm_model.predict(test_lstm_mfcc)
self.score = self.lstm_model.evaluate(test_lstm_mfcc, test_lstm_label)
print("test loss: ", self.score[0], ", mse: ", self.score[1], ", accuracy", self.score[2])
def end2end_model(self, int_to_labels, learning_rate, epoch, batchsize, Lstm_hidden_num, whether_Adam,
Momentum_gamma, weight_decay, whether_load):
"""
LSTM-based end to end model
:param int_to_labels: dictionary convert id number to prompt label
:param learning_rate
:param epoch
:param batchsize
:param Lstm_hidden_num: number of hidden units of LSTM layer
:param whether_Adam: whether to perform Adam optimiser, if not perform Momentum
:param Momentum gamma: a variable of Momentum
:param weight_decay: weight decay for Momentum
:param whether_load: whether to load trained end2end model in if it exists (or cover it)
"""
# encoder and decoder model test data
encoder_test = self.train_mfcc
encoder_test = encoder_test.reshape(encoder_test.shape[0], 1, encoder_test.shape[1], encoder_test.shape[2])
decoder_test_output = self.train_label
decoder_size = 10
decoder_test_input = np.zeros([1,1,decoder_size])
if (isfile("model/end2end_encoder.hdf5") and isfile("model/end2end_decoder.hdf5") and whether_load):
self.Model_end2end_encoder = load_model("model/end2end_encoder.hdf5")
self.Model_end2end_decoder = load_model("model/end2end_decoder.hdf5")
else:
#data prerproprocess for encoder
encoder_train = self.test_mfcc.reshape(self.test_mfcc.shape[0], self.test_mfcc.shape[1], self.test_mfcc.shape[2])
encoder_val = self.validate_mfcc.reshape(self.validate_mfcc.shape[0], self.validate_mfcc.shape[1], self.validate_mfcc.shape[2])
#data prerproprocess for decoder
decoder_train_input = np.zeros([encoder_train.shape[0],1,decoder_size])
decoder_val_input = np.zeros([encoder_val.shape[0],1,decoder_size])
decoder_train_output = self.test_label
decoder_val_output = self.validate_label
# train model
# encoder
encoder_input = Input(shape=(None, encoder_train.shape[2]))
encoder_output, encoder_h, encoder_c = LSTM(Lstm_hidden_num, return_state=True, return_sequences=True)(encoder_input)
encoder_h = Dropout(0.2)(encoder_h)
encoder_c = Dropout(0.2)(encoder_c)
encoder_h = BatchNormalization()(encoder_h)
encoder_c = BatchNormalization()(encoder_c)
# decoder
decoder_input = Input(shape=(None, decoder_train_input.shape[2]))
decoder_lstm = LSTM(Lstm_hidden_num, return_state=True, return_sequences=True)
decoder_output, decoder_h, decoder_c = decoder_lstm(decoder_input, initial_state=[encoder_h, encoder_c])
decoder_output = Dropout(0.2)(decoder_output)
decoder_dense = Dense(decoder_train_output.shape[1], activation='softmax')
output = decoder_dense(decoder_output)
self.Model_end2end_train = Model([encoder_input, decoder_input], output)
# optimizer
if whether_Adam:
optimizer = optimizers.Adam(lr=learning_rate, beta_1 = Momentum_gamma, decay=weight_decay)
else:
optimizer = optimizers.SGD(lr=learning_rate, momentum=Momentum_gamma, nesterov=True, decay=weight_decay)
self.Model_end2end_train.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['mse', 'accuracy'])
start = time.time()
self.history = self.Model_end2end_train.fit([encoder_train, decoder_train_input], decoder_train_output, epochs=epoch, batch_size=batchsize, validation_data=[[encoder_val, decoder_val_input], decoder_val_output])
self.training_time = time.time() - start
# encoder model
self.Model_end2end_encoder = Model(encoder_input, [encoder_h, encoder_c])
self.Model_end2end_encoder.save("model/end2end_encoder.hdf5")
# decoder model
decoder_h_input = Input(shape=(Lstm_hidden_num,))
decoder_c_input = Input(shape=(Lstm_hidden_num,))
decoder_output, decoder_h_output, decoder_c_output = decoder_lstm(decoder_input, initial_state=[decoder_h_input, decoder_c_input])
output = decoder_dense(decoder_output)
self.Model_end2end_decoder = Model([decoder_input, decoder_h_input, decoder_c_input], output)
self.Model_end2end_decoder.save("model/end2end_decoder.hdf5")
# prediction
# test accuracy counting
num_trueclassify = 0
for i in range(len(encoder_test)):
[h, c] = self.Model_end2end_encoder.predict(encoder_test[i])
current_decoder_train_output = self.Model_end2end_decoder.predict([decoder_test_input, h, c])
output_hat = np.argmax(current_decoder_train_output)
output_hat = int_to_labels[output_hat]
output = np.argmax(decoder_test_output[i])
output = int_to_labels[output]
if(output_hat==output):
num_trueclassify+=1
print("prediction:", output_hat)
print("actually:", output,'\n')
self.score = num_trueclassify / len(encoder_test)
print("test accuracy:", self.score)
def Attention_model(self, int_to_labels, learning_rate, epoch, batchsize, Lstm_hidden_num, whether_Adam,
Momentum_gamma, weight_decay, whether_load):
"""
LSTM-based end to end model with global attention
:param int_to_labels: dictionary convert id number to prompt label
:param learning_rate:
:param epoch:
:param batchsize:
:param Lstm_hidden_num: number of hidden units of LSTM layer
:param whether_Adam: whether to perform Adam optimiser, if not perform Momentum
:param Momentum gamma: a variable of Momentum
:param weight_decay: weight decay for Momentum
:param whether_load: whether to load trained Attention model in if it exists (or cover it)
:return:
"""
# encoder and decoder model test data
encoder_test = self.train_mfcc
encoder_test = encoder_test.reshape(encoder_test.shape[0], 1, encoder_test.shape[1], encoder_test.shape[2])
decoder_test_output = self.train_label
decoder_size = 10
decoder_test_input = np.zeros([1,1,decoder_size])
if (isfile("model/end2end_encoder.hdf5") and isfile("model/end2end_decoder.hdf5") and whether_load):
self.Model_end2end_encoder = load_model("model/end2end_encoder.hdf5")
self.Model_end2end_decoder = load_model("model/end2end_decoder.hdf5")
else:
#data prerproprocess for encoder
encoder_train = self.test_mfcc.reshape(self.test_mfcc.shape[0], self.test_mfcc.shape[1], self.test_mfcc.shape[2])
encoder_val = self.validate_mfcc.reshape(self.validate_mfcc.shape[0], self.validate_mfcc.shape[1], self.validate_mfcc.shape[2])
#data prerproprocess for decoder
decoder_train_input = np.zeros([encoder_train.shape[0],1,decoder_size])
decoder_val_input = np.zeros([encoder_val.shape[0],1,decoder_size])
decoder_train_output = self.test_label
decoder_val_output = self.validate_label
# train model
# encoder
encoder_input = Input(shape=(None, encoder_train.shape[2]))
encoder_output, encoder_h, encoder_c = LSTM(Lstm_hidden_num, return_state=True, return_sequences=True)(encoder_input)
encoder_h = Dropout(0.2)(encoder_h)
encoder_c = Dropout(0.2)(encoder_c)
encoder_output = Dropout(0.2)(encoder_output)
encoder_h = BatchNormalization()(encoder_h)
encoder_c = BatchNormalization()(encoder_c)
encoder_output = BatchNormalization()(encoder_output)
# decoder
decoder_input = Input(shape=(None, decoder_train_input.shape[2]))
decoder_lstm = LSTM(Lstm_hidden_num, return_state=True, return_sequences=True)
decoder_output, decoder_h, decoder_c = decoder_lstm(decoder_input, initial_state=[encoder_h, encoder_c])
decoder_output = Dropout(0.2)(decoder_output)
# Global Attention layer
attention_lstm = Attention()
attention_distribute = attention_lstm([encoder_output, decoder_output])
# global average pooling among attention distribution
GlobalAveragePooling1D_attetion_layer = GlobalAveragePooling1D()
attention_output = GlobalAveragePooling1D_attetion_layer(attention_distribute)
# global average pooling among decoder output
GlobalAveragePooling1D_decoder_layer = GlobalAveragePooling1D()
attention_output_new = GlobalAveragePooling1D_decoder_layer(decoder_output)
# Concatenate attention output and decoder output
Concaten_lstm = Concatenate()
Concaten_output = Concaten_lstm([attention_output_new, attention_output])
Concaten_output = Dropout(0.2)(Concaten_output)
decoder_dense = Dense(decoder_train_output.shape[1], activation='softmax')
output = decoder_dense(Concaten_output)
self.Model_Attetion_train = Model([encoder_input, decoder_input], output)
# optimizer
if whether_Adam:
optimizer = optimizers.Adam(lr=learning_rate, beta_1 = Momentum_gamma, decay=weight_decay)
else:
optimizer = optimizers.SGD(lr=learning_rate, momentum=Momentum_gamma, nesterov=True, decay=weight_decay)
self.Model_Attetion_train.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['mse', 'accuracy'])
start = time.time()
self.history = self.Model_Attetion_train.fit([encoder_train, decoder_train_input], decoder_train_output, epochs=epoch, batch_size=batchsize, validation_data=[[encoder_val, decoder_val_input], decoder_val_output])
self.training_time = time.time() - start
# encoder model
self.Model_Attetion_encoder = Model(encoder_input, [encoder_output, encoder_h, encoder_c])
self.Model_Attetion_encoder.save("model/Attention_encoder.hdf5")
# decoder model
decoder_h_input = Input(shape=(Lstm_hidden_num,))
decoder_c_input = Input(shape=(Lstm_hidden_num,))
encoder_output_for_attention = Input(shape = (None, Lstm_hidden_num))
decoder_output, decoder_h_output, decoder_c_output = decoder_lstm(decoder_input, initial_state=[decoder_h_input, decoder_c_input])
attention_distribute = attention_lstm([encoder_output_for_attention, decoder_output])
attention_output = GlobalAveragePooling1D_attetion_layer(attention_distribute)
decoder_output_new = GlobalAveragePooling1D_decoder_layer(decoder_output)
Concaten_output = Concaten_lstm([decoder_output_new, attention_output])
output = decoder_dense(Concaten_output)
self.Model_Attetion_decoder = Model([decoder_input, decoder_h_input, decoder_c_input, encoder_output_for_attention], output)
self.Model_Attetion_decoder.save( "model/Attention_decoder.hdf5")
# prediction
# test accuracy counting
num_trueclassify = 0
for i in range(len(encoder_test)):
[encoder_out, h, c] = self.Model_Attetion_encoder.predict(encoder_test[i])
current_decoder_train_output = self.Model_Attetion_decoder.predict([decoder_test_input, h, c, encoder_out])
output_hat = np.argmax(current_decoder_train_output)
output_hat = int_to_labels[output_hat]
output = np.argmax(decoder_test_output[i])
output = int_to_labels[output]
if(output_hat==output):
num_trueclassify+=1
#print("prediction:", output_hat)
#print("actually:", output,'\n')
self.score = num_trueclassify / len(encoder_test)
print("test accuracy:", self.score)
# plt.plot(hist.history['acc'])
# plt.plot(hist.history['val_acc'])
# plt.title('loss & acc')
# plt.ylabel('loss, acc')
# plt.xlabel('epoch')
# plt.legend(['train loss', 'val loss', 'train acc', 'val acc'])
# plt.show()
# save and load model
model.save('../data/h5/k67_1_male_female.h5')
# model = load_model('../data/h5/k67_1_male_female.h5')
#########################################################################################
# evaluate
loss, acc = model.evaluate(X_test, y_test, batch_size=32)
print("DIMENSION :", DIMENSION, "\tCHANNEL :", CHANNEL, "\tROUNDS :", ROUNDS,
"\tNODE", NODE)
print("loss :", loss)
print("acc :", acc)
# predict
y_pred = model.predict(
X_test) # 마지막 레이어 sigmoid 함수를 통해 0 ~ 1 사이의 값을 반환, shape==(337, 1)
#[[4.47866887e-06]
# [3.17825680e-03] (이하 생략)
y_pred = [(y[0] >= 0.5).astype(np.uint8) for y in y_pred]
# 반복문 한 줄로 해서 리스트 형식으로 변수에 저장
# 0.5보다 크면 1 (True), 작으면 0 (False), unit8 "unsigned 8bit int"
'''
def obtain_features(model_prefix,
features_prefix,
labels_prefix,
data_prefix,
training_percentage,
am_filling_percentage,
experiment,
occlusion=None,
bars_type=None):
""" Generate features for images.
Uses the previously trained neural networks for generating the features corresponding
to the images. It may introduce occlusions.
"""
(data, labels) = get_data(experiment, occlusion, bars_type)
total = len(data)
step = int(total / constants.training_stages)
# Amount of data used for training the networks
trdata = int(total * training_percentage)
# Amount of data used for testing memories
tedata = step
n = 0
histories = []
for i in range(0, total, step):
j = (i + tedata) % total
if j > i:
testing_data = data[i:j]
testing_labels = labels[i:j]
other_data = np.concatenate((data[0:i], data[j:total]), axis=0)
other_labels = np.concatenate((labels[0:i], labels[j:total]),
axis=0)
training_data = other_data[:trdata]
training_labels = other_labels[:trdata]
filling_data = other_data[trdata:]
filling_labels = other_labels[trdata:]
else:
testing_data = np.concatenate((data[0:j], data[i:total]), axis=0)
testing_labels = np.concatenate((labels[0:j], labels[i:total]),
axis=0)
training_data = data[j:j + trdata]
training_labels = labels[j:j + trdata]
filling_data = data[j + trdata:i]
filling_labels = labels[j + trdata:i]
# Recreate the exact same model, including its weights and the optimizer
model = tf.keras.models.load_model(
constants.model_filename(model_prefix, n))
# Drop the autoencoder and the last layers of the full connected neural network part.
classifier = Model(model.input, model.output[0])
no_hot = to_categorical(testing_labels)
classifier.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics='accuracy')
history = classifier.evaluate(testing_data,
no_hot,
batch_size=batch_size,
verbose=1,
return_dict=True)
print(history)
histories.append(history)
model = Model(classifier.input, classifier.layers[-4].output)
model.summary()
training_features = model.predict(training_data)
if len(filling_data) > 0:
filling_features = model.predict(filling_data)
else:
r, c = training_features.shape
filling_features = np.zeros((0, c))
testing_features = model.predict(testing_data)
dict = {
constants.training_suffix:
(training_data, training_features, training_labels),
constants.filling_suffix:
(filling_data, filling_features, filling_labels),
constants.testing_suffix:
(testing_data, testing_features, testing_labels)
}
for suffix in dict:
data_fn = constants.data_filename(data_prefix + suffix, n)
features_fn = constants.data_filename(features_prefix + suffix, n)
labels_fn = constants.data_filename(labels_prefix + suffix, n)
d, f, l = dict[suffix]
np.save(data_fn, d)
np.save(features_fn, f)
np.save(labels_fn, l)
n += 1
return histories
optimizer='adam',
loss=tf.losses.BinaryCrossentropy(from_logits=True),
metrics=[
tf.metrics.BinaryAccuracy(threshold=0.5, name='accuracy')
])
callbacks = [
keras.callbacks.EarlyStopping(monitor="accuracy",
patience=15,
verbose=1,
mode="min",
restore_best_weights=True),
keras.callbacks.ModelCheckpoint(
filepath="Models/fakeNewsDetector_Advanced.hdf5",
verbose=1,
save_best_only=True)
]
history = model_classifier.fit([partial_x_train, verified_train],
partial_y_train,
epochs=40,
batch_size=256,
validation_data=([x_val,
verified_val], y_val),
verbose=1,
callbacks=callbacks)
else:
results = model_classifier.evaluate([X, verified], y)
print(f'Accuracy on all data is: {results[1]}')
class Dependency_Parser:
def __init__(self,
hidden_neurons=512,
epochs=1,
batch_size=32,
verbose=1,
max_len=50,
n_tags=51,
load_f=False,
loadFile="tmp/model.h5"):
self.hidden_neurons = hidden_neurons
self.epochs = epochs
self.batch_size = batch_size
self.verbose = verbose
sess = tf.Session()
K.set_session(sess)
input_text = Input(shape=(max_len, ), dtype=tf.string)
# embed = ElmoLayer()
# embedding = embed(input_text)
embedding = ElmoLayer()(input_text)
# Don't know why but it needs initialization after ElmoLayer
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
# print_emb = tf.Print(embedding, [embedding])
x = Bidirectional(
LSTM(units=hidden_neurons,
return_sequences=True,
recurrent_dropout=0.2,
dropout=0.2))(embedding)
# x = Bidirectional(LSTM(units=hidden_neurons, return_sequences=True,
# recurrent_dropout=0.2, dropout=0.2))(print_emb)
x_rnn = Bidirectional(
LSTM(units=hidden_neurons,
return_sequences=True,
recurrent_dropout=0.2,
dropout=0.2))(x)
x = add([x, x_rnn]) # residual connection to the first biLSTM
out = TimeDistributed(Dense(n_tags, activation="softmax"))(x)
self.model = Model(input_text, out)
self.model.compile(optimizer="adam",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
def fit(self, X_tr, y_tr, val):
checkpoint_path = "trained/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
cp_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_path, save_weights_only=True, verbose=1)
return self.model.fit(x=X_tr,
y=y_tr,
batch_size=self.batch_size,
epochs=self.epochs,
verbose=self.verbose,
validation_data=val,
callbacks=[cp_callback])
def evaluate(self, X_test, y_test):
return self.model.evaluate(x=X_test,
y=y_test,
batch_size=self.batch_size,
verbose=self.verbose)
def load(self, checkpoint_path):
return self.model.load_weights(checkpoint_path)
def predict(self, x):
return self.model.predict(x)
def save(self, checkpoint="trained/model.ckpt"):
return self.model.save_weights(checkpoint)
callbacks = []
callbacks.append(
tf.keras.callbacks.LearningRateScheduler(lr_scheduler, verbose=0))
if args.logging:
callbacks.append(
tf.keras.callbacks.CSVLogger(
os.path.join(
logdir, '{0}_{1}_{2}_{3}-{4}.log'.format(
args.task_id, args.training_set_size, args.encodings_type,
args.hops, args.random_state))))
model.fit(x=x_train,
y=y_train,
epochs=args.epochs,
validation_split=args.validation_split,
batch_size=batch_size,
callbacks=callbacks,
verbose=args.verbose)
callbacks = []
if args.logging:
callbacks.append(
MyCSVLogger(
os.path.join(
logdir, '{0}_{1}_{2}_{3}-{4}.log'.format(
args.task_id, args.training_set_size, args.encodings_type,
args.hops, args.random_state))))
model.evaluate(x=x_test, y=y_test, callbacks=callbacks, verbose=2)
predict = Dense(10, activation="softmax")(flatten)
vgg16 = Model(inputs=vgg_model.input, outputs=predict)
check_point_path = "./vgg_check_point.hdf5"
model_path = "vgg_model.hdf5"
check_point = ModelCheckpoint(check_point_path,
save_best_only=True,
monitor='val_accuracy')
tensorboard = TensorBoard(log_dir="logs/vgg")
epochs = 10
if os.path.exists(check_point_path):
vgg16.load_weights(check_point_path)
vgg16.compile(
optimizer=keras.optimizers.Adam(),
loss=keras.losses.CategoricalCrossentropy(label_smoothing=0.1),
metrics=["accuracy"])
vgg16_acc = vgg16.evaluate(valid_gen, steps=valid_steps)[1]
if vgg16_acc < 0.8:
vgg16.fit(train_gen,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=valid_gen,
validation_steps=valid_steps,
callbacks=[check_point, tensorboard])
'''
prune on vgg16, the layer index can be found by model.summary()
'''
state = 0
base_model = vgg16
base_model.evaluate(valid_gen, steps=valid_steps)
new_model = Model(inputs=vgg_model.input, outputs=predict)
layer_index = 17
labels_hold_out = labels[hold_out_nodes]
print(labels_hold_out)
hold_out_targets = target_encoding.transform(labels_hold_out)
hold_out_gen = generator.flow(hold_out_nodes, hold_out_targets)
# Make predictions for the held out set
hold_out_predictions = model.predict(hold_out_gen)
# Look at the held out predictions
hold_out_predictions = target_encoding.inverse_transform(hold_out_predictions)
results = pd.Series(hold_out_predictions, index=hold_out_nodes)
df = pd.DataFrame({"Predicted": results, "True": labels_hold_out})
print(df.head(10))
hold_out_metrics = model.evaluate(hold_out_gen)
print("\nHold Out Set Metrics:")
for name, val in zip(model.metrics_names, hold_out_metrics):
print("\t{}: {:0.4f}".format(name, val))
# Make TSNE plot of the held out node embeddings
embedding_model = Model(inputs=x_inp, outputs=x_out)
emb = embedding_model.predict(hold_out_gen)
X = emb
y = np.argmax(target_encoding.transform(labels_hold_out), axis=1)
if X.shape[1] > 2:
transform = TSNE # PCA
trans = transform(n_components=2)
emb_transformed = pd.DataFrame(trans.fit_transform(X),
index=hold_out_nodes)
# model fit_generator
STEP_SIZE_TRAIN = train_gen.n // train_gen.batch_size
STEP_SIZE_VALID = valid_gen.n // valid_gen.batch_size
transfer_learning_history = model.fit_generator(
generator=train_gen,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_gen,
validation_steps=STEP_SIZE_VALID,
epochs=20,
callbacks=callbacks,
class_weight='auto',
)
# model evaluate with validation set
model.evaluate(valid_gen, steps=STEP_SIZE_VALID, verbose=1)
#STEP_SIZE_TEST=test_gen.n//test_gen.batch_size
#test_gen.reset()
#pred=model.predict(test_gen,
#steps=STEP_SIZE_TEST,
#verbose=1)
#predicted_class_indices=np.argmax(pred,axis=1)
# CSV file for kaggle submission
#labels = train_gen.class_indices
#labels = dict((v,k) for k,v in labels.items())
#predictions = [k for k in predicted_class_indices]
callbacks=[es, rlrop])
model_json = model.to_json()
with open('model.json', 'w') as json_file:
json_file.write(model_json)
model.save_weights('model.h5')
#print('Loading weights...')
#model.load_weights('model.h5')
#print('Model loaded')
#print('Generating test data')
test = cags.test.map(CAGS.parse)
x_test = []
for x in test:
x_test.append(x['image'].numpy())
x_test = np.array(x_test)
scores = model.evaluate(x_dev, y_dev, batch_size=64, verbose=1)
print(f'Test result: {scores[1]*100} loss:{scores[0]}')
print('Generating predictions...')
# Generate test set annotations, but in args.logdir to allow parallel execution.
with open("cags_classification.txt", "w", encoding="utf-8") as out_file:
# TODO: Predict the probabilities on the test set
test_probabilities = model.predict(x_test, batch_size=args.batch_size)
for probs in test_probabilities:
print(np.argmax(probs), file=out_file)
checkpoint_path = "./checkpoints/"
checkpoint_dir = os.path.dirname(checkpoint_path)
cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,
verbose=1,
period=1)
#model.summary()
model.compile(optimizer=tf.keras.optimizers.Adam(0.03),
loss=custom_loss,
metrics=[auc])
model.fit((train_image1, train_image2), train_label,
epochs=10) #, callbacks = [cp_callback])
#4
loss, auc = model.evaluate((test_image1, test_image2), test_label)
print("saved model, loss: {:5.2f}, acc: {:5.2f}".format(loss, auc))
e_w = model.predict((test_image1, test_image2))
#print(type(e_w)) #<class 'tensorflow.python.framework.ops.EagerTensor'>
threshs = [0.2, 0.5, 0.7, 1., 1.3, 1.5, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0]
area_list = []
for thresh in threshs:
predictions = (e_w > thresh) #e_w=e_w/max(e_w)
precision, recall, th = metrics.precision_recall_curve(
test_label, predictions)
area_list.append(metrics.auc(recall, precision))
plt.plot(recall, precision, linewidth=1.0, label='thresh=' + str(thresh))
plt.plot([0.5, 1], [0.5, 1], linewidth=1.0, label='equal')
plt.title("precision and recall curve")
plt.legend()
# train_label,
# epochs=num_epochs,
# batch_size=num_batchsize,
# verbose=2,
# validation_data=(val_data, val_label),
# callbacks=[])
# load the saved best model
# model = models.load_model('best_model.h5')
# evaluate model
test_pred = model.predict(test_data)
t_start = process_time()
_, acc = model.evaluate(test_data,
test_label,
batch_size=num_batchsize,
verbose=2)
t_end = process_time()
t_cost = t_end - t_start
print(f"Test Accuracy: {acc:.4f}, Inference time: {t_cost:.2f}s")
plot_learncurve("CNN", history=history)
if 'lr' in lr_scheduler.history:
plt.plot(lr_scheduler.history['lr'])
plt.xlabel('iterations')
plt.ylabel('learning rate')
plt.title('Learning Rate Schedule')
plt.show()
else:
raise ValueError("no lr info in history.")
class EQLDIV:
"""
EQL-div function learning network
# Arguments
inputSize: number of input variables to model. Integer.
outputSize: number of variables outputted by model. Integer.
numLayers: number of layers in model. A layer is either a fully-
connected linear map and a nonlinear map (hidden layer), or just a
fully-connected linear map (final layer). The Keras Input layer
doesn't count.
layers: list of Keras layers containing all of the layer tensor outputs
(not the layer object itself) in the EQL model (including the Keras
Input layer).
hypothesisSet: list of 2-tuples, first element of tuples is tensorflow
R -> R function to be applied element-wise in nonlinear layer
components, second element is the corresponding sympy function for
use in printing out learned equations. In practice, usually
contains identity, sine, cosine, and sigmoid.
nonlinearInfo: list with rows equal to number of hidden layers and 2
columns. First column is number of unary functions in each hidden
layer, second column is number of binary functions in each hidden
layer
energyInfo: if energy regularization is to be used, is a list with
length three containing (1) a python function which uses tensorflow
methods to compute the Hamiltonian associated with each member of
a batch of predicted state, (2) a float value giving the actual
Hamilton of the data (NOTE: ENERGYINFO SHOULD ONLY BE USED WITH
TIMESERIES DATA OR OTHER CONSTANT ENERGY DATA), and (3) a
coefficient for scaling the energy error in the loss function
(10^-5 recommended, only activated during the second training
phase)
learningRate: optimizer learning rate.
divThreshold: float, value which denominator must be greater than in
order for division to occur (division returns 0 otherwise)
name: TensorFlow scope name (for TensorBoard)
changeOfVariables: a list with length two containing (1) a python
function which can transform an arbitrarily large numpy array
training data set to a different set of variables and (2) the
number of components per data point in the transformed data that
the function in the first component outputs
# References
- [Learning Equations for Extrapolation and Control](
https://arxiv.org/abs/1806.07259)
"""
def __init__(self, inputSize, outputSize, numLayers=3, hypothesisSet=None,
nonlinearInfo=None, energyInfo=None, changeOfVariables=None,
name='EQLDIV'):
self.inputSize = changeOfVariables[1] or inputSize
self.outputSize = outputSize
self.numLayers = numLayers
self.layers = []
self.hypothesisSet = hypothesisSet or [
[tf.identity, tf.sin, tf.cos, tf.sigmoid],
[sympy.Id, sympy.sin, sympy.cos, sympy.Function("sigm")]]
self.nonlinearInfo = nonlinearInfo or getNonlinearInfo(
self.numLayers-1, [4], 4)
self.energyInfo = energyInfo
self.changeOfVariables = changeOfVariables
self.name = name
self.unaryFunctions = [
[j % len(self.hypothesisSet[0])
for j in range(self.nonlinearInfo[i][0])]
for i in range(self.numLayers-1)]
def build(self, divThreshold=0.001, learningRate=0.001, method=Adam,
loss='mse', metrics=[rmse]):
"""
__init__ defines all of the variables this model depends upon, this
function actual creates the tensorflow Model variable
# Arguments
learningRate: model learning rate
method: model learning method
loss: loss function used to penalize model based on accuracy
metrics: metric used for gauging model performance when evaluated
on data points post-training
"""
self.layers.append(Input((self.inputSize,)))
inp = int(self.layers[-1].shape[1])
for i in range(self.numLayers):
if (i == self.numLayers - 1):
stddev = np.sqrt(1 / (self.outputSize * 2 * inp))
randNorm = RandomNormal(0, stddev=stddev)
self.layers.append(DivLayer(self.outputSize,
threshold=divThreshold,
kernel_initializer=randNorm,
)(self.layers[-1]))
break
u, v = self.nonlinearInfo[i] or None
out = u + 2 * v or None
stddev = np.sqrt(1 / (inp * out))
randNorm = RandomNormal(0, stddev=stddev)
self.layers.append(EqlLayer(self.nonlinearInfo[i],
self.hypothesisSet[0],
self.unaryFunctions[i],
kernel_initializer=randNorm,
)(self.layers[-1]))
inp = u + v
# Optimizer
optimizer = method(lr=learningRate)
# Model
self.model = Model(inputs=self.layers[0],
outputs=self.layers[-1])
# Compilation
self.model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
def fit(self, predictors, labels, numEpoch, regStrength=10**-3,
batchSize=20, normThreshold=0.001, verbose=0):
"""
Trains EQL model on a dataset following the training schedule defined
in the reference.
# Arguments
predictors: ? x inputSize array containing data to be trained on
labels: ? x outputSize array containing corresponding correct
output for predictors, compared with model output
numEpoch: integer, number of epochs
reg: regularization (in the range [10^-4, 10^-2.5] is usually
ideal)
batchSize: number of datapoints trained on per gradient descent
update
normThreshold: float, weight/bias elements below this value are
kept at zero during the final training phase
verbose: 0, 1, or 2, determines whether Keras is silent, prints a
progress bar, or prints a line every epoch.
"""
if self.changeOfVariables:
predictors = self.changeOfVariables[0](predictors)
# Division threshold callbacks
def phase1DivisionThresholdSchedule(epoch, logs):
K.set_value(self.model.layers[-1].threshold,
1 / np.sqrt(epoch + 1))
def phase2DivisionThresholdSchedule(epoch, logs):
K.set_value(self.model.layers[-1].threshold,
1 / np.sqrt(int(numEpoch * (1 / 4)) + epoch + 1))
def phase3DivisionThresholdSchedule(epoch, logs):
K.set_value(
self.model.layers[-1].threshold,
1 / np.sqrt(int(numEpoch * (7/10))
+ int(numEpoch * (1/4))
+ epoch + 1))
# PHASE 1: NO REGULARIZATION (T/4)
for i in range(1, self.numLayers+1):
K.set_value(self.model.layers[i].regularization, 0.)
K.set_value(self.model.layers[i].Wtrimmer,
np.ones(self.model.layers[i].W.shape))
K.set_value(self.model.layers[i].btrimmer,
np.ones(self.model.layers[i].b.shape))
dynamicDivisionThreshold = LambdaCallback(
on_epoch_begin=phase1DivisionThresholdSchedule)
self.model.fit(predictors, labels, epochs=int(numEpoch*(1/4)),
batch_size=batchSize, verbose=verbose,
callbacks=[dynamicDivisionThreshold])
# PHASE 2: REGULARIZATION (7T/10)
if self.energyInfo is not None:
K.set_value(self.model.layers[-1].loss.coef, self.coef)
for i in range(1, self.numLayers+1):
K.set_value(self.model.layers[i].regularization, regStrength)
dynamicDivisionThreshold = LambdaCallback(
on_epoch_begin=phase2DivisionThresholdSchedule)
self.model.fit(predictors, labels, epochs=int(numEpoch*(7/10)),
batch_size=batchSize, verbose=verbose,
callbacks=[dynamicDivisionThreshold])
# PHASE 3: NO REGULARIZATION, L0 NORM PRESERVATION (T/20)
for i in range(1, self.numLayers+1):
W, b = self.model.layers[i].get_weights()[:2]
K.set_value(self.model.layers[i].regularization, 0.)
K.set_value(self.model.layers[i].Wtrimmer,
(np.abs(W) > normThreshold).astype(float))
K.set_value(self.model.layers[i].btrimmer,
(np.abs(b) > normThreshold).astype(float))
dynamicDivisionThreshold = LambdaCallback(
on_epoch_begin=phase3DivisionThresholdSchedule)
self.model.fit(predictors, labels, epochs=int(numEpoch*(1/20)),
batch_size=batchSize, verbose=verbose,
callbacks=[dynamicDivisionThreshold])
# Final maintenance
K.set_value(self.model.layers[-1].threshold, 0.001)
def evaluate(self, predictors, labels, batchSize=10, verbose=0):
"""Evaluates trained model on data"""
return self.model.evaluate(predictors, labels, batch_size=batchSize,
verbose=verbose)[1]
def getEquation(self):
"""Prints learned equation of a trained model."""
# prepares lists for weights and biases
weights = []
bias = []
# pulls/separates weights and biases from a model
for i in range(1, self.numLayers+1):
weights.append(self.model.layers[i].get_weights()[0])
bias.append(self.model.layers[i].get_weights()[1])
# creates generic input vector
X = make_symbolic(1, self.inputSize)
for i, _ in enumerate(weights):
# computes the result of the next linear layer
W = sympy.Matrix(weights[i])
b = sympy.Transpose(sympy.Matrix(bias[i]))
Y = sympy.zeros(1, b.cols)
X = X*W + b
# computes the result of the next nonlinear layer, if applicable
if i != (len(weights) - 1):
u, v = self.nonlinearInfo[i]
# computes the result of the unary component of the nonlinear
# layer
# iterating over unary input
for j in range(u):
Y[0, j] = self.hypothesisSet[1][self.unaryFunctions[i][j]](
X[0, j])
# computes the result of the binary component of the nonlinear
# layer
# iterating over binary input
for j in range(v):
Y[0, j+u] = X[0, j * 2 + u] * X[0, j * 2 + u + 1]
# removes final v rows which are now outdated
for j in range(u + v, Y.cols):
Y.col_del(u + v)
X = Y
if i == (len(weights) - 1):
# computes the result of the binary component of the nonlinear
# layer
# iterating over binary input
for j in range(int(X.cols/2)):
if sympy.Abs(X[0, j*2+1]) == 0:
Y[0, j] = 0
else:
Y[0, j] = X[0, j*2] / X[0, j*2+1]
for j in range(int(X.cols/2)):
Y.col_del(int(X.cols/2))
X = Y
return X
def plotSlice(self, function, xmin=-2, xmax=2, step=0.01, width=10,
height=10, settings=dict(), save=False):
"""
Plots the x_1 = ... = x_n slice of the learned function in each output
variable.
"""
# x values
X = np.asarray(
[[(i * step) + xmin for i in range(int((xmax - xmin)/step))]
for j in range(self.inputSize)])
# goal function values
F_Y = np.apply_along_axis(function, 0, X)
# model predictions
model_Y = self.model.predict(np.transpose(X))
model_Y = np.transpose(model_Y)
settings['figure.figsize'] = (width, height)
with plt.rc_context(settings):
fig, axs = plt.subplots(self.outputSize, figsize=(width, height))
if self.outputSize == 1:
axs = [axs]
for i in range(self.outputSize):
axs[i].plot(X[0], F_Y[i], linestyle='-', label='Goal Function')
axs[i].plot(X[0], model_Y[i], linestyle=':',
label='Learned Function')
plt.legend()
if save:
plt.savefig(self.name + '.png', bbox_inches='tight', dpi=300)
def percentError(self, predictors, labels):
"""
Returns the average percent error in each variable of a trained model
with respect to a testing data set
"""
labels = np.reshape(labels, (-1, self.outputSize))
predictions = self.model.predict(predictors)
error = np.divide(np.abs(predictions - labels), np.abs(labels))
error = np.sum(error, 0)
error *= (100 / labels.shape[0])
return error
def sparsity(self, minMag=0.01):
"""Returns the sparsity of a trained model (number of active nodes)"""
vec = np.ones((1, self.inputSize))
weights = self.model.get_weights()
sparsity = 0
for i in range(self.numLayers - 1):
u, v = self.nonlinearInfo[i]
w, b = weights[i*2], weights[i*2+1]
vec = np.dot(vec, w) + b
vec = np.concatenate(
(vec[:, :u], vec[:, u::2] * vec[:, u+1::2]), axis=1)
sparsity += np.sum(
(np.abs(vec) > np.full_like(vec, minMag)).astype(int))
w, b = weights[len(weights)-2], weights[len(weights)-1]
vec = np.dot(vec, w) + b
vec = vec[0, ::2] * vec[0, 1::2]
sparsity += np.sum(
(np.abs(vec) > np.full_like(vec, minMag)).astype(int))
return sparsity
def setPipeline(self, pipeline):
"""
Saves the scikit_learn pipeline used to modify training data so that
the same pipeline can be applied to testing data
"""
self.pipeline = pipeline
def applyPipeline(self, x):
"""Applies a saved scikit_learn pipeline to a dataset"""
if self.pipeline is not None:
for op in self.pipeline:
x = op.transform(x)
return x
def odecompat(self, t, x):
"""Wrapper for Keras' function, solve_ivp compatible"""
# if statement is bad hack for experimentations with feature
# engineering for double pendulum
# if self.inputSize == 7:
# if len(np.array(x).shape) == 2:
# y = [x[0, 0] - x[0, 2], x[0, 1]**2, x[0, 3]**2]
# x = np.append(x, y)
# else:
# y = [x[0] - x[2], x[1]**2, x[3]**2]
# x = np.append(x, y)
prediction = self.model.predict(np.reshape(x, (1, -1)))
return prediction
def printJacobian(self, x):
"""
Prints the Jacobian of the learned function, evaluated at a point
# Arguments
x: point at which Jacobian is evaluated, array-like with
self.inputSize elements
"""
x = np.reshape(x, (1, 1, self.inputSize)).tolist()
gradients = [tf.gradients(self.model.output[:, i], self.model.input)[0]
for i in range(4)]
funcs = [K.function((self.model.input, ), [g]) for g in gradients]
jacobian = np.concatenate([func(x)[0] for func in funcs], axis=0)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(jacobian)
class EQL:
"""
EQL function learning network
# Arguments
inputSize: number of input variables to model. Integer.
outputSize: number of variables outputted by model. Integer.
numLayers: number of layers in model. A layer is either a fully-
connected linear map and a nonlinear map (hidden layer), or just a
fully-connected linear map (final layer). The Keras Input layer
doesn't count.
layers: list of Keras layers containing all of the layers in the
EQL model (including the Keras Input layer).
hypothesisSet: 2 x ? list, first row contains tensorflow R -> R
function to be applied element-wise in nonlinear layer components,
second row contains the corresponding sympy functions for use in
printing out learned equations. In practice, usually contains
identity, sine, cosine, and sigmoid.
nonlinearInfo: list with rows equal to number of hidden layers and 2
columns. First column is number of unary functions in each hidden
layer, second column is number of binary functions in each hidden
layer
learningRate: optimizer learning rate.
name: TensorFlow scope name (for TensorBoard)
# References
- [Extrapolation and learning equations](
https://arxiv.org/abs/1610.02995)
"""
def __init__(self, inputSize, outputSize, numLayers=4, hypothesisSet=None,
nonlinearInfo=None, name='EQL'):
self.inputSize = inputSize
self.outputSize = outputSize
self.numLayers = numLayers
self.layers = []
self.hypothesisSet = hypothesisSet or [
[tf.identity, tf.sin, tf.cos, tf.sigmoid],
[sympy.Id, sympy.sin, sympy.cos, sympy.Function("sigm")]]
self.nonlinearInfo = nonlinearInfo or getNonlinearInfo(
self.numLayers-1, [4], 4)
self.name = name
self.unaryFunctions = [
[j % len(self.hypothesisSet[0])
for j in range(self.nonlinearInfo[i][0])]
for i in range(numLayers-1)]
def build(self, learningRate=0.001, method=Adam, loss='mse',
metrics=[rmse]):
"""
__init__ defines all of the variables this model depends upon, this
function actual creates the tensorflow Model variable
# Arguments
learningRate: model learning rate
method: model learning method
loss: loss function used to penalize model based on accuracy
metrics: metric used for gauging model performance when evaluated
on data points post-training
"""
self.layers.append(Input((self.inputSize,)))
inp = int(self.layers[-1].shape[1])
for i in range(self.numLayers - 1):
u, v = self.nonlinearInfo[i]
out = u + 2 * v
stddev = np.sqrt(1 / (inp * out))
randNorm = RandomNormal(0, stddev=stddev)
self.layers.append(EqlLayer(self.nonlinearInfo[i],
self.hypothesisSet[0],
self.unaryFunctions[i],
kernel_initializer=randNorm,
)(self.layers[-1]))
inp = u + v
stddev = np.sqrt(1 / (self.outputSize * inp))
randNorm = RandomNormal(0, stddev=stddev)
self.layers.append(Connected(self.outputSize,
kernel_initializer=randNorm
)(self.layers[-1]))
# Optimizer
optimizer = method(lr=learningRate)
# Model
self.model = Model(inputs=self.layers[0],
outputs=self.layers[-1])
# Compilation
self.model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
def fit(self, predictors, labels, numEpoch, reg=10**-3, batchSize=20,
threshold=0.1, verbose=0):
"""
Trains EQL model on a dataset following the training schedule defined
in the reference.
# Arguments
predictors: ? x inputSize array containing data to be trained on
labels: ? x outputSize array containing corresponding correct
output for predictors, compared with model output
numEpoch: integer, number of epochs
reg: regularization (in the range [10^-4, 10^-2.5] is usually
ideal)
batchSize: number of datapoints trained on per gradient descent
update
threshold: float, weight/bias elements below this value are kept
at zero during the final training phase
"""
# PHASE 1: NO REGULARIZATION (T/4)
for i in range(1, self.numLayers+1):
K.set_value(self.model.layers[i].regularization, 0.)
K.set_value(self.model.layers[i].Wtrimmer,
np.ones(self.model.layers[i].W.shape))
K.set_value(self.model.layers[i].btrimmer,
np.ones(self.model.layers[i].b.shape))
self.model.fit(predictors, labels, epochs=int(numEpoch*(1/4)),
batch_size=batchSize, verbose=verbose)
# PHASE 2: REGULARIZATION (7T/10)
for i in range(1, self.numLayers+1):
K.set_value(self.model.layers[i].regularization, reg)
self.model.fit(predictors, labels, epochs=int(numEpoch*(7/10)),
batch_size=batchSize, verbose=verbose)
# PHASE 3: NO REGULARIZATION, L0 NORM PRESERVATION (T/20)
for i in range(1, self.numLayers+1):
W, b = self.model.layers[i].get_weights()[:2]
K.set_value(self.model.layers[i].regularization, 0.)
K.set_value(self.model.layers[i].Wtrimmer,
(np.abs(W) > threshold).astype(float))
K.set_value(self.model.layers[i].btrimmer,
(np.abs(b) > threshold).astype(float))
self.model.fit(predictors, labels, epochs=int(numEpoch*(1/20)),
batch_size=batchSize, verbose=verbose)
def evaluate(self, predictors, labels, batchSize=10, verbose=0):
"""Evaluates trained model on data"""
return self.model.evaluate(predictors, labels, batch_size=batchSize,
verbose=verbose)[1]
def getEquation(self):
"""Prints learned equation of a trained model."""
# prepares lists for weights and biases
weights = []
bias = []
# pulls/separates weights and biases from a model
for i in range(1, self.numLayers+1):
weights.append(self.model.layers[i].get_weights()[0])
bias.append(self.model.layers[i].get_weights()[1])
print(np.array(weights).shape, np.array(self.nonlinearInfo).shape)
# creates generic input vector
X = make_symbolic(1, self.inputSize)
for i, _ in enumerate(weights):
# computes the result of the next linear layer
W = sympy.Matrix(weights[i])
b = sympy.Transpose(sympy.Matrix(bias[i]))
Y = sympy.zeros(1, b.cols)
X = X*W + b
# computes the result of the next nonlinear layer, if applicable
if i != (len(weights) - 1):
u, v = self.nonlinearInfo[i]
# computes the result of the unary component of the nonlinear
# layer
# iterating over unary input
for j in range(u):
Y[0, j] = self.hypothesisSet[1][self.unaryFunctions[i][j]](
X[0, j])
# computes the result of the binary component of the nonlinear
# layer
# iterating over binary input
for j in range(v):
Y[0, j+u] = X[0, j * 2 + u] * X[0, j * 2 + u + 1]
# removes final v rows which are now outdated
for j in range(u + v, Y.cols):
Y.col_del(u + v)
X = Y
return X
def plotSlice(self, function, xmin=-2, xmax=2, step=0.01, width=10,
height=10, settings=dict(), save=False):
"""
Plots the x_1 = ... = x_n slice of the learned function in each output
variable.
"""
# x values
X = np.asarray(
[[(i * step) + xmin for i in range(int((xmax - xmin)/step))]
for j in range(self.inputSize)])
# goal function values
F_Y = np.apply_along_axis(function, 0, X)
# model predictions
model_Y = self.model.predict(np.transpose(X))
model_Y = np.transpose(model_Y)
settings['figure.figsize'] = (width, height)
with plt.rc_context(settings):
fig, axs = plt.subplots(self.outputSize, figsize=(width, height))
if self.outputSize == 1:
axs = [axs]
for i in range(self.outputSize):
axs[i].plot(X[0], F_Y[i], linestyle='-', label='Goal Function')
axs[i].plot(X[0], model_Y[i], linestyle=':',
label='Learned Function')
plt.legend()
if save:
plt.savefig(self.name + '.png', bbox_inches='tight', dpi=300)
def percentError(self, predictors, labels):
"""
Returns the average percent error in each variable of a trained model
with respect to a testing data set
"""
labels = np.reshape(labels, (-1, self.outputSize))
predictions = self.model.predict(predictors)
error = np.divide(np.abs(predictions - labels), np.abs(labels))
error = np.sum(error, 0)
error *= (100 / labels.shape[0])
return error
def sparsity(self, minMag=0.01):
"""Returns the sparsity of a trained model (number of active nodes)"""
vec = np.ones((1, self.inputSize))
weights = self.model.get_weights()
sparsity = 0
for i in range(self.numLayers - 1):
u, v = self.nonlinearInfo[i]
w, b = weights[i*2], weights[i*2+1]
vec = np.dot(vec, w) + b
vec = np.concatenate(
(vec[:, :u], vec[:, u::2] * vec[:, u+1::2]), axis=1)
sparsity += np.sum(
(np.abs(vec) > np.full_like(vec, minMag)).astype(int))
w, b = weights[len(weights)-2], weights[len(weights)-1]
vec = np.dot(vec, w) + b
sparsity += np.sum(
(np.abs(vec) > np.full_like(vec, minMag)).astype(int))
return sparsity
def odecompat(self, t, x):
"""Wrapper for Keras' predict function, solve_ivp compatible"""
prediction = self.model.predict(np.reshape(x, (1, len(x))))
return prediction
def printJacobian(self, x):
"""
Prints the Jacobian of the learned function, evaluated at a point
# Arguments
x: point at which Jacobian is evaluated, array-like with
self.inputSize elements
"""
x = np.reshape(x, (1, 1, self.inputSize)).tolist()
gradients = [tf.gradients(self.model.output[:, i], self.model.input)[0]
for i in range(4)]
funcs = [K.function((self.model.input, ), [g]) for g in gradients]
jacobian = np.concatenate([func(x)[0] for func in funcs], axis=0)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(jacobian)
save_best_only=True,
mode='min')
callbacks_list = [checkpoint]
designed_model.fit(train_data,
dx_train,
class_weight=class_weights,
epochs=epoch,
batch_size=batchSize,
verbose=1,
validation_split=0.15,
callbacks=callbacks_list)
#history = designed_model.fit_generator(datagen.flow(train_data,dx_train, batch_size=64),validation_data = (val_data,dx_val), epochs = epoch,verbose = 1, steps_per_epoch=32)
print(test_data.shape)
print(dx_test.shape)
score = designed_model.evaluate(test_data, dx_test, verbose=1)
print("score:", score)
pred = designed_model.predict(test_data)
print(pred.shape)
pred_out = np.argmax(pred, axis=1)
test_out = np.argmax(dx_test, axis=1)
print(pred_out.shape)
print(pred_out)
conf = confusion_matrix(test_out, pred_out)
print(conf)
# 마지막 단계에서 어떤 함수를 쓰던 accuracy 같은 성능 면이 달라지는 것은 아니지만
# softmax 는 결과값을 확률값으로 바꿔주어서 보기 편하게 만드는 역할을 한다.
# create a model
# param
vgg16 = Model(inputs, outputs, name='vgg16')
vgg16.summary()
# compile the model
# 모델 학습과정 설정 | 손실 함수 및 최적화 방법
# optimizer:
optimizer = tf.keras.optimizers.Adam(
learning_rate=0.0001) # lower learning rate, better performance
vgg16.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
# train the model
# 모델 학습
vgg16.fit(x_train, y_train_onehot, epochs=30)
# evaluate the model with test set
# 모델 평가하기
vgg16.evaluate(x_test, y_test_onehot, verbose=2)
# predict the model with test set
# 모델 사용하기
# vgg16.predict(x) : x라는 input을 넣어서 output 예측 생성
# loss: 3.0565 - accuracy: 0.3367
elif epoch <=4:
return 0.0002 * strategy.num_replicas_in_sync
else:
return 0.0001 * strategy.num_replicas_in_sync
#callbacks = [ tf.keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=1, profile_batch = 3)]#, cb ]
callbacks = []
callbacks = [tf.keras.callbacks.LearningRateScheduler(scheduler)]
print('starting at ', time())
history = model.fit(train, steps_per_epoch=steps_per_epoch, \
epochs=num_epochs, callbacks=callbacks, verbose=1)
print("finishing at:", time())
print("evaluating:", time())
model.evaluate(test, steps=validation_steps)
print("evaluated at:", time())
model_full_path= "/tmp/mymodel" + str(worker_number) + ".h5"
print("Training finished, now saving the model in h5 format to: " + model_full_path)
model.save(model_full_path, save_format="h5")
print("model saved.\n")
#print("..saving the model in tf format (TF 2.0) to: " + model_full_path)
#tf.keras.models.save_model(model, "/tmp/mymodel"+ str(worker_number) + ".tf", save_format='tf')
#print("model saved.\n")
exit()
dense1 = Dense(64, activation='relu')(dense1)
dense1 = Dense(64, activation='relu')(dense1)
output1 = Dense(1)(dense1)
model = Model(inputs=input1, outputs=output1)
#3. compile and fit
model.compile(optimizer='adam', loss='mse', metrics=['mae'])
model.fit(x_train,
y_train,
batch_size=4,
epochs=150,
verbose=1,
validation_split=0.2)
#4. evalutate and predict
mse, mae = model.evaluate(x_test, y_test, batch_size=4)
print("mse :", mse, "\nmae :", mae)
y_predict = model.predict(x_test)
# RMSE 구하기
from sklearn.metrics import mean_squared_error
def RMSE(y_test, y_predict):
return np.sqrt(mean_squared_error(y_test, y_predict))
print('RMSE :', RMSE(y_test, y_predict))
# R2 구하기
model.summary()
# In[171]:
# We train our model.
history = model.fit(train_images, train_labels, epochs=5, batch_size=128, callbacks=callbacks)
# In[172]:
# Evaluate the test data.
test_loss, test_acc = model.evaluate(test_images, test_labels)
# In[173]:
# Plot accuracy and loss of your training epochs.
loss = history.history['loss']
acc = history.history['accuracy']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'b', label='Training loss')
plt.plot(epochs, acc, 'r', label='Training acc')
plt.title('Training loss and accuracy')
plt.xlabel('Epochs')
plt.ylabel('Loss')
def train_delta(matrix):
import numpy as np
import tensorflow as tf
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras import Model, Input
from tensorflow.keras.layers import (Dense, Dropout, GRU, Flatten,
GaussianNoise, concatenate)
from tensorflow.keras.models import load_model
from tensorflow.keras.callbacks import Callback
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.optimizers import Adam
import supplemental_functions
from supplemental_functions import (sampling_fix, prepareinput,
prepareinput_nozero, prepareoutput)
tf.keras.backend.clear_session()
[
newdim, percent_drilled, start, stop, inc_layer1, inc_layer2,
data_layer1, data_layer2, dense_layer, range_max, memory, predictions,
drop1, drop2, lr, bs, ensemble_count
] = matrix
drop1 = drop1 / 100
drop2 = drop2 / 100
inc_layer2 = inc_layer2 / 1000
lr = lr / 10000
percent_drilled = percent_drilled / 100
df = pd.read_csv('F9ADepth.csv')
df_target = df.copy()
droplist = [
'nameWellbore', 'name', 'Pass Name unitless',
'MWD Continuous Inclination dega', 'Measured Depth m',
'MWD Continuous Azimuth dega', "Unnamed: 0", "Unnamed: 0.1"
]
for i in droplist:
df = df.drop(i, 1)
for i in list(df):
if df[i].count() < 1000:
del df[i]
info(f'dropped {i}')
start = start
stop = stop
step = 0.230876
X = np.arange(start, stop, step)
X = X.reshape(X.shape[0], 1)
X = np.arange(start, stop, step)
X = X.reshape(X.shape[0], 1)
my_data1 = sampling_fix(df_target, 'MWD Continuous Inclination dega',
start, stop, 1.7, 1, 0).predict(X)
data_array = []
for i in list(df):
sampled = sampling_fix(df_target, i, start, stop, 1.7, 3, 0).predict(X)
if np.isnan(np.sum(sampled)) == False:
data_array.append(sampled)
info(f'Using {i}')
data_array = np.asarray(data_array)
dftemp = pd.DataFrame()
dftemp['dinc'] = my_data1
dftemp['dinc'] = dftemp['dinc'].diff(1).rolling(3, center=True).mean()
my_data1 = dftemp['dinc'].ffill().bfill()
data_array = data_array.T
pre_PCA_scaler = MinMaxScaler()
data_array = pre_PCA_scaler.fit_transform(data_array)
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
# =============================================================================
# pca = PCA().fit(data_array)
# plt.plot(np.cumsum(pca.explained_variance_ratio_))
# plt.xlabel('number of components')
# plt.ylabel('cumulative explained variance');
#
# plt.show()
# =============================================================================
sampcount = int(len(data_array) * percent_drilled)
pca = TSNE(n_components=newdim).fit(data_array[:sampcount])
projected = pca.transform(data_array)
my_data = []
for i in range(newdim):
my_data.append(projected[:, i])
my_data1 = my_data1[:, np.newaxis]
my_data_newaxis = []
for i in my_data:
my_data_newaxis.append(i[:, np.newaxis])
temp_data1 = pd.DataFrame(my_data1.flatten())
temp_data1 = pd.DataFrame(my_data1)
range1 = temp_data1[0].diff(memory + predictions)
range2 = np.amax(range1)
RNN_scaler = MinMaxScaler()
my_data1 = RNN_scaler.fit_transform(my_data1)
my_data_scaled = []
for i in my_data_newaxis:
my_data_scaled.append(MinMaxScaler().fit_transform(i))
X1 = prepareinput(my_data1, memory)
Xdata = []
for i in my_data_scaled:
Xn = prepareinput_nozero(i, memory, predictions)
Xdata.append(Xn)
y_temp = prepareoutput(my_data1, memory, predictions)
stack = []
for i in range(memory):
stack.append(np.roll(my_data1, -i))
X_temp = np.hstack(stack)
y = y_temp
data_length = len(my_data1) - memory - predictions
testing_cutoff = 0.80
border1 = int((data_length) * (percent_drilled * 0.8))
border2 = int((data_length) * (percent_drilled))
border3 = int((data_length) * (percent_drilled + 0.2))
X1_train = X1[:border1]
X1_test = X1[border1:border2]
X1_test2 = X1[border2:border3]
Xdata_train = []
Xdata_test = []
Xdata_test2 = []
for i in Xdata:
Xdata_train.append(i[:border1])
Xdata_test.append(i[border1:border2])
Xdata_test2.append(i[border2:border3])
y_train, y_test, y_test2 = y[:border1], y[border1:border2], y[
border2:border3]
X1_train = X1_train.reshape((X1_train.shape[0], X1_train.shape[1], 1))
X1_test = X1_test.reshape((X1_test.shape[0], X1_test.shape[1], 1))
X1_test2 = X1_test2.reshape((X1_test2.shape[0], X1_test2.shape[1], 1))
Xdata_train_r = []
Xdata_test_r = []
Xdata_test2_r = []
for i in range(newdim):
Xdata_train_r.append(Xdata_train[i].reshape(
(Xdata_train[i].shape[0], Xdata_train[i].shape[1], 1)))
Xdata_test_r.append(Xdata_test[i].reshape(
(Xdata_test[i].shape[0], Xdata_test[i].shape[1], 1)))
Xdata_test2_r.append(Xdata_test2[i].reshape(
(Xdata_test2[i].shape[0], Xdata_test2[i].shape[1], 1)))
X_train_con = np.concatenate(Xdata_train_r, axis=2)
X_test_con = np.concatenate(Xdata_test_r, axis=2)
X_test2_con = np.concatenate(Xdata_test2_r, axis=2)
X_train = [X1_train, X_train_con]
X_test = [X1_test, X_test_con]
X_test2 = [X1_test2, X_test2_con]
input1 = Input(shape=(memory, 1))
input2 = Input(shape=(memory + predictions, newdim))
x1 = GaussianNoise(inc_layer2, input_shape=(memory, 1))(input1)
x1 = GRU(units=inc_layer1,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
kernel_regularizer='l2',
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
return_sequences=False,
return_state=False,
stateful=False)(x1)
x1 = Dropout(drop1)(x1)
x1 = Model(inputs=input1, outputs=x1)
x2 = Dense(data_layer1, input_shape=(memory + predictions, 3))(input2)
x2 = Dropout(drop2)(x2)
x2 = Flatten()(x2)
x2 = Dense(data_layer2)(x2)
x2 = Model(inputs=input2, outputs=x2)
combined = concatenate([x1.output, x2.output])
z = Dense(dense_layer, activation="relu")(combined)
z = Dense(predictions, activation="linear")(z)
#define the model
myadam = Adam(learning_rate=lr, beta_1=0.9, beta_2=0.999, amsgrad=False)
class PlotResuls(Callback):
def on_train_begin(self, logs={}):
self.i = 0
self.x = []
self.losses = []
self.val_losses = []
#self.fig = plt.figure()
self.logs = []
def on_epoch_end(self, epoch, logs={}):
self.logs.append(logs)
self.x.append(self.i)
self.losses.append(logs.get('loss'))
self.val_losses.append(logs.get('val_loss'))
self.i += 1
#print (".", end = '')
if (epoch % 14999 == 0) & (epoch > 0):
print(epoch)
plt.plot(self.x, np.log(self.losses), label="loss")
plt.plot(self.x, np.log(self.val_losses), label="val_loss")
plt.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2)
plt.title("Loss")
plt.legend()
plt.show()
#mymanyplots(epoch, data, model)
#data = [X1, X2, X3, X4, y, X1_train,X_train, X_test, X1_test, border1, border2, y_train, y_test, memory, y_temp, predictions]
plot_results = PlotResuls()
es = EarlyStopping(monitor='val_loss', mode='min', verbose=0, patience=25)
ens_val_array = np.zeros(ensemble_count)
ens_test_array = np.zeros(ensemble_count)
for ens_no in range(ensemble_count):
tf.keras.backend.clear_session()
mc = ModelCheckpoint(f'best_model_ens_{ens_no}.h5',
monitor='val_loss',
mode='min',
save_best_only=True,
verbose=0)
model = Model(inputs=[x1.input, x2.input], outputs=z)
model.compile(optimizer=myadam, loss='mean_squared_error')
history = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=2000,
verbose=0,
batch_size=bs,
callbacks=[plot_results, es, mc])
model = load_model(f'best_model_ens_{ens_no}.h5')
valresult = np.log(model.evaluate(x=X_test, y=y_test, verbose=0))
testresult = np.log(model.evaluate(x=X_test2, y=y_test2, verbose=0))
ens_val_array[ens_no] = valresult
ens_test_array[ens_no] = testresult
winner = ens_val_array.argmin()
model = load_model(f'best_model_ens_{winner}.h5')
info(ens_val_array)
info(ens_test_array)
info(f'Validation winner {winner}')
sample_count = len(X_test2[0])
y_pred = model.predict(X_test2)
plt.plot(np.log(history.history['loss']), label='loss')
plt.plot(np.log(history.history['val_loss']), label='test')
plt.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2)
plt.legend()
plt.clf()
plt.show()
for i in range(5):
rand = np.random.randint(0, len(X_test[0]))
y_test_descaled = RNN_scaler.inverse_transform(y_test[rand,
np.newaxis])
y_in_descaled = RNN_scaler.inverse_transform(X_test[0][rand, :])
y_in_descaled = y_in_descaled.flatten()
y_test_descaled = y_test_descaled.flatten()
y_pred = model.predict(X_test)
y_pred_descaled = RNN_scaler.inverse_transform(y_pred[rand,
np.newaxis])
y_pred_descaled = y_pred_descaled.flatten()
plt.plot(y_test_descaled, label="true")
plt.plot(y_pred_descaled, label="predicted")
plt.title('Inclination delta')
#plt.ylim(0,1)
plt.legend()
plt.show()
plt.figure(figsize=(5, 4))
x_after = np.linspace(0, 23, 100)
x_before = np.linspace(-23, -0.23, 100)
plt.plot(x_before,
np.cumsum(y_in_descaled),
label="measured",
linestyle="-",
c="black")
commonpoint = np.cumsum(y_in_descaled)[-1]
plt.plot(x_after,
commonpoint + np.cumsum(y_test_descaled),
label="actual",
linestyle='-.',
c='black')
plt.plot(x_after,
commonpoint + np.cumsum(y_pred_descaled),
label="predicted",
linestyle=':',
c='black')
#plt.title('')
plt.ylim(-1, 7)
plt.grid()
plt.tight_layout()
#plt.hlines(0, -23, 23, linewidth=0.5)
plt.xlabel("Distance to sensor [m]")
plt.ylabel("Inclination, local coordinates, [deg]")
plt.legend()
plt.tight_layout()
plt.savefig(f'Sample, {percent_drilled}, no.{i}.pdf')
plt.show()
# #### Different ensemble, voting ######
# ypred_array = []
# for i in range(ensemble_count):
# model = load_model(f'best_model_ens_{i}.h5')
# y_pred = model.predict(X_test2)
# ypred_array.append(y_pred)
# y_pred = np.average(ypred_array, axis=0)
# ######## Different ensemble ends here #
y_test_descaled = RNN_scaler.inverse_transform(y_test2)
y_pred = model.predict(X_test2)
y_pred_descaled = RNN_scaler.inverse_transform(y_pred)
error_matrix = np.cumsum(y_pred_descaled, axis=1) - np.cumsum(
y_test_descaled, axis=1)
def rand_jitter(arr):
stdev = .004 * (max(arr) - min(arr))
return arr + np.random.randn(len(arr)) * stdev
def jitter(x,
y,
s=20,
c='b',
marker='o',
cmap=None,
norm=None,
vmin=None,
vmax=None,
alpha=None,
linewidths=None,
verts=None,
**kwargs):
return plt.scatter(rand_jitter(x),
rand_jitter(y),
s=s,
c=c,
marker=marker,
cmap=cmap,
norm=norm,
vmin=vmin,
vmax=vmax,
alpha=alpha,
linewidths=linewidths,
verts=verts,
**kwargs)
plt.figure(figsize=(5, 5), dpi=200)
for i in range(sample_count):
_ = jitter(x_after,
error_matrix[i],
alpha=1,
s=0.5,
marker=".",
c="black")
plt.title(f"delta, drilled {percent_drilled}")
plt.xlabel("Distance to sensor [m]")
plt.ylabel("Prediction error [deg]")
plt.grid()
plt.tight_layout()
plt.savefig(f'Birdflock, {percent_drilled}.pdf')
plt.show()
#plt.plot(np.median(error_matrix, axis=0), linewidth=8, alpha=1, c="white")
#plt.plot(np.median(error_matrix, axis=0), linewidth=2, alpha=1, c="black")
plt.scatter(np.arange(0, 100, 1),
np.average(np.abs(error_matrix), axis=0),
marker="o",
s=40,
alpha=0.7,
c="white",
zorder=2)
c_array = np.empty(100, dtype=object)
aae = np.average(np.abs(error_matrix), axis=0)
for i in range(100):
if aae[i] <= 0.4:
c_array[i] = "green"
elif aae[i] <= 0.8:
c_array[i] = "orange"
else:
c_array[i] = "red"
plt.scatter(np.arange(0, 100, 1),
aae,
marker=".",
s=20,
alpha=1,
c=c_array,
zorder=3,
label="Average Absolute Error")
plt.ylim((-3, 3))
plt.axhline(y=0, xmin=0, xmax=1, linewidth=2, c="black")
plt.axhline(y=0.4, xmin=0, xmax=1, linewidth=1, c="black")
plt.axhline(y=0.8, xmin=0, xmax=1, linewidth=1, c="black")
plt.legend()
plt.show()
model = load_model('best_model.h5')
#mymanyplots(-1, data, model)
#myerrorplots(data, model)
valresult = np.log(model.evaluate(x=X_test, y=y_test, verbose=0))
testresult = np.log(model.evaluate(x=X_test2, y=y_test2, verbose=0))
return valresult, testresult, aae
class RelevanceModel:
def __init__(
self,
feature_config: FeatureConfig,
tfrecord_type: str,
file_io: FileIO,
scorer: Optional[ScorerBase] = None,
metrics: List[Union[Type[kmetrics.Metric], str]] = [],
optimizer: Optional[Optimizer] = None,
model_file: Optional[str] = None,
compile_keras_model: bool = False,
output_name: str = "score",
logger=None,
):
"""Use this constructor to define a custom scorer"""
self.feature_config: FeatureConfig = feature_config
self.logger: Logger = logger
self.output_name = output_name
self.scorer = scorer
self.tfrecord_type = tfrecord_type
self.file_io = file_io
if scorer:
self.max_sequence_size = scorer.interaction_model.max_sequence_size
else:
self.max_sequence_size = 0
# Load/Build Model
if model_file and not compile_keras_model:
"""
If a model file is specified, load it without compiling into a keras model
NOTE:
This will allow the model to be only used for inference and
cannot be used for retraining.
"""
self.model: Model = self.load(model_file)
self.is_compiled = False
else:
"""
Specify inputs to the model
Individual input nodes are defined for each feature
Each data point represents features for all records in a single query
"""
inputs: Dict[str, Input] = feature_config.define_inputs()
scores, train_features, metadata_features = scorer(inputs)
# Create model with functional Keras API
self.model = Model(inputs=inputs, outputs={self.output_name: scores})
# Get loss fn
loss_fn = scorer.loss.get_loss_fn(**metadata_features)
# Get metric objects
metrics_impl: List[Union[str, kmetrics.Metric]] = get_metrics_impl(
metrics=metrics, feature_config=feature_config, metadata_features=metadata_features
)
# Compile model
"""
NOTE:
Related Github issue: https://github.com/tensorflow/probability/issues/519
"""
self.model.compile(
optimizer=optimizer,
loss=loss_fn,
metrics=metrics_impl,
experimental_run_tf_function=False,
)
# Write model summary to logs
model_summary = list()
self.model.summary(print_fn=lambda x: model_summary.append(x))
self.logger.info("\n".join(model_summary))
if model_file:
"""
If model file is specified, load the weights from the SavedModel
NOTE:
The architecture, loss and metrics of self.model need to
be the same as the loaded SavedModel
"""
self.load_weights(model_file)
self.is_compiled = True
@classmethod
def from_relevance_scorer(
cls,
interaction_model: InteractionModel,
model_config: dict,
feature_config: FeatureConfig,
loss: RelevanceLossBase,
metrics: List[Union[kmetrics.Metric, str]],
optimizer: Optimizer,
tfrecord_type: str,
file_io: FileIO,
model_file: Optional[str] = None,
compile_keras_model: bool = False,
output_name: str = "score",
logger=None,
):
"""Use this as constructor to define a custom InteractionModel with RelevanceScorer"""
assert isinstance(interaction_model, InteractionModel)
assert isinstance(loss, RelevanceLossBase)
scorer: ScorerBase = RelevanceScorer(
model_config=model_config,
interaction_model=interaction_model,
loss=loss,
output_name=output_name,
)
return cls(
scorer=scorer,
feature_config=feature_config,
metrics=metrics,
optimizer=optimizer,
tfrecord_type=tfrecord_type,
model_file=model_file,
compile_keras_model=compile_keras_model,
output_name=output_name,
file_io=file_io,
logger=logger,
)
@classmethod
def from_univariate_interaction_model(
cls,
model_config,
feature_config: FeatureConfig,
tfrecord_type: str,
loss: RelevanceLossBase,
metrics: List[Union[kmetrics.Metric, str]],
optimizer: Optimizer,
feature_layer_keys_to_fns: dict = {},
model_file: Optional[str] = None,
compile_keras_model: bool = False,
output_name: str = "score",
max_sequence_size: int = 0,
file_io: FileIO = None,
logger=None,
):
"""Use this as constructor to use UnivariateInteractionModel and RelevanceScorer"""
interaction_model: InteractionModel = UnivariateInteractionModel(
feature_config=feature_config,
feature_layer_keys_to_fns=feature_layer_keys_to_fns,
tfrecord_type=tfrecord_type,
max_sequence_size=max_sequence_size,
)
return cls.from_relevance_scorer(
interaction_model=interaction_model,
model_config=model_config,
feature_config=feature_config,
loss=loss,
metrics=metrics,
optimizer=optimizer,
tfrecord_type=tfrecord_type,
model_file=model_file,
compile_keras_model=compile_keras_model,
output_name=output_name,
file_io=file_io,
logger=logger,
)
def fit(
self,
dataset: RelevanceDataset,
num_epochs: int,
models_dir: str,
logs_dir: Optional[str] = None,
logging_frequency: int = 25,
monitor_metric: str = "",
monitor_mode: str = "",
patience=2,
):
"""
Trains model for defined number of epochs
Args:
dataset: an instance of RankingDataset
num_epochs: int value specifying number of epochs to train for
models_dir: directory to save model checkpoints
logs_dir: directory to save model logs
logging_frequency: every #batches to log results
"""
if not monitor_metric.startswith("val_"):
monitor_metric = "val_{}".format(monitor_metric)
callbacks_list: list = self._build_callback_hooks(
models_dir=models_dir,
logs_dir=logs_dir,
is_training=True,
logging_frequency=logging_frequency,
monitor_mode=monitor_mode,
monitor_metric=monitor_metric,
patience=patience,
)
if self.is_compiled:
self.model.fit(
x=dataset.train,
validation_data=dataset.validation,
epochs=num_epochs,
verbose=True,
callbacks=callbacks_list,
)
else:
raise NotImplementedError(
"The model could not be trained. Check if the model was compiled correctly. Training loaded SavedModel is not currently supported."
)
def predict(
self,
test_dataset: data.TFRecordDataset,
inference_signature: str = "serving_default",
additional_features: dict = {},
logs_dir: Optional[str] = None,
logging_frequency: int = 25,
):
"""
Predict the labels for the trained model
Args:
test_dataset: an instance of tf.data.dataset
inference_signature: If using a SavedModel for prediction, specify the inference signature
logging_frequency: integer representing how often(in batches) to log status
Returns:
ranking scores or new ranks for each record in a query
"""
if logs_dir:
outfile = os.path.join(logs_dir, RelevanceModelConstants.MODEL_PREDICTIONS_CSV_FILE)
# Delete file if it exists
self.file_io.rm_file(outfile)
_predict_fn = get_predict_fn(
model=self.model,
tfrecord_type=self.tfrecord_type,
feature_config=self.feature_config,
inference_signature=inference_signature,
is_compiled=self.is_compiled,
output_name=self.output_name,
features_to_return=self.feature_config.get_features_to_log(),
additional_features=additional_features,
max_sequence_size=self.max_sequence_size,
)
predictions_df_list = list()
batch_count = 0
for predictions_dict in test_dataset.map(_predict_fn).take(-1):
predictions_df = pd.DataFrame(predictions_dict)
if logs_dir:
if os.path.isfile(outfile):
predictions_df.to_csv(outfile, mode="a", header=False, index=False)
else:
# If writing first time, write headers to CSV file
predictions_df.to_csv(outfile, mode="w", header=True, index=False)
else:
predictions_df_list.append(predictions_df)
batch_count += 1
if batch_count % logging_frequency == 0:
self.logger.info("Finished predicting scores for {} batches".format(batch_count))
predictions_df = None
if logs_dir:
self.logger.info("Model predictions written to -> {}".format(outfile))
else:
predictions_df = pd.concat(predictions_df_list)
return predictions_df
def evaluate(
self,
test_dataset: data.TFRecordDataset,
inference_signature: str = None,
additional_features: dict = {},
group_metrics_min_queries: int = 50,
logs_dir: Optional[str] = None,
logging_frequency: int = 25,
):
"""
Evaluate the ranking model
Args:
test_dataset: an instance of tf.data.dataset
inference_signature: If using a SavedModel for prediction, specify the inference signature
logging_frequency: integer representing how often(in batches) to log status
metric_group_keys: list of fields to compute group based metrics on
save_to_file: set to True to save predictions to file like self.predict()
Returns:
metrics and groupwise metrics as pandas DataFrames
NOTE:
Only if the keras model is compiled, you can directly do a model.evaluate()
Override this method to implement your own evaluation metrics.
"""
if self.is_compiled:
return self.model.evaluate(test_dataset)
else:
raise NotImplementedError
def save(
self,
models_dir: str,
preprocessing_keys_to_fns={},
postprocessing_fn=None,
required_fields_only: bool = True,
pad_sequence: bool = False,
):
"""
Save tf.keras model to models_dir
Args:
models_dir: path to directory to save the model
"""
model_file = os.path.join(models_dir, "final")
# Save model with default signature
self.model.save(filepath=os.path.join(model_file, "default"))
"""
Save model with custom signatures
Currently supported
- signature to read TFRecord SequenceExample inputs
"""
self.model.save(
filepath=os.path.join(model_file, "tfrecord"),
signatures=define_serving_signatures(
model=self.model,
tfrecord_type=self.tfrecord_type,
feature_config=self.feature_config,
preprocessing_keys_to_fns=preprocessing_keys_to_fns,
postprocessing_fn=postprocessing_fn,
required_fields_only=required_fields_only,
pad_sequence=pad_sequence,
max_sequence_size=self.max_sequence_size,
),
)
self.logger.info("Final model saved to : {}".format(model_file))
def load(self, model_file: str) -> Model:
"""
Loads model from the SavedModel file specified
Args:
model_file: path to file with saved tf keras model
Returns:
loaded tf keras model
"""
"""
NOTE:
There is currently a bug in Keras Model with saving/loading
models with custom losses and metrics.
Therefore, we are currently loading the SavedModel with compile=False
The saved model signatures can be used for inference at serving time
NOTE: Retraining currently not supported!
Would require compiling the model with the right loss and optimizer states
Ref:
https://github.com/keras-team/keras/issues/5916
https://github.com/tensorflow/tensorflow/issues/32348
https://github.com/keras-team/keras/issues/3977
"""
model = tf.keras.models.load_model(model_file, compile=False)
self.logger.info("Successfully loaded SavedModel from {}".format(model_file))
self.logger.warning("Retraining is not yet supported. Model is loaded with compile=False")
return model
def load_weights(self, model_file: str):
# Load saved model with compile=False
loaded_model = self.load(model_file)
# Set weights of Keras model from the loaded model weights
self.model.set_weights(loaded_model.get_weights())
self.logger.info("Weights have been set from SavedModel. RankingModel can now be trained.")
def _build_callback_hooks(
self,
models_dir: str,
logs_dir: Optional[str] = None,
is_training=True,
logging_frequency=25,
monitor_metric: str = "",
monitor_mode: str = "",
patience=2,
):
"""
Build callback hooks for the training loop
Returns:
callbacks_list: list of callbacks
"""
callbacks_list: list = list()
if is_training:
# Model checkpoint
if models_dir and monitor_metric:
checkpoints_path = os.path.join(
models_dir, RelevanceModelConstants.CHECKPOINT_FNAME
)
cp_callback = callbacks.ModelCheckpoint(
filepath=checkpoints_path,
save_weights_only=False,
verbose=1,
save_best_only=True,
mode=monitor_mode,
monitor=monitor_metric,
)
callbacks_list.append(cp_callback)
# Early Stopping
if monitor_metric:
early_stopping_callback = callbacks.EarlyStopping(
monitor=monitor_metric,
mode=monitor_mode,
patience=patience,
verbose=1,
restore_best_weights=True,
)
callbacks_list.append(early_stopping_callback)
# TensorBoard
if logs_dir:
tensorboard_callback = callbacks.TensorBoard(
log_dir=logs_dir, histogram_freq=1, update_freq=5
)
callbacks_list.append(tensorboard_callback)
# Debugging/Logging
logger = self.logger
class DebuggingCallback(callbacks.Callback):
def __init__(self, patience=0):
super(DebuggingCallback, self).__init__()
self.epoch = 0
def on_train_batch_end(self, batch, logs=None):
if batch % logging_frequency == 0:
logger.info("[epoch: {} | batch: {}] {}".format(self.epoch, batch, logs))
def on_epoch_end(self, epoch, logs=None):
logger.info("End of Epoch {}".format(self.epoch))
logger.info(logs)
def on_epoch_begin(self, epoch, logs=None):
self.epoch = epoch + 1
logger.info("Starting Epoch : {}".format(self.epoch))
logger.info(logs)
def on_train_begin(self, logs):
logger.info("Training Model")
def on_test_begin(self, logs):
logger.info("Evaluating Model")
def on_predict_begin(self, logs):
logger.info("Predicting scores using model")
def on_train_end(self, logs):
logger.info("Completed training model")
logger.info(logs)
def on_test_end(self, logs):
logger.info("Completed evaluating model")
logger.info(logs)
def on_predict_end(self, logs):
logger.info("Completed Predicting scores using model")
logger.info(logs)
callbacks_list.append(DebuggingCallback())
# Add more here
return callbacks_list
out= model.train_on_batch(x_batch, y_batch)
loss_val = out[0]
acc = out[1]*100
train_losses.append(loss_val)
train_accuracies.append(acc)
pbar.update(1)
pbar.set_postfix_str(f"Loss: {loss_val:.4f} ({np.mean(train_losses):.4f}) Acc: {acc:.3f} ({np.mean(train_accuracies):.3f})")
with tqdm_notebook(total=VAL_STEPS, desc=f"Test_ Epoch {epoch+1}") as pbar:
test_losses = []
test_accuracies = []
for s in range(VAL_STEPS):
x_batch_val, y_batch_val = getBatch(test_size, "val")
evaluation = model.evaluate(x_batch_val, y_batch_val)
loss_val= evaluation[0]
acc = evaluation[1]*100
test_losses.append(loss_val)
test_accuracies.append(acc)
pbar.update(1)
pbar.set_postfix_str(f"Loss: {loss_val:.4f} ({np.mean(test_losses):.4f}) Acc: {acc:.3f} ({np.mean(test_accuracies):.3f})")
model.save_weights(model_name+'.h5', overwrite=True)
# Sample outputs from validation set
LABELS_LIST = "airplane automobile bird cat deer dog frog horse ship truck".split(" ")
test_gen = generator.flow(test_subjects.index, test_targets)
history = model.fit(
train_gen,
epochs = 20,
validation_data = test_gen,
verbose = 2,
shuffle = False
)
# Plot training/validation accuracy/loss
print(sg.utils.plot_history(history, return_figure = True))
plt.savefig("testsave.png")
# plt.show()
# Test run model on the testing generator (again :) )
test_metrics = model.evaluate(test_gen)
print("\nTest Set Metrics:")
for name, val in zip(model.metrics_names, test_metrics):
print("\t{}: {:0.4f}".format(name, val))
# Make predictions using the model
all_nodes = node_subjects.index
all_mapper = generator.flow(all_nodes)
all_predictions = model.predict(all_mapper)
# Use inverse_transform of the LabelBinarizer to turn the
# one-hot encodings back into their textual labels
node_predictions = target_encoding.inverse_transform(all_predictions)
df = pd.DataFrame({"Predicted": node_predictions, "True": node_subjects})
print(df.head(10))
def run_prediction():
'''Procedure to run prediction model in jupyter notebook.
'''
# load dataset
X_train, X_test, y_train, y_test = datasets_v0.load_data()
frame = framework_xception(batch_size=32)
input_size = (128, 128, 3)
# Prepare Datasets
frame.load_datagenerators(X_train,
y_train,
X_test,
y_test,
input_size=(128, 128))
# load base model
base_model = Xception(weights='imagenet',
include_top=False,
input_shape=(128, 128, 3))
# load top model
top_model = base_model.output
top_model = Flatten()(top_model)
top_model = Dense(256, activation='relu')(top_model)
top_model = Dropout(0.5)(top_model)
predictions = Dense(1, activation='relu')(top_model)
# stack
model = Model(inputs=base_model.input, outputs=predictions)
#print(model.summary())
# set trainable layer to top layers only
change_trainable_layers(model, 132)
# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
#optimizer = optimizers.SGD(lr=1e-4, momentum=0.9)
optimizer = optimizers.Adam(learning_rate=1e-4,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False,
name='Adam')
model.compile(loss='mean_squared_error',
optimizer=optimizer,
metrics=[soft_acc,
tf.keras.metrics.RootMeanSquaredError()])
print('\nFitting the model ... ...')
log_dir = "../logs/fit/xception/" + dt.datetime.now().strftime(
"%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
history = model.fit(
x=frame.train_generator,
#y=self.train_df['label'].values,
#generator = train_generator,
batch_size=None,
epochs=20,
verbose=1,
validation_data=frame.valid_generator,
#shuffle=False,
#class_weight=None,
#sample_weight=None,
#initial_epoch=0,
steps_per_epoch=frame.STEP_SIZE_TRAIN,
validation_steps=frame.STEP_SIZE_VALID,
#validation_freq=1,
max_queue_size=frame.batch_size * 8,
#workers=4,
use_multiprocessing=False,
callbacks=[tensorboard_callback])
print('\nValidationg the model ... ...')
log_dir = "../logs/validation/" + dt.datetime.now().strftime(
"%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
model.evaluate(
x=frame.test_generator,
y=None,
batch_size=None,
verbose=1,
sample_weight=None,
steps=frame.STEP_SIZE_TEST,
max_queue_size=frame.batch_size * 8,
#workers=1,
use_multiprocessing=False,
callbacks=[tensorboard_callback])
pred = model.predict(
x=frame.test_generator,
steps=frame.STEP_SIZE_TEST,
max_queue_size=frame.batch_size * 8,
#workers=8,
use_multiprocessing=False,
verbose=True)
input_dataset = tf.data.Dataset.from_generator(
lambda: data_generator(input_fns, input_gt_fns, validation=False),
output_types=(tf.float32, tf.uint16),
output_shapes=(tf.TensorShape([None, None, None, None]),
tf.TensorShape([None, None, None, None])))
# construct validation dataset generator
validation_dataset = tf.data.Dataset.from_generator(
lambda: data_generator(validation_fns, validation_gt_fns, validation=True),
output_types=(tf.float32, tf.uint16),
output_shapes=(tf.TensorShape([None, None, None, None]),
tf.TensorShape([None, None, None, None])))
if TESTING:
# # record test loss (MAE)
[loss, acc] = model.evaluate(input_dataset)
print('Loss: ' + str(loss))
# predict output images on test set
out = model.predict(input_dataset, verbose=1)
psnr = []
ssim = []
# record metrics and also save images to disk
for idx, im in enumerate(out):
# get corresponding ground truth image
gt_image = gt_map[input_gt_fns[idx]]
# Rescale ground truth back to 0-255 and convert to uint8
onetensor = tf.ones(shape=tf.shape(y_true))
temploss = (-1)*tf.reduce_sum(tf.math.multiply(y_true, tf.math.log(y_pred))) - tf.reduce_sum(tf.math.multiply((onetensor - y_true), \
tf.math.log(onetensor - y_pred)))
return temploss
##
model.compile(optimizer='adam', loss=binaryloss)
# fit the model to the training data and evaluate with the test data
tic = time.perf_counter()
history = model.fit(obs_train, lab_train, epochs=1, batch_size=32, verbose=1)
toc = time.perf_counter()
model.evaluate(obs_test, lab_test)
print('Total training time: ' + str(toc - tic) + ' seconds.')
# save off the model and model history
# model.save('model0ones_15dB_fixedH')
# with open('F:/Comm Studies/PermutationSimulations/UnlabeledSensing/TrainingHistories/trainHistoryDict_model0ones_15dB_fixedH', 'wb') as file_pi:
# pickle.dump(history.history, file_pi)
# -----------------------------------------------------------------------------
# ---------- MISCELLANEOUS TESTING CODE - IGNORE BELOW THIS POINT -------------
# -----------------------------------------------------------------------------
#---------------------------US test-------------------------------------------
#model = tf.keras.models.load_model('model0_100dB_fixedH')
def train_networks(training_percentage, filename, experiment):
stages = constants.training_stages
(data, labels) = get_data(experiment, one_hot=True)
total = len(data)
step = total / stages
# Amount of training data, from which a percentage is used for
# validation.
training_size = int(total * training_percentage)
n = 0
histories = []
for k in range(stages):
i = k * step
j = int(i + training_size) % total
i = int(i)
if j > i:
training_data = data[i:j]
training_labels = labels[i:j]
testing_data = np.concatenate((data[0:i], data[j:total]), axis=0)
testing_labels = np.concatenate((labels[0:i], labels[j:total]),
axis=0)
else:
training_data = np.concatenate((data[i:total], data[0:j]), axis=0)
training_labels = np.concatenate((labels[i:total], labels[0:j]),
axis=0)
testing_data = data[j:i]
testing_labels = labels[j:i]
training_data, training_labels = expand_data(training_data,
training_labels)
truly_training = int(training_size * truly_training_percentage)
validation_data = training_data[truly_training:]
validation_labels = training_labels[truly_training:]
training_data = training_data[:truly_training]
training_labels = training_labels[:truly_training]
input_img = Input(shape=(img_columns, img_rows, img_colors))
encoded = get_encoder(input_img)
classified = get_classifier(encoded)
decoded = get_decoder(encoded)
model = Model(inputs=input_img, outputs=[classified, decoded])
model.compile(loss=['categorical_crossentropy', 'binary_crossentropy'],
optimizer='adam',
metrics='accuracy')
model.summary()
history = model.fit(training_data, (training_labels, training_data),
batch_size=batch_size,
epochs=epochs,
validation_data=(validation_data, {
'classification': validation_labels,
'autoencoder': validation_data
}),
callbacks=[EarlyStoppingAtLossCrossing(patience)],
verbose=2)
histories.append(history)
history = model.evaluate(testing_data, (testing_labels, testing_data),
return_dict=True)
histories.append(history)
model.save(constants.model_filename(filename, n))
n += 1
return histories
print(hist.history.keys())
print('train loss: ', hist.history['loss'][-1])
print('train acc: ', hist.history['accuracy'][-1])
print('val acc: ', hist.history['val_loss'][-1])
print('val acc: ', hist.history['val_accuracy'][-1])
# %matplotlib inline
import matplotlib.pyplot as plt
plt.plot(hist.history['loss'])
plt.xlabel('epoch')
plt.ylabel('loss')
plt.show()
"""- 모델 평가(Test Model)"""
test_loss = model.evaluate([test_df.userid, test_df.shop_id], test_df.rating)
print('test loss: ', test_loss)
"""- ## 학습된 머신을 활용한 예측"""
pd.options.display.float_format = '{:.2f}'.format # 출력 포매팅 설정
# ratings_df[(ratings_df['userid'] == 249) & (ratings_df['shop_Id'] == 70)]
# print(df_backup.head(30))
print(df_backup[df_backup['userid'] == 'user000777'])
# print(df.head())
userid = 777 # 1 ~ 610
shop_id = 67
target = df_backup[(df_backup['userid'] == ('user' +
f'{str(userid).zfill(6)}'))
& (df_backup['shop_id'] == shop_id)]
conv2 = layers.Conv2D(64, kernel_size=(3,3), activation='relu')(pool1)
noise2 = layers.GaussianNoise(test_dict[2])(conv2, training=True)
pool2 = layers.MaxPooling2D(pool_size=(2, 2))(noise2)
conv3 = layers.Conv2D(64, kernel_size=(3,3), activation='relu')(pool2)
noise1 = layers.GaussianNoise(test_dict[3])(conv3, training=True)
flat = layers.Flatten()(noise1)
hidden1 = layers.Dense(64, activation='relu')(flat)
output = layers.Dense(10)(hidden1)
model1 = Model(inputs=visible, outputs=output)
model1.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model1.set_weights(weights1)
test_loss, test_acc = model1.evaluate(test_images, test_labels, verbose=0)
fintest_trial.append(test_acc)
parameters[name]=fintest_trial
#testing clear model
name=str(m)+'clearmodelLayer'+str(l)
fintest_trial=[]
for n in range(0, 2):
print('n', n)
del(model1)
tf.compat.v1.reset_default_graph()
if n==0:
test_dict={1: 0, 2: 0, 3: 0}
else:
test_dict=noise_dict
visible = layers.Input(shape=(32,32,3))
def lra_framework(model: Model, lra_algorithm, x_train, x_test, y_test, dataset, model_name):
scores = []
arch = model_name.split('_')[0]
dir = os.path.join(arch, model_name)
if not os.path.exists(dir):
os.makedirs(dir)
logger_path = os.path.join(dir, 'log_file_model_{}'.format(model_name))
if os.path.exists(logger_path + ".log"):
os.remove(logger_path + ".log")
logger = get_logger(logger_path)
samples = x_train # np.concatennum_of_paramsate((x_train, x_test))
initial_score = model.evaluate(x_test, y_test, verbose=0)
score = np.copy(initial_score)
score_to_plot = []
compression_ratio_to_plot = []
score_to_plot.append(score[-1])
compression_ratio_to_plot.append(0)
opt = tf.keras.optimizers.Adam()
temp_model = clone_model(model)
temp_model.compile(
loss=tf.keras.losses.categorical_crossentropy,
optimizer=opt,
metrics=['accuracy'])
lra_model = clone_model(model)
lra_model.set_weights(model.get_weights())
lra_model.compile(
loss=tf.keras.losses.categorical_crossentropy,
optimizer=opt,
metrics=['accuracy'])
relevant_layers, relevant_layers_index_in_model = get_relevant_layers(model)
initial_num_of_params = get_initial_number_of_params(relevant_layers)
print('\n\n')
it = 0
while (initial_score[1] - score[1] < cfg.accuracy_tolerance):
klds = []
print("Start of Iteration {0}:".format(it))
logger.info("Start of Iteration {0}:".format(it))
curr_num_of_params = 0
for i, layer_index in enumerate(relevant_layers_index_in_model):
temp_model.set_weights(lra_model.get_weights())
temp_model, _, _, num_of_params_layer_i = lra_per_layer(temp_model, layer_index=layer_index,
algorithm=lra_algorithm, update_memory=False)
curr_num_of_params += num_of_params_layer_i
# kld_per_layer = evaluate_kld_for_each_layer(model, temp_model, samples)
kld_per_layer = evaluate_kld_for_last_layer(model, temp_model, samples)
print('{} ({}): KLD per layer {:.4f}'.format(relevant_layers[i].name, layer_index, kld_per_layer))
# klds.append(sum(kld_per_layer))
klds.append(kld_per_layer)
print("Compression ratio : {}. \n number of params: \n \t lra_model {:,} \n \t initial model {:,}"
.format(1 - curr_num_of_params / initial_num_of_params, curr_num_of_params, initial_num_of_params))
logger.info("Compression ratio : {}. number of params: \t lra_model {:,} \t initial model {:,}"
.format(1 - curr_num_of_params / initial_num_of_params, curr_num_of_params, initial_num_of_params))
if curr_num_of_params < initial_num_of_params:
score_to_plot.append(score[-1])
compression_ratio_to_plot.append(1 - curr_num_of_params / initial_num_of_params) # so the graph will start from 0 and go to 1
min_kld_index = np.argmin(klds)
layer_with_min_kld = relevant_layers[min_kld_index]
layer_index_in_model_with_min_kld = relevant_layers_index_in_model[min_kld_index]
# print('---------------- Start Compression with {0} for layer {1}!) ----------------'.format(lra_algorithm,
# layer_with_min_kld.name))
lra_model, truncated, full_svs, _ = lra_per_layer(lra_model, layer_index=layer_index_in_model_with_min_kld,
algorithm=lra_algorithm, update_memory=True)
if 'tsvd' in lra_algorithm:
print('Approximate {0} {1} using {2}/{3} singular values'.format(layer_with_min_kld.name,
layer_index_in_model_with_min_kld,
truncated, full_svs))
logger.info('Approximate {0} {1} using {2}/{3} singular values'.format(layer_with_min_kld.name,
layer_index_in_model_with_min_kld,
truncated, full_svs))
if 'rrqr' in lra_algorithm:
print('Approximate {0} {1} using {2}/{3} rank ratio '.format(layer_with_min_kld.name,
layer_index_in_model_with_min_kld,
truncated, full_svs))
logger.info('Approximate {0} {1} using {2}/{3} rank ratio'.format(layer_with_min_kld.name,
layer_index_in_model_with_min_kld,
truncated, full_svs))
# print('---------------- Done Compression with {0} for layer {1}!) ----------------'.format(lra_algorithm,
# layer_with_min_kld.name))
score = lra_model.evaluate(x_test, y_test, verbose=0)
scores.append(score)
print("End of Iteration {}:\ntest loss = {:.3f}\ntest accuracy = {:.3f}\n\n\n".format(it, score[0], score[1]))
logger.info("End of Iteration {}:\ntest loss = {:.3f}\ntest accuracy = {:.3f}\n\n\n".format(it, score[0], score[1]))
it += 1
save_model_path = os.path.join(dir, '{0}_{1}_lra.h5'.format(model_name, dataset))
print('Saving model to: ', save_model_path)
logger.info('Saving model to: {}'.format(save_model_path))
save_model(lra_model, save_model_path, include_optimizer=False, save_format='h5')
score_to_plot = np.asarray(score_to_plot)
compression_ratio_to_plot = np.asarray(compression_ratio_to_plot)
np.save(os.path.join(dir, 'score_{}'.format(model_name)), score_to_plot)
np.save(os.path.join(dir, 'compression_{}'.format(model_name)), compression_ratio_to_plot)
plot_score_versus_compression(save_dir=dir, score_data=score_to_plot,
compression_data=compression_ratio_to_plot, model_name=model_name, algo=lra_algorithm)
