class model: """ Model to recognised prompt (label) based on the input MFCC feature CNN-model Lstm-model End2end-model Attention-model """ def __init__(self, train_mfcc, train_label, test_mfcc, test_label, validate_mfcc, validate_label): self.train_mfcc = train_mfcc self.train_label = train_label self.test_mfcc = test_mfcc self.test_label = test_label self.validate_mfcc = validate_mfcc self.validate_label = validate_label self.history = None def CNN_model(self, learning_rate, epoch, batchsize, whether_Adam, Momentum_gamma, weight_decay, whether_load, cnn_type): """ Resnet model :param learning_rate :param epoch :param batchsize :param whether_Adam: whether to perform Adam optimiser, if not perform Momentum :param Momentum gamma: a variable of Momentum :param weight_decay: weight decay for Momentum :param whether_load: whether to load trained Resnet model in if it exists (or cover it) """ test_cnn_mfcc = self.train_mfcc test_cnn_label = self.train_label if(isfile("model/resnet_label.hdf5") and whether_load): self.cnn_model = load_model("model/resnet_label.hdf5") else: train_cnn_mfcc = self.test_mfcc train_cnn_label = self.test_label val_cnn_mfcc = self.validate_mfcc val_cnn_label = self.validate_label # input input = Input(shape=(self.test_mfcc.shape[1], self.test_mfcc.shape[2], 1)) # Concatenate -1 dimension to be three channels, to fit the input need in ResNet50 input_concate = Concatenate()([input,input,input]) # CNN series network (VGG+Resnet) # reference: https://keras.io/api/applications/ if(cnn_type == 'ResNet50'): from tensorflow.keras.applications import ResNet50 cnn_output = ResNet50(pooling = 'avg')(input_concate) elif(cnn_type == 'ResNet101'): from tensorflow.keras.applications import ResNet101 cnn_output = ResNet101(pooling = 'avg')(input_concate) elif(cnn_type == 'ResNet152'): from tensorflow.keras.applications import ResNet152 cnn_output = ResNet152(pooling = 'avg')(input_concate) elif(cnn_type == 'ResNet50V2'): from tensorflow.keras.applications import ResNet50V2 cnn_output = ResNet50V2(pooling = 'avg')(input_concate) elif(cnn_type == 'ResNet101V2'): from tensorflow.keras.applications import ResNet101V2 cnn_output = ResNet101V2(pooling = 'avg')(input_concate) elif(cnn_type == 'ResNet152V2'): from tensorflow.keras.applications import ResNet152V2 cnn_output = ResNet152V2(pooling = 'avg')(input_concate) elif(cnn_type == 'VGG16'): # width and height should not smaller than 32 from tensorflow.keras.applications import VGG16 cnn_output = VGG16(include_top = False, pooling = 'avg')(input_concate) cnn_output = Flatten()(cnn_output) elif(cnn_type == 'VGG19'): # width and height should not smaller than 32 from tensorflow.keras.applications import VGG19 cnn_output = VGG19(include_top = False, pooling = 'avg')(input_concate) cnn_output = Flatten()(cnn_output) else: # CNN layers we design print("No recognised CNN network. The CNN layers we designed are performed") # convolution layers conv_output1 = Conv2D(filters=32, strides=(1, 1), kernel_size=5, activation='relu')(input) # pool_output1 = MaxPool2D(pool_size=(2, 2))(conv_output1) conv_output2 = Conv2D(filters=8, strides=(2, 2), kernel_size=4, activation='relu')(conv_output1) conv_output2 = Dropout(0.2)(conv_output2) conv_output2_batch = BatchNormalization()(conv_output2) cnn_output = Flatten()(conv_output2_batch) cnn_output = Flatten()(cnn_output) # dense with sigmoid Dense_sigmoid = Dense(24, activation='sigmoid')(cnn_output) Dense_sigmoid = Dropout(0.2)(Dense_sigmoid) # dense output output = Dense(self.test_label.shape[1], activation='softmax')(Dense_sigmoid) # cnn model for labels recognision self.cnn_model = Model(input, output) # optimizer if whether_Adam: optimizer = optimizers.Adam(lr=learning_rate, beta_1 = Momentum_gamma, decay=weight_decay) else: optimizer = optimizers.SGD(lr=learning_rate, momentum=Momentum_gamma, nesterov=True, decay=weight_decay) self.cnn_model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['mse', 'accuracy']) start = time.time() self.history = self.cnn_model.fit(train_cnn_mfcc, train_cnn_label, epochs=epoch, batch_size=batchsize, validation_data=[val_cnn_mfcc,val_cnn_label]) self.training_time = time.time() - start self.cnn_model.save("model/resnet_label.hdf5") # model evaluation self.cnn_model.predict(test_cnn_mfcc) self.score = self.cnn_model.evaluate(test_cnn_mfcc, test_cnn_label) print("test loss: ", self.score[0], ", mse: ", self.score[1], ", accuracy", self.score[2]) def LSTM_model(self, learning_rate, epoch, batchsize, Lstm_hidden_num, whether_Adam, Momentum_gamma, weight_decay, whether_load, bidirectional = True): """ LSTM with concated hidden states connected to output layer :param learning_rate :param epoch :param batchsize :param Lstm_hidden_num: number of hidden units of LSTM layer :param whether_Adam: whether to perform Adam optimiser, if not perform Momentum :param Momentum gamma: a variable of Momentum :param weight_decay: weight decay for Momentum :param whether_load: whether to load trained LSTM model in if it exists (or cover it) :param bidirectional: whether to use bidirectional LSTM or a normal one """ test_lstm_mfcc = self.train_mfcc.reshape(self.train_mfcc.shape[0],self.train_mfcc.shape[1],self.train_mfcc.shape[2]) test_lstm_label = self.train_label if (isfile("model/lstm_label.hdf5") and whether_load): self.lstm_model = load_model("model/lstm_label.hdf5") else: train_lstm_mfcc = self.test_mfcc.reshape(self.test_mfcc.shape[0], self.test_mfcc.shape[1], self.test_mfcc.shape[2]) train_lstm_label = self.test_label val_lstm_mfcc = self.validate_mfcc.reshape(self.validate_mfcc.shape[0], self.validate_mfcc.shape[1], self.validate_mfcc.shape[2]) val_lstm_label = self.validate_label # input input = Input(shape=(None, self.test_mfcc.shape[2])) lstm_layer = LSTM(Lstm_hidden_num, return_state=True) if(bidirectional): bi_lstm_layer = Bidirectional(lstm_layer) _, forword_h, forword_c, backword_h, backword_c = bi_lstm_layer(input) Concate = Concatenate()([forword_h, forword_c, backword_h, backword_c]) else: _, h, c = lstm_layer(input) Concate = Concatenate()([h, c]) Concate = Dropout(0.2)(Concate) Concate_batchnorm = BatchNormalization()(Concate) # dense output output = Dense(self.test_label.shape[1], activation='softmax')(Concate_batchnorm) # cnn model for labels recognision self.lstm_model = Model(input, output) # optimizer if whether_Adam: optimizer = optimizers.Adam(lr=learning_rate, beta_1 = Momentum_gamma, decay=weight_decay) else: optimizer = optimizers.SGD(lr=learning_rate, momentum=Momentum_gamma, nesterov=True, decay=weight_decay) self.lstm_model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['mse', 'accuracy']) start = time.time() self.history = self.lstm_model.fit(train_lstm_mfcc, train_lstm_label, epochs=epoch, batch_size=batchsize, validation_data=[val_lstm_mfcc, val_lstm_label]) self.training_time = time.time() - start self.lstm_model.save("model/lstm_label.hdf5") # model evaluation self.lstm_model.predict(test_lstm_mfcc) self.score = self.lstm_model.evaluate(test_lstm_mfcc, test_lstm_label) print("test loss: ", self.score[0], ", mse: ", self.score[1], ", accuracy", self.score[2]) def end2end_model(self, int_to_labels, learning_rate, epoch, batchsize, Lstm_hidden_num, whether_Adam, Momentum_gamma, weight_decay, whether_load): """ LSTM-based end to end model :param int_to_labels: dictionary convert id number to prompt label :param learning_rate :param epoch :param batchsize :param Lstm_hidden_num: number of hidden units of LSTM layer :param whether_Adam: whether to perform Adam optimiser, if not perform Momentum :param Momentum gamma: a variable of Momentum :param weight_decay: weight decay for Momentum :param whether_load: whether to load trained end2end model in if it exists (or cover it) """ # encoder and decoder model test data encoder_test = self.train_mfcc encoder_test = encoder_test.reshape(encoder_test.shape[0], 1, encoder_test.shape[1], encoder_test.shape[2]) decoder_test_output = self.train_label decoder_size = 10 decoder_test_input = np.zeros([1,1,decoder_size]) if (isfile("model/end2end_encoder.hdf5") and isfile("model/end2end_decoder.hdf5") and whether_load): self.Model_end2end_encoder = load_model("model/end2end_encoder.hdf5") self.Model_end2end_decoder = load_model("model/end2end_decoder.hdf5") else: #data prerproprocess for encoder encoder_train = self.test_mfcc.reshape(self.test_mfcc.shape[0], self.test_mfcc.shape[1], self.test_mfcc.shape[2]) encoder_val = self.validate_mfcc.reshape(self.validate_mfcc.shape[0], self.validate_mfcc.shape[1], self.validate_mfcc.shape[2]) #data prerproprocess for decoder decoder_train_input = np.zeros([encoder_train.shape[0],1,decoder_size]) decoder_val_input = np.zeros([encoder_val.shape[0],1,decoder_size]) decoder_train_output = self.test_label decoder_val_output = self.validate_label # train model # encoder encoder_input = Input(shape=(None, encoder_train.shape[2])) encoder_output, encoder_h, encoder_c = LSTM(Lstm_hidden_num, return_state=True, return_sequences=True)(encoder_input) encoder_h = Dropout(0.2)(encoder_h) encoder_c = Dropout(0.2)(encoder_c) encoder_h = BatchNormalization()(encoder_h) encoder_c = BatchNormalization()(encoder_c) # decoder decoder_input = Input(shape=(None, decoder_train_input.shape[2])) decoder_lstm = LSTM(Lstm_hidden_num, return_state=True, return_sequences=True) decoder_output, decoder_h, decoder_c = decoder_lstm(decoder_input, initial_state=[encoder_h, encoder_c]) decoder_output = Dropout(0.2)(decoder_output) decoder_dense = Dense(decoder_train_output.shape[1], activation='softmax') output = decoder_dense(decoder_output) self.Model_end2end_train = Model([encoder_input, decoder_input], output) # optimizer if whether_Adam: optimizer = optimizers.Adam(lr=learning_rate, beta_1 = Momentum_gamma, decay=weight_decay) else: optimizer = optimizers.SGD(lr=learning_rate, momentum=Momentum_gamma, nesterov=True, decay=weight_decay) self.Model_end2end_train.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['mse', 'accuracy']) start = time.time() self.history = self.Model_end2end_train.fit([encoder_train, decoder_train_input], decoder_train_output, epochs=epoch, batch_size=batchsize, validation_data=[[encoder_val, decoder_val_input], decoder_val_output]) self.training_time = time.time() - start # encoder model self.Model_end2end_encoder = Model(encoder_input, [encoder_h, encoder_c]) self.Model_end2end_encoder.save("model/end2end_encoder.hdf5") # decoder model decoder_h_input = Input(shape=(Lstm_hidden_num,)) decoder_c_input = Input(shape=(Lstm_hidden_num,)) decoder_output, decoder_h_output, decoder_c_output = decoder_lstm(decoder_input, initial_state=[decoder_h_input, decoder_c_input]) output = decoder_dense(decoder_output) self.Model_end2end_decoder = Model([decoder_input, decoder_h_input, decoder_c_input], output) self.Model_end2end_decoder.save("model/end2end_decoder.hdf5") # prediction # test accuracy counting num_trueclassify = 0 for i in range(len(encoder_test)): [h, c] = self.Model_end2end_encoder.predict(encoder_test[i]) current_decoder_train_output = self.Model_end2end_decoder.predict([decoder_test_input, h, c]) output_hat = np.argmax(current_decoder_train_output) output_hat = int_to_labels[output_hat] output = np.argmax(decoder_test_output[i]) output = int_to_labels[output] if(output_hat==output): num_trueclassify+=1 print("prediction:", output_hat) print("actually:", output,'\n') self.score = num_trueclassify / len(encoder_test) print("test accuracy:", self.score) def Attention_model(self, int_to_labels, learning_rate, epoch, batchsize, Lstm_hidden_num, whether_Adam, Momentum_gamma, weight_decay, whether_load): """ LSTM-based end to end model with global attention :param int_to_labels: dictionary convert id number to prompt label :param learning_rate: :param epoch: :param batchsize: :param Lstm_hidden_num: number of hidden units of LSTM layer :param whether_Adam: whether to perform Adam optimiser, if not perform Momentum :param Momentum gamma: a variable of Momentum :param weight_decay: weight decay for Momentum :param whether_load: whether to load trained Attention model in if it exists (or cover it) :return: """ # encoder and decoder model test data encoder_test = self.train_mfcc encoder_test = encoder_test.reshape(encoder_test.shape[0], 1, encoder_test.shape[1], encoder_test.shape[2]) decoder_test_output = self.train_label decoder_size = 10 decoder_test_input = np.zeros([1,1,decoder_size]) if (isfile("model/end2end_encoder.hdf5") and isfile("model/end2end_decoder.hdf5") and whether_load): self.Model_end2end_encoder = load_model("model/end2end_encoder.hdf5") self.Model_end2end_decoder = load_model("model/end2end_decoder.hdf5") else: #data prerproprocess for encoder encoder_train = self.test_mfcc.reshape(self.test_mfcc.shape[0], self.test_mfcc.shape[1], self.test_mfcc.shape[2]) encoder_val = self.validate_mfcc.reshape(self.validate_mfcc.shape[0], self.validate_mfcc.shape[1], self.validate_mfcc.shape[2]) #data prerproprocess for decoder decoder_train_input = np.zeros([encoder_train.shape[0],1,decoder_size]) decoder_val_input = np.zeros([encoder_val.shape[0],1,decoder_size]) decoder_train_output = self.test_label decoder_val_output = self.validate_label # train model # encoder encoder_input = Input(shape=(None, encoder_train.shape[2])) encoder_output, encoder_h, encoder_c = LSTM(Lstm_hidden_num, return_state=True, return_sequences=True)(encoder_input) encoder_h = Dropout(0.2)(encoder_h) encoder_c = Dropout(0.2)(encoder_c) encoder_output = Dropout(0.2)(encoder_output) encoder_h = BatchNormalization()(encoder_h) encoder_c = BatchNormalization()(encoder_c) encoder_output = BatchNormalization()(encoder_output) # decoder decoder_input = Input(shape=(None, decoder_train_input.shape[2])) decoder_lstm = LSTM(Lstm_hidden_num, return_state=True, return_sequences=True) decoder_output, decoder_h, decoder_c = decoder_lstm(decoder_input, initial_state=[encoder_h, encoder_c]) decoder_output = Dropout(0.2)(decoder_output) # Global Attention layer attention_lstm = Attention() attention_distribute = attention_lstm([encoder_output, decoder_output]) # global average pooling among attention distribution GlobalAveragePooling1D_attetion_layer = GlobalAveragePooling1D() attention_output = GlobalAveragePooling1D_attetion_layer(attention_distribute) # global average pooling among decoder output GlobalAveragePooling1D_decoder_layer = GlobalAveragePooling1D() attention_output_new = GlobalAveragePooling1D_decoder_layer(decoder_output) # Concatenate attention output and decoder output Concaten_lstm = Concatenate() Concaten_output = Concaten_lstm([attention_output_new, attention_output]) Concaten_output = Dropout(0.2)(Concaten_output) decoder_dense = Dense(decoder_train_output.shape[1], activation='softmax') output = decoder_dense(Concaten_output) self.Model_Attetion_train = Model([encoder_input, decoder_input], output) # optimizer if whether_Adam: optimizer = optimizers.Adam(lr=learning_rate, beta_1 = Momentum_gamma, decay=weight_decay) else: optimizer = optimizers.SGD(lr=learning_rate, momentum=Momentum_gamma, nesterov=True, decay=weight_decay) self.Model_Attetion_train.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['mse', 'accuracy']) start = time.time() self.history = self.Model_Attetion_train.fit([encoder_train, decoder_train_input], decoder_train_output, epochs=epoch, batch_size=batchsize, validation_data=[[encoder_val, decoder_val_input], decoder_val_output]) self.training_time = time.time() - start # encoder model self.Model_Attetion_encoder = Model(encoder_input, [encoder_output, encoder_h, encoder_c]) self.Model_Attetion_encoder.save("model/Attention_encoder.hdf5") # decoder model decoder_h_input = Input(shape=(Lstm_hidden_num,)) decoder_c_input = Input(shape=(Lstm_hidden_num,)) encoder_output_for_attention = Input(shape = (None, Lstm_hidden_num)) decoder_output, decoder_h_output, decoder_c_output = decoder_lstm(decoder_input, initial_state=[decoder_h_input, decoder_c_input]) attention_distribute = attention_lstm([encoder_output_for_attention, decoder_output]) attention_output = GlobalAveragePooling1D_attetion_layer(attention_distribute) decoder_output_new = GlobalAveragePooling1D_decoder_layer(decoder_output) Concaten_output = Concaten_lstm([decoder_output_new, attention_output]) output = decoder_dense(Concaten_output) self.Model_Attetion_decoder = Model([decoder_input, decoder_h_input, decoder_c_input, encoder_output_for_attention], output) self.Model_Attetion_decoder.save( "model/Attention_decoder.hdf5") # prediction # test accuracy counting num_trueclassify = 0 for i in range(len(encoder_test)): [encoder_out, h, c] = self.Model_Attetion_encoder.predict(encoder_test[i]) current_decoder_train_output = self.Model_Attetion_decoder.predict([decoder_test_input, h, c, encoder_out]) output_hat = np.argmax(current_decoder_train_output) output_hat = int_to_labels[output_hat] output = np.argmax(decoder_test_output[i]) output = int_to_labels[output] if(output_hat==output): num_trueclassify+=1 #print("prediction:", output_hat) #print("actually:", output,'\n') self.score = num_trueclassify / len(encoder_test) print("test accuracy:", self.score)
# plt.plot(hist.history['acc']) # plt.plot(hist.history['val_acc']) # plt.title('loss & acc') # plt.ylabel('loss, acc') # plt.xlabel('epoch') # plt.legend(['train loss', 'val loss', 'train acc', 'val acc']) # plt.show() # save and load model model.save('../data/h5/k67_1_male_female.h5') # model = load_model('../data/h5/k67_1_male_female.h5') ######################################################################################### # evaluate loss, acc = model.evaluate(X_test, y_test, batch_size=32) print("DIMENSION :", DIMENSION, "\tCHANNEL :", CHANNEL, "\tROUNDS :", ROUNDS, "\tNODE", NODE) print("loss :", loss) print("acc :", acc) # predict y_pred = model.predict( X_test) # 마지막 레이어 sigmoid 함수를 통해 0 ~ 1 사이의 값을 반환, shape==(337, 1) #[[4.47866887e-06] # [3.17825680e-03] (이하 생략) y_pred = [(y[0] >= 0.5).astype(np.uint8) for y in y_pred] # 반복문 한 줄로 해서 리스트 형식으로 변수에 저장 # 0.5보다 크면 1 (True), 작으면 0 (False), unit8 "unsigned 8bit int" '''
def obtain_features(model_prefix, features_prefix, labels_prefix, data_prefix, training_percentage, am_filling_percentage, experiment, occlusion=None, bars_type=None): """ Generate features for images. Uses the previously trained neural networks for generating the features corresponding to the images. It may introduce occlusions. """ (data, labels) = get_data(experiment, occlusion, bars_type) total = len(data) step = int(total / constants.training_stages) # Amount of data used for training the networks trdata = int(total * training_percentage) # Amount of data used for testing memories tedata = step n = 0 histories = [] for i in range(0, total, step): j = (i + tedata) % total if j > i: testing_data = data[i:j] testing_labels = labels[i:j] other_data = np.concatenate((data[0:i], data[j:total]), axis=0) other_labels = np.concatenate((labels[0:i], labels[j:total]), axis=0) training_data = other_data[:trdata] training_labels = other_labels[:trdata] filling_data = other_data[trdata:] filling_labels = other_labels[trdata:] else: testing_data = np.concatenate((data[0:j], data[i:total]), axis=0) testing_labels = np.concatenate((labels[0:j], labels[i:total]), axis=0) training_data = data[j:j + trdata] training_labels = labels[j:j + trdata] filling_data = data[j + trdata:i] filling_labels = labels[j + trdata:i] # Recreate the exact same model, including its weights and the optimizer model = tf.keras.models.load_model( constants.model_filename(model_prefix, n)) # Drop the autoencoder and the last layers of the full connected neural network part. classifier = Model(model.input, model.output[0]) no_hot = to_categorical(testing_labels) classifier.compile(optimizer='adam', loss='categorical_crossentropy', metrics='accuracy') history = classifier.evaluate(testing_data, no_hot, batch_size=batch_size, verbose=1, return_dict=True) print(history) histories.append(history) model = Model(classifier.input, classifier.layers[-4].output) model.summary() training_features = model.predict(training_data) if len(filling_data) > 0: filling_features = model.predict(filling_data) else: r, c = training_features.shape filling_features = np.zeros((0, c)) testing_features = model.predict(testing_data) dict = { constants.training_suffix: (training_data, training_features, training_labels), constants.filling_suffix: (filling_data, filling_features, filling_labels), constants.testing_suffix: (testing_data, testing_features, testing_labels) } for suffix in dict: data_fn = constants.data_filename(data_prefix + suffix, n) features_fn = constants.data_filename(features_prefix + suffix, n) labels_fn = constants.data_filename(labels_prefix + suffix, n) d, f, l = dict[suffix] np.save(data_fn, d) np.save(features_fn, f) np.save(labels_fn, l) n += 1 return histories
optimizer='adam', loss=tf.losses.BinaryCrossentropy(from_logits=True), metrics=[ tf.metrics.BinaryAccuracy(threshold=0.5, name='accuracy') ]) callbacks = [ keras.callbacks.EarlyStopping(monitor="accuracy", patience=15, verbose=1, mode="min", restore_best_weights=True), keras.callbacks.ModelCheckpoint( filepath="Models/fakeNewsDetector_Advanced.hdf5", verbose=1, save_best_only=True) ] history = model_classifier.fit([partial_x_train, verified_train], partial_y_train, epochs=40, batch_size=256, validation_data=([x_val, verified_val], y_val), verbose=1, callbacks=callbacks) else: results = model_classifier.evaluate([X, verified], y) print(f'Accuracy on all data is: {results[1]}')
class Dependency_Parser: def __init__(self, hidden_neurons=512, epochs=1, batch_size=32, verbose=1, max_len=50, n_tags=51, load_f=False, loadFile="tmp/model.h5"): self.hidden_neurons = hidden_neurons self.epochs = epochs self.batch_size = batch_size self.verbose = verbose sess = tf.Session() K.set_session(sess) input_text = Input(shape=(max_len, ), dtype=tf.string) # embed = ElmoLayer() # embedding = embed(input_text) embedding = ElmoLayer()(input_text) # Don't know why but it needs initialization after ElmoLayer sess.run(tf.global_variables_initializer()) sess.run(tf.tables_initializer()) # print_emb = tf.Print(embedding, [embedding]) x = Bidirectional( LSTM(units=hidden_neurons, return_sequences=True, recurrent_dropout=0.2, dropout=0.2))(embedding) # x = Bidirectional(LSTM(units=hidden_neurons, return_sequences=True, # recurrent_dropout=0.2, dropout=0.2))(print_emb) x_rnn = Bidirectional( LSTM(units=hidden_neurons, return_sequences=True, recurrent_dropout=0.2, dropout=0.2))(x) x = add([x, x_rnn]) # residual connection to the first biLSTM out = TimeDistributed(Dense(n_tags, activation="softmax"))(x) self.model = Model(input_text, out) self.model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"]) def fit(self, X_tr, y_tr, val): checkpoint_path = "trained/cp.ckpt" checkpoint_dir = os.path.dirname(checkpoint_path) cp_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_path, save_weights_only=True, verbose=1) return self.model.fit(x=X_tr, y=y_tr, batch_size=self.batch_size, epochs=self.epochs, verbose=self.verbose, validation_data=val, callbacks=[cp_callback]) def evaluate(self, X_test, y_test): return self.model.evaluate(x=X_test, y=y_test, batch_size=self.batch_size, verbose=self.verbose) def load(self, checkpoint_path): return self.model.load_weights(checkpoint_path) def predict(self, x): return self.model.predict(x) def save(self, checkpoint="trained/model.ckpt"): return self.model.save_weights(checkpoint)
callbacks = [] callbacks.append( tf.keras.callbacks.LearningRateScheduler(lr_scheduler, verbose=0)) if args.logging: callbacks.append( tf.keras.callbacks.CSVLogger( os.path.join( logdir, '{0}_{1}_{2}_{3}-{4}.log'.format( args.task_id, args.training_set_size, args.encodings_type, args.hops, args.random_state)))) model.fit(x=x_train, y=y_train, epochs=args.epochs, validation_split=args.validation_split, batch_size=batch_size, callbacks=callbacks, verbose=args.verbose) callbacks = [] if args.logging: callbacks.append( MyCSVLogger( os.path.join( logdir, '{0}_{1}_{2}_{3}-{4}.log'.format( args.task_id, args.training_set_size, args.encodings_type, args.hops, args.random_state)))) model.evaluate(x=x_test, y=y_test, callbacks=callbacks, verbose=2)
predict = Dense(10, activation="softmax")(flatten) vgg16 = Model(inputs=vgg_model.input, outputs=predict) check_point_path = "./vgg_check_point.hdf5" model_path = "vgg_model.hdf5" check_point = ModelCheckpoint(check_point_path, save_best_only=True, monitor='val_accuracy') tensorboard = TensorBoard(log_dir="logs/vgg") epochs = 10 if os.path.exists(check_point_path): vgg16.load_weights(check_point_path) vgg16.compile( optimizer=keras.optimizers.Adam(), loss=keras.losses.CategoricalCrossentropy(label_smoothing=0.1), metrics=["accuracy"]) vgg16_acc = vgg16.evaluate(valid_gen, steps=valid_steps)[1] if vgg16_acc < 0.8: vgg16.fit(train_gen, steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=valid_gen, validation_steps=valid_steps, callbacks=[check_point, tensorboard]) ''' prune on vgg16, the layer index can be found by model.summary() ''' state = 0 base_model = vgg16 base_model.evaluate(valid_gen, steps=valid_steps) new_model = Model(inputs=vgg_model.input, outputs=predict) layer_index = 17
labels_hold_out = labels[hold_out_nodes] print(labels_hold_out) hold_out_targets = target_encoding.transform(labels_hold_out) hold_out_gen = generator.flow(hold_out_nodes, hold_out_targets) # Make predictions for the held out set hold_out_predictions = model.predict(hold_out_gen) # Look at the held out predictions hold_out_predictions = target_encoding.inverse_transform(hold_out_predictions) results = pd.Series(hold_out_predictions, index=hold_out_nodes) df = pd.DataFrame({"Predicted": results, "True": labels_hold_out}) print(df.head(10)) hold_out_metrics = model.evaluate(hold_out_gen) print("\nHold Out Set Metrics:") for name, val in zip(model.metrics_names, hold_out_metrics): print("\t{}: {:0.4f}".format(name, val)) # Make TSNE plot of the held out node embeddings embedding_model = Model(inputs=x_inp, outputs=x_out) emb = embedding_model.predict(hold_out_gen) X = emb y = np.argmax(target_encoding.transform(labels_hold_out), axis=1) if X.shape[1] > 2: transform = TSNE # PCA trans = transform(n_components=2) emb_transformed = pd.DataFrame(trans.fit_transform(X), index=hold_out_nodes)
# model fit_generator STEP_SIZE_TRAIN = train_gen.n // train_gen.batch_size STEP_SIZE_VALID = valid_gen.n // valid_gen.batch_size transfer_learning_history = model.fit_generator( generator=train_gen, steps_per_epoch=STEP_SIZE_TRAIN, validation_data=valid_gen, validation_steps=STEP_SIZE_VALID, epochs=20, callbacks=callbacks, class_weight='auto', ) # model evaluate with validation set model.evaluate(valid_gen, steps=STEP_SIZE_VALID, verbose=1) #STEP_SIZE_TEST=test_gen.n//test_gen.batch_size #test_gen.reset() #pred=model.predict(test_gen, #steps=STEP_SIZE_TEST, #verbose=1) #predicted_class_indices=np.argmax(pred,axis=1) # CSV file for kaggle submission #labels = train_gen.class_indices #labels = dict((v,k) for k,v in labels.items()) #predictions = [k for k in predicted_class_indices]
callbacks=[es, rlrop]) model_json = model.to_json() with open('model.json', 'w') as json_file: json_file.write(model_json) model.save_weights('model.h5') #print('Loading weights...') #model.load_weights('model.h5') #print('Model loaded') #print('Generating test data') test = cags.test.map(CAGS.parse) x_test = [] for x in test: x_test.append(x['image'].numpy()) x_test = np.array(x_test) scores = model.evaluate(x_dev, y_dev, batch_size=64, verbose=1) print(f'Test result: {scores[1]*100} loss:{scores[0]}') print('Generating predictions...') # Generate test set annotations, but in args.logdir to allow parallel execution. with open("cags_classification.txt", "w", encoding="utf-8") as out_file: # TODO: Predict the probabilities on the test set test_probabilities = model.predict(x_test, batch_size=args.batch_size) for probs in test_probabilities: print(np.argmax(probs), file=out_file)
checkpoint_path = "./checkpoints/" checkpoint_dir = os.path.dirname(checkpoint_path) cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, verbose=1, period=1) #model.summary() model.compile(optimizer=tf.keras.optimizers.Adam(0.03), loss=custom_loss, metrics=[auc]) model.fit((train_image1, train_image2), train_label, epochs=10) #, callbacks = [cp_callback]) #4 loss, auc = model.evaluate((test_image1, test_image2), test_label) print("saved model, loss: {:5.2f}, acc: {:5.2f}".format(loss, auc)) e_w = model.predict((test_image1, test_image2)) #print(type(e_w)) #<class 'tensorflow.python.framework.ops.EagerTensor'> threshs = [0.2, 0.5, 0.7, 1., 1.3, 1.5, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0] area_list = [] for thresh in threshs: predictions = (e_w > thresh) #e_w=e_w/max(e_w) precision, recall, th = metrics.precision_recall_curve( test_label, predictions) area_list.append(metrics.auc(recall, precision)) plt.plot(recall, precision, linewidth=1.0, label='thresh=' + str(thresh)) plt.plot([0.5, 1], [0.5, 1], linewidth=1.0, label='equal') plt.title("precision and recall curve") plt.legend()
# train_label, # epochs=num_epochs, # batch_size=num_batchsize, # verbose=2, # validation_data=(val_data, val_label), # callbacks=[]) # load the saved best model # model = models.load_model('best_model.h5') # evaluate model test_pred = model.predict(test_data) t_start = process_time() _, acc = model.evaluate(test_data, test_label, batch_size=num_batchsize, verbose=2) t_end = process_time() t_cost = t_end - t_start print(f"Test Accuracy: {acc:.4f}, Inference time: {t_cost:.2f}s") plot_learncurve("CNN", history=history) if 'lr' in lr_scheduler.history: plt.plot(lr_scheduler.history['lr']) plt.xlabel('iterations') plt.ylabel('learning rate') plt.title('Learning Rate Schedule') plt.show() else: raise ValueError("no lr info in history.")
class EQLDIV: """ EQL-div function learning network # Arguments inputSize: number of input variables to model. Integer. outputSize: number of variables outputted by model. Integer. numLayers: number of layers in model. A layer is either a fully- connected linear map and a nonlinear map (hidden layer), or just a fully-connected linear map (final layer). The Keras Input layer doesn't count. layers: list of Keras layers containing all of the layer tensor outputs (not the layer object itself) in the EQL model (including the Keras Input layer). hypothesisSet: list of 2-tuples, first element of tuples is tensorflow R -> R function to be applied element-wise in nonlinear layer components, second element is the corresponding sympy function for use in printing out learned equations. In practice, usually contains identity, sine, cosine, and sigmoid. nonlinearInfo: list with rows equal to number of hidden layers and 2 columns. First column is number of unary functions in each hidden layer, second column is number of binary functions in each hidden layer energyInfo: if energy regularization is to be used, is a list with length three containing (1) a python function which uses tensorflow methods to compute the Hamiltonian associated with each member of a batch of predicted state, (2) a float value giving the actual Hamilton of the data (NOTE: ENERGYINFO SHOULD ONLY BE USED WITH TIMESERIES DATA OR OTHER CONSTANT ENERGY DATA), and (3) a coefficient for scaling the energy error in the loss function (10^-5 recommended, only activated during the second training phase) learningRate: optimizer learning rate. divThreshold: float, value which denominator must be greater than in order for division to occur (division returns 0 otherwise) name: TensorFlow scope name (for TensorBoard) changeOfVariables: a list with length two containing (1) a python function which can transform an arbitrarily large numpy array training data set to a different set of variables and (2) the number of components per data point in the transformed data that the function in the first component outputs # References - [Learning Equations for Extrapolation and Control]( https://arxiv.org/abs/1806.07259) """ def __init__(self, inputSize, outputSize, numLayers=3, hypothesisSet=None, nonlinearInfo=None, energyInfo=None, changeOfVariables=None, name='EQLDIV'): self.inputSize = changeOfVariables[1] or inputSize self.outputSize = outputSize self.numLayers = numLayers self.layers = [] self.hypothesisSet = hypothesisSet or [ [tf.identity, tf.sin, tf.cos, tf.sigmoid], [sympy.Id, sympy.sin, sympy.cos, sympy.Function("sigm")]] self.nonlinearInfo = nonlinearInfo or getNonlinearInfo( self.numLayers-1, [4], 4) self.energyInfo = energyInfo self.changeOfVariables = changeOfVariables self.name = name self.unaryFunctions = [ [j % len(self.hypothesisSet[0]) for j in range(self.nonlinearInfo[i][0])] for i in range(self.numLayers-1)] def build(self, divThreshold=0.001, learningRate=0.001, method=Adam, loss='mse', metrics=[rmse]): """ __init__ defines all of the variables this model depends upon, this function actual creates the tensorflow Model variable # Arguments learningRate: model learning rate method: model learning method loss: loss function used to penalize model based on accuracy metrics: metric used for gauging model performance when evaluated on data points post-training """ self.layers.append(Input((self.inputSize,))) inp = int(self.layers[-1].shape[1]) for i in range(self.numLayers): if (i == self.numLayers - 1): stddev = np.sqrt(1 / (self.outputSize * 2 * inp)) randNorm = RandomNormal(0, stddev=stddev) self.layers.append(DivLayer(self.outputSize, threshold=divThreshold, kernel_initializer=randNorm, )(self.layers[-1])) break u, v = self.nonlinearInfo[i] or None out = u + 2 * v or None stddev = np.sqrt(1 / (inp * out)) randNorm = RandomNormal(0, stddev=stddev) self.layers.append(EqlLayer(self.nonlinearInfo[i], self.hypothesisSet[0], self.unaryFunctions[i], kernel_initializer=randNorm, )(self.layers[-1])) inp = u + v # Optimizer optimizer = method(lr=learningRate) # Model self.model = Model(inputs=self.layers[0], outputs=self.layers[-1]) # Compilation self.model.compile(optimizer=optimizer, loss=loss, metrics=metrics) def fit(self, predictors, labels, numEpoch, regStrength=10**-3, batchSize=20, normThreshold=0.001, verbose=0): """ Trains EQL model on a dataset following the training schedule defined in the reference. # Arguments predictors: ? x inputSize array containing data to be trained on labels: ? x outputSize array containing corresponding correct output for predictors, compared with model output numEpoch: integer, number of epochs reg: regularization (in the range [10^-4, 10^-2.5] is usually ideal) batchSize: number of datapoints trained on per gradient descent update normThreshold: float, weight/bias elements below this value are kept at zero during the final training phase verbose: 0, 1, or 2, determines whether Keras is silent, prints a progress bar, or prints a line every epoch. """ if self.changeOfVariables: predictors = self.changeOfVariables[0](predictors) # Division threshold callbacks def phase1DivisionThresholdSchedule(epoch, logs): K.set_value(self.model.layers[-1].threshold, 1 / np.sqrt(epoch + 1)) def phase2DivisionThresholdSchedule(epoch, logs): K.set_value(self.model.layers[-1].threshold, 1 / np.sqrt(int(numEpoch * (1 / 4)) + epoch + 1)) def phase3DivisionThresholdSchedule(epoch, logs): K.set_value( self.model.layers[-1].threshold, 1 / np.sqrt(int(numEpoch * (7/10)) + int(numEpoch * (1/4)) + epoch + 1)) # PHASE 1: NO REGULARIZATION (T/4) for i in range(1, self.numLayers+1): K.set_value(self.model.layers[i].regularization, 0.) K.set_value(self.model.layers[i].Wtrimmer, np.ones(self.model.layers[i].W.shape)) K.set_value(self.model.layers[i].btrimmer, np.ones(self.model.layers[i].b.shape)) dynamicDivisionThreshold = LambdaCallback( on_epoch_begin=phase1DivisionThresholdSchedule) self.model.fit(predictors, labels, epochs=int(numEpoch*(1/4)), batch_size=batchSize, verbose=verbose, callbacks=[dynamicDivisionThreshold]) # PHASE 2: REGULARIZATION (7T/10) if self.energyInfo is not None: K.set_value(self.model.layers[-1].loss.coef, self.coef) for i in range(1, self.numLayers+1): K.set_value(self.model.layers[i].regularization, regStrength) dynamicDivisionThreshold = LambdaCallback( on_epoch_begin=phase2DivisionThresholdSchedule) self.model.fit(predictors, labels, epochs=int(numEpoch*(7/10)), batch_size=batchSize, verbose=verbose, callbacks=[dynamicDivisionThreshold]) # PHASE 3: NO REGULARIZATION, L0 NORM PRESERVATION (T/20) for i in range(1, self.numLayers+1): W, b = self.model.layers[i].get_weights()[:2] K.set_value(self.model.layers[i].regularization, 0.) K.set_value(self.model.layers[i].Wtrimmer, (np.abs(W) > normThreshold).astype(float)) K.set_value(self.model.layers[i].btrimmer, (np.abs(b) > normThreshold).astype(float)) dynamicDivisionThreshold = LambdaCallback( on_epoch_begin=phase3DivisionThresholdSchedule) self.model.fit(predictors, labels, epochs=int(numEpoch*(1/20)), batch_size=batchSize, verbose=verbose, callbacks=[dynamicDivisionThreshold]) # Final maintenance K.set_value(self.model.layers[-1].threshold, 0.001) def evaluate(self, predictors, labels, batchSize=10, verbose=0): """Evaluates trained model on data""" return self.model.evaluate(predictors, labels, batch_size=batchSize, verbose=verbose)[1] def getEquation(self): """Prints learned equation of a trained model.""" # prepares lists for weights and biases weights = [] bias = [] # pulls/separates weights and biases from a model for i in range(1, self.numLayers+1): weights.append(self.model.layers[i].get_weights()[0]) bias.append(self.model.layers[i].get_weights()[1]) # creates generic input vector X = make_symbolic(1, self.inputSize) for i, _ in enumerate(weights): # computes the result of the next linear layer W = sympy.Matrix(weights[i]) b = sympy.Transpose(sympy.Matrix(bias[i])) Y = sympy.zeros(1, b.cols) X = X*W + b # computes the result of the next nonlinear layer, if applicable if i != (len(weights) - 1): u, v = self.nonlinearInfo[i] # computes the result of the unary component of the nonlinear # layer # iterating over unary input for j in range(u): Y[0, j] = self.hypothesisSet[1][self.unaryFunctions[i][j]]( X[0, j]) # computes the result of the binary component of the nonlinear # layer # iterating over binary input for j in range(v): Y[0, j+u] = X[0, j * 2 + u] * X[0, j * 2 + u + 1] # removes final v rows which are now outdated for j in range(u + v, Y.cols): Y.col_del(u + v) X = Y if i == (len(weights) - 1): # computes the result of the binary component of the nonlinear # layer # iterating over binary input for j in range(int(X.cols/2)): if sympy.Abs(X[0, j*2+1]) == 0: Y[0, j] = 0 else: Y[0, j] = X[0, j*2] / X[0, j*2+1] for j in range(int(X.cols/2)): Y.col_del(int(X.cols/2)) X = Y return X def plotSlice(self, function, xmin=-2, xmax=2, step=0.01, width=10, height=10, settings=dict(), save=False): """ Plots the x_1 = ... = x_n slice of the learned function in each output variable. """ # x values X = np.asarray( [[(i * step) + xmin for i in range(int((xmax - xmin)/step))] for j in range(self.inputSize)]) # goal function values F_Y = np.apply_along_axis(function, 0, X) # model predictions model_Y = self.model.predict(np.transpose(X)) model_Y = np.transpose(model_Y) settings['figure.figsize'] = (width, height) with plt.rc_context(settings): fig, axs = plt.subplots(self.outputSize, figsize=(width, height)) if self.outputSize == 1: axs = [axs] for i in range(self.outputSize): axs[i].plot(X[0], F_Y[i], linestyle='-', label='Goal Function') axs[i].plot(X[0], model_Y[i], linestyle=':', label='Learned Function') plt.legend() if save: plt.savefig(self.name + '.png', bbox_inches='tight', dpi=300) def percentError(self, predictors, labels): """ Returns the average percent error in each variable of a trained model with respect to a testing data set """ labels = np.reshape(labels, (-1, self.outputSize)) predictions = self.model.predict(predictors) error = np.divide(np.abs(predictions - labels), np.abs(labels)) error = np.sum(error, 0) error *= (100 / labels.shape[0]) return error def sparsity(self, minMag=0.01): """Returns the sparsity of a trained model (number of active nodes)""" vec = np.ones((1, self.inputSize)) weights = self.model.get_weights() sparsity = 0 for i in range(self.numLayers - 1): u, v = self.nonlinearInfo[i] w, b = weights[i*2], weights[i*2+1] vec = np.dot(vec, w) + b vec = np.concatenate( (vec[:, :u], vec[:, u::2] * vec[:, u+1::2]), axis=1) sparsity += np.sum( (np.abs(vec) > np.full_like(vec, minMag)).astype(int)) w, b = weights[len(weights)-2], weights[len(weights)-1] vec = np.dot(vec, w) + b vec = vec[0, ::2] * vec[0, 1::2] sparsity += np.sum( (np.abs(vec) > np.full_like(vec, minMag)).astype(int)) return sparsity def setPipeline(self, pipeline): """ Saves the scikit_learn pipeline used to modify training data so that the same pipeline can be applied to testing data """ self.pipeline = pipeline def applyPipeline(self, x): """Applies a saved scikit_learn pipeline to a dataset""" if self.pipeline is not None: for op in self.pipeline: x = op.transform(x) return x def odecompat(self, t, x): """Wrapper for Keras' function, solve_ivp compatible""" # if statement is bad hack for experimentations with feature # engineering for double pendulum # if self.inputSize == 7: # if len(np.array(x).shape) == 2: # y = [x[0, 0] - x[0, 2], x[0, 1]**2, x[0, 3]**2] # x = np.append(x, y) # else: # y = [x[0] - x[2], x[1]**2, x[3]**2] # x = np.append(x, y) prediction = self.model.predict(np.reshape(x, (1, -1))) return prediction def printJacobian(self, x): """ Prints the Jacobian of the learned function, evaluated at a point # Arguments x: point at which Jacobian is evaluated, array-like with self.inputSize elements """ x = np.reshape(x, (1, 1, self.inputSize)).tolist() gradients = [tf.gradients(self.model.output[:, i], self.model.input)[0] for i in range(4)] funcs = [K.function((self.model.input, ), [g]) for g in gradients] jacobian = np.concatenate([func(x)[0] for func in funcs], axis=0) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) print(jacobian)
class EQL: """ EQL function learning network # Arguments inputSize: number of input variables to model. Integer. outputSize: number of variables outputted by model. Integer. numLayers: number of layers in model. A layer is either a fully- connected linear map and a nonlinear map (hidden layer), or just a fully-connected linear map (final layer). The Keras Input layer doesn't count. layers: list of Keras layers containing all of the layers in the EQL model (including the Keras Input layer). hypothesisSet: 2 x ? list, first row contains tensorflow R -> R function to be applied element-wise in nonlinear layer components, second row contains the corresponding sympy functions for use in printing out learned equations. In practice, usually contains identity, sine, cosine, and sigmoid. nonlinearInfo: list with rows equal to number of hidden layers and 2 columns. First column is number of unary functions in each hidden layer, second column is number of binary functions in each hidden layer learningRate: optimizer learning rate. name: TensorFlow scope name (for TensorBoard) # References - [Extrapolation and learning equations]( https://arxiv.org/abs/1610.02995) """ def __init__(self, inputSize, outputSize, numLayers=4, hypothesisSet=None, nonlinearInfo=None, name='EQL'): self.inputSize = inputSize self.outputSize = outputSize self.numLayers = numLayers self.layers = [] self.hypothesisSet = hypothesisSet or [ [tf.identity, tf.sin, tf.cos, tf.sigmoid], [sympy.Id, sympy.sin, sympy.cos, sympy.Function("sigm")]] self.nonlinearInfo = nonlinearInfo or getNonlinearInfo( self.numLayers-1, [4], 4) self.name = name self.unaryFunctions = [ [j % len(self.hypothesisSet[0]) for j in range(self.nonlinearInfo[i][0])] for i in range(numLayers-1)] def build(self, learningRate=0.001, method=Adam, loss='mse', metrics=[rmse]): """ __init__ defines all of the variables this model depends upon, this function actual creates the tensorflow Model variable # Arguments learningRate: model learning rate method: model learning method loss: loss function used to penalize model based on accuracy metrics: metric used for gauging model performance when evaluated on data points post-training """ self.layers.append(Input((self.inputSize,))) inp = int(self.layers[-1].shape[1]) for i in range(self.numLayers - 1): u, v = self.nonlinearInfo[i] out = u + 2 * v stddev = np.sqrt(1 / (inp * out)) randNorm = RandomNormal(0, stddev=stddev) self.layers.append(EqlLayer(self.nonlinearInfo[i], self.hypothesisSet[0], self.unaryFunctions[i], kernel_initializer=randNorm, )(self.layers[-1])) inp = u + v stddev = np.sqrt(1 / (self.outputSize * inp)) randNorm = RandomNormal(0, stddev=stddev) self.layers.append(Connected(self.outputSize, kernel_initializer=randNorm )(self.layers[-1])) # Optimizer optimizer = method(lr=learningRate) # Model self.model = Model(inputs=self.layers[0], outputs=self.layers[-1]) # Compilation self.model.compile(optimizer=optimizer, loss=loss, metrics=metrics) def fit(self, predictors, labels, numEpoch, reg=10**-3, batchSize=20, threshold=0.1, verbose=0): """ Trains EQL model on a dataset following the training schedule defined in the reference. # Arguments predictors: ? x inputSize array containing data to be trained on labels: ? x outputSize array containing corresponding correct output for predictors, compared with model output numEpoch: integer, number of epochs reg: regularization (in the range [10^-4, 10^-2.5] is usually ideal) batchSize: number of datapoints trained on per gradient descent update threshold: float, weight/bias elements below this value are kept at zero during the final training phase """ # PHASE 1: NO REGULARIZATION (T/4) for i in range(1, self.numLayers+1): K.set_value(self.model.layers[i].regularization, 0.) K.set_value(self.model.layers[i].Wtrimmer, np.ones(self.model.layers[i].W.shape)) K.set_value(self.model.layers[i].btrimmer, np.ones(self.model.layers[i].b.shape)) self.model.fit(predictors, labels, epochs=int(numEpoch*(1/4)), batch_size=batchSize, verbose=verbose) # PHASE 2: REGULARIZATION (7T/10) for i in range(1, self.numLayers+1): K.set_value(self.model.layers[i].regularization, reg) self.model.fit(predictors, labels, epochs=int(numEpoch*(7/10)), batch_size=batchSize, verbose=verbose) # PHASE 3: NO REGULARIZATION, L0 NORM PRESERVATION (T/20) for i in range(1, self.numLayers+1): W, b = self.model.layers[i].get_weights()[:2] K.set_value(self.model.layers[i].regularization, 0.) K.set_value(self.model.layers[i].Wtrimmer, (np.abs(W) > threshold).astype(float)) K.set_value(self.model.layers[i].btrimmer, (np.abs(b) > threshold).astype(float)) self.model.fit(predictors, labels, epochs=int(numEpoch*(1/20)), batch_size=batchSize, verbose=verbose) def evaluate(self, predictors, labels, batchSize=10, verbose=0): """Evaluates trained model on data""" return self.model.evaluate(predictors, labels, batch_size=batchSize, verbose=verbose)[1] def getEquation(self): """Prints learned equation of a trained model.""" # prepares lists for weights and biases weights = [] bias = [] # pulls/separates weights and biases from a model for i in range(1, self.numLayers+1): weights.append(self.model.layers[i].get_weights()[0]) bias.append(self.model.layers[i].get_weights()[1]) print(np.array(weights).shape, np.array(self.nonlinearInfo).shape) # creates generic input vector X = make_symbolic(1, self.inputSize) for i, _ in enumerate(weights): # computes the result of the next linear layer W = sympy.Matrix(weights[i]) b = sympy.Transpose(sympy.Matrix(bias[i])) Y = sympy.zeros(1, b.cols) X = X*W + b # computes the result of the next nonlinear layer, if applicable if i != (len(weights) - 1): u, v = self.nonlinearInfo[i] # computes the result of the unary component of the nonlinear # layer # iterating over unary input for j in range(u): Y[0, j] = self.hypothesisSet[1][self.unaryFunctions[i][j]]( X[0, j]) # computes the result of the binary component of the nonlinear # layer # iterating over binary input for j in range(v): Y[0, j+u] = X[0, j * 2 + u] * X[0, j * 2 + u + 1] # removes final v rows which are now outdated for j in range(u + v, Y.cols): Y.col_del(u + v) X = Y return X def plotSlice(self, function, xmin=-2, xmax=2, step=0.01, width=10, height=10, settings=dict(), save=False): """ Plots the x_1 = ... = x_n slice of the learned function in each output variable. """ # x values X = np.asarray( [[(i * step) + xmin for i in range(int((xmax - xmin)/step))] for j in range(self.inputSize)]) # goal function values F_Y = np.apply_along_axis(function, 0, X) # model predictions model_Y = self.model.predict(np.transpose(X)) model_Y = np.transpose(model_Y) settings['figure.figsize'] = (width, height) with plt.rc_context(settings): fig, axs = plt.subplots(self.outputSize, figsize=(width, height)) if self.outputSize == 1: axs = [axs] for i in range(self.outputSize): axs[i].plot(X[0], F_Y[i], linestyle='-', label='Goal Function') axs[i].plot(X[0], model_Y[i], linestyle=':', label='Learned Function') plt.legend() if save: plt.savefig(self.name + '.png', bbox_inches='tight', dpi=300) def percentError(self, predictors, labels): """ Returns the average percent error in each variable of a trained model with respect to a testing data set """ labels = np.reshape(labels, (-1, self.outputSize)) predictions = self.model.predict(predictors) error = np.divide(np.abs(predictions - labels), np.abs(labels)) error = np.sum(error, 0) error *= (100 / labels.shape[0]) return error def sparsity(self, minMag=0.01): """Returns the sparsity of a trained model (number of active nodes)""" vec = np.ones((1, self.inputSize)) weights = self.model.get_weights() sparsity = 0 for i in range(self.numLayers - 1): u, v = self.nonlinearInfo[i] w, b = weights[i*2], weights[i*2+1] vec = np.dot(vec, w) + b vec = np.concatenate( (vec[:, :u], vec[:, u::2] * vec[:, u+1::2]), axis=1) sparsity += np.sum( (np.abs(vec) > np.full_like(vec, minMag)).astype(int)) w, b = weights[len(weights)-2], weights[len(weights)-1] vec = np.dot(vec, w) + b sparsity += np.sum( (np.abs(vec) > np.full_like(vec, minMag)).astype(int)) return sparsity def odecompat(self, t, x): """Wrapper for Keras' predict function, solve_ivp compatible""" prediction = self.model.predict(np.reshape(x, (1, len(x)))) return prediction def printJacobian(self, x): """ Prints the Jacobian of the learned function, evaluated at a point # Arguments x: point at which Jacobian is evaluated, array-like with self.inputSize elements """ x = np.reshape(x, (1, 1, self.inputSize)).tolist() gradients = [tf.gradients(self.model.output[:, i], self.model.input)[0] for i in range(4)] funcs = [K.function((self.model.input, ), [g]) for g in gradients] jacobian = np.concatenate([func(x)[0] for func in funcs], axis=0) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) print(jacobian)
save_best_only=True, mode='min') callbacks_list = [checkpoint] designed_model.fit(train_data, dx_train, class_weight=class_weights, epochs=epoch, batch_size=batchSize, verbose=1, validation_split=0.15, callbacks=callbacks_list) #history = designed_model.fit_generator(datagen.flow(train_data,dx_train, batch_size=64),validation_data = (val_data,dx_val), epochs = epoch,verbose = 1, steps_per_epoch=32) print(test_data.shape) print(dx_test.shape) score = designed_model.evaluate(test_data, dx_test, verbose=1) print("score:", score) pred = designed_model.predict(test_data) print(pred.shape) pred_out = np.argmax(pred, axis=1) test_out = np.argmax(dx_test, axis=1) print(pred_out.shape) print(pred_out) conf = confusion_matrix(test_out, pred_out) print(conf)
# 마지막 단계에서 어떤 함수를 쓰던 accuracy 같은 성능 면이 달라지는 것은 아니지만 # softmax 는 결과값을 확률값으로 바꿔주어서 보기 편하게 만드는 역할을 한다. # create a model # param vgg16 = Model(inputs, outputs, name='vgg16') vgg16.summary() # compile the model # 모델 학습과정 설정 | 손실 함수 및 최적화 방법 # optimizer: optimizer = tf.keras.optimizers.Adam( learning_rate=0.0001) # lower learning rate, better performance vgg16.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) # train the model # 모델 학습 vgg16.fit(x_train, y_train_onehot, epochs=30) # evaluate the model with test set # 모델 평가하기 vgg16.evaluate(x_test, y_test_onehot, verbose=2) # predict the model with test set # 모델 사용하기 # vgg16.predict(x) : x라는 input을 넣어서 output 예측 생성 # loss: 3.0565 - accuracy: 0.3367
elif epoch <=4: return 0.0002 * strategy.num_replicas_in_sync else: return 0.0001 * strategy.num_replicas_in_sync #callbacks = [ tf.keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=1, profile_batch = 3)]#, cb ] callbacks = [] callbacks = [tf.keras.callbacks.LearningRateScheduler(scheduler)] print('starting at ', time()) history = model.fit(train, steps_per_epoch=steps_per_epoch, \ epochs=num_epochs, callbacks=callbacks, verbose=1) print("finishing at:", time()) print("evaluating:", time()) model.evaluate(test, steps=validation_steps) print("evaluated at:", time()) model_full_path= "/tmp/mymodel" + str(worker_number) + ".h5" print("Training finished, now saving the model in h5 format to: " + model_full_path) model.save(model_full_path, save_format="h5") print("model saved.\n") #print("..saving the model in tf format (TF 2.0) to: " + model_full_path) #tf.keras.models.save_model(model, "/tmp/mymodel"+ str(worker_number) + ".tf", save_format='tf') #print("model saved.\n") exit()
dense1 = Dense(64, activation='relu')(dense1) dense1 = Dense(64, activation='relu')(dense1) output1 = Dense(1)(dense1) model = Model(inputs=input1, outputs=output1) #3. compile and fit model.compile(optimizer='adam', loss='mse', metrics=['mae']) model.fit(x_train, y_train, batch_size=4, epochs=150, verbose=1, validation_split=0.2) #4. evalutate and predict mse, mae = model.evaluate(x_test, y_test, batch_size=4) print("mse :", mse, "\nmae :", mae) y_predict = model.predict(x_test) # RMSE 구하기 from sklearn.metrics import mean_squared_error def RMSE(y_test, y_predict): return np.sqrt(mean_squared_error(y_test, y_predict)) print('RMSE :', RMSE(y_test, y_predict)) # R2 구하기
model.summary() # In[171]: # We train our model. history = model.fit(train_images, train_labels, epochs=5, batch_size=128, callbacks=callbacks) # In[172]: # Evaluate the test data. test_loss, test_acc = model.evaluate(test_images, test_labels) # In[173]: # Plot accuracy and loss of your training epochs. loss = history.history['loss'] acc = history.history['accuracy'] epochs = range(1, len(loss) + 1) plt.plot(epochs, loss, 'b', label='Training loss') plt.plot(epochs, acc, 'r', label='Training acc') plt.title('Training loss and accuracy') plt.xlabel('Epochs') plt.ylabel('Loss')
def train_delta(matrix): import numpy as np import tensorflow as tf import pandas as pd import matplotlib.pyplot as plt from sklearn.preprocessing import MinMaxScaler from tensorflow.keras import Model, Input from tensorflow.keras.layers import (Dense, Dropout, GRU, Flatten, GaussianNoise, concatenate) from tensorflow.keras.models import load_model from tensorflow.keras.callbacks import Callback from tensorflow.keras.callbacks import EarlyStopping from tensorflow.keras.callbacks import ModelCheckpoint from tensorflow.keras.optimizers import Adam import supplemental_functions from supplemental_functions import (sampling_fix, prepareinput, prepareinput_nozero, prepareoutput) tf.keras.backend.clear_session() [ newdim, percent_drilled, start, stop, inc_layer1, inc_layer2, data_layer1, data_layer2, dense_layer, range_max, memory, predictions, drop1, drop2, lr, bs, ensemble_count ] = matrix drop1 = drop1 / 100 drop2 = drop2 / 100 inc_layer2 = inc_layer2 / 1000 lr = lr / 10000 percent_drilled = percent_drilled / 100 df = pd.read_csv('F9ADepth.csv') df_target = df.copy() droplist = [ 'nameWellbore', 'name', 'Pass Name unitless', 'MWD Continuous Inclination dega', 'Measured Depth m', 'MWD Continuous Azimuth dega', "Unnamed: 0", "Unnamed: 0.1" ] for i in droplist: df = df.drop(i, 1) for i in list(df): if df[i].count() < 1000: del df[i] info(f'dropped {i}') start = start stop = stop step = 0.230876 X = np.arange(start, stop, step) X = X.reshape(X.shape[0], 1) X = np.arange(start, stop, step) X = X.reshape(X.shape[0], 1) my_data1 = sampling_fix(df_target, 'MWD Continuous Inclination dega', start, stop, 1.7, 1, 0).predict(X) data_array = [] for i in list(df): sampled = sampling_fix(df_target, i, start, stop, 1.7, 3, 0).predict(X) if np.isnan(np.sum(sampled)) == False: data_array.append(sampled) info(f'Using {i}') data_array = np.asarray(data_array) dftemp = pd.DataFrame() dftemp['dinc'] = my_data1 dftemp['dinc'] = dftemp['dinc'].diff(1).rolling(3, center=True).mean() my_data1 = dftemp['dinc'].ffill().bfill() data_array = data_array.T pre_PCA_scaler = MinMaxScaler() data_array = pre_PCA_scaler.fit_transform(data_array) from sklearn.decomposition import PCA from sklearn.manifold import TSNE # ============================================================================= # pca = PCA().fit(data_array) # plt.plot(np.cumsum(pca.explained_variance_ratio_)) # plt.xlabel('number of components') # plt.ylabel('cumulative explained variance'); # # plt.show() # ============================================================================= sampcount = int(len(data_array) * percent_drilled) pca = TSNE(n_components=newdim).fit(data_array[:sampcount]) projected = pca.transform(data_array) my_data = [] for i in range(newdim): my_data.append(projected[:, i]) my_data1 = my_data1[:, np.newaxis] my_data_newaxis = [] for i in my_data: my_data_newaxis.append(i[:, np.newaxis]) temp_data1 = pd.DataFrame(my_data1.flatten()) temp_data1 = pd.DataFrame(my_data1) range1 = temp_data1[0].diff(memory + predictions) range2 = np.amax(range1) RNN_scaler = MinMaxScaler() my_data1 = RNN_scaler.fit_transform(my_data1) my_data_scaled = [] for i in my_data_newaxis: my_data_scaled.append(MinMaxScaler().fit_transform(i)) X1 = prepareinput(my_data1, memory) Xdata = [] for i in my_data_scaled: Xn = prepareinput_nozero(i, memory, predictions) Xdata.append(Xn) y_temp = prepareoutput(my_data1, memory, predictions) stack = [] for i in range(memory): stack.append(np.roll(my_data1, -i)) X_temp = np.hstack(stack) y = y_temp data_length = len(my_data1) - memory - predictions testing_cutoff = 0.80 border1 = int((data_length) * (percent_drilled * 0.8)) border2 = int((data_length) * (percent_drilled)) border3 = int((data_length) * (percent_drilled + 0.2)) X1_train = X1[:border1] X1_test = X1[border1:border2] X1_test2 = X1[border2:border3] Xdata_train = [] Xdata_test = [] Xdata_test2 = [] for i in Xdata: Xdata_train.append(i[:border1]) Xdata_test.append(i[border1:border2]) Xdata_test2.append(i[border2:border3]) y_train, y_test, y_test2 = y[:border1], y[border1:border2], y[ border2:border3] X1_train = X1_train.reshape((X1_train.shape[0], X1_train.shape[1], 1)) X1_test = X1_test.reshape((X1_test.shape[0], X1_test.shape[1], 1)) X1_test2 = X1_test2.reshape((X1_test2.shape[0], X1_test2.shape[1], 1)) Xdata_train_r = [] Xdata_test_r = [] Xdata_test2_r = [] for i in range(newdim): Xdata_train_r.append(Xdata_train[i].reshape( (Xdata_train[i].shape[0], Xdata_train[i].shape[1], 1))) Xdata_test_r.append(Xdata_test[i].reshape( (Xdata_test[i].shape[0], Xdata_test[i].shape[1], 1))) Xdata_test2_r.append(Xdata_test2[i].reshape( (Xdata_test2[i].shape[0], Xdata_test2[i].shape[1], 1))) X_train_con = np.concatenate(Xdata_train_r, axis=2) X_test_con = np.concatenate(Xdata_test_r, axis=2) X_test2_con = np.concatenate(Xdata_test2_r, axis=2) X_train = [X1_train, X_train_con] X_test = [X1_test, X_test_con] X_test2 = [X1_test2, X_test2_con] input1 = Input(shape=(memory, 1)) input2 = Input(shape=(memory + predictions, newdim)) x1 = GaussianNoise(inc_layer2, input_shape=(memory, 1))(input1) x1 = GRU(units=inc_layer1, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer='l2', recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, return_sequences=False, return_state=False, stateful=False)(x1) x1 = Dropout(drop1)(x1) x1 = Model(inputs=input1, outputs=x1) x2 = Dense(data_layer1, input_shape=(memory + predictions, 3))(input2) x2 = Dropout(drop2)(x2) x2 = Flatten()(x2) x2 = Dense(data_layer2)(x2) x2 = Model(inputs=input2, outputs=x2) combined = concatenate([x1.output, x2.output]) z = Dense(dense_layer, activation="relu")(combined) z = Dense(predictions, activation="linear")(z) #define the model myadam = Adam(learning_rate=lr, beta_1=0.9, beta_2=0.999, amsgrad=False) class PlotResuls(Callback): def on_train_begin(self, logs={}): self.i = 0 self.x = [] self.losses = [] self.val_losses = [] #self.fig = plt.figure() self.logs = [] def on_epoch_end(self, epoch, logs={}): self.logs.append(logs) self.x.append(self.i) self.losses.append(logs.get('loss')) self.val_losses.append(logs.get('val_loss')) self.i += 1 #print (".", end = '') if (epoch % 14999 == 0) & (epoch > 0): print(epoch) plt.plot(self.x, np.log(self.losses), label="loss") plt.plot(self.x, np.log(self.val_losses), label="val_loss") plt.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2) plt.title("Loss") plt.legend() plt.show() #mymanyplots(epoch, data, model) #data = [X1, X2, X3, X4, y, X1_train,X_train, X_test, X1_test, border1, border2, y_train, y_test, memory, y_temp, predictions] plot_results = PlotResuls() es = EarlyStopping(monitor='val_loss', mode='min', verbose=0, patience=25) ens_val_array = np.zeros(ensemble_count) ens_test_array = np.zeros(ensemble_count) for ens_no in range(ensemble_count): tf.keras.backend.clear_session() mc = ModelCheckpoint(f'best_model_ens_{ens_no}.h5', monitor='val_loss', mode='min', save_best_only=True, verbose=0) model = Model(inputs=[x1.input, x2.input], outputs=z) model.compile(optimizer=myadam, loss='mean_squared_error') history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=2000, verbose=0, batch_size=bs, callbacks=[plot_results, es, mc]) model = load_model(f'best_model_ens_{ens_no}.h5') valresult = np.log(model.evaluate(x=X_test, y=y_test, verbose=0)) testresult = np.log(model.evaluate(x=X_test2, y=y_test2, verbose=0)) ens_val_array[ens_no] = valresult ens_test_array[ens_no] = testresult winner = ens_val_array.argmin() model = load_model(f'best_model_ens_{winner}.h5') info(ens_val_array) info(ens_test_array) info(f'Validation winner {winner}') sample_count = len(X_test2[0]) y_pred = model.predict(X_test2) plt.plot(np.log(history.history['loss']), label='loss') plt.plot(np.log(history.history['val_loss']), label='test') plt.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2) plt.legend() plt.clf() plt.show() for i in range(5): rand = np.random.randint(0, len(X_test[0])) y_test_descaled = RNN_scaler.inverse_transform(y_test[rand, np.newaxis]) y_in_descaled = RNN_scaler.inverse_transform(X_test[0][rand, :]) y_in_descaled = y_in_descaled.flatten() y_test_descaled = y_test_descaled.flatten() y_pred = model.predict(X_test) y_pred_descaled = RNN_scaler.inverse_transform(y_pred[rand, np.newaxis]) y_pred_descaled = y_pred_descaled.flatten() plt.plot(y_test_descaled, label="true") plt.plot(y_pred_descaled, label="predicted") plt.title('Inclination delta') #plt.ylim(0,1) plt.legend() plt.show() plt.figure(figsize=(5, 4)) x_after = np.linspace(0, 23, 100) x_before = np.linspace(-23, -0.23, 100) plt.plot(x_before, np.cumsum(y_in_descaled), label="measured", linestyle="-", c="black") commonpoint = np.cumsum(y_in_descaled)[-1] plt.plot(x_after, commonpoint + np.cumsum(y_test_descaled), label="actual", linestyle='-.', c='black') plt.plot(x_after, commonpoint + np.cumsum(y_pred_descaled), label="predicted", linestyle=':', c='black') #plt.title('') plt.ylim(-1, 7) plt.grid() plt.tight_layout() #plt.hlines(0, -23, 23, linewidth=0.5) plt.xlabel("Distance to sensor [m]") plt.ylabel("Inclination, local coordinates, [deg]") plt.legend() plt.tight_layout() plt.savefig(f'Sample, {percent_drilled}, no.{i}.pdf') plt.show() # #### Different ensemble, voting ###### # ypred_array = [] # for i in range(ensemble_count): # model = load_model(f'best_model_ens_{i}.h5') # y_pred = model.predict(X_test2) # ypred_array.append(y_pred) # y_pred = np.average(ypred_array, axis=0) # ######## Different ensemble ends here # y_test_descaled = RNN_scaler.inverse_transform(y_test2) y_pred = model.predict(X_test2) y_pred_descaled = RNN_scaler.inverse_transform(y_pred) error_matrix = np.cumsum(y_pred_descaled, axis=1) - np.cumsum( y_test_descaled, axis=1) def rand_jitter(arr): stdev = .004 * (max(arr) - min(arr)) return arr + np.random.randn(len(arr)) * stdev def jitter(x, y, s=20, c='b', marker='o', cmap=None, norm=None, vmin=None, vmax=None, alpha=None, linewidths=None, verts=None, **kwargs): return plt.scatter(rand_jitter(x), rand_jitter(y), s=s, c=c, marker=marker, cmap=cmap, norm=norm, vmin=vmin, vmax=vmax, alpha=alpha, linewidths=linewidths, verts=verts, **kwargs) plt.figure(figsize=(5, 5), dpi=200) for i in range(sample_count): _ = jitter(x_after, error_matrix[i], alpha=1, s=0.5, marker=".", c="black") plt.title(f"delta, drilled {percent_drilled}") plt.xlabel("Distance to sensor [m]") plt.ylabel("Prediction error [deg]") plt.grid() plt.tight_layout() plt.savefig(f'Birdflock, {percent_drilled}.pdf') plt.show() #plt.plot(np.median(error_matrix, axis=0), linewidth=8, alpha=1, c="white") #plt.plot(np.median(error_matrix, axis=0), linewidth=2, alpha=1, c="black") plt.scatter(np.arange(0, 100, 1), np.average(np.abs(error_matrix), axis=0), marker="o", s=40, alpha=0.7, c="white", zorder=2) c_array = np.empty(100, dtype=object) aae = np.average(np.abs(error_matrix), axis=0) for i in range(100): if aae[i] <= 0.4: c_array[i] = "green" elif aae[i] <= 0.8: c_array[i] = "orange" else: c_array[i] = "red" plt.scatter(np.arange(0, 100, 1), aae, marker=".", s=20, alpha=1, c=c_array, zorder=3, label="Average Absolute Error") plt.ylim((-3, 3)) plt.axhline(y=0, xmin=0, xmax=1, linewidth=2, c="black") plt.axhline(y=0.4, xmin=0, xmax=1, linewidth=1, c="black") plt.axhline(y=0.8, xmin=0, xmax=1, linewidth=1, c="black") plt.legend() plt.show() model = load_model('best_model.h5') #mymanyplots(-1, data, model) #myerrorplots(data, model) valresult = np.log(model.evaluate(x=X_test, y=y_test, verbose=0)) testresult = np.log(model.evaluate(x=X_test2, y=y_test2, verbose=0)) return valresult, testresult, aae
class RelevanceModel: def __init__( self, feature_config: FeatureConfig, tfrecord_type: str, file_io: FileIO, scorer: Optional[ScorerBase] = None, metrics: List[Union[Type[kmetrics.Metric], str]] = [], optimizer: Optional[Optimizer] = None, model_file: Optional[str] = None, compile_keras_model: bool = False, output_name: str = "score", logger=None, ): """Use this constructor to define a custom scorer""" self.feature_config: FeatureConfig = feature_config self.logger: Logger = logger self.output_name = output_name self.scorer = scorer self.tfrecord_type = tfrecord_type self.file_io = file_io if scorer: self.max_sequence_size = scorer.interaction_model.max_sequence_size else: self.max_sequence_size = 0 # Load/Build Model if model_file and not compile_keras_model: """ If a model file is specified, load it without compiling into a keras model NOTE: This will allow the model to be only used for inference and cannot be used for retraining. """ self.model: Model = self.load(model_file) self.is_compiled = False else: """ Specify inputs to the model Individual input nodes are defined for each feature Each data point represents features for all records in a single query """ inputs: Dict[str, Input] = feature_config.define_inputs() scores, train_features, metadata_features = scorer(inputs) # Create model with functional Keras API self.model = Model(inputs=inputs, outputs={self.output_name: scores}) # Get loss fn loss_fn = scorer.loss.get_loss_fn(**metadata_features) # Get metric objects metrics_impl: List[Union[str, kmetrics.Metric]] = get_metrics_impl( metrics=metrics, feature_config=feature_config, metadata_features=metadata_features ) # Compile model """ NOTE: Related Github issue: https://github.com/tensorflow/probability/issues/519 """ self.model.compile( optimizer=optimizer, loss=loss_fn, metrics=metrics_impl, experimental_run_tf_function=False, ) # Write model summary to logs model_summary = list() self.model.summary(print_fn=lambda x: model_summary.append(x)) self.logger.info("\n".join(model_summary)) if model_file: """ If model file is specified, load the weights from the SavedModel NOTE: The architecture, loss and metrics of self.model need to be the same as the loaded SavedModel """ self.load_weights(model_file) self.is_compiled = True @classmethod def from_relevance_scorer( cls, interaction_model: InteractionModel, model_config: dict, feature_config: FeatureConfig, loss: RelevanceLossBase, metrics: List[Union[kmetrics.Metric, str]], optimizer: Optimizer, tfrecord_type: str, file_io: FileIO, model_file: Optional[str] = None, compile_keras_model: bool = False, output_name: str = "score", logger=None, ): """Use this as constructor to define a custom InteractionModel with RelevanceScorer""" assert isinstance(interaction_model, InteractionModel) assert isinstance(loss, RelevanceLossBase) scorer: ScorerBase = RelevanceScorer( model_config=model_config, interaction_model=interaction_model, loss=loss, output_name=output_name, ) return cls( scorer=scorer, feature_config=feature_config, metrics=metrics, optimizer=optimizer, tfrecord_type=tfrecord_type, model_file=model_file, compile_keras_model=compile_keras_model, output_name=output_name, file_io=file_io, logger=logger, ) @classmethod def from_univariate_interaction_model( cls, model_config, feature_config: FeatureConfig, tfrecord_type: str, loss: RelevanceLossBase, metrics: List[Union[kmetrics.Metric, str]], optimizer: Optimizer, feature_layer_keys_to_fns: dict = {}, model_file: Optional[str] = None, compile_keras_model: bool = False, output_name: str = "score", max_sequence_size: int = 0, file_io: FileIO = None, logger=None, ): """Use this as constructor to use UnivariateInteractionModel and RelevanceScorer""" interaction_model: InteractionModel = UnivariateInteractionModel( feature_config=feature_config, feature_layer_keys_to_fns=feature_layer_keys_to_fns, tfrecord_type=tfrecord_type, max_sequence_size=max_sequence_size, ) return cls.from_relevance_scorer( interaction_model=interaction_model, model_config=model_config, feature_config=feature_config, loss=loss, metrics=metrics, optimizer=optimizer, tfrecord_type=tfrecord_type, model_file=model_file, compile_keras_model=compile_keras_model, output_name=output_name, file_io=file_io, logger=logger, ) def fit( self, dataset: RelevanceDataset, num_epochs: int, models_dir: str, logs_dir: Optional[str] = None, logging_frequency: int = 25, monitor_metric: str = "", monitor_mode: str = "", patience=2, ): """ Trains model for defined number of epochs Args: dataset: an instance of RankingDataset num_epochs: int value specifying number of epochs to train for models_dir: directory to save model checkpoints logs_dir: directory to save model logs logging_frequency: every #batches to log results """ if not monitor_metric.startswith("val_"): monitor_metric = "val_{}".format(monitor_metric) callbacks_list: list = self._build_callback_hooks( models_dir=models_dir, logs_dir=logs_dir, is_training=True, logging_frequency=logging_frequency, monitor_mode=monitor_mode, monitor_metric=monitor_metric, patience=patience, ) if self.is_compiled: self.model.fit( x=dataset.train, validation_data=dataset.validation, epochs=num_epochs, verbose=True, callbacks=callbacks_list, ) else: raise NotImplementedError( "The model could not be trained. Check if the model was compiled correctly. Training loaded SavedModel is not currently supported." ) def predict( self, test_dataset: data.TFRecordDataset, inference_signature: str = "serving_default", additional_features: dict = {}, logs_dir: Optional[str] = None, logging_frequency: int = 25, ): """ Predict the labels for the trained model Args: test_dataset: an instance of tf.data.dataset inference_signature: If using a SavedModel for prediction, specify the inference signature logging_frequency: integer representing how often(in batches) to log status Returns: ranking scores or new ranks for each record in a query """ if logs_dir: outfile = os.path.join(logs_dir, RelevanceModelConstants.MODEL_PREDICTIONS_CSV_FILE) # Delete file if it exists self.file_io.rm_file(outfile) _predict_fn = get_predict_fn( model=self.model, tfrecord_type=self.tfrecord_type, feature_config=self.feature_config, inference_signature=inference_signature, is_compiled=self.is_compiled, output_name=self.output_name, features_to_return=self.feature_config.get_features_to_log(), additional_features=additional_features, max_sequence_size=self.max_sequence_size, ) predictions_df_list = list() batch_count = 0 for predictions_dict in test_dataset.map(_predict_fn).take(-1): predictions_df = pd.DataFrame(predictions_dict) if logs_dir: if os.path.isfile(outfile): predictions_df.to_csv(outfile, mode="a", header=False, index=False) else: # If writing first time, write headers to CSV file predictions_df.to_csv(outfile, mode="w", header=True, index=False) else: predictions_df_list.append(predictions_df) batch_count += 1 if batch_count % logging_frequency == 0: self.logger.info("Finished predicting scores for {} batches".format(batch_count)) predictions_df = None if logs_dir: self.logger.info("Model predictions written to -> {}".format(outfile)) else: predictions_df = pd.concat(predictions_df_list) return predictions_df def evaluate( self, test_dataset: data.TFRecordDataset, inference_signature: str = None, additional_features: dict = {}, group_metrics_min_queries: int = 50, logs_dir: Optional[str] = None, logging_frequency: int = 25, ): """ Evaluate the ranking model Args: test_dataset: an instance of tf.data.dataset inference_signature: If using a SavedModel for prediction, specify the inference signature logging_frequency: integer representing how often(in batches) to log status metric_group_keys: list of fields to compute group based metrics on save_to_file: set to True to save predictions to file like self.predict() Returns: metrics and groupwise metrics as pandas DataFrames NOTE: Only if the keras model is compiled, you can directly do a model.evaluate() Override this method to implement your own evaluation metrics. """ if self.is_compiled: return self.model.evaluate(test_dataset) else: raise NotImplementedError def save( self, models_dir: str, preprocessing_keys_to_fns={}, postprocessing_fn=None, required_fields_only: bool = True, pad_sequence: bool = False, ): """ Save tf.keras model to models_dir Args: models_dir: path to directory to save the model """ model_file = os.path.join(models_dir, "final") # Save model with default signature self.model.save(filepath=os.path.join(model_file, "default")) """ Save model with custom signatures Currently supported - signature to read TFRecord SequenceExample inputs """ self.model.save( filepath=os.path.join(model_file, "tfrecord"), signatures=define_serving_signatures( model=self.model, tfrecord_type=self.tfrecord_type, feature_config=self.feature_config, preprocessing_keys_to_fns=preprocessing_keys_to_fns, postprocessing_fn=postprocessing_fn, required_fields_only=required_fields_only, pad_sequence=pad_sequence, max_sequence_size=self.max_sequence_size, ), ) self.logger.info("Final model saved to : {}".format(model_file)) def load(self, model_file: str) -> Model: """ Loads model from the SavedModel file specified Args: model_file: path to file with saved tf keras model Returns: loaded tf keras model """ """ NOTE: There is currently a bug in Keras Model with saving/loading models with custom losses and metrics. Therefore, we are currently loading the SavedModel with compile=False The saved model signatures can be used for inference at serving time NOTE: Retraining currently not supported! Would require compiling the model with the right loss and optimizer states Ref: https://github.com/keras-team/keras/issues/5916 https://github.com/tensorflow/tensorflow/issues/32348 https://github.com/keras-team/keras/issues/3977 """ model = tf.keras.models.load_model(model_file, compile=False) self.logger.info("Successfully loaded SavedModel from {}".format(model_file)) self.logger.warning("Retraining is not yet supported. Model is loaded with compile=False") return model def load_weights(self, model_file: str): # Load saved model with compile=False loaded_model = self.load(model_file) # Set weights of Keras model from the loaded model weights self.model.set_weights(loaded_model.get_weights()) self.logger.info("Weights have been set from SavedModel. RankingModel can now be trained.") def _build_callback_hooks( self, models_dir: str, logs_dir: Optional[str] = None, is_training=True, logging_frequency=25, monitor_metric: str = "", monitor_mode: str = "", patience=2, ): """ Build callback hooks for the training loop Returns: callbacks_list: list of callbacks """ callbacks_list: list = list() if is_training: # Model checkpoint if models_dir and monitor_metric: checkpoints_path = os.path.join( models_dir, RelevanceModelConstants.CHECKPOINT_FNAME ) cp_callback = callbacks.ModelCheckpoint( filepath=checkpoints_path, save_weights_only=False, verbose=1, save_best_only=True, mode=monitor_mode, monitor=monitor_metric, ) callbacks_list.append(cp_callback) # Early Stopping if monitor_metric: early_stopping_callback = callbacks.EarlyStopping( monitor=monitor_metric, mode=monitor_mode, patience=patience, verbose=1, restore_best_weights=True, ) callbacks_list.append(early_stopping_callback) # TensorBoard if logs_dir: tensorboard_callback = callbacks.TensorBoard( log_dir=logs_dir, histogram_freq=1, update_freq=5 ) callbacks_list.append(tensorboard_callback) # Debugging/Logging logger = self.logger class DebuggingCallback(callbacks.Callback): def __init__(self, patience=0): super(DebuggingCallback, self).__init__() self.epoch = 0 def on_train_batch_end(self, batch, logs=None): if batch % logging_frequency == 0: logger.info("[epoch: {} | batch: {}] {}".format(self.epoch, batch, logs)) def on_epoch_end(self, epoch, logs=None): logger.info("End of Epoch {}".format(self.epoch)) logger.info(logs) def on_epoch_begin(self, epoch, logs=None): self.epoch = epoch + 1 logger.info("Starting Epoch : {}".format(self.epoch)) logger.info(logs) def on_train_begin(self, logs): logger.info("Training Model") def on_test_begin(self, logs): logger.info("Evaluating Model") def on_predict_begin(self, logs): logger.info("Predicting scores using model") def on_train_end(self, logs): logger.info("Completed training model") logger.info(logs) def on_test_end(self, logs): logger.info("Completed evaluating model") logger.info(logs) def on_predict_end(self, logs): logger.info("Completed Predicting scores using model") logger.info(logs) callbacks_list.append(DebuggingCallback()) # Add more here return callbacks_list
out= model.train_on_batch(x_batch, y_batch) loss_val = out[0] acc = out[1]*100 train_losses.append(loss_val) train_accuracies.append(acc) pbar.update(1) pbar.set_postfix_str(f"Loss: {loss_val:.4f} ({np.mean(train_losses):.4f}) Acc: {acc:.3f} ({np.mean(train_accuracies):.3f})") with tqdm_notebook(total=VAL_STEPS, desc=f"Test_ Epoch {epoch+1}") as pbar: test_losses = [] test_accuracies = [] for s in range(VAL_STEPS): x_batch_val, y_batch_val = getBatch(test_size, "val") evaluation = model.evaluate(x_batch_val, y_batch_val) loss_val= evaluation[0] acc = evaluation[1]*100 test_losses.append(loss_val) test_accuracies.append(acc) pbar.update(1) pbar.set_postfix_str(f"Loss: {loss_val:.4f} ({np.mean(test_losses):.4f}) Acc: {acc:.3f} ({np.mean(test_accuracies):.3f})") model.save_weights(model_name+'.h5', overwrite=True) # Sample outputs from validation set LABELS_LIST = "airplane automobile bird cat deer dog frog horse ship truck".split(" ")
test_gen = generator.flow(test_subjects.index, test_targets) history = model.fit( train_gen, epochs = 20, validation_data = test_gen, verbose = 2, shuffle = False ) # Plot training/validation accuracy/loss print(sg.utils.plot_history(history, return_figure = True)) plt.savefig("testsave.png") # plt.show() # Test run model on the testing generator (again :) ) test_metrics = model.evaluate(test_gen) print("\nTest Set Metrics:") for name, val in zip(model.metrics_names, test_metrics): print("\t{}: {:0.4f}".format(name, val)) # Make predictions using the model all_nodes = node_subjects.index all_mapper = generator.flow(all_nodes) all_predictions = model.predict(all_mapper) # Use inverse_transform of the LabelBinarizer to turn the # one-hot encodings back into their textual labels node_predictions = target_encoding.inverse_transform(all_predictions) df = pd.DataFrame({"Predicted": node_predictions, "True": node_subjects}) print(df.head(10))
def run_prediction(): '''Procedure to run prediction model in jupyter notebook. ''' # load dataset X_train, X_test, y_train, y_test = datasets_v0.load_data() frame = framework_xception(batch_size=32) input_size = (128, 128, 3) # Prepare Datasets frame.load_datagenerators(X_train, y_train, X_test, y_test, input_size=(128, 128)) # load base model base_model = Xception(weights='imagenet', include_top=False, input_shape=(128, 128, 3)) # load top model top_model = base_model.output top_model = Flatten()(top_model) top_model = Dense(256, activation='relu')(top_model) top_model = Dropout(0.5)(top_model) predictions = Dense(1, activation='relu')(top_model) # stack model = Model(inputs=base_model.input, outputs=predictions) #print(model.summary()) # set trainable layer to top layers only change_trainable_layers(model, 132) # compile the model with a SGD/momentum optimizer # and a very slow learning rate. #optimizer = optimizers.SGD(lr=1e-4, momentum=0.9) optimizer = optimizers.Adam(learning_rate=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False, name='Adam') model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=[soft_acc, tf.keras.metrics.RootMeanSquaredError()]) print('\nFitting the model ... ...') log_dir = "../logs/fit/xception/" + dt.datetime.now().strftime( "%Y%m%d-%H%M%S") tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) history = model.fit( x=frame.train_generator, #y=self.train_df['label'].values, #generator = train_generator, batch_size=None, epochs=20, verbose=1, validation_data=frame.valid_generator, #shuffle=False, #class_weight=None, #sample_weight=None, #initial_epoch=0, steps_per_epoch=frame.STEP_SIZE_TRAIN, validation_steps=frame.STEP_SIZE_VALID, #validation_freq=1, max_queue_size=frame.batch_size * 8, #workers=4, use_multiprocessing=False, callbacks=[tensorboard_callback]) print('\nValidationg the model ... ...') log_dir = "../logs/validation/" + dt.datetime.now().strftime( "%Y%m%d-%H%M%S") tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) model.evaluate( x=frame.test_generator, y=None, batch_size=None, verbose=1, sample_weight=None, steps=frame.STEP_SIZE_TEST, max_queue_size=frame.batch_size * 8, #workers=1, use_multiprocessing=False, callbacks=[tensorboard_callback]) pred = model.predict( x=frame.test_generator, steps=frame.STEP_SIZE_TEST, max_queue_size=frame.batch_size * 8, #workers=8, use_multiprocessing=False, verbose=True)
input_dataset = tf.data.Dataset.from_generator( lambda: data_generator(input_fns, input_gt_fns, validation=False), output_types=(tf.float32, tf.uint16), output_shapes=(tf.TensorShape([None, None, None, None]), tf.TensorShape([None, None, None, None]))) # construct validation dataset generator validation_dataset = tf.data.Dataset.from_generator( lambda: data_generator(validation_fns, validation_gt_fns, validation=True), output_types=(tf.float32, tf.uint16), output_shapes=(tf.TensorShape([None, None, None, None]), tf.TensorShape([None, None, None, None]))) if TESTING: # # record test loss (MAE) [loss, acc] = model.evaluate(input_dataset) print('Loss: ' + str(loss)) # predict output images on test set out = model.predict(input_dataset, verbose=1) psnr = [] ssim = [] # record metrics and also save images to disk for idx, im in enumerate(out): # get corresponding ground truth image gt_image = gt_map[input_gt_fns[idx]] # Rescale ground truth back to 0-255 and convert to uint8
onetensor = tf.ones(shape=tf.shape(y_true)) temploss = (-1)*tf.reduce_sum(tf.math.multiply(y_true, tf.math.log(y_pred))) - tf.reduce_sum(tf.math.multiply((onetensor - y_true), \ tf.math.log(onetensor - y_pred))) return temploss ## model.compile(optimizer='adam', loss=binaryloss) # fit the model to the training data and evaluate with the test data tic = time.perf_counter() history = model.fit(obs_train, lab_train, epochs=1, batch_size=32, verbose=1) toc = time.perf_counter() model.evaluate(obs_test, lab_test) print('Total training time: ' + str(toc - tic) + ' seconds.') # save off the model and model history # model.save('model0ones_15dB_fixedH') # with open('F:/Comm Studies/PermutationSimulations/UnlabeledSensing/TrainingHistories/trainHistoryDict_model0ones_15dB_fixedH', 'wb') as file_pi: # pickle.dump(history.history, file_pi) # ----------------------------------------------------------------------------- # ---------- MISCELLANEOUS TESTING CODE - IGNORE BELOW THIS POINT ------------- # ----------------------------------------------------------------------------- #---------------------------US test------------------------------------------- #model = tf.keras.models.load_model('model0_100dB_fixedH')
def train_networks(training_percentage, filename, experiment): stages = constants.training_stages (data, labels) = get_data(experiment, one_hot=True) total = len(data) step = total / stages # Amount of training data, from which a percentage is used for # validation. training_size = int(total * training_percentage) n = 0 histories = [] for k in range(stages): i = k * step j = int(i + training_size) % total i = int(i) if j > i: training_data = data[i:j] training_labels = labels[i:j] testing_data = np.concatenate((data[0:i], data[j:total]), axis=0) testing_labels = np.concatenate((labels[0:i], labels[j:total]), axis=0) else: training_data = np.concatenate((data[i:total], data[0:j]), axis=0) training_labels = np.concatenate((labels[i:total], labels[0:j]), axis=0) testing_data = data[j:i] testing_labels = labels[j:i] training_data, training_labels = expand_data(training_data, training_labels) truly_training = int(training_size * truly_training_percentage) validation_data = training_data[truly_training:] validation_labels = training_labels[truly_training:] training_data = training_data[:truly_training] training_labels = training_labels[:truly_training] input_img = Input(shape=(img_columns, img_rows, img_colors)) encoded = get_encoder(input_img) classified = get_classifier(encoded) decoded = get_decoder(encoded) model = Model(inputs=input_img, outputs=[classified, decoded]) model.compile(loss=['categorical_crossentropy', 'binary_crossentropy'], optimizer='adam', metrics='accuracy') model.summary() history = model.fit(training_data, (training_labels, training_data), batch_size=batch_size, epochs=epochs, validation_data=(validation_data, { 'classification': validation_labels, 'autoencoder': validation_data }), callbacks=[EarlyStoppingAtLossCrossing(patience)], verbose=2) histories.append(history) history = model.evaluate(testing_data, (testing_labels, testing_data), return_dict=True) histories.append(history) model.save(constants.model_filename(filename, n)) n += 1 return histories
print(hist.history.keys()) print('train loss: ', hist.history['loss'][-1]) print('train acc: ', hist.history['accuracy'][-1]) print('val acc: ', hist.history['val_loss'][-1]) print('val acc: ', hist.history['val_accuracy'][-1]) # %matplotlib inline import matplotlib.pyplot as plt plt.plot(hist.history['loss']) plt.xlabel('epoch') plt.ylabel('loss') plt.show() """- 모델 평가(Test Model)""" test_loss = model.evaluate([test_df.userid, test_df.shop_id], test_df.rating) print('test loss: ', test_loss) """- ## 학습된 머신을 활용한 예측""" pd.options.display.float_format = '{:.2f}'.format # 출력 포매팅 설정 # ratings_df[(ratings_df['userid'] == 249) & (ratings_df['shop_Id'] == 70)] # print(df_backup.head(30)) print(df_backup[df_backup['userid'] == 'user000777']) # print(df.head()) userid = 777 # 1 ~ 610 shop_id = 67 target = df_backup[(df_backup['userid'] == ('user' + f'{str(userid).zfill(6)}')) & (df_backup['shop_id'] == shop_id)]
conv2 = layers.Conv2D(64, kernel_size=(3,3), activation='relu')(pool1) noise2 = layers.GaussianNoise(test_dict[2])(conv2, training=True) pool2 = layers.MaxPooling2D(pool_size=(2, 2))(noise2) conv3 = layers.Conv2D(64, kernel_size=(3,3), activation='relu')(pool2) noise1 = layers.GaussianNoise(test_dict[3])(conv3, training=True) flat = layers.Flatten()(noise1) hidden1 = layers.Dense(64, activation='relu')(flat) output = layers.Dense(10)(hidden1) model1 = Model(inputs=visible, outputs=output) model1.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) model1.set_weights(weights1) test_loss, test_acc = model1.evaluate(test_images, test_labels, verbose=0) fintest_trial.append(test_acc) parameters[name]=fintest_trial #testing clear model name=str(m)+'clearmodelLayer'+str(l) fintest_trial=[] for n in range(0, 2): print('n', n) del(model1) tf.compat.v1.reset_default_graph() if n==0: test_dict={1: 0, 2: 0, 3: 0} else: test_dict=noise_dict visible = layers.Input(shape=(32,32,3))
def lra_framework(model: Model, lra_algorithm, x_train, x_test, y_test, dataset, model_name): scores = [] arch = model_name.split('_')[0] dir = os.path.join(arch, model_name) if not os.path.exists(dir): os.makedirs(dir) logger_path = os.path.join(dir, 'log_file_model_{}'.format(model_name)) if os.path.exists(logger_path + ".log"): os.remove(logger_path + ".log") logger = get_logger(logger_path) samples = x_train # np.concatennum_of_paramsate((x_train, x_test)) initial_score = model.evaluate(x_test, y_test, verbose=0) score = np.copy(initial_score) score_to_plot = [] compression_ratio_to_plot = [] score_to_plot.append(score[-1]) compression_ratio_to_plot.append(0) opt = tf.keras.optimizers.Adam() temp_model = clone_model(model) temp_model.compile( loss=tf.keras.losses.categorical_crossentropy, optimizer=opt, metrics=['accuracy']) lra_model = clone_model(model) lra_model.set_weights(model.get_weights()) lra_model.compile( loss=tf.keras.losses.categorical_crossentropy, optimizer=opt, metrics=['accuracy']) relevant_layers, relevant_layers_index_in_model = get_relevant_layers(model) initial_num_of_params = get_initial_number_of_params(relevant_layers) print('\n\n') it = 0 while (initial_score[1] - score[1] < cfg.accuracy_tolerance): klds = [] print("Start of Iteration {0}:".format(it)) logger.info("Start of Iteration {0}:".format(it)) curr_num_of_params = 0 for i, layer_index in enumerate(relevant_layers_index_in_model): temp_model.set_weights(lra_model.get_weights()) temp_model, _, _, num_of_params_layer_i = lra_per_layer(temp_model, layer_index=layer_index, algorithm=lra_algorithm, update_memory=False) curr_num_of_params += num_of_params_layer_i # kld_per_layer = evaluate_kld_for_each_layer(model, temp_model, samples) kld_per_layer = evaluate_kld_for_last_layer(model, temp_model, samples) print('{} ({}): KLD per layer {:.4f}'.format(relevant_layers[i].name, layer_index, kld_per_layer)) # klds.append(sum(kld_per_layer)) klds.append(kld_per_layer) print("Compression ratio : {}. \n number of params: \n \t lra_model {:,} \n \t initial model {:,}" .format(1 - curr_num_of_params / initial_num_of_params, curr_num_of_params, initial_num_of_params)) logger.info("Compression ratio : {}. number of params: \t lra_model {:,} \t initial model {:,}" .format(1 - curr_num_of_params / initial_num_of_params, curr_num_of_params, initial_num_of_params)) if curr_num_of_params < initial_num_of_params: score_to_plot.append(score[-1]) compression_ratio_to_plot.append(1 - curr_num_of_params / initial_num_of_params) # so the graph will start from 0 and go to 1 min_kld_index = np.argmin(klds) layer_with_min_kld = relevant_layers[min_kld_index] layer_index_in_model_with_min_kld = relevant_layers_index_in_model[min_kld_index] # print('---------------- Start Compression with {0} for layer {1}!) ----------------'.format(lra_algorithm, # layer_with_min_kld.name)) lra_model, truncated, full_svs, _ = lra_per_layer(lra_model, layer_index=layer_index_in_model_with_min_kld, algorithm=lra_algorithm, update_memory=True) if 'tsvd' in lra_algorithm: print('Approximate {0} {1} using {2}/{3} singular values'.format(layer_with_min_kld.name, layer_index_in_model_with_min_kld, truncated, full_svs)) logger.info('Approximate {0} {1} using {2}/{3} singular values'.format(layer_with_min_kld.name, layer_index_in_model_with_min_kld, truncated, full_svs)) if 'rrqr' in lra_algorithm: print('Approximate {0} {1} using {2}/{3} rank ratio '.format(layer_with_min_kld.name, layer_index_in_model_with_min_kld, truncated, full_svs)) logger.info('Approximate {0} {1} using {2}/{3} rank ratio'.format(layer_with_min_kld.name, layer_index_in_model_with_min_kld, truncated, full_svs)) # print('---------------- Done Compression with {0} for layer {1}!) ----------------'.format(lra_algorithm, # layer_with_min_kld.name)) score = lra_model.evaluate(x_test, y_test, verbose=0) scores.append(score) print("End of Iteration {}:\ntest loss = {:.3f}\ntest accuracy = {:.3f}\n\n\n".format(it, score[0], score[1])) logger.info("End of Iteration {}:\ntest loss = {:.3f}\ntest accuracy = {:.3f}\n\n\n".format(it, score[0], score[1])) it += 1 save_model_path = os.path.join(dir, '{0}_{1}_lra.h5'.format(model_name, dataset)) print('Saving model to: ', save_model_path) logger.info('Saving model to: {}'.format(save_model_path)) save_model(lra_model, save_model_path, include_optimizer=False, save_format='h5') score_to_plot = np.asarray(score_to_plot) compression_ratio_to_plot = np.asarray(compression_ratio_to_plot) np.save(os.path.join(dir, 'score_{}'.format(model_name)), score_to_plot) np.save(os.path.join(dir, 'compression_{}'.format(model_name)), compression_ratio_to_plot) plot_score_versus_compression(save_dir=dir, score_data=score_to_plot, compression_data=compression_ratio_to_plot, model_name=model_name, algo=lra_algorithm)