class LSTM(object): """Long Short Term Memory Regressor Class""" def __init__(self, layers, pct_dropout=0.2): """Build computational graph model Parameters ---------- layers: list | [input, hidden_1, hidden_2, output] Dimensions of each layer pct_dropout: float | 0.0 to 1.0 Percentage of dropout for hidden LSTM layers Returns ------- model: keras.Model Compiled keras sequential model """ if not isinstance(layers, list): raise TypeError( 'layers was expected to be of type %s, received %s' % (type([]), type(layers))) if len(layers) != 4: raise ValueError('4 layer dimentions required, received only %d' % len(layers)) self.model = Sequential() self.model.add( _LSTM(layers[1], input_shape=(layers[1], layers[0]), return_sequences=True, dropout=pct_dropout)) self.model.add( _LSTM(layers[2], return_sequences=False, dropout=pct_dropout)) self.model.add(Dense(layers[3], activation='linear')) self.model.compile(loss="mse", optimizer="rmsprop") def fit(self, X, y, **kwargs): """Train the model""" self.model.fit(X, y, **kwargs) def predict(self, series): """Prediction using provided series""" return self.model.predict(series)
class Model(object): def __init__(self): self.model = Sequential() self.model.add( Conv2D(32, (3, 3), input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3))) self.model.add(Activation('relu')) self.model.add(MaxPooling2D(pool_size=(2, 2))) self.model.add(Conv2D(32, (3, 3))) self.model.add(Activation('relu')) self.model.add(MaxPooling2D(pool_size=(2, 2))) self.model.add(Conv2D(64, (3, 3))) self.model.add(Activation('relu')) self.model.add(MaxPooling2D(pool_size=(2, 2))) self.model.add(Flatten()) self.model.add(Dense(64)) self.model.add(Activation('relu')) self.model.add(Dropout(0.5)) self.model.add(Dense(1)) self.model.add(Activation('sigmoid')) def load(self, file_path=FILE_PATH): print('Model Loaded.') self.model.load_weights(file_path) def predict(self, image): # 预测样本分类 img = image.resize((1, IMAGE_SIZE, IMAGE_SIZE, 3)) img = image.astype('float32') img /= 255 #归一化 result = self.model.predict(img) print(result) # 概率 result = self.model.predict_classes(img) print(result) # 0/1 return result[0]
classifier.add(Dense(units=6, kernel_initializer='uniform', activation='relu')) # Adding the output layer classifier.add( Dense(units=1, kernel_initializer='uniform', activation='sigmoid')) # Compiling the ANN classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) # Fitting the ANN to the Training set classifier.fit(X_train, y_train, batch_size=10, epochs=100, validation_split=0.1) # Part 3 - Making predictions and evaluating the model # Predicting the Test set results y_pred = classifier.predict(X_test) y_pred = (y_pred > 0.5) # Making the Confusion Matrix from sklearn.metrics import confusion_matrix cm = confusion_matrix(y_test, y_pred) print(cm) backend.clear_session()
class AmazonKerasClassifier: def __init__(self): self.losses = [] self.classifier = Sequential() def add_conv_layer(self, img_size=(32, 32), img_channels=3): self.classifier.add( BatchNormalization(input_shape=(*img_size, img_channels))) self.classifier.add( Conv2D(32, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(32, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add( Conv2D(64, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(64, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add( Conv2D(128, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(128, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add( Conv2D(256, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(256, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) def add_flatten_layer(self): self.classifier.add(Flatten()) def add_ann_layer(self, output_size): self.classifier.add(Dense(512, activation='relu')) self.classifier.add(BatchNormalization()) self.classifier.add(Dropout(0.5)) self.classifier.add(Dense(output_size, activation='sigmoid')) def _get_fbeta_score(self, classifier, X_valid, y_valid): p_valid = classifier.predict(X_valid) return fbeta_score(y_valid, np.array(p_valid) > 0.2, beta=2, average='samples') def train_model(self, x_train, y_train, learn_rate=0.001, epoch=5, batch_size=128, validation_split_size=0.2, train_callbacks=()): history = LossHistory() X_train, X_valid, y_train, y_valid = train_test_split( x_train, y_train, test_size=validation_split_size) opt = Nadam(lr=learn_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) self.classifier.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) # early stopping will auto-stop training process if model stops learning after 3 epochs earlyStopping = EarlyStopping(monitor='val_loss', patience=3, verbose=0, mode='auto') for i in range(epoch): self.classifier.fit( X_train, y_train, batch_size=batch_size, epochs=1, verbose=2, validation_data=(X_valid, y_valid), callbacks=[history, *train_callbacks, earlyStopping]) fbeta_score = self._get_fbeta_score(self.classifier, X_valid, y_valid) print('fbeta score: %s' % fbeta_score) return [history.train_losses, history.val_losses, fbeta_score] def save_weights(self, weight_file_path): self.classifier.save_weights(weight_file_path) def load_weights(self, weight_file_path): self.classifier.load_weights(weight_file_path) def predict(self, x_test): predictions = self.classifier.predict(x_test) return predictions def map_predictions(self, predictions, labels_map, thresholds): """ Return the predictions mapped to their labels :param predictions: the predictions from the predict() method :param labels_map: the map :param thresholds: The threshold of each class to be considered as existing or not existing :return: the predictions list mapped to their labels """ predictions_labels = [] for prediction in predictions: labels = [ labels_map[i] for i, value in enumerate(prediction) if value > thresholds[i] ] predictions_labels.append(labels) return predictions_labels def close(self): backend.clear_session()
class KerasModel: def __init__(self, img_size, img_channels=3, output_size=17): self.losses = [] self.model = Sequential() self.model.add( BatchNormalization(input_shape=(img_size[0], img_size[1], img_channels))) self.model.add(Conv2D(32, (3, 3), padding='same', activation='relu')) self.model.add(Conv2D(32, (3, 3), activation='relu')) self.model.add(MaxPooling2D(pool_size=2)) self.model.add(Dropout(0.3)) self.model.add(Conv2D(64, (3, 3), padding='same', activation='relu')) self.model.add(Conv2D(64, (3, 3), activation='relu')) self.model.add(MaxPooling2D(pool_size=2)) self.model.add(Dropout(0.3)) self.model.add(Conv2D(128, (3, 3), padding='same', activation='relu')) self.model.add(Conv2D(128, (3, 3), activation='relu')) self.model.add(MaxPooling2D(pool_size=2)) self.model.add(Dropout(0.3)) self.model.add(Conv2D(256, (3, 3), padding='same', activation='relu')) self.model.add(Conv2D(256, (3, 3), activation='relu')) self.model.add(MaxPooling2D(pool_size=2)) self.model.add(Dropout(0.3)) self.model.add(Conv2D(512, (3, 3), padding='same', activation='relu')) self.model.add(Conv2D(512, (3, 3), activation='relu')) self.model.add(MaxPooling2D(pool_size=2)) self.model.add(Dropout(0.3)) self.model.add(Flatten()) self.model.add(Dense(512, activation='relu')) self.model.add(BatchNormalization()) self.model.add(Dropout(0.5)) self.model.add(Dense(output_size, activation='sigmoid')) def get_fbeta_score(self, validation_data, verbose=True): p_valid = self.model.predict(validation_data[0]) thresholds = optimise_f2_thresholds(validation_data[1], p_valid, verbose=verbose) return fbeta_score(validation_data[1], np.array(p_valid) > thresholds, beta=2, average='samples'), thresholds def fit(self, flow, epochs, lr, validation_data, train_callbacks=[], batches=300): history = LossHistory() fbeta = Fbeta(validation_data) opt = Adam(lr=lr) self.model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) earlyStopping = EarlyStopping(monitor='val_loss', patience=3, verbose=0, mode='auto') self.model.fit_generator(flow, steps_per_epoch=batches, epochs=epochs, callbacks=[history, earlyStopping, fbeta] + train_callbacks, validation_data=validation_data) fb_score, thresholds = self.get_fbeta_score(validation_data, verbose=False) return [ fbeta.fbeta, history.train_losses, history.val_losses, fb_score, thresholds ] def save_weights(self, weight_file_path): self.model.save_weights(weight_file_path) def load_weights(self, weight_file_path): self.model.load_weights(weight_file_path) def predict_image(self, image): img = Image.fromarray(np.uint8(image * 255)) images = [img.copy().rotate(i) for i in [-90, 90, 180]] images.append(img) images = np.asarray([ np.asarray(image.convert("RGB"), dtype=np.float32) / 255 for image in images ]) return sum(self.model.predict(images)) / 4 def predict(self, x_test): return [self.predict_image(img) for img in tqdm(x_test)] def map_predictions(self, predictions, labels_map, thresholds): predictions_labels = [] for prediction in predictions: labels = [ labels_map[i] for i, value in enumerate(prediction) if value > thresholds[i] ] predictions_labels.append(labels) return predictions_labels def close(self): backend.clear_session()
import numpy as np from tensorflow.contrib.keras.api.keras.models import Sequential from tensorflow.contrib.keras.api.keras.layers import Dense, Activation from tensorflow.contrib.keras.api.keras.optimizers import SGD, Adam #import matplotlib.pyplot as plt data = np.loadtxt('sin.csv', delimiter=',', unpack=True) x = data[0] y = data[1] model = Sequential() model.add(Dense(30, input_shape=(1, ))) model.add(Activation('sigmoid')) model.add(Dense(40)) model.add(Activation('sigmoid')) model.add(Dense(1)) sgd = Adam(lr=0.1) model.compile(loss='mean_squared_error', optimizer=sgd) model.fit(x, y, epochs=1000, batch_size=20, verbose=0) print('save model') model.save('sin_model.h5') predictions = model.predict(x) print(np.mean(np.square(predictions - y))) preds = model.predict(x) plt.plot(x, y, 'b', x, preds, 'r--') plt.show()
classifier.save(model_path) print("Model saved to", model_path) # Saving loss history to file : lossLog_path = 'dataset/loss_history.log' myFile = open(lossLog_path, 'w+') myFile.write(history.losses) myFile.close() # Clearing session : backend.clear_session() # Confirming class indices: print("The model class indices are:", training_set.class_indices) # Predicting a new image : import numpy as np from keras.preprocessing import image test_image = image.load_img('dataset/single_prediction/cat_or_dog_2.jpg', target_size = input_size) test_image = image.img_to_array(test_image) test_image = np.expand_dims(test_image, axis = 0) #Adding an extra dimension to the image as the .predict() function expects a 4D array w/ the 4th dimension corresponding to the batch. result = classifier.predict(test_image) if result[0][0] == 1: # The .predict() function returns a 2D array, thus [0][0] corresponds to the first element. prediction = 'dog' else: prediction = 'cat' print ("This is a",prediction)
def onBeginTraining(self): ue.log("starting mnist keras cnn training") model_file_name = "mnistKerasCNN" model_directory = ue.get_content_dir() + "/Scripts/" model_sess_path = model_directory + model_file_name + ".tfsess" model_json_path = model_directory + model_file_name + ".json" my_file = Path(model_json_path) #reset the session each time we get training calls K.clear_session() #let's train batch_size = 128 num_classes = 10 epochs = 8 # input image dimensions img_rows, img_cols = 28, 28 # the data, shuffled and split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 ue.log('x_train shape:' + str(x_train.shape)) ue.log(str(x_train.shape[0]) + 'train samples') ue.log(str(x_test.shape[0]) + 'test samples') # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) model = Sequential() model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) # model.add(Dropout(0.2)) # model.add(Flatten()) # model.add(Dense(512, activation='relu')) # model.add(Dropout(0.2)) # model.add(Dense(num_classes, activation='softmax')) #model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test), callbacks=[self.stopcallback]) score = model.evaluate(x_test, y_test, verbose=0) ue.log("mnist keras cnn training complete.") ue.log('Test loss:' + str(score[0])) ue.log('Test accuracy:' + str(score[1])) self.session = K.get_session() self.model = model stored = {'model':model, 'session': self.session} #run a test evaluation ue.log(x_test.shape) result_test = model.predict(np.reshape(x_test[500],(1,28,28,1))) ue.log(result_test) #flush the architecture model data to disk #with open(model_json_path, "w") as json_file: # json_file.write(model.to_json()) #flush the whole model and weights to disk #saver = tf.train.Saver() #save_path = saver.save(K.get_session(), model_sess_path) #model.save(model_path) return stored
# Predict single cases test_image_1 = image.load_img('dataset/single_prediction/cat_or_dog_1.jpg', target_size=input_size) test_image_2 = image.load_img('dataset/single_prediction/cat_or_dog_2.jpg', target_size=input_size) test_image_1 = image.img_to_array(test_image_1) test_image_2 = image.img_to_array(test_image_2) # adding a 4th dimension for predict method # this dimension corresponds to the batch, cannot accept single inputs # only batch of size single or greater inputs test_image_1 = np.expand_dims(test_image_1, axis=0) test_image_2 = np.expand_dims(test_image_2, axis=0) predict_1 = classifier.predict(test_image_1) predict_2 = classifier.predict(test_image_2) # to understand output of 0 or 1, cats are 0, dogs are 1 training_set.class_indices if predict_1[0][0] == 1: prediction = 'dog' else: prediction = 'cat' print("The image 1 is a ", prediction) if predict_2[0][0] == 1: prediction = 'dog' else: prediction = 'cat'
regressor.fit(X_train, y_train, batch_size=32, epochs=200) # Part 3 - Making the predictions and visualising the results script_dir = os.path.dirname(__file__) test_set_path = os.path.join(script_dir, '../dataset/Google_Stock_Price_Test.csv') # Getting the real stock price of 2017 test_set = pd.read_csv(test_set_path) real_stock_price = test_set.iloc[:, 1:2].values # Getting the predicted stock price of 2017 inputs = real_stock_price inputs = sc.transform(inputs) inputs = np.reshape(inputs, (20, 1, 1)) predicted_stock_price = regressor.predict(inputs) predicted_stock_price = sc.inverse_transform(predicted_stock_price) # Align real values with their predictions predicted_stock_price = predicted_stock_price[:-1] real_stock_price = real_stock_price[1:] # Visualising the results plt.plot(real_stock_price, color='red', label='Real Google Stock Price') plt.plot(predicted_stock_price, color='blue', label='Predicted Google Stock Price') plt.title('Google Stock Price Prediction') plt.xlabel('Time') plt.ylabel('Google Stock Price') plt.legend() plt.show()
class AmazonKerasClassifier: def __init__(self): self.losses = [] self.classifier = Sequential() def add_conv_layer(self, img_size=(32, 32), img_channels=3): self.classifier.add( BatchNormalization(input_shape=(*img_size, img_channels))) self.classifier.add(Conv2D(32, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=(2, 2))) self.classifier.add(Dropout(0.25)) self.classifier.add(Conv2D(64, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=(2, 2))) self.classifier.add(Dropout(0.25)) self.classifier.add(Conv2D(16, (2, 2), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=(2, 2))) self.classifier.add(Dropout(0.25)) def add_flatten_layer(self): self.classifier.add(Flatten()) def add_ann_layer(self, output_size): self.classifier.add(Dense(256, activation='relu')) self.classifier.add(Dropout(0.25)) self.classifier.add(Dense(128, activation='relu')) self.classifier.add(Dropout(0.25)) self.classifier.add(Dense(output_size, activation='sigmoid')) def _get_fbeta_score(self, classifier, X_valid, y_valid): p_valid = classifier.predict(X_valid) #print ('p_valid') #print(p_valid.shape) #print(p_valid) return fbeta_score(y_valid, np.array(p_valid) > 0.2, beta=2, average='samples') def train_model(self, x_train, y_train, epoch=5, batch_size=128, validation_split_size=0.2, train_callbacks=()): history = LossHistory() X_train, X_valid, y_train, y_valid = train_test_split( x_train, y_train, test_size=validation_split_size) adam = Adam(lr=0.01, decay=1e-6) rms = RMSprop(lr=0.0001, decay=1e-6) self.classifier.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print('X_train.shape[0]') print(X_train.shape[0]) checkpointer = ModelCheckpoint(filepath="weights.best.hdf5", verbose=1, save_best_only=True) datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization= False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 0, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range= 0.1, # randomly shift images horizontally (fraction of total width) height_shift_range= 0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False) # randomly flip images datagen.fit(X_train) self.classifier.fit_generator( datagen.flow(X_train, y_train, batch_size=batch_size), steps_per_epoch=X_train.shape[0] // batch_size, epochs=epoch, validation_data=(X_valid, y_valid), callbacks=[history, *train_callbacks, checkpointer]) fbeta_score = self._get_fbeta_score(self.classifier, X_valid, y_valid) print(fbeta_score) return [history.train_losses, history.val_losses, fbeta_score] def load_weight(self): self.classifier.load_weights("weights.best.hdf5") def predict(self, x_test): predictions = self.classifier.predict(x_test) #print('predictions') #print(predictions.shape) #print(predictions) return predictions def map_predictions(self, predictions, labels_map, thresholds): """ Return the predictions mapped to their labels :param predictions: the predictions from the predict() method :param labels_map: the map :param thresholds: The threshold of each class to be considered as existing or not existing :return: the predictions list mapped to their labels """ predictions_labels = [] for prediction in predictions: labels = [ labels_map[i] for i, value in enumerate(prediction) if value > thresholds[i] ] predictions_labels.append(labels) return predictions_labels def close(self): backend.clear_session()
import tensorflow.contrib.keras.api.keras as keras from tensorflow.contrib.keras.api.keras.models import Sequential from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation from tensorflow.contrib.keras.api.keras.optimizers import SGD import numpy as np x_train = np.random.random((1000, 20)) y_train = keras.utils.to_categorical(np.random.randint(10, size=(1000, 1)), num_classes=10) x_test = np.random.random((100, 20)) y_test = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10) model = Sequential() model.add(Dense(64, activation='relu', input_dim=20)) model.add(Dropout(0.5)) model.add(Dense(64, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(10, activation='softmax')) sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=20, batch_size=128) score = model.evaluate(x_test, y_test, batch_size=128) y_pred = model.predict(np.random.random((1, 20)), batch_size=128) print(y_pred)
#Keras LSTM层的工作方式是通过接收3维(N,W,F)的数字阵列,其中N是训练序列的数目,W是序列长度,F是每个序列的特征数目。 TIME_STEPS = 30 INPUT_SIZE = 1 #model.add(LSTM(1,batch_input_shape=(None, TIME_STEPS, INPUT_SIZE))) model.add(LSTM(1,input_shape=(TIME_STEPS,INPUT_SIZE))) model.add(Dropout(0.2)) model.add(Dense(1)) model.add(Activation("linear")) start = time.time() model.compile(loss="mse", optimizer="rmsprop") print("Compilation Time : ", time.time() - start) tbCallBack.set_model(model) model.fit(train_x,train_y,batch_size=32,epochs=5) score = model.evaluate(train_x, train_y, batch_size=32) #model.save_weights('w1.hdf5') predicted = model.predict(test_x,batch_size=32,verbose=2) predicted = np.reshape(predicted, (predicted.size,)) print(predicted) print(score) plot_results(predicted,test_y) ''' model.add(Dense(128,activation='relu',input_shape=[None,5],input_dim=2)) model.add(Dense(3,activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) tbCallBack.set_model(model)
class Model(object): def __init__(self): self.model = Sequential() self.model.add( Conv2D(32, (3, 3), input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3))) self.model.add(Activation('relu')) self.model.add(MaxPooling2D(pool_size=(2, 2))) self.model.add(Conv2D(32, (3, 3))) self.model.add(Activation('relu')) self.model.add(MaxPooling2D(pool_size=(2, 2))) self.model.add(Conv2D(64, (3, 3))) self.model.add(Activation('relu')) self.model.add(MaxPooling2D(pool_size=(2, 2))) self.model.add(Flatten()) self.model.add(Dense(64)) self.model.add(Activation('relu')) self.model.add(Dropout(0.5)) self.model.add(Dense(1)) self.model.add(Activation('sigmoid')) def train(self, dataset, batch_size=batch_size, nb_epoch=epochs): self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) self.model.fit_generator( dataset.train, steps_per_epoch=nb_train_samples // batch_size, epochs=epochs, validation_data=dataset.valid, validation_steps=nb_validation_samples // batch_size) def save(self, file_path=FILE_PATH): print('Model Saved.') self.model.save_weights(file_path) def load(self, file_path=FILE_PATH): print('Model Loaded.') self.model.load_weights(file_path) def predict(self, image): # 预测样本分类 img = image.resize((1, IMAGE_SIZE, IMAGE_SIZE, 3)) img = image.astype('float32') img /= 255 #归一化 result = self.model.predict(img) print(result) # 概率 result = self.model.predict_classes(img) print(result) # 0/1 return result[0] def evaluate(self, dataset): # 测试样本准确率 score = self.model.evaluate_generator(dataset.valid, steps=2) print("样本准确率%s: %.2f%%" % (self.model.metrics_names[1], score[1] * 100))
Dense(units=1, kernel_initializer='uniform', activation='sigmoid')) # Compiling the ANN #sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) #classifier.compile(loss='mean_squared_error', optimizer=sgd) classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) # Fitting the ANN to the Training set classifier.fit(X_train, y_train, batch_size=5, epochs=10) # Part 3 - Making predictions and evaluating the model # Predicting the Test set results y_pred = classifier.predict(X_test) y_pred = (y_pred > 0.5) # Making the Confusion Matrix from sklearn.metrics import confusion_matrix cm = confusion_matrix(y_test, y_pred) print(cm) backend.clear_session() #predict a single dataset/single observation new_prediction = classifier.predict( sc.transform(np.array([[9867856477, 641.9, 1519689600]]))) new_prediction = (new_prediction > 0.5) #Part 4 Evaluating, Improving and Tuning the ANN
class AmazonKerasClassifier: def __init__(self): self.losses = [] self.classifier = Sequential() self.x_vail = [] self.y_vail = [] self.train_filepath = '' self.train_img_filepath = '' self.valid_filepath = '' self.valid_img_filepath = '' self.test_img_filepath = '' self.test_addition_img_filepath = '' self.test_img_name_list = '' self.y_map = {} def setTrainFilePath(self, value): self.train_filepath = value def getTrainFilePath(self): return self.train_filepath def setValidFilePath(self, value): self.valid_filepath = value def getValidFilePath(self): return self.valid_filepath def setTrainImgFilePath(self, value): self.train_img_filepath = value def getTrainImgFilePath(self): return self.train_img_filepath def setValidImgFilePath(self, value): self.valid_img_filepath = value def getValidImgFilePath(self): return self.valid_img_filepath def setTestImgFilePath(self, value): self.test_img_filepath = value def getTestImgFilePath(self): return self.test_img_filepath def setTestAdditionImgFilePath(self, value): self.test_addition_img_filepath = value def getTestAdditionImgFilePath(self): return self.test_addition_img_filepath def getTestImgNameList(self): return self.test_img_name_list def getYMap(self): return self.y_map def vgg(self, type=16, bn=False, img_size=(224, 224), img_channels=3, output_size=1000): if type == 16 and bn == False: layer_list = vgg.vgg16(num_classes=output_size) elif type == 16 and bn == True: layer_list = vgg.vgg16_bn(num_classes=output_size) elif type == 11 and bn == False: layer_list = vgg.vgg11(num_classes=output_size) elif type == 11 and bn == True: layer_list = vgg.vgg11_bn(num_classes=output_size) elif type == 13 and bn == False: layer_list = vgg.vgg13(num_classes=output_size) elif type == 13 and bn == True: layer_list = vgg.vgg13_bn(num_classes=output_size) elif type == 19 and bn == False: layer_list = vgg.vgg19(num_classes=output_size) elif type == 19 and bn == True: layer_list = vgg.vgg19_bn(num_classes=output_size) else: print("请输入11,13,16,19这四个数字中的一个!") self.classifier.add( BatchNormalization(input_shape=(*img_size, img_channels))) for i, value in enumerate(layer_list): self.classifier.add(eval(value)) def squeezenet(self, type, img_size=(64, 64), img_channels=3, output_size=1000): input_shape = Input(shape=(*img_size, img_channels)) if type == 1: x = squeezenet.squeezenet1_0(input_shape, num_classes=output_size) elif type == 1.1: x = squeezenet.squeezenet1_1(input_shape, num_classes=output_size) else: print("请输入1,1.0这两个数字中的一个!") model = Model(inputs=input_shape, outputs=x) self.classifier = model def resnet(self, type, img_size=(64, 64), img_channels=3, output_size=1000): input_shape = Input(shape=(*img_size, img_channels)) if type == 18: x = resnet.resnet18(input_shape, num_classes=output_size) elif type == 34: x = resnet.resnet34(input_shape, num_classes=output_size) elif type == 50: x = resnet.resnet50(input_shape, num_classes=output_size) elif type == 101: x = resnet.resnet101(input_shape, num_classes=output_size) elif type == 152: x = resnet.resnet152(input_shape, num_classes=output_size) else: print("请输入18,34,50,101,152这五个数字中的一个!") return model = Model(inputs=input_shape, outputs=x) self.classifier = model def inception(self, img_size=(299, 299), img_channels=3, output_size=1000): input_shape = Input(shape=(*img_size, img_channels)) x = inception.inception_v3(input_shape, num_classes=output_size, aux_logits=True, transform_input=False) model = Model(inputs=input_shape, outputs=x) self.classifier = model def densenet(self, type, img_size=(299, 299), img_channels=3, output_size=1000): input_shape = Input(shape=(*img_size, img_channels)) if type == 161: x = densenet.densenet161(input_shape, num_classes=output_size) elif type == 121: x = densenet.densenet121(input_shape, num_classes=output_size) elif type == 169: x = densenet.densenet169(input_shape, num_classes=output_size) elif type == 201: x = densenet.densenet201(input_shape, num_classes=output_size) else: print("请输入161,121,169,201这四个数字中的一个!") return model = Model(inputs=input_shape, outputs=x) self.classifier = model def alexnet(self, img_size=(299, 299), img_channels=3, output_size=1000): input_shape = Input(shape=(*img_size, img_channels)) x = alexnet.alexnet(input_shape, num_classes=output_size) model = Model(inputs=input_shape, outputs=x) self.classifier = model def add_conv_layer(self, img_size=(32, 32), img_channels=3): self.classifier.add( BatchNormalization(input_shape=(*img_size, img_channels))) self.classifier.add( Conv2D(32, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(32, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add( Conv2D(64, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(64, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add( Conv2D(128, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(128, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add( Conv2D(256, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(256, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) def add_flatten_layer(self): self.classifier.add(Flatten()) def add_ann_layer(self, output_size): self.classifier.add(Dense(512, activation='relu')) self.classifier.add(BatchNormalization()) self.classifier.add(Dropout(0.5)) self.classifier.add(Dense(output_size, activation='sigmoid')) def _get_fbeta_score2(self, classifier, X_valid, y_valid): p_valid = classifier.predict(X_valid) result_threshold_list_final, score_result = self.grid_search_best_threshold( y_valid, np.array(p_valid)) return result_threshold_list_final, score_result def _get_fbeta_score(self, classifier, X_valid, y_valid): p_valid = classifier.predict(X_valid) return fbeta_score(y_valid, np.array(p_valid) > 0.2, beta=2, average='samples') def grid_search_best_threshold(self, y_valid, p_valid): threshold_list = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] result_threshold_list_temp = [ 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2 ] result_threshold_list_final = [ 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2 ] for i in range(17): score_result = 0 for j in range(9): result_threshold_list_temp[i] = threshold_list[j] score_temp = fbeta_score(y_valid, p_valid > result_threshold_list_temp, beta=2, average='samples') if score_result < score_temp: score_result = score_temp result_threshold_list_final[i] = threshold_list[j] result_threshold_list_temp[i] = result_threshold_list_final[i] return result_threshold_list_final, score_result def train_model(self, x_train, y_train, learn_rate=0.001, epoch=5, batch_size=128, validation_split_size=0.2, train_callbacks=()): history = LossHistory() X_train, X_valid, y_train, y_valid = train_test_split( x_train, y_train, test_size=validation_split_size) self.x_vail = X_valid self.y_vail = y_valid opt = Adam(lr=learn_rate) self.classifier.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) earlyStopping = EarlyStopping(monitor='val_loss', patience=3, verbose=0, mode='auto') self.classifier.fit( X_train, y_train, batch_size=batch_size, epochs=epoch, verbose=1, validation_data=(X_valid, y_valid), callbacks=[history, *train_callbacks, earlyStopping]) fbeta_score = self._get_fbeta_score(self.classifier, X_valid, y_valid) return [history.train_losses, history.val_losses, fbeta_score] def train_model_generator(self, generator_train, generator_valid, learn_rate=0.001, epoch=5, batchSize=128, steps=32383, validation_steps=8096, train_callbacks=()): history = LossHistory() #valid 8096 32383 opt = Adam(lr=learn_rate) steps = steps / batchSize + 1 - 9 validation_steps = validation_steps / batchSize + 1 if steps % batchSize == 0: steps = steps / batchSize - 9 if validation_steps % batchSize == 0: validation_steps = validation_steps / batchSize print(steps, validation_steps) self.classifier.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) earlyStopping = EarlyStopping(monitor='val_loss', patience=3, verbose=0, mode='auto') self.classifier.fit_generator( generator_train, steps_per_epoch=steps, epochs=epoch, verbose=1, validation_data=generator_valid, validation_steps=validation_steps, callbacks=[history, *train_callbacks, earlyStopping]) fbeta_score = self._get_fbeta_score(self.classifier, X_valid, y_valid) return [history.train_losses, history.val_losses, fbeta_score] def generate_trainOrValid_img_from_file(self, train_set_folder, train_csv_file, img_resize=(32, 32), batchSize=128, process_count=cpu_count()): labels_df = pd.read_csv(train_csv_file) labels = sorted( set( chain.from_iterable( [tags.split(" ") for tags in labels_df['tags'].values]))) labels_map = {l: i for i, l in enumerate(labels)} files_path = [] tags_list = [] for file_name, tags in labels_df.values: files_path.append('{}/{}.jpg'.format(train_set_folder, file_name)) tags_list.append(tags) X = [] Y = [] iter_num = 1 self.y_map = {v: k for k, v in labels_map.items()} with ThreadPoolExecutor(process_count) as pool: for img_array, targets in tqdm(pool.map( self._train_transform_to_matrices, [(file_path, tag, labels_map, img_resize) for file_path, tag in zip(files_path, tags_list)]), total=len(files_path)): if iter_num % batchSize == 0: X = [] Y = [] iter_num = 0 X.append(img_array) Y.append(targets) iter_num += 1 if iter_num == batchSize: print(iter_num) yield (np.array(X), np.array(Y)) def _train_transform_to_matrices(self, *args): file_path, tags, labels_map, img_resize = list(args[0]) img = Image.open(file_path) img.thumbnail(img_resize) img_array = np.asarray(img.convert("RGB"), dtype=np.float32) / 255 targets = np.zeros(len(labels_map)) for t in tags.split(' '): targets[labels_map[t]] = 1 return img_array, targets def generate_test_img_from_file(self, test_set_folder, img_resize=(32, 32), batchSize=128, process_count=cpu_count()): x_test = [] x_test_filename = [] files_name = os.listdir(test_set_folder) X = [] Y = [] iter_num = 1 with ThreadPoolExecutor(process_count) as pool: for img_array, file_name in tqdm(pool.map( _test_transform_to_matrices, [(test_set_folder, file_name, img_resize) for file_name in files_name]), total=len(files_name)): x_test.append(img_array) x_test_filename.append(file_name) self.test_img_name_list = x_test_filename if iter_num % batchSize == 0: X = [] Y = [] iter_num = 0 X.append(img_array) Y.append(targets) iter_num += 1 if iter_num == batchSize: print(iter_num) yield (np.array(X), np.array(Y)) def _test_transform_to_matrices(self, *args): test_set_folder, file_name, img_resize = list(args[0]) img = Image.open('{}/{}'.format(test_set_folder, file_name)) img.thumbnail(img_resize) # Convert to RGB and normalize img_array = np.array(img.convert("RGB"), dtype=np.float32) / 255 return img_array, file_name def save_weights(self, weight_file_path): self.classifier.save_weights(weight_file_path) def load_weights(self, weight_file_path): self.classifier.load_weights(weight_file_path) def setBestThreshold(self): result_threshold_list_final, score_result = self._get_fbeta_score2( self.classifier, self.x_vail, self.y_vail) print('最好得分:{}'.format(score_result)) print('最好的阈值:{}'.format(result_threshold_list_final)) return result_threshold_list_final def predict(self, x_test): predictions = self.classifier.predict(x_test) return predictions def predict_generator(self, generator): predictions = self.classifier.predcit_generator(generator) return predictions def map_predictions(self, predictions, labels_map, thresholds): predictions_labels = [] for prediction in predictions: labels = [ labels_map[i] for i, value in enumerate(prediction) if value > thresholds[i] ] predictions_labels.append(labels) return predictions_labels def close(self): backend.clear_session()
class AmazonKerasClassifier: def __init__(self): self.losses = [] self.classifier = Sequential() def add_conv_layer(self, img_size=(32, 32), img_channels=3): self.classifier.add(BatchNormalization(input_shape=(*img_size, img_channels))) self.classifier.add(Conv2D(32, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=(2, 2))) self.classifier.add(Dropout(0.25)) self.classifier.add(Conv2D(64, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=(2, 2))) self.classifier.add(Dropout(0.25)) self.classifier.add(Conv2D(16, (2, 2), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=(2, 2))) self.classifier.add(Dropout(0.25)) def add_flatten_layer(self): self.classifier.add(Flatten()) def add_ann_layer(self, output_size): self.classifier.add(Dense(256, activation='relu')) self.classifier.add(Dropout(0.5)) self.classifier.add(Dense(512, activation='relu')) self.classifier.add(Dropout(0.5)) self.classifier.add(Dense(output_size, activation='sigmoid')) def _get_fbeta_score(self, classifier, X_valid, y_valid): p_valid = classifier.predict(X_valid) return fbeta_score(y_valid, np.array(p_valid) > 0.2, beta=2, average='samples') def train_model(self, x_train, y_train, epoch=5, batch_size=128, validation_split_size=0.2, train_callbacks=()): history = LossHistory() X_train, X_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=validation_split_size) self.classifier.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) self.classifier.fit(X_train, y_train, batch_size=batch_size, epochs=epoch, verbose=1, validation_data=(X_valid, y_valid), callbacks=[history, *train_callbacks]) fbeta_score = self._get_fbeta_score(self.classifier, X_valid, y_valid) return [history.train_losses, history.val_losses, fbeta_score] def predict(self, x_test): predictions = self.classifier.predict(x_test) return predictions def map_predictions(self, predictions, labels_map, thresholds): """ Return the predictions mapped to their labels :param predictions: the predictions from the predict() method :param labels_map: the map :param thresholds: The threshold of each class to be considered as existing or not existing :return: the predictions list mapped to their labels """ predictions_labels = [] for prediction in predictions: labels = [labels_map[i] for i, value in enumerate(prediction) if value > thresholds[i]] predictions_labels.append(labels) return predictions_labels def close(self): backend.clear_session()
# Compiling the ANN classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) # Fitting the ANN to the Training set classifier.fit(X_train, y_train, batch_size=10, epochs=100, validation_split=0.1) # Part 3 - Making predictions and evaluating the model # Predicting the Test set results y_pred = classifier.predict(X_test) y_pred = (y_pred > 0.5) # Making the Confusion Matrix from sklearn.metrics import confusion_matrix cm = confusion_matrix(y_test, y_pred) print(cm) # Predicting a new customer with the following paramteres as per exercise # """Geography: France Credit Score: 600 Gender: Male Age: 40 years old Tenure: 3 years
class AmazonKerasClassifier: def __init__(self): self.losses = [] self.classifier = Sequential() def add_conv_layer(self, img_size=(32, 32), img_channels=3): self.classifier.add(BatchNormalization(input_shape=(img_size, img_channels))) self.classifier.add(Conv2D(32, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(32, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add(Conv2D(64, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(64, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add(Conv2D(128, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(128, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) self.classifier.add(Conv2D(256, (3, 3), padding='same', activation='relu')) self.classifier.add(Conv2D(256, (3, 3), activation='relu')) self.classifier.add(MaxPooling2D(pool_size=2)) self.classifier.add(Dropout(0.25)) def add_flatten_layer(self): self.classifier.add(Flatten()) def add_ann_layer(self, output_size): self.classifier.add(Dense(512, activation='relu')) self.classifier.add(BatchNormalization()) self.classifier.add(Dropout(0.5)) self.classifier.add(Dense(output_size, activation='sigmoid')) def _get_fbeta_score(self, classifier, X_valid, y_valid): p_valid = classifier.predict(X_valid) return fbeta_score(y_valid, np.array(p_valid) > 0.2, beta=2, average='samples') def train_model(self, x_train, y_train, learn_rate=0.001, epoch=5, batch_size=128, validation_split_size=0.2, train_callbacks=()): history = LossHistory() X_train, X_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=validation_split_size) opt = Adam(lr=learn_rate) self.classifier.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) # early stopping will auto-stop training process if model stops learning after 3 epochs earlyStopping = EarlyStopping(monitor='val_loss', patience=3, verbose=0, mode='auto') self.classifier.fit(X_train, y_train, batch_size=batch_size, epochs=epoch, verbose=1, validation_data=(X_valid, y_valid), callbacks=[history, *train_callbacks, earlyStopping]) fbeta_score = self._get_fbeta_score(self.classifier, X_valid, y_valid) return [history.train_losses, history.val_losses, fbeta_score] def save_weights(self, weight_file_path): self.classifier.save_weights(weight_file_path) def load_weights(self, weight_file_path): self.classifier.load_weights(weight_file_path) def predict(self, x_test): predictions = self.classifier.predict(x_test) return predictions def map_predictions(self, predictions, labels_map, thresholds): """ Return the predictions mapped to their labels :param predictions: the predictions from the predict() method :param labels_map: the map :param thresholds: The threshold of each class to be considered as existing or not existing :return: the predictions list mapped to their labels """ predictions_labels = [] for prediction in predictions: labels = [labels_map[i] for i, value in enumerate(prediction) if value > thresholds[i]] predictions_labels.append(labels) return predictions_labels def close(self): backend.clear_session()
loss='binary_crossentropy', metrics=['accuracy']) # Fitting the ANN to the Training set #batch size= no of observations after which you wish to update the weights # epochs= no. of vibrations classifier.fit(X_train, y_train, batch_size=10, epochs=100, validation_split=0.1) # Part 3 - Making predictions and evaluating the model # Predicting the Test set results y_pred = classifier.predict(X_test) #this makes y_pred a boolean type one, which return true if y_pred is greteater than 0.5 and false if y_pred is less than 0.5 y_pred = (y_pred > 0.5) # Predicting a single new observation """Predict if the customer with the following informations will leave the bank: Geography: France Credit Score: 600 Gender: Male Age: 40 Tenure: 3 Balance: 60000 Number of Products: 2 Has Credit Card: Yes Is Active Member: Yes
regressor.fit(X_train, y_train, epochs=100, batch_size=32) # Part 3 - Making the predictions and visualising the results # Getting the real stock price for February 1st 2012 - January 31st 2017 path = os.path.join(script_dir, '../dataset/Google_Stock_Price_Test.csv') dataset_test = pd.read_csv(path) test_set = dataset_test.iloc[:, 1:2].values real_stock_price = np.concatenate((training_set[0:1258], test_set), axis=0) # Getting the predicted stock price of 2017 scaled_real_stock_price = sc.fit_transform(real_stock_price) inputs = [] for i in range(1258, 1278): inputs.append(scaled_real_stock_price[i - 60:i, 0]) inputs = np.array(inputs) inputs = np.reshape(inputs, (inputs.shape[0], inputs.shape[1], 1)) predicted_stock_price = regressor.predict(inputs) predicted_stock_price = sc.inverse_transform(predicted_stock_price) # Visualising the results plt.plot(real_stock_price[1258:], color='red', label='Real Google Stock Price') plt.plot(predicted_stock_price, color='blue', label='Predicted Google Stock Price') plt.title('Google Stock Price Prediction') plt.xlabel('Time') plt.ylabel('Google Stock Price') plt.legend() plt.show()
def model_inference_critisism(model_name, Bayesian, seq_array1, seq_array2, label_array1, label_array2, sequence_length, N, D, H=100, H1=100, H2=50): less_50 = label_array2 <= 50 if not Bayesian: model = Sequential() if model_name == 'Fully Connected Layer': model.add(Dense(H, input_shape=[D], activation='tanh')) elif model_name == 'Simple RNN': model.add(SimpleRNN(input_shape=(sequence_length, D), units=H)) elif model_name == 'LSTM': model.add(LSTM(input_shape=(sequence_length, D), units=H)) elif model_name == 'GRU': model.add(GRU(input_shape=(sequence_length, D), units=H)) model.add(Dense(units=1)) elif model_name == 'Two Layer Simple RNN': model.add( SimpleRNN(input_shape=(sequence_length, D), units=H1, return_sequences=True)) model.add(SimpleRNN(units=H2)) elif model_name == 'Two Layer LSTM': model.add( LSTM(input_shape=(sequence_length, D), units=H1, return_sequences=True)) model.add(LSTM(units=H2)) elif model_name == 'Two Layer GRU': model.add( GRU(input_shape=(sequence_length, D), units=H1, return_sequences=True)) model.add(GRU(units=H2)) else: raise Exception('Please specify a valid model!') model.add(Dense(units=1)) if model_name == 'Fully Connected Layer': # inference nadam = Nadam(lr=0.05) model.compile(loss='mean_squared_error', optimizer=nadam, metrics=['mean_squared_error']) model.fit(seq_array1[:, sequence_length - 1, :], label_array1, epochs=300, batch_size=200, validation_data=(seq_array2[:, sequence_length - 1, :], label_array2), verbose=0) y_pred = np.squeeze( model.predict(seq_array2[:, sequence_length - 1, :])) else: model.compile(loss='mean_squared_error', optimizer='nadam', metrics=['mean_squared_error']) model.fit(seq_array1, label_array1, epochs=300, batch_size=200, validation_data=(seq_array2, label_array2), verbose=0) y_pred = np.squeeze(model.predict(seq_array2)) # Critisim # Histogram sns.distplot(y_pred) plt.title('Histogram of RUL, Frequentist {}'.format(model_name)) plt.xlabel('RUL') plt.show() # RMSE print('Validation RMSE: {}'.format( np.sqrt(mean_squared_error(label_array2, y_pred)))) print('Validation RMSE for RUL under 50: {}'.format( np.sqrt(mean_squared_error(label_array2[less_50], y_pred[less_50])))) # Prediction time series pd.DataFrame([label_array2, y_pred]).transpose().rename(columns={ 0: 'True', 1: 'Pred' })[-1500:].plot() plt.title('Prediction of RUL, Frequentist {}'.format(model_name)) plt.xlabel('RUL') plt.show() elif Bayesian: if model_name == 'Fully Connected Layer': W_0 = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) W_1 = Normal(loc=tf.zeros([H, 1]), scale=tf.ones([H, 1])) b_0 = Normal(loc=tf.zeros(H), scale=tf.ones(H)) b_1 = Normal(loc=tf.zeros(1), scale=tf.ones(1)) x = tf.placeholder(tf.float32, [N, D]) y = Normal(loc=neural_network_with_2_layers(x, W_0, W_1, b_0, b_1), scale=tf.ones(N) * 0.1) # constant noise # BACKWARD MODEL A q_W_0 = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_W_1 = Normal(loc=tf.Variable(tf.random_normal([H, 1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, 1])))) q_b_0 = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_b_1 = Normal(loc=tf.Variable(tf.random_normal([1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([1])))) # INFERENCE A # this will take a couple of minutes inference = ed.KLqp(latent_vars={ W_0: q_W_0, b_0: q_b_0, W_1: q_W_1, b_1: q_b_1 }, data={ x: seq_array1[:, sequence_length - 1, :], y: label_array1 }) inference.run(n_samples=5, n_iter=25000) xp = tf.placeholder(tf.float32, seq_array2[:, sequence_length - 1, :].shape) y_preds = [ sess.run( neural_network_with_2_layers(xp, q_W_0, q_W_1, q_b_0, q_b_1), {xp: seq_array2[:, sequence_length - 1, :]}) for _ in range(50) ] elif model_name == 'Simple RNN': Wh = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H])) Wx = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) Wy = Normal(loc=tf.zeros([H, 1]), scale=tf.ones([H, 1])) bh = Normal(loc=tf.zeros(H), scale=tf.ones(H)) by = Normal(loc=tf.zeros(1), scale=tf.ones(1)) X = tf.placeholder(tf.float32, [N, sequence_length, D]) # X = tf.placeholder(tf.float32,[sequence_length,N,D]) y = Normal(loc=rnn_layer(X, Wh, Wx, bh, Wy, by, H), scale=1.) # BACKWARD MODEL A q_Wh = Normal(loc=tf.Variable(tf.random_normal([H, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H])))) q_Wx = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_Wy = Normal(loc=tf.Variable(tf.random_normal([H, 1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, 1])))) q_bh = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_by = Normal(loc=tf.Variable(tf.random_normal([1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([1])))) # INFERENCE A # this will take a couple of minutes inference = ed.KLqp(latent_vars={ Wh: q_Wh, bh: q_bh, Wx: q_Wx, Wy: q_Wy, by: q_by }, data={ X: seq_array1, y: label_array1 }) inference.run(n_samples=5, n_iter=2500) Xp = tf.placeholder(tf.float32, seq_array2.shape) y_preds = [ sess.run(rnn_layer(Xp, q_Wh, q_Wx, q_bh, q_Wy, q_by, H), {Xp: seq_array2}) for _ in range(50) ] elif model_name == 'LSTM': Wf = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H])) Uf = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) Wi = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H])) Ui = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) Wo = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H])) Uo = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) Wc = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H])) Uc = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) Wy = Normal(loc=tf.zeros([H, 1]), scale=tf.ones([H, 1])) bf = Normal(loc=tf.zeros(H), scale=tf.ones(H)) bi = Normal(loc=tf.zeros(H), scale=tf.ones(H)) bo = Normal(loc=tf.zeros(H), scale=tf.ones(H)) bc = Normal(loc=tf.zeros(H), scale=tf.ones(H)) by = Normal(loc=tf.zeros(1), scale=tf.ones(1)) X = tf.placeholder(tf.float32, [N, sequence_length, D]) y = Normal(loc=LSTM_layer(X, Wf, Uf, Wi, Ui, Wo, Uo, Wc, Uc, bf, bi, bo, bc, Wy, by, H), scale=1.) # BACKWARD MODEL A q_Wf = Normal(loc=tf.Variable(tf.random_normal([H, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H])))) q_Uf = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_Wi = Normal(loc=tf.Variable(tf.random_normal([H, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H])))) q_Ui = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_Wo = Normal(loc=tf.Variable(tf.random_normal([H, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H])))) q_Uo = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_Wc = Normal(loc=tf.Variable(tf.random_normal([H, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H])))) q_Uc = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_Wy = Normal(loc=tf.Variable(tf.random_normal([H, 1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, 1])))) q_bf = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_bi = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_bo = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_bc = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_by = Normal(loc=tf.Variable(tf.random_normal([1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([1])))) # INFERENCE A # this will take a couple of minutes inference = ed.KLqp(latent_vars={ Wf: q_Wf, Uf: q_Uf, Wi: q_Wi, Ui: q_Ui, Wo: q_Wo, Uo: q_Uo, Wc: q_Wc, Uc: q_Uc, bf: q_bf, bi: q_bi, bo: q_bo, bc: q_bc, Wy: q_Wy, by: q_by }, data={ X: seq_array1, y: label_array1 }) inference.run(n_samples=5, n_iter=2500) Xp = tf.placeholder(tf.float32, seq_array2.shape) y_preds = [ sess.run( LSTM_layer(Xp, q_Wf, q_Uf, q_Wi, q_Ui, q_Wo, q_Uo, q_Wc, q_Uc, q_bf, q_bi, q_bo, q_bc, q_Wy, q_by, H), {Xp: seq_array2}) for _ in range(50) ] elif model_name == 'GRU': Wz = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H])) Uz = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) Wr = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H])) Ur = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) Wh = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H])) Uh = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H])) Wy = Normal(loc=tf.zeros([H, 1]), scale=tf.ones([H, 1])) bz = Normal(loc=tf.zeros(H), scale=tf.ones(H)) br = Normal(loc=tf.zeros(H), scale=tf.ones(H)) bh = Normal(loc=tf.zeros(H), scale=tf.ones(H)) by = Normal(loc=tf.zeros(1), scale=tf.ones(1)) X = tf.placeholder(tf.float32, [N, sequence_length, D]) y = Normal(loc=GRU_layer(X, Wz, Uz, Wr, Ur, Wh, Uh, bz, br, bh, Wy, by, H), scale=1.) # BACKWARD MODEL A q_Wz = Normal(loc=tf.Variable(tf.random_normal([H, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H])))) q_Uz = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_Wr = Normal(loc=tf.Variable(tf.random_normal([H, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H])))) q_Ur = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_Wh = Normal(loc=tf.Variable(tf.random_normal([H, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H])))) q_Uh = Normal(loc=tf.Variable(tf.random_normal([D, H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H])))) q_Wy = Normal(loc=tf.Variable(tf.random_normal([H, 1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, 1])))) q_bz = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_br = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_bh = Normal(loc=tf.Variable(tf.random_normal([H])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H])))) q_by = Normal(loc=tf.Variable(tf.random_normal([1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([1])))) # INFERENCE A # this will take a couple of minutes inference = ed.KLqp(latent_vars={ Wz: q_Wz, Uz: q_Uz, Wr: q_Wr, Ur: q_Ur, Wh: q_Wh, Uh: q_Uh, bz: q_bz, bz: q_bz, bh: q_bh, Wy: q_Wy, by: q_by }, data={ X: seq_array1, y: label_array1 }) inference.run(n_samples=5, n_iter=2500) Xp = tf.placeholder(tf.float32, seq_array2.shape) y_preds = [ sess.run( GRU_layer(Xp, q_Wz, q_Uz, q_Wr, q_Ur, q_Wh, q_Uh, q_bz, q_br, q_bh, q_Wy, q_by, H), {Xp: seq_array2}) for _ in range(50) ] elif model_name == 'Two Layer Simple RNN': Wh1 = Normal(loc=tf.zeros([H1, H1]), scale=tf.ones([H1, H1])) Wx1 = Normal(loc=tf.zeros([D, H1]), scale=tf.ones([D, H1])) Wh2 = Normal(loc=tf.zeros([H2, H2]), scale=tf.ones([H2, H2])) Wx2 = Normal(loc=tf.zeros([H1, H2]), scale=tf.ones([H1, H2])) Wy = Normal(loc=tf.zeros([H2, 1]), scale=tf.ones([H2, 1])) bh1 = Normal(loc=tf.zeros(H1), scale=tf.ones(H1)) bh2 = Normal(loc=tf.zeros(H2), scale=tf.ones(H2)) by = Normal(loc=tf.zeros(1), scale=tf.ones(1)) X = tf.placeholder(tf.float32, [N, sequence_length, D]) y = Normal(loc=two_rnn_layer(X, Wh1, Wx1, bh1, Wh2, Wx2, bh2, Wy, by, H1, H2), scale=1.) # BACKWARD MODEL A q_Wh1 = Normal(loc=tf.Variable(tf.random_normal([H1, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H1])))) q_Wx1 = Normal(loc=tf.Variable(tf.random_normal([D, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H1])))) q_Wh2 = Normal(loc=tf.Variable(tf.random_normal([H2, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, H2])))) q_Wx2 = Normal(loc=tf.Variable(tf.random_normal([H1, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H2])))) q_Wy = Normal(loc=tf.Variable(tf.random_normal([H2, 1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, 1])))) q_bh1 = Normal(loc=tf.Variable(tf.random_normal([H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1])))) q_bh2 = Normal(loc=tf.Variable(tf.random_normal([H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2])))) q_by = Normal(loc=tf.Variable(tf.random_normal([1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([1])))) # INFERENCE A # this will take a couple of minutes inference = ed.KLqp(latent_vars={ Wh1: q_Wh1, bh1: q_bh1, Wh2: q_Wh2, bh2: q_bh2, Wx1: q_Wx1, Wx2: q_Wx2, Wy: q_Wy, by: q_by }, data={ X: seq_array1, y: label_array1 }) inference.run(n_samples=5, n_iter=2500) Xp = tf.placeholder(tf.float32, seq_array2.shape) y_preds = [ sess.run( two_rnn_layer(Xp, q_Wh1, q_Wx1, q_bh1, q_Wh2, q_Wx2, q_bh2, q_Wy, q_by, H1, H2), {Xp: seq_array2}) for _ in range(50) ] elif model_name == 'Two Layer LSTM': Wf1 = Normal(loc=tf.zeros([H1, H1]), scale=tf.ones([H1, H1])) Uf1 = Normal(loc=tf.zeros([D, H1]), scale=tf.ones([D, H1])) Wi1 = Normal(loc=tf.zeros([H1, H1]), scale=tf.ones([H1, H1])) Ui1 = Normal(loc=tf.zeros([D, H1]), scale=tf.ones([D, H1])) Wo1 = Normal(loc=tf.zeros([H1, H1]), scale=tf.ones([H1, H1])) Uo1 = Normal(loc=tf.zeros([D, H1]), scale=tf.ones([D, H1])) Wc1 = Normal(loc=tf.zeros([H1, H1]), scale=tf.ones([H1, H1])) Uc1 = Normal(loc=tf.zeros([D, H1]), scale=tf.ones([D, H1])) Wf2 = Normal(loc=tf.zeros([H2, H2]), scale=tf.ones([H2, H2])) Uf2 = Normal(loc=tf.zeros([H1, H2]), scale=tf.ones([H1, H2])) Wi2 = Normal(loc=tf.zeros([H2, H2]), scale=tf.ones([H2, H2])) Ui2 = Normal(loc=tf.zeros([H1, H2]), scale=tf.ones([H1, H2])) Wo2 = Normal(loc=tf.zeros([H2, H2]), scale=tf.ones([H2, H2])) Uo2 = Normal(loc=tf.zeros([H1, H2]), scale=tf.ones([H1, H2])) Wc2 = Normal(loc=tf.zeros([H2, H2]), scale=tf.ones([H2, H2])) Uc2 = Normal(loc=tf.zeros([H1, H2]), scale=tf.ones([H1, H2])) Wy = Normal(loc=tf.zeros([H2, 1]), scale=tf.ones([H2, 1])) bf1 = Normal(loc=tf.zeros(H1), scale=tf.ones(H1)) bi1 = Normal(loc=tf.zeros(H1), scale=tf.ones(H1)) bo1 = Normal(loc=tf.zeros(H1), scale=tf.ones(H1)) bc1 = Normal(loc=tf.zeros(H1), scale=tf.ones(H1)) bf2 = Normal(loc=tf.zeros(H2), scale=tf.ones(H2)) bi2 = Normal(loc=tf.zeros(H2), scale=tf.ones(H2)) bo2 = Normal(loc=tf.zeros(H2), scale=tf.ones(H2)) bc2 = Normal(loc=tf.zeros(H2), scale=tf.ones(H2)) by = Normal(loc=tf.zeros(1), scale=tf.ones(1)) X = tf.placeholder(tf.float32, [N, sequence_length, D]) y = Normal(loc=two_LSTM_layer(X, Wf1, Uf1, Wi1, Ui1, Wo1, Uo1, Wc1, Uc1, bf1, bi1, bo1, bc1, Wf2, Uf2, Wi2, Ui2, Wo2, Uo2, Wc2, Uc2, bf2, bi2, bo2, bc2, Wy, by, H1, H2), scale=1.) # BACKWARD MODEL A q_Wf1 = Normal(loc=tf.Variable(tf.random_normal([H1, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H1])))) q_Uf1 = Normal(loc=tf.Variable(tf.random_normal([D, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H1])))) q_Wi1 = Normal(loc=tf.Variable(tf.random_normal([H1, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H1])))) q_Ui1 = Normal(loc=tf.Variable(tf.random_normal([D, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H1])))) q_Wo1 = Normal(loc=tf.Variable(tf.random_normal([H1, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H1])))) q_Uo1 = Normal(loc=tf.Variable(tf.random_normal([D, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H1])))) q_Wc1 = Normal(loc=tf.Variable(tf.random_normal([H1, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H1])))) q_Uc1 = Normal(loc=tf.Variable(tf.random_normal([D, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H1])))) q_Wf2 = Normal(loc=tf.Variable(tf.random_normal([H2, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, H2])))) q_Uf2 = Normal(loc=tf.Variable(tf.random_normal([H1, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H2])))) q_Wi2 = Normal(loc=tf.Variable(tf.random_normal([H2, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, H2])))) q_Ui2 = Normal(loc=tf.Variable(tf.random_normal([H1, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H2])))) q_Wo2 = Normal(loc=tf.Variable(tf.random_normal([H2, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, H2])))) q_Uo2 = Normal(loc=tf.Variable(tf.random_normal([H1, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H2])))) q_Wc2 = Normal(loc=tf.Variable(tf.random_normal([H2, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, H2])))) q_Uc2 = Normal(loc=tf.Variable(tf.random_normal([H1, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H2])))) q_Wy = Normal(loc=tf.Variable(tf.random_normal([H2, 1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, 1])))) q_bf1 = Normal(loc=tf.Variable(tf.random_normal([H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1])))) q_bi1 = Normal(loc=tf.Variable(tf.random_normal([H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1])))) q_bo1 = Normal(loc=tf.Variable(tf.random_normal([H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1])))) q_bc1 = Normal(loc=tf.Variable(tf.random_normal([H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1])))) q_bf2 = Normal(loc=tf.Variable(tf.random_normal([H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2])))) q_bi2 = Normal(loc=tf.Variable(tf.random_normal([H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2])))) q_bo2 = Normal(loc=tf.Variable(tf.random_normal([H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2])))) q_bc2 = Normal(loc=tf.Variable(tf.random_normal([H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2])))) q_by = Normal(loc=tf.Variable(tf.random_normal([1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([1])))) # INFERENCE A # this will take a couple of minutes inference = ed.KLqp(latent_vars={ Wf1: q_Wf1, Uf1: q_Uf1, Wi1: q_Wi1, Ui1: q_Ui1, Wo1: q_Wo1, Uo1: q_Uo1, Wc1: q_Wc1, Uc1: q_Uc1, Wf2: q_Wf2, Uf2: q_Uf2, Wi2: q_Wi2, Ui2: q_Ui2, Wo2: q_Wo2, Uo2: q_Uo2, Wc2: q_Wc2, Uc2: q_Uc2, bf1: q_bf1, bi1: q_bi1, bo1: q_bo1, bc1: q_bc1, bf2: q_bf2, bi2: q_bi2, bo2: q_bo2, bc2: q_bc2, Wy: q_Wy, by: q_by }, data={ X: seq_array1, y: label_array1 }) inference.run(n_samples=5, n_iter=2500) Xp = tf.placeholder(tf.float32, seq_array2.shape) y_preds = [ sess.run( two_LSTM_layer(Xp, q_Wf1, q_Uf1, q_Wi1, q_Ui1, q_Wo1, q_Uo1, q_Wc1, q_Uc1, q_bf1, q_bi1, q_bo1, q_bc1, q_Wf2, q_Uf2, q_Wi2, q_Ui2, q_Wo2, q_Uo2, q_Wc2, q_Uc2, q_bf2, q_bi2, q_bo2, q_bc2, q_Wy, q_by, H1, H2), {Xp: seq_array2}) for _ in range(50) ] elif model_name == 'Two Layer GRU': Wz1 = Normal(loc=tf.zeros([H1, H1]), scale=tf.ones([H1, H1])) Uz1 = Normal(loc=tf.zeros([D, H1]), scale=tf.ones([D, H1])) Wr1 = Normal(loc=tf.zeros([H1, H1]), scale=tf.ones([H1, H1])) Ur1 = Normal(loc=tf.zeros([D, H1]), scale=tf.ones([D, H1])) Wh1 = Normal(loc=tf.zeros([H1, H1]), scale=tf.ones([H1, H1])) Uh1 = Normal(loc=tf.zeros([D, H1]), scale=tf.ones([D, H1])) Wz2 = Normal(loc=tf.zeros([H2, H2]), scale=tf.ones([H2, H2])) Uz2 = Normal(loc=tf.zeros([H1, H2]), scale=tf.ones([H1, H2])) Wr2 = Normal(loc=tf.zeros([H2, H2]), scale=tf.ones([H2, H2])) Ur2 = Normal(loc=tf.zeros([H1, H2]), scale=tf.ones([H1, H2])) Wh2 = Normal(loc=tf.zeros([H2, H2]), scale=tf.ones([H2, H2])) Uh2 = Normal(loc=tf.zeros([H1, H2]), scale=tf.ones([H1, H2])) Wy = Normal(loc=tf.zeros([H2, 1]), scale=tf.ones([H2, 1])) bz1 = Normal(loc=tf.zeros(H1), scale=tf.ones(H1)) br1 = Normal(loc=tf.zeros(H1), scale=tf.ones(H1)) bh1 = Normal(loc=tf.zeros(H1), scale=tf.ones(H1)) bz2 = Normal(loc=tf.zeros(H2), scale=tf.ones(H2)) br2 = Normal(loc=tf.zeros(H2), scale=tf.ones(H2)) bh2 = Normal(loc=tf.zeros(H2), scale=tf.ones(H2)) by = Normal(loc=tf.zeros(1), scale=tf.ones(1)) X = tf.placeholder(tf.float32, [N, sequence_length, D]) y = Normal(loc=two_GRU_layer(X, Wz1, Uz1, Wr1, Ur1, Wh1, Uh1, bz1, br1, bh1, Wz2, Uz2, Wr2, Ur2, Wh2, Uh2, bz2, br2, bh2, Wy, by, H1, H2), scale=1.) # BACKWARD MODEL A q_Wz1 = Normal(loc=tf.Variable(tf.random_normal([H1, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H1])))) q_Uz1 = Normal(loc=tf.Variable(tf.random_normal([D, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H1])))) q_Wr1 = Normal(loc=tf.Variable(tf.random_normal([H1, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H1])))) q_Ur1 = Normal(loc=tf.Variable(tf.random_normal([D, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H1])))) q_Wh1 = Normal(loc=tf.Variable(tf.random_normal([H1, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H1])))) q_Uh1 = Normal(loc=tf.Variable(tf.random_normal([D, H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H1])))) q_Wz2 = Normal(loc=tf.Variable(tf.random_normal([H2, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, H2])))) q_Uz2 = Normal(loc=tf.Variable(tf.random_normal([H1, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H2])))) q_Wr2 = Normal(loc=tf.Variable(tf.random_normal([H2, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, H2])))) q_Ur2 = Normal(loc=tf.Variable(tf.random_normal([H1, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H2])))) q_Wh2 = Normal(loc=tf.Variable(tf.random_normal([H2, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, H2])))) q_Uh2 = Normal(loc=tf.Variable(tf.random_normal([H1, H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1, H2])))) q_Wy = Normal(loc=tf.Variable(tf.random_normal([H2, 1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2, 1])))) q_bz1 = Normal(loc=tf.Variable(tf.random_normal([H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1])))) q_br1 = Normal(loc=tf.Variable(tf.random_normal([H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1])))) q_bh1 = Normal(loc=tf.Variable(tf.random_normal([H1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H1])))) q_bz2 = Normal(loc=tf.Variable(tf.random_normal([H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2])))) q_br2 = Normal(loc=tf.Variable(tf.random_normal([H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2])))) q_bh2 = Normal(loc=tf.Variable(tf.random_normal([H2])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H2])))) q_by = Normal(loc=tf.Variable(tf.random_normal([1])), scale=tf.nn.softplus( tf.Variable(tf.random_normal([1])))) # INFERENCE A # this will take a couple of minutes inference = ed.KLqp(latent_vars={ Wz1: q_Wz1, Uz1: q_Uz1, Wr1: q_Wr1, Ur1: q_Ur1, Wh1: q_Wh1, Uh1: q_Uh1, Wz2: q_Wz2, Uz2: q_Uz2, Wr2: q_Wr2, Ur2: q_Ur2, Wh2: q_Wh2, Uh2: q_Uh2, bz1: q_bz1, bz1: q_bz1, bh1: q_bh1, bz2: q_bz2, bz2: q_bz2, bh2: q_bh2, Wy: q_Wy, by: q_by }, data={ X: seq_array1, y: label_array1 }) inference.run(n_samples=5, n_iter=2500) Xp = tf.placeholder(tf.float32, seq_array2.shape) y_preds = [ sess.run( two_GRU_layer(Xp, q_Wz1, q_Uz1, q_Wr1, q_Ur1, q_Wh1, q_Uh1, q_bz1, q_br1, q_bh1, q_Wz2, q_Uz2, q_Wr2, q_Ur2, q_Wh2, q_Uh2, q_bz2, q_br2, q_bh2, q_Wy, q_by, H1, H2), {Xp: seq_array2}) for _ in range(50) ] else: raise Exception('Please specify a valid model!') # Critisism # Histogram sns.distplot(y_preds[0]) plt.title('Histogram of RUL, Bayesian {}'.format(model_name)) plt.xlabel('RUL') plt.show() # RMSE print('Average Validation RMSE: {}'.format( np.mean([ np.sqrt(mean_squared_error(label_array2, y_pred)) for y_pred in y_preds ]))) print('Average Validation RMSE for RUL under 50: {}'.format( np.mean([ np.sqrt( mean_squared_error(label_array2[less_50], y_pred[less_50])) for y_pred in y_preds ]))) # Prediction time series pd.DataFrame([label_array2, y_preds[0]]).transpose().rename(columns={ 0: 'True', 1: 'Pred' })[-1500:].plot() plt.title('Prediction of RUL, Bayesian {}'.format(model_name)) plt.xlabel('RUL') plt.show() # Posterior prediction distribution 1500 [ plt.plot(y_pred[-1500:], color='black', alpha=0.1) for y_pred in y_preds ] plt.plot(label_array2[-1500:]) plt.title('Distribution of Prediction of RUL, Bayesian {}'.format( model_name)) plt.xlabel('RUL(last 1500 days)') plt.show() # Posterior prediction distribution 150 [ plt.plot(y_pred[-150:], color='black', alpha=0.1) for y_pred in y_preds ] plt.plot(label_array2[-150:]) plt.title('Distribution of Prediction of RUL, Bayesian {}'.format( model_name)) plt.xlabel('RUL(last 150 days)') plt.show() else: raise Exception( 'Please specify a boolean for use Bayesian inference or not!')