This is the major change we have to make while defining the model architecture for solving a multi-label image classification problem. ''' new_model.add(Dense(<number of classes>, activation='sigmoid')) # Next our new model is required to be compiled to have the changes made on it to take effect new_model.compile(RMSprop(lr=0.0001, decay=1e-6),loss="binary_crossentropy",metrics=["accuracy"]) # Train the model history = new_model.fit_generator(generator=generator_train, steps_per_epoch=STEP_SIZE_TRAIN, validation_data=generator_valid, validation_steps=STEP_SIZE_VALID, epochs=10 ) # Validation of model performance # Pre-process the test image before using for model performance validation img = image.load_img('<path of test image>',target_size=(224,224,3)) img = image.img_to_array(img) img = img/255 # The model will predict the probability for each class present in test image and we will take the top 3 predictions classes = np.array(columns) proba = new_model.predict(img.reshape(1,224,224,3)) top_3 = np.argsort(proba[0])[:-4:-1] for i in range(3): print("{}".format(classes[top_3[i]])+" ({:.3})".format(proba[0][top_3[i]])) plt.imshow(img)
data = csvreader.read('memocode_adder_inputs.csv') labels = data[['output']].copy() features = data[['data1', 'data2']].copy() Data_f = {'data1': [10000, 30000], 'data2': [20000, -5000]} test_features = DataFrame(Data_f, columns=['data1', 'data2']) Data_l = {'output': [26897, 27837]} test_labels = DataFrame(Data_l, columns=['output']) model = Sequential() model.add(Dense(5, input_dim=2, activation='linear', use_bias=False)) model.add(Dense(1, activation='linear', use_bias=False)) model.summary() model.compile(loss='mse', optimizer='adam', metrics=['mse', 'mae']) model.fit(features, labels) # serialize model to JSON model_json = model.to_json() with open("memocode_adder.json", "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 model.save_weights("memocode_adder.h5") print("Saved model to disk") print(test_features) predictions = model.predict(test_features) print(predictions)
test_size=0.2, random_state=124) print('X_train', X_train.shape) print('X_test', X_test.shape) print('Y_train', Y_train.shape) print('Y_test', Y_test.shape) n_cols = X_train.shape[1] #nbr of columns in predictors # model model = Sequential() # Add the first layer model.add(Dense(50, activation='relu', input_shape=(n_cols, ))) # Add the second layer model.add(Dense(100, activation='relu')) # Add the output layer model.add(Dense(2)) # Compile the model model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) history = model.fit(X_train, Y_train, epochs=40, validation_split=0.2) Y_pred_test = model.predict(X_test) print('the short-term score is {:.3f}'.format( spearman_corr(Y_test[:, 0], Y_pred_test[:, 0]))) print('the long-term score is {:.3f}'.format( spearman_corr(Y_test[:, 1], Y_pred_test[:, 1])))
model.add(Dense(100, activation='relu')) # Add the output layer model.add(Dense(2)) # Compile the model model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) # Fit the model r=model.fit(X_train, Y_train, validation_split=0.3, epochs=40) # Compile the model model.compile(optimizer='adam', loss = 'mean_squared_error') # Print the loss print("Loss function: " + model.loss) # Fit the model r = model.fit(X_train,Y_train,epochs=40,validation_data=(X_test,Y_test)) predictions = model.predict(X_test) print(predictions) Get_score(predictions, Y_test) # Spearman scores print('the short-term score is {:.3f}'.format(spearman_corr(Y_test[:,0],predictions[:,0]))) print('the long-term score is {:.3f}'.format(spearman_corr(Y_test[:,1],predictions[:,1])))
validation_data=validation_data, callbacks=callbacks) result = model.evaluate(x=np.expand_dims(x_test_scaled, axis=0), y=np.expand_dims(y_test_scaled, axis=0)) start_idx = 0 length = 10000 target_names = target_output y_pred, y_true,x = ml.plot_comparison(x_test_scaled,y_test,y_scaler, target_names,model) mse_vect = list() for i in range(len(y_pred)): mse_vect.append((y_pred[i]-y_true[i])**2) mse = sum(mse_vect) plt.figure() plt.plot(mse_vect) data_size = 144 training_size = 144 new = np.array(data.solar_irradiance).reshape(8760,2) new_expand = np.expand_dims(new,axis=0) new_pred = model.predict(new_expand) new_pred_rescaled =y_scaler.inverse_transform(new_pred[0]) plt.plot(new_pred_rescaled[0:23]*(data_size/training_size)) plt.plot(data.iloc[:,0][0:23])
x_data = data[:, 0:x_column] y_data = data[:, x_column:] model = Sequential() model.add(Dense(units=y_column, input_dim=x_column, activation='sigmoid')) learning_rate = 0.1 # 학습율 sgd = tf.keras.optimizers.SGD(lr=learning_rate) model.compile(optimizer=sgd, loss='binary_crossentropy') model.fit(x=x_data, y=y_data, epochs=2000, verbose=0) x_result = model.predict(x_data) print(x_result) print('-' * 30) # 0 : 강아지, 1 : 고양이 x_test = [[2, 1], [6, 5], [11, 6]] def getCategory(mydata): mylist = ['강아지', '고양이'] print('예측 : %s, %s' % (mydata, mylist[mydata[0]])) # flatten() : 차원을 1차원으로 만들어 주는 함수 for item in x_test: H = model.predict(np.array([item]))
for tok in start[0]: str += rc.token_to_words([int(tok)]) + ' ' str = str.upper() + ": " # Recreate the generator test_gen = rc.generate_train() wc = 0 for in_array, _ in test_gen: if wc >= TEST_WORDS: break wc += batch_size predict = model.predict(in_array) for j in range(batch_size): predict_word = np.argmax(predict[j, look_back - 1, :]) str += " " + rc.token_to_words([predict_word]) print("Epoch %d. Story: %s." % (i, str)) print("EPOCH %d of %d." % (i, num_epochs)) rc.set_params(batch_size=batch_size, look_back=look_back, skip=skip, loop=True) model.fit_generator(rc.generate_train(), tr_count, 1, validation_data=rc.generate_test(),
model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dense(num_classes, activation='softmax')) from tensorflow.python.keras.optimizers import Adam optimizer = Adam(lr=1e-3) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) model.fit(x=data.train.images, y=data.train.labels, epochs=1, batch_size=128) # evaluation: result = model.evaluate(x=data.test.images, y=data.test.labels) for name, value in zip(model.metrics_names, result): print(name, value) print("{0}:{1:.2%}".format(model.metrics_names[1], result[1])) # prediction images = data.test.images[:9] cls_true = data.test.cls[:9] y_pred = model.predict(x=images) cls_pred = np.argmax(y_pred, axis=1) plot_images(images=images, cls_true=cls_true, cls_pred=cls_pred) y_pred = model.predict(x=data.test.images) cls_pred = np.argmax(y_pred, axis=1) plot_example_errors(cls_pred)
y = data[:, x_column:] # 정답이 가질 수 있는 클래스 개수 NB_CLASSES = 3 x_train, x_test, y_train, y_test = train_test_split(x, y, train_size=0.7) print(y_train) # 원핫 인코딩 적용 전 # np_utils : 원핫 인코딩 수행 y_train = np_utils.to_categorical(y_train, num_classes=NB_CLASSES, dtype='float32') print(y_train) model = Sequential() model.add(Dense(input_shape=(x_column, ), units=3, activation='softmax')) # 모델의 간략한 정보를 출력해준다. model.summary() # 'sgd' : 확률적 경사 하강법 model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=1000, verbose=0) print(history) for i in range(len(x_test)): H = model.predict(np.array([x_test[i]])) pred = np.argmax(H, axis=-1) print(f'예측값 : {pred}, 정답 : {y_test[i]}') print(f'가설정보 : {H.flatten()}')
def Cnn(dataset, testdataset): model = Sequential() model.add( Conv2D(filters=8, kernel_size=(3, 3), strides=(1, 1), padding='same', input_shape=(54, 54, 1), activation='relu')) # Create Max-Pool 1 model.add(MaxPooling2D(pool_size=(2, 2))) # Create CN layer 2 model.add( Conv2D(filters=16, kernel_size=(3, 3), strides=(1, 1), padding='same', activation='relu')) # Create Max-Pool 2 model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(filters=32, kernel_size=(3, 3), strides=(2, 2), padding='same')) model.add(Flatten()) model.add(Dense(Label_size, activation='softmax')) sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) #################### # 設定模型的訓練方式 #################### model.compile( loss= 'categorical_crossentropy' # 設定 Loss 損失函數 為 categorical_crossentropy , optimizer='adam' # 設定 Optimizer 最佳化方法 為 adam , metrics=['accuracy'] # 設定 Model 評估準確率方法 為 accuracy ) history = model.fit(dataset, epochs=30, steps_per_epoch=1000, validation_data=testdataset, validation_steps=1, shuffle=True) model.save('I:\dataSet/test_model_unknow_hahaha.h5') score = model.evaluate(testdataset, steps=1) print('Test loss:', score[0]) print('Test accuracy:', score[1]) predicted = model.predict(testdataset, steps=1) print(predicted) y_classes = predicted df = pd.DataFrame(y_classes) print(df) show_train_history(history, 'acc', 'val_acc') show_train_history(history, 'loss', 'val_loss') return model
class RNNGRU(object): """Support vector machine class.""" def __init__(self, batch_size=64, sequence_length=20, warmup_steps=50, epochs=20, display=False): """Instantiate the class. Args: batch_size: Size of batch sequence_length: Length of vectors for for each target save: Save charts if True Returns: None """ # Initialize key variables self.target_names = ['Temp', 'WindSpeed', 'Pressure'] self.warmup_steps = warmup_steps self.epochs = epochs self.batch_size = batch_size self.display = display # Get data x_data, y_data = self.data() print('\n> Numpy Data Type: {}'.format(type(x_data))) print("> Numpy Data Shape: {}".format(x_data.shape)) print("> Numpy Data Row[0]: {}".format(x_data[0])) print('> Numpy Targets Type: {}'.format(type(y_data))) print("> Numpy Targets Shape: {}".format(y_data.shape)) ''' This is the number of observations (aka. data-points or samples) in the data-set: ''' num_data = len(x_data) ''' This is the fraction of the data-set that will be used for the training-set: ''' train_split = 0.9 ''' This is the number of observations in the training-set: ''' self.num_train = int(train_split * num_data) ''' This is the number of observations in the test-set: ''' num_test = num_data - self.num_train print('> Number of Samples: {}'.format(num_data)) print("> Number of Training Samples: {}".format(self.num_train)) print("> Number of Test Samples: {}".format(num_test)) print("> Batch Size: {}".format(batch_size)) steps_per_epoch = int(self.num_train / batch_size) print("> Recommended Epoch Steps: {:.2f}".format(steps_per_epoch)) # Create test and training data x_train = x_data[0:self.num_train] x_test = x_data[self.num_train:] self.y_train = y_data[0:self.num_train] self.y_test = y_data[self.num_train:] self.num_x_signals = x_data.shape[1] self.num_y_signals = y_data.shape[1] print("> Training Minimum Value:", np.min(x_train)) print("> Training Maximum Value:", np.max(x_train)) ''' The neural network works best on values roughly between -1 and 1, so we need to scale the data before it is being input to the neural network. We can use scikit-learn for this. We first create a scaler-object for the input-signals. Then we detect the range of values from the training-data and scale the training-data. ''' x_scaler = MinMaxScaler() self.x_train_scaled = x_scaler.fit_transform(x_train) print('> Scaled Training Minimum Value: {}'.format( np.min(self.x_train_scaled))) print('> Scaled Training Maximum Value: {}'.format( np.max(self.x_train_scaled))) self.x_test_scaled = x_scaler.transform(x_test) ''' The target-data comes from the same data-set as the input-signals, because it is the weather-data for one of the cities that is merely time-shifted. But the target-data could be from a different source with different value-ranges, so we create a separate scaler-object for the target-data. ''' self.y_scaler = MinMaxScaler() self.y_train_scaled = self.y_scaler.fit_transform(self.y_train) y_test_scaled = self.y_scaler.transform(self.y_test) # Data Generator ''' The data-set has now been prepared as 2-dimensional numpy arrays. The training-data has almost 300k observations, consisting of 20 input-signals and 3 output-signals. These are the array-shapes of the input and output data: ''' print('> Scaled Training Data Shape: {}'.format( self.x_train_scaled.shape)) print('> Scaled Training Targets Shape: {}'.format( self.y_train_scaled.shape)) # We then create the batch-generator. generator = self.batch_generator(batch_size, sequence_length) # Validation Set ''' The neural network trains quickly so we can easily run many training epochs. But then there is a risk of overfitting the model to the training-set so it does not generalize well to unseen data. We will therefore monitor the model's performance on the test-set after each epoch and only save the model's weights if the performance is improved on the test-set. The batch-generator randomly selects a batch of short sequences from the training-data and uses that during training. But for the validation-data we will instead run through the entire sequence from the test-set and measure the prediction accuracy on that entire sequence. ''' validation_data = (np.expand_dims(self.x_test_scaled, axis=0), np.expand_dims(y_test_scaled, axis=0)) # Create the Recurrent Neural Network self.model = Sequential() ''' We can now add a Gated Recurrent Unit (GRU) to the network. This will have 512 outputs for each time-step in the sequence. Note that because this is the first layer in the model, Keras needs to know the shape of its input, which is a batch of sequences of arbitrary length (indicated by None), where each observation has a number of input-signals (num_x_signals). ''' self.model.add( GRU(units=512, return_sequences=True, input_shape=( None, self.num_x_signals, ))) ''' The GRU outputs a batch of sequences of 512 values. We want to predict 3 output-signals, so we add a fully-connected (or dense) layer which maps 512 values down to only 3 values. The output-signals in the data-set have been limited to be between 0 and 1 using a scaler-object. So we also limit the output of the neural network using the Sigmoid activation function, which squashes the output to be between 0 and 1.''' self.model.add(Dense(self.num_y_signals, activation='sigmoid')) ''' A problem with using the Sigmoid activation function, is that we can now only output values in the same range as the training-data. For example, if the training-data only has temperatures between -20 and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1. So if we limit the output of the neural network to be between 0 and 1 using the Sigmoid function, this can only be mapped back to temperature values between -20 and +30. We can use a linear activation function on the output instead. This allows for the output to take on arbitrary values. It might work with the standard initialization for a simple network architecture, but for more complicated network architectures e.g. with more layers, it might be necessary to initialize the weights with smaller values to avoid NaN values during training. You may need to experiment with this to get it working. ''' if False: # Maybe use lower init-ranges. init = RandomUniform(minval=-0.05, maxval=0.05) self.model.add( Dense(self.num_y_signals, activation='linear', kernel_initializer=init)) # Compile Model ''' This is the optimizer and the beginning learning-rate that we will use. We then compile the Keras model so it is ready for training. ''' optimizer = RMSprop(lr=1e-3) self.model.compile(loss=self.loss_mse_warmup, optimizer=optimizer) ''' This is a very small model with only two layers. The output shape of (None, None, 3) means that the model will output a batch with an arbitrary number of sequences, each of which has an arbitrary number of observations, and each observation has 3 signals. This corresponds to the 3 target signals we want to predict. ''' print('> Model Summary:\n') print(self.model.summary()) # Callback Functions ''' During training we want to save checkpoints and log the progress to TensorBoard so we create the appropriate callbacks for Keras. This is the callback for writing checkpoints during training. ''' path_checkpoint = '/tmp/23_checkpoint.keras' callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True) ''' This is the callback for stopping the optimization when performance worsens on the validation-set. ''' callback_early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1) ''' This is the callback for writing the TensorBoard log during training. ''' callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/', histogram_freq=0, write_graph=False) ''' This callback reduces the learning-rate for the optimizer if the validation-loss has not improved since the last epoch (as indicated by patience=0). The learning-rate will be reduced by multiplying it with the given factor. We set a start learning-rate of 1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4. We don't want the learning-rate to go any lower than this. ''' callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-4, patience=0, verbose=1) callbacks = [ callback_early_stopping, callback_checkpoint, callback_tensorboard, callback_reduce_lr ] # Train the Recurrent Neural Network '''We can now train the neural network. Note that a single "epoch" does not correspond to a single processing of the training-set, because of how the batch-generator randomly selects sub-sequences from the training-set. Instead we have selected steps_per_epoch so that one "epoch" is processed in a few minutes. With these settings, each "epoch" took about 2.5 minutes to process on a GTX 1070. After 14 "epochs" the optimization was stopped because the validation-loss had not decreased for 5 "epochs". This optimization took about 35 minutes to finish. Also note that the loss sometimes becomes NaN (not-a-number). This is often resolved by restarting and running the Notebook again. But it may also be caused by your neural network architecture, learning-rate, batch-size, sequence-length, etc. in which case you may have to modify those settings. ''' self.model.fit_generator(generator=generator, epochs=self.epochs, steps_per_epoch=steps_per_epoch, validation_data=validation_data, callbacks=callbacks) # Load Checkpoint ''' Because we use early-stopping when training the model, it is possible that the model's performance has worsened on the test-set for several epochs before training was stopped. We therefore reload the last saved checkpoint, which should have the best performance on the test-set. ''' try: self.model.load_weights(path_checkpoint) except Exception as error: print('\n> Error trying to load checkpoint.\n\n{}'.format(error)) sys.exit(0) # Performance on Test-Set ''' We can now evaluate the model's performance on the test-set. This function expects a batch of data, but we will just use one long time-series for the test-set, so we just expand the array-dimensionality to create a batch with that one sequence. ''' result = self.model.evaluate(x=np.expand_dims(self.x_test_scaled, axis=0), y=np.expand_dims(y_test_scaled, axis=0)) print('> Loss (test-set): {}'.format(result)) # If you have several metrics you can use this instead. if False: for res, metric in zip(result, self.model.metrics_names): print('{0}: {1:.3e}'.format(metric, res)) def batch_generator(self, batch_size, sequence_length): """Generator function for creating random batches of training-data. Args: batch_size: Size of batch sequence_length: Length of sequence Returns: (x_batch, y_batch) """ # Infinite loop. while True: # Allocate a new array for the batch of input-signals. x_shape = (batch_size, sequence_length, self.num_x_signals) x_batch = np.zeros(shape=x_shape, dtype=np.float16) # Allocate a new array for the batch of output-signals. y_shape = (batch_size, sequence_length, self.num_y_signals) y_batch = np.zeros(shape=y_shape, dtype=np.float16) # Fill the batch with random sequences of data. for i in range(batch_size): # Get a random start-index. # This points somewhere into the training-data. idx = np.random.randint(self.num_train - sequence_length) # Copy the sequences of data starting at this index. x_batch[i] = self.x_train_scaled[idx:idx + sequence_length] y_batch[i] = self.y_train_scaled[idx:idx + sequence_length] yield (x_batch, y_batch) def plot_comparison(self, start_idx, length=100, train=True): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. train: Boolean whether to use training- or test-set. Returns: None """ if train: # Use training-data. x_values = self.x_train_scaled y_true = self.y_train shim = 'Train' else: # Use test-data. x_values = self.x_test_scaled y_true = self.y_test shim = 'Test' # End-index for the sequences. end_idx = start_idx + length # Select the sequences from the given start-index and # of the given length. x_values = x_values[start_idx:end_idx] y_true = y_true[start_idx:end_idx] # Input-signals for the model. x_values = np.expand_dims(x_values, axis=0) # Use the model to predict the output-signals. y_pred = self.model.predict(x_values) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. y_pred_rescaled = self.y_scaler.inverse_transform(y_pred[0]) # For each output-signal. for signal in range(len(self.target_names)): # Create a filename filename = ( '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format( self.batch_size, self.epochs, self.num_train, signal, int(time.time()), shim) # Get the output-signal predicted by the model. signal_pred = y_pred_rescaled[:, signal] # Get the true output-signal from the data-set. signal_true = y_true[:, signal] # Make the plotting-canvas bigger. plt.figure(figsize=(15, 5)) # Plot and compare the two signals. plt.plot(signal_true, label='true') plt.plot(signal_pred, label='pred') # Plot grey box for warmup-period. _ = plt.axvspan(0, self.warmup_steps, facecolor='black', alpha=0.15) # Plot labels etc. plt.ylabel(self.target_names[signal]) plt.legend() # Show and save the image if self.display is True: plt.savefig(filename, bbox_inches='tight') plt.show() else: plt.savefig(filename, bbox_inches='tight') print('> Saving file: {}'.format(filename)) def data(self): """Get data to analyze. Args: None Returns: (x_data, y_data): X and Y values as numpy arrays """ # Download data weather.maybe_download_and_extract() # Import data into Pandas dataframe pandas_df = weather.load_resampled_data() print('\n> First Rows of Data:\n\n{}'.format(pandas_df.head(3))) # Print the cities cities = weather.cities print('\n> Cities: {}'.format(cities)) # Print dataframe shape print('> Dataframe shape (Original): {}'.format( pandas_df.values.shape)) # The two signals that have missing data. (Columns with Nans) pandas_df.drop(('Esbjerg', 'Pressure'), axis=1, inplace=True) pandas_df.drop(('Roskilde', 'Pressure'), axis=1, inplace=True) # Print dataframe shape print('> Dataframe shape (New): {}'.format(pandas_df.values.shape)) # Verify that the columns have been dropped print('\n> First Rows of Updated Data:\n\n{}'.format( pandas_df.head(1))) # Add Data ''' We can add some input-signals to the data that may help our model in making predictions. For example, given just a temperature of 10 degrees Celcius the model wouldn't know whether that temperature was measured during the day or the night, or during summer or winter. The model would have to infer this from the surrounding data-points which might not be very accurate for determining whether it's an abnormally warm winter, or an abnormally cold summer, or whether it's day or night. So having this information could make a big difference in how accurately the model can predict the next output. Although the data-set does contain the date and time information for each observation, it is only used in the index so as to order the data. We will therefore add separate input-signals to the data-set for the day-of-year (between 1 and 366) and the hour-of-day (between 0 and 23). ''' pandas_df['Various', 'Day'] = pandas_df.index.dayofyear pandas_df['Various', 'Hour'] = pandas_df.index.hour # Target Data for Prediction ''' We will try and predict the future weather-data for this city. ''' target_city = 'Odense' ''' We will try and predict these signals. ''' self.target_names = ['Temp', 'WindSpeed', 'Pressure'] ''' The following is the number of time-steps that we will shift the target-data. Our data-set is resampled to have an observation for each hour, so there are 24 observations for 24 hours. If we want to predict the weather 24 hours into the future, we shift the data 24 time-steps. If we want to predict the weather 7 days into the future, we shift the data 7 * 24 time-steps. ''' shift_days = 1 shift_steps = shift_days * 24 # Number of hours. # Create a new data-frame with the time-shifted data. ''' Note the negative time-shift! We want the future state targets to line up with the timestamp of the last value of each sample set. ''' df_targets = pandas_df[target_city][self.target_names].shift( -shift_steps) ''' WARNING! You should double-check that you have shifted the data in the right direction! We want to predict the future, not the past! The shifted data-frame is confusing because Pandas keeps the original This is the first shift_steps + 5 rows of the original data-frame: ''' explanatory_hours = shift_steps + 5 print('\n> First Rows of Updated Data ({} hours):\n\n{}'.format( explanatory_hours, pandas_df[target_city][self.target_names].head(explanatory_hours))) ''' The following is the first 5 rows of the time-shifted data-frame. This should be identical to the last 5 rows shown above from the original data, except for the time-stamp. ''' print('\n> First Rows of Shifted Data - Target Labels ' '(Notice 1980 Dates):\n\n{}'.format(df_targets.head(5))) ''' The time-shifted data-frame has the same length as the original data-frame, but the last observations are NaN (not a number) because the data has been shifted backwards so we are trying to shift data that does not exist in the original data-frame. ''' print('\n> Last Rows of Shifted Data - Target Labels ' '(Notice 2018 Dates):\n\n{}'.format(df_targets.tail())) # NumPy Arrays ''' We now convert the Pandas data-frames to NumPy arrays that can be input to the neural network. We also remove the last part of the numpy arrays, because the target-data has NaN for the shifted period, and we only want to have valid data and we need the same array-shapes for the input- and output-data. These are the input-signals: ''' x_data = pandas_df.values[0:-shift_steps] y_data = df_targets.values[:-shift_steps] # Return return (x_data, y_data) def loss_mse_warmup(self, y_true, y_pred): """Calculate the Mean Squared Errror. Calculate the Mean Squared Error between y_true and y_pred, but ignore the beginning "warmup" part of the sequences. We will use Mean Squared Error (MSE) as the loss-function that will be minimized. This measures how closely the model's output matches the true output signals. However, at the beginning of a sequence, the model has only seen input-signals for a few time-steps, so its generated output may be very inaccurate. Using the loss-value for the early time-steps may cause the model to distort its later output. We therefore give the model a "warmup-period" of 50 time-steps where we don't use its accuracy in the loss-function, in hope of improving the accuracy for later time-steps Args: y_true: Desired output. y_pred: Model's output. Returns: loss_mean: Mean Squared Error """ warmup_steps = self.warmup_steps # The shape of both input tensors are: # [batch_size, sequence_length, num_y_signals]. # Ignore the "warmup" parts of the sequences # by taking slices of the tensors. y_true_slice = y_true[:, warmup_steps:, :] y_pred_slice = y_pred[:, warmup_steps:, :] # These sliced tensors both have this shape: # [batch_size, sequence_length - warmup_steps, num_y_signals] # Calculate the MSE loss for each value in these tensors. # This outputs a 3-rank tensor of the same shape. loss = tf.losses.mean_squared_error(labels=y_true_slice, predictions=y_pred_slice) # Keras may reduce this across the first axis (the batch) # but the semantics are unclear, so to be sure we use # the loss across the entire tensor, we reduce it to a # single scalar with the mean function. loss_mean = tf.reduce_mean(loss) return loss_mean
def onBeginTraining(self): ue.log("starting mnist keras cnn training") model_file_name = "mnistKerasCNN" model_directory = ue.get_content_dir() + "/Scripts/" model_sess_path = model_directory + model_file_name + ".tfsess" model_json_path = model_directory + model_file_name + ".json" my_file = Path(model_json_path) #reset the session each time we get training calls K.clear_session() #let's train batch_size = 128 num_classes = 10 epochs = 8 # input image dimensions img_rows, img_cols = 28, 28 # the data, shuffled and split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 ue.log('x_train shape:' + str(x_train.shape)) ue.log(str(x_train.shape[0]) + 'train samples') ue.log(str(x_test.shape[0]) + 'test samples') # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) model = Sequential() model.add( Conv2D(64, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) # model.add(Dropout(0.2)) # model.add(Flatten()) # model.add(Dense(512, activation='relu')) # model.add(Dropout(0.2)) # model.add(Dense(num_classes, activation='softmax')) #model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test), callbacks=[self.stopcallback]) score = model.evaluate(x_test, y_test, verbose=0) ue.log("mnist keras cnn training complete.") ue.log('Test loss:' + str(score[0])) ue.log('Test accuracy:' + str(score[1])) self.session = K.get_session() self.model = model stored = {'model': model, 'session': self.session} #run a test evaluation ue.log(x_test.shape) result_test = model.predict(np.reshape(x_test[500], (1, 28, 28, 1))) ue.log(result_test) #flush the architecture model data to disk #with open(model_json_path, "w") as json_file: # json_file.write(model.to_json()) #flush the whole model and weights to disk #saver = tf.train.Saver() #save_path = saver.save(K.get_session(), model_sess_path) #model.save(model_path) return stored
print(learning_data.stats) # validate the model validation_loss = model.evaluate(xs_validation, ys_validation) print() print("Validation loss & accuracy: " + str(validation_loss)) # compute precision and recall with different thresholds # for reporting anomalies # assumption: correct and incorrect arguments are alternating # in list of x-y pairs threshold_to_correct = Counter() threshold_to_incorrect = Counter() threshold_to_found_seeded_bugs = Counter() threshold_to_warnings_in_orig_code = Counter() ys_prediction = model.predict(xs_validation) poss_anomalies = [] for idx in range(0, len(xs_validation), 2): # probab(original code should be changed), expect 0 y_prediction_orig = ys_prediction[idx][0] # probab(changed code should be changed), expect 1 y_prediction_changed = ys_prediction[idx + 1][0] # higher means more likely to be anomaly in current code anomaly_score = learning_data.anomaly_score( y_prediction_orig, y_prediction_changed) # higher means more likely to be correct in current code normal_score = learning_data.normal_score( y_prediction_orig, y_prediction_changed) is_anomaly = False for threshold_raw in range(1, 20, 1): threshold = threshold_raw / 20.0
class NeuralNetwork(Model): # X represents the features, Y represents the labels X = None Y = None prediction = None model = None path = 'models/nn.h5' regressor = None classifier = None def __init__(self): Model.__init__(self) def __init__(self, feature_headers, label_headers, type='regressor', epochs=150, batch_size=50, X=None, Y=None, cfg=False, pca=None): if X is not None: self.X = X if Y is not None: self.Y = Y self.feature_headers = feature_headers self.label_headers = label_headers if pca is not None: self.no_inputs = pca['n_components'] else: self.no_inputs = len(feature_headers) self.no_outputs = len(label_headers) self.epochs = epochs self.batch_size = batch_size self.mapping_dict = None self.type = type self.cfg = cfg self.model = Sequential() self.init_model() def init_model(self): self.model.add( Dense(self.no_inputs * 2 + 1, input_dim=self.no_inputs, kernel_initializer='normal', activation='sigmoid')) self.model.add(Dense(self.no_outputs, activation='sigmoid')) if self.type == 'regressor': output_activation = 'linear' else: output_activation = 'tanh' self.model.add(Dense(self.no_outputs, activation=output_activation)) self.model.compile(loss='mse', optimizer='adam', metrics=['mse', 'mae']) def summary(self): self.model.summary() def compile(self, loss='mae', optimizer='adam', metrics=['mse', 'mae']): self.model.compile(loss=loss, optimizer=optimizer, metrics=metrics) def fit(self, X=None, Y=None): if X is not None: self.X = X if Y is not None: self.Y = Y.copy() if self.type == 'classifier': self.Y = self.map_str_to_number(self.Y) print('Neural Network Train started............') self.model.fit(self.X.values, self.Y.values, epochs=self.epochs, batch_size=self.batch_size, verbose=0, validation_split=0.2) print('Neural Network Train completed..........') return self.model def save(self, filename='nn_model_test.h5'): if self.cfg: with open('ann_configs.txt', 'w') as f: self.model.summary(print_fn=lambda x: f.write(x + '\n')) self.model.save(filename) def predict(self, test_X): predictions = self.model.predict(test_X.values) if self.type == 'classifier': predictions = predictions.round() return predictions[:, 0] def score(self, test_X, test_Y): y_pred = self.predict(test_X) r2s = r2_score(test_Y, y_pred, multioutput='variance_weighted') return r2s def map_str_to_number(self, Y): mapping_flag = False if self.mapping_dict is not None: for label_header in self.label_headers: Y[label_header] = Y[label_header].map(self.mapping_dict) return Y mapping_dict = None for label_header in self.label_headers: check_list = pd.Series(Y[label_header]) for item in check_list: if type(item) == str: mapping_flag = True break if mapping_flag: classes = Y[label_header].unique() mapping_dict = {} index = 0 for c in classes: mapping_dict[c] = index index += 1 Y[label_header] = Y[label_header].map(mapping_dict) mapping_flag = False self.mapping_dict = mapping_dict return Y def map_number_to_str(self, Y, classes): Y = Y.round() Y = Y.astype(int) if self.mapping_dict is not None: mapping_dict = self.mapping_dict else: mapping_dict = {} index = 0 for c in classes: mapping_dict[index] = c index += 1 inv_map = {v: k for k, v in mapping_dict.items()} return Y.map(inv_map) def getAccuracy(self, test_labels, predictions, origin=0, hitmissr=0.8): if self.type == 'classifier': correct = 0 df = pd.DataFrame(data=predictions.flatten()) test_labels = self.map_str_to_number(test_labels.copy()) for i in range(len(df)): if (df.values[i] == test_labels.values[i]): correct = correct + 1 else: correct = 0 df = pd.DataFrame(data=predictions.flatten()) for i in range(len(df)): if 1 - abs(df.values[i] - test_labels.values[i]) / abs( df.values[i]) >= hitmissr: correct = correct + 1 return float(correct) / len(df) def getConfusionMatrix(self, test_labels, predictions, label_headers): df = pd.DataFrame(data=predictions.flatten()) if self.type == 'classifier': index = 0 for label_header in label_headers: classes = test_labels[label_header].unique() df_tmp = self.map_number_to_str(df.ix[:, index], classes) title = 'Normalized confusion matrix for NeuralNetwork (' + label_header + ')' self.plot_confusion_matrix(test_labels.ix[:, index], df_tmp, classes=classes, normalize=True, title=title) index = index + 1 else: return 'No Confusion Matrix for Regression' def getROC(self, test_labels, predictions, label_headers): predictions = pd.DataFrame(data=predictions.flatten()) predictions.columns = test_labels.columns.values if self.type == 'classifier': test_labels = self.map_str_to_number(test_labels) fpr, tpr, _ = roc_curve(test_labels, predictions) plt.figure(1) plt.plot([0, 1], [0, 1], 'k--') plt.plot(fpr, tpr) plt.xlabel('False positive rate') plt.ylabel('True positive rate') plt.title('ROC curve') plt.show() else: return 'No Confusion Matrix for Regression' def featureImportance(self): return '' def getRSquare(self, test_labels, predictions, mode='single'): df = pd.DataFrame(data=predictions.flatten()) if self.type == 'regressor': if mode == 'multiple': errors = r2_score(test_labels, df, multioutput='variance_weighted') else: errors = r2_score(test_labels, df) return errors else: return 'No RSquare for Classification' def getMSE(self, test_labels, predictions): df = pd.DataFrame(data=predictions.flatten()) if self.type == 'regressor': errors = mean_squared_error(test_labels, df) return errors else: return 'No MSE for Classification' def getMAPE(self, test_labels, predictions): df = pd.DataFrame(data=predictions.flatten()) if self.type == 'regressor': errors = np.mean(np.abs( (test_labels - df.values) / test_labels)) * 100 return errors.values[0] else: return 'No MAPE for Classification' def getRMSE(self, test_labels, predictions): df = pd.DataFrame(data=predictions.flatten()) if self.type == 'regressor': errors = sqrt(mean_squared_error(test_labels, df)) return errors else: return 'No RMSE for Classification' def load(self, path, type): self.model = load_model(path) self.type = type return self.model
class KerasCNN(object): """Support vector machine class.""" # Convolutional Layer 1. filter_size1 = 5 # Convolution filters are 5 x 5 pixels. num_filters1 = 16 # There are 16 of these filters. # Convolutional Layer 2. filter_size2 = 5 # Convolution filters are 5 x 5 pixels. num_filters2 = 36 # There are 36 of these filters. # Fully-connected layer. fc_size = 128 # Number of neurons in fully-connected laye # Get data from files data = MNIST(data_dir='/tmp/data/MNIST/') # The number of pixels in each dimension of an image. img_size = data.img_size # The images are stored in one-dimensional arrays of this length. img_size_flat = data.img_size_flat # Tuple with height and width of images used to reshape arrays. img_shape = data.img_shape # Tuple with height, width and depth used to reshape arrays. # This is used for reshaping in Keras. img_shape_full = data.img_shape_full # Number of classes, one class for each of 10 digits. num_classes = data.num_classes # Number of colour channels for the images: 1 channel for gray-scale. num_channels = data.num_channels def __init__(self): """Instantiate the class. Args: train_batch_size: Training batch size Returns: None """ # Initialize variables epochs = 2 """ print('{0: <{1}} {2}'.format('Encoded X image:', fill, self.x_image)) """ # Start construction of the Keras Sequential model. self.model = Sequential() # Add an input layer which is similar to a feed_dict in TensorFlow. # Note that the input-shape must be a tuple containing the image-size. self.model.add(InputLayer(input_shape=(self.img_size_flat,))) # The input is a flattened array with 784 elements, # but the convolutional layers expect images with shape (28, 28, 1) self.model.add(Reshape(self.img_shape_full)) # First convolutional layer with ReLU-activation and max-pooling. self.model.add( Conv2D(kernel_size=5, strides=1, filters=16, padding='same', activation='relu', name='layer_conv1')) self.model.add(MaxPooling2D(pool_size=2, strides=2)) # Second convolutional layer with ReLU-activation and max-pooling. self.model.add( Conv2D(kernel_size=5, strides=1, filters=36, padding='same', activation='relu', name='layer_conv2')) self.model.add(MaxPooling2D(pool_size=2, strides=2)) # Flatten the 4-rank output of the convolutional layers # to 2-rank that can be input to a fully-connected / dense layer. self.model.add(Flatten()) # First fully-connected / dense layer with ReLU-activation. self.model.add(Dense(128, activation='relu')) # Last fully-connected / dense layer with softmax-activation # for use in classification. self.model.add(Dense(self.num_classes, activation='softmax')) # Model Compilation ''' The Neural Network has now been defined and must be finalized by adding a loss-function, optimizer and performance metrics. This is called model "compilation" in Keras. We can either define the optimizer using a string, or if we want more control of its parameters then we need to instantiate an object. For example, we can set the learning-rate. ''' optimizer = Adam(lr=1e-3) ''' For a classification-problem such as MNIST which has 10 possible classes, we need to use the loss-function called categorical_crossentropy. The performance metric we are interested in is the classification accuracy. ''' self.model.compile( optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) # Training ''' Now that the model has been fully defined with loss-function and optimizer, we can train it. This function takes numpy-arrays and performs the given number of training epochs using the given batch-size. An epoch is one full use of the entire training-set. So for 10 epochs we would iterate randomly over the entire training-set 10 times. ''' self.model.fit(x=self.data.x_train, y=self.data.y_train, epochs=epochs, batch_size=128) # Evaluation ''' Now that the model has been trained we can test its performance on the test-set. This also uses numpy-arrays as input. ''' result = self.model.evaluate(x=self.data.x_test, y=self.data.y_test) ''' Print actual versus predicted values ''' print('\nActual vs Predicted X values') start = 0 stop = 300 predictions = self.model.predict(self.data.x_test[start:stop]) for pointer in range(start, stop): predicted = np.argmax(predictions[pointer]) actual = np.argmax(self.data.y_test[pointer]) print( '{}: Actual: {}\tPredicted: {}\tMatch: {}'.format( str(pointer).zfill(3), predicted, actual, predicted == actual)) ''' We can print all the performance metrics for the test-set. ''' print('\nPerfomance metrics') for name, value in zip(self.model.metrics_names, result): print('{} {}'.format(name, value)) ''' Print the model summary ''' print('\n\nModel Summary\n\n{}'.format(self.model.summary())) def plot_example_errors(self, cls_pred): """Plot 9 images in a 3x3 grid. Function used to plot 9 images in a 3x3 grid, and writing the true and predicted classes below each image. Args: cls_pred: Array of the predicted class-number for all images in the test-set. Returns: None """ # Boolean array whether the predicted class is incorrect. incorrect = (cls_pred != self.data.y_test_cls) # Get the images from the test-set that have been # incorrectly classified. images = self.data.x_test[incorrect] # Get the predicted classes for those images. cls_pred = cls_pred[incorrect] # Get the true classes for those images. cls_true = self.data.y_test_cls[incorrect] # Plot the first 9 images. plot_images( images[0:9], self.img_shape, cls_true[0:9], cls_pred=cls_pred[0:9])
model.add(LSTM(100)) model.add(Dense(9, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=Adam(0.005), metrics=['accuracy']) epochs = 2 batch_size = 100 history = model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, validation_split=0.1, callbacks=[ EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001) ]) accr = model.evaluate(X_test, Y_test) print('Test set\n Loss: {:0.3f}\n Accuracy: {:0.3f}'.format( accr[0], accr[1])) new_complaint = ['what is the load condition? .'] seq = tokenizer.texts_to_sequences(new_complaint) padded = pad_sequences(seq, maxlen=MAX_SEQUENCE_LENGTH) pred = model.predict(padded) print(pred) print("Prediction class: ", Y[np.argmax(pred)])
class RNNGRU(object): """Process data for ingestion.""" def __init__( self, data, sequence_length=20, warmup_steps=50, dropout=0, layers=1, patience=10, units=512, display=False): """Instantiate the class. Args: data: Tuple of (x_data, y_data, target_names) batch_size: Size of batch sequence_length: Length of vectors for for each target warmup_steps: Returns: None """ # Initialize key variables self._warmup_steps = warmup_steps self._data = data self.display = display path_checkpoint = '/tmp/checkpoint.keras' _layers = int(abs(layers)) # Delete any stale checkpoint file if os.path.exists(path_checkpoint) is True: os.remove(path_checkpoint) ################################### # TensorFlow wizardry config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a total of half the GPU memory to be allocated config.gpu_options.per_process_gpu_memory_fraction = 0.95 # Crash with DeadlineExceeded instead of hanging forever when your # queues get full/empty config.operation_timeout_in_ms = 60000 # Create a session with the above options specified. backend.tensorflow_backend.set_session(tf.Session(config=config)) ################################### # Get data self._y_current = self._data.close() # Create training arrays x_train = self._data.vectors_train() self._y_train = self._data.classes_train() # Create test arrays for VALIDATION and EVALUATION xv_test = self._data.vectors_test() self._yv_test = self._data.classes_test() (self.training_rows, self._training_vector_count) = x_train.shape (self.test_rows, _) = xv_test.shape (_, self._training_class_count) = self._y_train.shape # Print stuff print('\n> Numpy Data Type: {}'.format(type(x_train))) print("> Numpy Data Shape: {}".format(x_train.shape)) print("> Numpy Data Row[0]: {}".format(x_train[0])) print("> Numpy Data Row[Last]: {}".format(x_train[-1])) print('> Numpy Targets Type: {}'.format(type(self._y_train))) print("> Numpy Targets Shape: {}".format(self._y_train.shape)) print('> Number of Samples: {}'.format(self._y_current.shape[0])) print('> Number of Training Samples: {}'.format(x_train.shape[0])) print('> Number of Training Classes: {}'.format( self._training_class_count)) print('> Number of Test Samples: {}'.format(self.test_rows)) print("> Training Minimum Value:", np.min(x_train)) print("> Training Maximum Value:", np.max(x_train)) print('> Number X signals: {}'.format(self._training_vector_count)) print('> Number Y signals: {}'.format(self._training_class_count)) # Print epoch related data print('> Epochs:', self._data.epochs()) print('> Batch Size:', self._data.batch_size()) print('> Steps:', self._data.epoch_steps()) # Display estimated memory footprint of training data. print("> Data size: {:.2f} Bytes".format(x_train.nbytes)) ''' The neural network works best on values roughly between -1 and 1, so we need to scale the data before it is being input to the neural network. We can use scikit-learn for this. We first create a scaler-object for the input-signals. Then we detect the range of values from the training-data and scale the training-data. ''' self._x_scaler = MinMaxScaler() self._x_train_scaled = self._x_scaler.fit_transform(x_train) print('> Scaled Training Minimum Value: {}'.format( np.min(self._x_train_scaled))) print('> Scaled Training Maximum Value: {}'.format( np.max(self._x_train_scaled))) self._xv_test_scaled = self._x_scaler.transform(xv_test) ''' The target-data comes from the same data-set as the input-signals, because it is the weather-data for one of the cities that is merely time-shifted. But the target-data could be from a different source with different value-ranges, so we create a separate scaler-object for the target-data. ''' self._y_scaler = MinMaxScaler() self._y_train_scaled = self._y_scaler.fit_transform(self._y_train) yv_test_scaled = self._y_scaler.transform(self._yv_test) # Data Generator ''' The data-set has now been prepared as 2-dimensional numpy arrays. The training-data has almost 300k observations, consisting of 20 input-signals and 3 output-signals. These are the array-shapes of the input and output data: ''' print('> Scaled Training Data Shape: {}'.format( self._x_train_scaled.shape)) print('> Scaled Training Targets Shape: {}'.format( self._y_train_scaled.shape)) # We then create the batch-generator. generator = self._batch_generator( self._data.batch_size(), sequence_length) # Validation Set ''' The neural network trains quickly so we can easily run many training epochs. But then there is a risk of overfitting the model to the training-set so it does not generalize well to unseen data. We will therefore monitor the model's performance on the test-set after each epoch and only save the model's weights if the performance is improved on the test-set. The batch-generator randomly selects a batch of short sequences from the training-data and uses that during training. But for the validation-data we will instead run through the entire sequence from the test-set and measure the prediction accuracy on that entire sequence. ''' validation_data = (np.expand_dims(self._xv_test_scaled, axis=0), np.expand_dims(yv_test_scaled, axis=0)) # Create the Recurrent Neural Network self._model = Sequential() ''' We can now add a Gated Recurrent Unit (GRU) to the network. This will have 512 outputs for each time-step in the sequence. Note that because this is the first layer in the model, Keras needs to know the shape of its input, which is a batch of sequences of arbitrary length (indicated by None), where each observation has a number of input-signals (num_x_signals). ''' self._model.add(GRU( units=units, return_sequences=True, recurrent_dropout=dropout, input_shape=(None, self._training_vector_count,))) for _ in range(0, _layers): self._model.add(GRU( units=units, recurrent_dropout=dropout, return_sequences=True)) ''' The GRU outputs a batch of sequences of 512 values. We want to predict 3 output-signals, so we add a fully-connected (or dense) layer which maps 512 values down to only 3 values. The output-signals in the data-set have been limited to be between 0 and 1 using a scaler-object. So we also limit the output of the neural network using the Sigmoid activation function, which squashes the output to be between 0 and 1.''' self._model.add( Dense(self._training_class_count, activation='sigmoid')) ''' A problem with using the Sigmoid activation function, is that we can now only output values in the same range as the training-data. For example, if the training-data only has temperatures between -20 and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1. So if we limit the output of the neural network to be between 0 and 1 using the Sigmoid function, this can only be mapped back to temperature values between -20 and +30. We can use a linear activation function on the output instead. This allows for the output to take on arbitrary values. It might work with the standard initialization for a simple network architecture, but for more complicated network architectures e.g. with more layers, it might be necessary to initialize the weights with smaller values to avoid NaN values during training. You may need to experiment with this to get it working. ''' if False: # Maybe use lower init-ranges. # init = RandomUniform(minval=-0.05, maxval=0.05) init = RandomUniform(minval=-0.05, maxval=0.05) self._model.add(Dense( self._training_class_count, activation='linear', kernel_initializer=init)) # Compile Model ''' This is the optimizer and the beginning learning-rate that we will use. We then compile the Keras model so it is ready for training. ''' optimizer = RMSprop(lr=1e-3) self._model.compile( loss=self._loss_mse_warmup, optimizer=optimizer, metrics=['accuracy']) ''' This is a very small model with only two layers. The output shape of (None, None, 3) means that the model will output a batch with an arbitrary number of sequences, each of which has an arbitrary number of observations, and each observation has 3 signals. This corresponds to the 3 target signals we want to predict. ''' print('> Model Summary:\n') print(self._model.summary()) # Callback Functions ''' During training we want to save checkpoints and log the progress to TensorBoard so we create the appropriate callbacks for Keras. This is the callback for writing checkpoints during training. ''' callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True) ''' This is the callback for stopping the optimization when performance worsens on the validation-set. ''' callback_early_stopping = EarlyStopping(monitor='val_loss', patience=patience, verbose=1) ''' This is the callback for writing the TensorBoard log during training. ''' callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/', histogram_freq=0, write_graph=False) ''' This callback reduces the learning-rate for the optimizer if the validation-loss has not improved since the last epoch (as indicated by patience=0). The learning-rate will be reduced by multiplying it with the given factor. We set a start learning-rate of 1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4. We don't want the learning-rate to go any lower than this. ''' callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-4, patience=0, verbose=1) callbacks = [callback_early_stopping, callback_checkpoint, callback_tensorboard, callback_reduce_lr] # Train the Recurrent Neural Network '''We can now train the neural network. Note that a single "epoch" does not correspond to a single processing of the training-set, because of how the batch-generator randomly selects sub-sequences from the training-set. Instead we have selected steps_per_epoch so that one "epoch" is processed in a few minutes. With these settings, each "epoch" took about 2.5 minutes to process on a GTX 1070. After 14 "epochs" the optimization was stopped because the validation-loss had not decreased for 5 "epochs". This optimization took about 35 minutes to finish. Also note that the loss sometimes becomes NaN (not-a-number). This is often resolved by restarting and running the Notebook again. But it may also be caused by your neural network architecture, learning-rate, batch-size, sequence-length, etc. in which case you may have to modify those settings. ''' print('\n> Starting data training\n') self._history = self._model.fit_generator( generator=generator, epochs=self._data.epochs(), steps_per_epoch=self._data.epoch_steps(), validation_data=validation_data, callbacks=callbacks) # Load Checkpoint ''' Because we use early-stopping when training the model, it is possible that the model's performance has worsened on the test-set for several epochs before training was stopped. We therefore reload the last saved checkpoint, which should have the best performance on the test-set. ''' print('> Loading model weights') if os.path.exists(path_checkpoint): self._model.load_weights(path_checkpoint) # Performance on Test-Set ''' We can now evaluate the model's performance on the test-set. This function expects a batch of data, but we will just use one long time-series for the test-set, so we just expand the array-dimensionality to create a batch with that one sequence. ''' result = self._model.evaluate( x=np.expand_dims(self._xv_test_scaled, axis=0), y=np.expand_dims(yv_test_scaled, axis=0)) print('> Loss (test-set): {}'.format(result)) # If you have several metrics you can use this instead. if False: for res, metric in zip(result, self._model.metrics_names): print('{0}: {1:.3e}'.format(metric, res)) def _batch_generator(self, batch_size, sequence_length): """Create generator function to create random batches of training-data. Args: batch_size: Size of batch sequence_length: Length of sequence Returns: (x_batch, y_batch) """ # Infinite loop. while True: # Allocate a new array for the batch of input-signals. x_shape = ( batch_size, sequence_length, self._training_vector_count) x_batch = np.zeros(shape=x_shape, dtype=np.float16) # Allocate a new array for the batch of output-signals. y_shape = (batch_size, sequence_length, self._training_class_count) y_batch = np.zeros(shape=y_shape, dtype=np.float16) # Fill the batch with random sequences of data. for i in range(batch_size): # Get a random start-index. # This points somewhere into the training-data. idx = np.random.randint( self.training_rows - sequence_length) # Copy the sequences of data starting at this index. x_batch[i] = self._x_train_scaled[idx:idx+sequence_length] y_batch[i] = self._y_train_scaled[idx:idx+sequence_length] yield (x_batch, y_batch) def _loss_mse_warmup(self, y_true, y_pred): """Calculate the Mean Squared Errror. Calculate the Mean Squared Error between y_true and y_pred, but ignore the beginning "warmup" part of the sequences. We will use Mean Squared Error (MSE) as the loss-function that will be minimized. This measures how closely the model's output matches the true output signals. However, at the beginning of a sequence, the model has only seen input-signals for a few time-steps, so its generated output may be very inaccurate. Using the loss-value for the early time-steps may cause the model to distort its later output. We therefore give the model a "warmup-period" of 50 time-steps where we don't use its accuracy in the loss-function, in hope of improving the accuracy for later time-steps Args: y_true: Desired output. y_pred: Model's output. Returns: loss_mean: Mean Squared Error """ warmup_steps = self._warmup_steps # The shape of both input tensors are: # [batch_size, sequence_length, num_y_signals]. # Ignore the "warmup" parts of the sequences # by taking slices of the tensors. y_true_slice = y_true[:, warmup_steps:, :] y_pred_slice = y_pred[:, warmup_steps:, :] # These sliced tensors both have this shape: # [batch_size, sequence_length - warmup_steps, num_y_signals] # Calculate the MSE loss for each value in these tensors. # This outputs a 3-rank tensor of the same shape. loss = tf.losses.mean_squared_error(labels=y_true_slice, predictions=y_pred_slice) # Keras may reduce this across the first axis (the batch) # but the semantics are unclear, so to be sure we use # the loss across the entire tensor, we reduce it to a # single scalar with the mean function. loss_mean = tf.reduce_mean(loss) return loss_mean def plot_train(self, start_idx, length=100): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. Returns: None """ # Plot self._plot_comparison(start_idx, length=length, train=True) def plot_test(self, start_idx, length=100): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. Returns: None """ # Plot self._plot_comparison(start_idx, length=length, train=False) def _plot_comparison(self, start_idx, length=100, train=True): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. train: Boolean whether to use training- or test-set. Returns: None """ # Initialize key variables datetimes = {} num_train = self.training_rows # End-index for the sequences. end_idx = start_idx + length # Variables for date formatting days = mdates.DayLocator() # Every day months = mdates.MonthLocator() # Every month months_format = mdates.DateFormatter('%b %Y') days_format = mdates.DateFormatter('%d') # Assign other variables dependent on the type of data we are plotting if train is True: # Use training-data. x_values = self._x_train_scaled[start_idx:end_idx] y_true = self._y_train[start_idx:end_idx] shim = 'Train' # Datetimes to use for training datetimes[shim] = self._data.datetime()[ :num_train][start_idx:end_idx] else: # Scale the data x_test_scaled = self._x_scaler.transform( self._data.vectors_test_all()) # Use test-data. x_values = x_test_scaled[start_idx:end_idx] y_true = self._yv_test[start_idx:end_idx] shim = 'Test' # Datetimes to use for testing datetimes[shim] = self._data.datetime()[ num_train:][start_idx:end_idx] # Input-signals for the model. x_values = np.expand_dims(x_values, axis=0) # Use the model to predict the output-signals. y_pred = self._model.predict(x_values) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. y_pred_rescaled = self._y_scaler.inverse_transform(y_pred[0]) # For each output-signal. for signal in range(len(self._data.labels())): # Assign other variables dependent on the type of data plot if train is True: # Only get current values that are a part of the training data current = self._y_current[:num_train][start_idx:end_idx] # The number of datetimes for the 'actual' plot must match # that of current values datetimes['actual'] = self._data.datetime()[ :num_train][start_idx:end_idx] else: # Only get current values that are a part of the test data. current = self._y_current[ num_train:][start_idx:] # The number of datetimes for the 'actual' plot must match # that of current values datetimes['actual'] = self._data.datetime()[ num_train:][start_idx:] # Create a filename filename = ( '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format( self._data.batch_size(), self._data.epochs(), num_train, signal, int(time.time()), shim) # Get the output-signal predicted by the model. signal_pred = y_pred_rescaled[:, signal] # Get the true output-signal from the data-set. signal_true = y_true[:, signal] # Create a new chart (fig, axis) = plt.subplots(figsize=(15, 5)) # Plot and compare the two signals. axis.plot( datetimes[shim][:len(signal_true)], signal_true, label='Current +{}'.format(self._data.labels()[signal])) axis.plot( datetimes[shim][:len(signal_pred)], signal_pred, label='Prediction') axis.plot(datetimes['actual'], current, label='Current') # Set plot labels and titles axis.set_title('{1}ing Forecast ({0} Future Intervals)'.format( self._data.labels()[signal], shim)) axis.set_ylabel('Values') axis.legend( bbox_to_anchor=(1.04, 0.5), loc='center left', borderaxespad=0) # Add gridlines and ticks ax = plt.gca() ax.grid(True) # Add major gridlines ax.xaxis.grid(which='major', color='black', alpha=0.2) ax.yaxis.grid(which='major', color='black', alpha=0.2) # Add minor ticks (They must be turned on first) ax.minorticks_on() ax.xaxis.grid(which='minor', color='black', alpha=0.1) ax.yaxis.grid(which='minor', color='black', alpha=0.1) # Format the tick labels ax.xaxis.set_major_locator(months) ax.xaxis.set_major_formatter(months_format) ax.xaxis.set_minor_locator(days) # Remove tick marks ax.tick_params(axis='both', which='both', length=0) # Print day numbers on xaxis for Test data only if train is False: ax.xaxis.set_minor_formatter(days_format) plt.setp(ax.xaxis.get_minorticklabels(), rotation=90) # Rotates and right aligns the x labels, and moves the bottom of # the axes up to make room for them fig.autofmt_xdate() # Plot grey box for warmup-period if we are working with training # data and the start is within the warmup-period if (0 < start_idx < self._warmup_steps): if train is True: plt.axvspan( datetimes[shim][start_idx], datetimes[shim][self._warmup_steps], facecolor='black', alpha=0.15) # Show and save the image if self.display is True: fig.savefig(filename, bbox_inches='tight') plt.show() else: fig.savefig(filename, bbox_inches='tight') print('> Saving file: {}'.format(filename)) # Close figure plt.close(fig=fig) def plot_accuracy(self): """Plot the predicted and true output-signals. Args: None Returns: None """ # Summarize history for accuracy plt.figure(figsize=(15, 5)) plt.plot(self._history.history['acc']) plt.plot(self._history.history['val_acc']) plt.title('Model Accuracy') plt.ylabel('Accuracy') plt.xlabel('Epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() # Summarize history for loss plt.figure(figsize=(15, 5)) plt.plot(self._history.history['loss']) plt.plot(self._history.history['val_loss']) plt.title('Model loss') plt.ylabel('Loss') plt.xlabel('Epoch') plt.legend(['Train', 'Test'], loc='upper left') plt.show()
Dense(10), Activation('softmax'), ]) # In[3]: fitur ektraksi no 8 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) print(model.summary()) # In[3]: fitur ektraksi, no 9 model.fit(train_input, train_labels, epochs=10, batch_size=32, validation_split=0.2) # In[3]: fitur ektraksi no 10 loss, acc = model.evaluate(test_input, test_labels, batch_size=32) # In[3]: fitur ektraksi print("Done!") print("Loss: %.4f, accuracy: %.4f" % (loss, acc)) # In[3]: fitur ektraksi no 11 model.predict(test_input[:1]) # Error Yang Di Dapat Merupakan Modul error dimana belum adanya module tersebut ModuleNotFoundError: No Module named 'keras.models' # Cara penanganannya adalah install modul keras # dengan perintah seperti berikut #dengan pip pip install keras #dengan conda conda install keras
# visualise Loss plt.plot(hist.history['loss']) plt.plot(hist.history['val_loss']) plt.title('Loss V.S. Val Loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'validation'], loc='upper left') plt.show() # Predictions - use the rest of data in the file to test the accuracy of predictions test_features = zeolite_13X_error_testset[1:11, 0:8] test_targets = zeolite_13X_error_testset[1:11, 12] # Using this way, all the values extrapolated will not form a vector length automatically test_features_modify = scaler_x.transform(test_features) predicted_test_targets = model.predict(test_features_modify) # Invert Normalise Final_test_targets = scaler_y.inverse_transform(predicted_test_targets) # Reshape the test targets to the same shape as the prediction length = test_targets.shape[0] test_targets = test_targets.reshape((length, 1)) targets = Final_test_targets.reshape((length, 1)) error = abs(test_targets - targets) print('Absolute errors between predicted Specific Energy (SE) and real SE \n', error, '\n') Precision = abs(1 - error/targets) print('Estimated Precision of model in SE Predicting \n', Precision, '\n') # Compute the average precision
def processANN(self, X_train_scaled, X_test_scaled, targetTrainX_scaled, numOfParameterInput, selectedTarget0_And_optimizer1, numberOfHidLayer, output1_Activation, numOfHidNeuron, actHidLayer, epoch, view): model = Sequential() #initializer = self.createRandom() #bias + weight random initializer = keras.initializers.Constant(0.039) #susun arsitektur yang diinputkan for i in range(numberOfHidLayer): if (i == 0): # input layer dan hidden layer pertama (gabungan) if actHidLayer[i] == "None": model.add( Dense(numOfHidNeuron[i], input_dim=numOfParameterInput, kernel_initializer=initializer)) else: model.add( Dense(numOfHidNeuron[i], activation=actHidLayer[i], input_dim=numOfParameterInput, kernel_initializer=initializer)) else: # hidden layer lanjutan if actHidLayer[i] == "None": model.add( Dense(numOfHidNeuron[i], kernel_initializer=initializer)) else: model.add( Dense(numOfHidNeuron[i], activation=actHidLayer[i], kernel_initializer=initializer)) #susun output layer sesuai input if output1_Activation == "None": model.add(Dense(1, kernel_initializer=initializer)) else: model.add( Dense(1, activation=output1_Activation, kernel_initializer=initializer)) model.compile(loss='mean_squared_error', optimizer=selectedTarget0_And_optimizer1[1], metrics=['accuracy']) start_time = time.time() #ambil waktu sebelum training #proses training history = model.fit(X_train_scaled, targetTrainX_scaled, validation_split=0.2, epochs=epoch, batch_size=self.batch, verbose=0, shuffle=False, callbacks=[CustomCallback(view, epoch)]) end_time = time.time() #ambil waktu setelah training #proses testing trainPredict = model.predict( X_train_scaled) #hasil ann training (skala [0,1]) testPredict = model.predict( X_test_scaled) #hasil ann dari data testing (skala [0,1]) self.plot_history(history) #print(model.layers[0].get_weights()[0]) #print(model.summary()) return trainPredict, testPredict, history, model, end_time - start_time
) model.summary() from tensorflow.python.keras.callbacks import LambdaCallback simple_logging = LambdaCallback(on_epoch_end = lambda e, l: print(e, end='.')) E = 20 h = model.fit( x_train, y_train, validation_split = 0.2, epochs = E, callbacks = [simple_logging], verbose = False ) # Commented out IPython magic to ensure Python compatibility. import matplotlib.pyplot as plt # %matplotlib inline plt.plot(range(E), h.history['acc'], label = 'Training') plt.plot(range(E), h.history['val_acc'], label = 'Validation') plt.legend() plt.show() loss, acc = model.evaluate(x_test, y_test) print('Test set accuracy: ', acc * 100) import numpy as np prediction = model.predict(np.expand_dims(x_test[0], axis = 0)) class_names = ['Negative', 'Positive'] print(class_names[np.argmax(prediction[0])]) #print(decode(x_test[0])) #[]
sl = SuperLearner([conv,rnn,rcnn],['cnn','rnn','rcnn'], loss='nloglik') # fit the super learner to learn the best coefficient num_batches = 0 num_samples = 0 y = [] y_pred = [] for batch_sample, batch_label in val_gen: num_batches += 1 if num_batches < 0: break if num_batches % 100 == 0: print ("processing %d batches" % num_batches) cnn_sample = batch_sample[:,-1,:].reshape((batch_size,1,-1,1)) y.append(utils.to_categorical(batch_label, num_classes=5)) cnn_predict = conv.predict(cnn_sample, batch_size = batch_size) rnn_predict = rnn.predict(batch_sample, batch_size = batch_size) rcnn_predict = rcnn.predict(batch_sample, batch_size = batch_size) predict = np.zeros((batch_size,3,5)) predict[:,0,:] = cnn_predict predict[:,1,:] = rnn_predict predict[:,2,:] = rcnn_predict y_pred.append(predict) y = np.asarray(y).reshape((-1,5)) y_pred = np.asarray(y_pred).reshape((-1,3,5)) sl.fit(y,y_pred) # test on the test set num_batches = 0 num_samples = 0
model.compile(loss=keras.losses.categorical_crossentropy,optimizer='adam',metrics=['accuracy']) history = LossHistory() ##fitting our sequential model model.fit(trainX, trainY, batch_size=100, epochs=100,#run with 50 epochs first to get 95% accuracy validation_split = 0.05, callbacks=[history]) history.loss_plot('epoch') history.acc_plot('epoch') test_dir = '../input/test/*.png' test_imgs=[] names = [] for timage in glob.glob(test_dir): img = Image.open(timage) names.append(timage.split('/')[-1]) test_imgs.append(ImageOps.fit(img,(32, 32),Image.ANTIALIAS).convert('RGB')) test_images = np.array([np.array(im) for im in test_imgs]) test_images_X = test_images.reshape(test_images.shape[0], 32, 32, 3) / 255 test_y = lb.inverse_transform(model.predict(test_images_X)) df = pd.DataFrame(data={'file': names, 'species': test_y}) df_sort = df.sort_values(by=['file']) df_sort.to_csv('results.csv', index=False) print(df_sort)
pr_label.append(ind[0]) pr_label=[(prt+1) for prt in pr_label] return pr_label # In[27]: model.load_weights("H5 weightLSTM/modeltsnippt.h5") # In[28]: pred = model.predict(x_train_pad) # In[29]: sniptrainlabel=pridct_label(pred) # In[30]: model.load_weights("H5 weightLSTM/descrips_text.h5") # In[31]:
x = np.linspace(0, value, N) dataset.append(x + noise) #dataset.append(x) (X_train, Y_train) = (np.asarray(dataset[0:int(ND * .8)]), slope[0:int(ND * .8)]) (X_test, Y_test) = (np.asarray(dataset[int(ND * .8):]), slope[int(ND * .8):]) model = Sequential() model.add( Dense(50, input_dim=N, kernel_initializer='normal', activation='relu')) model.add(Dense(10, activation='relu')) model.add(Dense(1, activation='linear')) model.summary() model.compile(loss='mse', optimizer='adam', metrics=['mse', 'mae']) history = model.fit(X_train, Y_train, epochs=20, batch_size=50, verbose=1) slope_cal = np.linspace(0, 1, 10) dataset_cal = [] for value in slope_cal: dataset_cal.append( np.linspace(0, value, N) + np.random.normal(0, Noise_Amp, N)) x_cal = np.asarray(dataset_cal) print('if working', slope_cal.reshape(-1, 1), 'will be the same as', model.predict(x_cal))
x_test = [[2, 1], [6, 5], [11, 6]] model = Sequential() model.add(Dense(units=y_column, input_dim=x_column, activation='sigmoid')) learning_rate = 0.1 sgd = SGD(lr=learning_rate) model.compile(loss='binary_crossentropy', optimizer=sgd) model.fit(x=x_train, y=y_train, epochs=2000, verbose=0) # 0: 강아지, 1: 고양이 def GetCategory(mydata): mylist = ['강아지', '고양이'] print(f'예측: {mydata}, {mylist[mydata[0]]}') # flatten() : 차원을 1차원으로 만들어 주는 함수 for item in x_test: H = model.predict(np.array([item])) print('예측 확률치:', H.flatten()) pred = (model.predict(np.array([item])) > 0.5).astype("int32") print('테스트 데이터:', np.array([item])) # print(pred.flatten()) GetCategory(pred.flatten())
def get_long_term(stock, no_of_days): mydb = mysql.connector.connect(host="localhost", user="******", passwd="<password>", database="appdata", auth_plugin='mysql_native_password') print("Stock: ", stock) mycursor = mydb.cursor() mycursor.execute("SELECT * from stocks where ticker='" + str(stock) + "';") result = mycursor.fetchall() print("Result: ", result) sid = result[0][0] mycursor.execute("SELECT * from historical where sid='" + str(sid) + "' order by dat desc limit 240;") result = mycursor.fetchall() result.reverse() window = 5 data = [] target = [] new_input = [] for i in range(len(result) - window): j = i input_batch = [] while j < i + window: input_batch.append(float(result[j][5])) j += 1 data.append(input_batch) target.append(float(result[j][5])) latest = result[-window:] for row in latest: new_input.append(float(row[5])) x = pd.DataFrame(data) y = pd.DataFrame(target) y = np.reshape(y, (-1, 1)) scaler_x = MinMaxScaler() scaler_y = MinMaxScaler() scaler_x.fit(x) xscale = scaler_x.transform(x) scaler_y.fit(y) yscale = scaler_y.transform(y) x_train, x_test, y_train, y_test = train_test_split(xscale, yscale) model = Sequential() model.add( Dense(10, input_dim=window, kernel_initializer='normal', activation='relu')) model.add(Dense(8, activation='relu')) model.add(Dense(no_of_days, activation='linear')) model.compile(loss='mse', optimizer='adam', metrics=['mse', 'mae']) history = model.fit(x_train, y_train, epochs=50, batch_size=50, verbose=0, validation_split=0.2) xnew = np.array([new_input]) xnew = scaler_x.transform(xnew) ynew = model.predict(xnew) ynew = scaler_y.inverse_transform(ynew) predictions = [] for y in ynew[0]: predictions.append(y) for prediction in predictions: target.append(prediction) ema_50 = calc_ema(target[-50:]) ema_200 = calc_ema(target[-200:]) macd = calc_macd(ema_50, ema_200) rsi = calc_rsi(target[-71:]) suggestion = get_suggestion(macd, rsi) prices, ema_vals, rsi_vals = [], [], [] for row in result: prices.append(float(row[5])) for p in predictions: prices.append(float(p)) i = 10 while i < len(prices): sample = prices[:i] ema_vals.append(calc_ema(sample)) rsi_vals.append(calc_rsi(sample)) i += 1 size = len(ema_vals) price = prices[-size:] return predictions, suggestion, price, ema_vals, rsi_vals
print('Iteration', iteration) model.fit(x, y, batch_size=128, epochs=1) start_index = random.randint(0, len(text) - maxlen - 1) for diversity in [0.2, 0.5, 1.0, 1.2]: print() print('----- diversity:', diversity) generated = '' sentence = text[start_index:start_index + maxlen] generated += sentence print('----- Generating with seed: "' + sentence + '"') sys.stdout.write(generated) for i in range(400): x_pred = np.zeros((1, maxlen, len(chars))) for t, char in enumerate(sentence): x_pred[0, t, char_indices[char]] = 1. preds = model.predict(x_pred, verbose=0)[0] next_index = sample(preds, diversity) next_char = indices_char[next_index] generated += next_char sentence = sentence[1:] + next_char sys.stdout.write(next_char) sys.stdout.flush() print()
return X_train, X_test, y_train, y_test X_train, X_test, y_train, y_test = train_test_split(df) dim = 36 model = Sequential() model.add(LSTM(dim, return_sequences=True, input_shape=(30, dim))) model.add(BatchNormalization()) model.add(LSTM(dim, return_sequences=True)) model.add(BatchNormalization()) # model.add(Dense(dim, activation='relu')) model.add(Dense(dim, activation='linear')) model.compile(loss="mean_squared_error", optimizer="rmsprop", metrics=['accuracy']) # model.fit(X_train, y_train, batch_size=20, nb_epoch=10, validation_split=0.05) model.fit(X_train, y_train, batch_size=10, epochs=5) print('len X_train: {}'.format(len(X_train))) print('len X_test: {}'.format(len(X_test))) predicted = model.predict(X_test) rmse = np.sqrt(((predicted - y_test)**2).mean(axis=0)) print('rmse: {}'.format(rmse)) print('len predictd: {}'.format(len(predicted))) print('len y_test: {}'.format(len(y_test)))
# Plot de mse plt.figure(figsize=(10, 8)) plt.plot(history.history['mean_squared_error']) #training plt.plot(history.history['val_mean_squared_error']) #validation plt.title('MSE') plt.ylabel("Mse") plt.yticks(np.arange(0, 900, 100)) plt.xlabel('Epoch') plt.legend(['train', 'validation'], loc='upper right') plt.savefig("Mse_5_12_12_1.png") plt.show() # Metrics evaluation mse_train = history.history["mean_squared_error"][-1] y_predict = ANN_Baseline.predict(X_test) from sklearn.metrics import mean_squared_error mse_test = mean_squared_error(y_test, y_predict) from sklearn.metrics import r2_score r2_test = r2_score(y_test, y_predict) r2_train = history.history["coeff_determination"][-1] metrics_data = np.array([mse_train, mse_test, r2_train, r2_test]).reshape(1, -1) metrics = pd.DataFrame( metrics_data, columns=["mse_train", "mse_test", "r2_train", "r2_test"])
very_first = first time_series_for_angle.append(first) time_series_for_angle.append(second) num += 1 number_times_to_run = len(time_series_for_angle) - 1 # build NN time series for angle NN_time_series = [] predict_single = np.zeros([1, 8]) predict_single[0] = very_first NN_time_series.append(very_first) x_1 = [] for i in range(number_times_to_run): predictions_1 = pendulum_NN_x1.predict(predict_single) predictions_2 = pendulum_NN_y1.predict(predict_single) predictions_3 = pendulum_NN_x2.predict(predict_single) predictions_4 = pendulum_NN_y2.predict(predict_single) predictions_5 = pendulum_NN_v1x.predict(predict_single) predictions_6 = pendulum_NN_v2x.predict(predict_single) predictions_7 = pendulum_NN_v1y.predict(predict_single) predictions_8 = pendulum_NN_v2y.predict(predict_single) predictions_output = [ predictions_1[0], predictions_2, predictions_3, predictions_4, predictions_5, predictions_6, predictions_7, predictions_8 ] predict_single[0] = predictions_output NN_time_series.append(predictions_output)
# # loss_and_metrics = model.evaluate(X_test, Y_test, verbose=2) # # print("Test Loss", loss_and_metrics[0]) # # print("Test Accuracy", loss_and_metrics[1] * 100, "%") new_model.save('MEXceptionModel.h5') print('model saved') # if (os.path.exists('MEvgg16Model.h5')): # new_model=load_model('MEvgg16Model.h5') # print('model loaded') print('predicting...') print(X_test.shape) # prediction = new_model.predict(test_data) prediction = new_model.predict(X_test) predicted_label = np.argmax(prediction, axis=1) predicted_label = predicted_label + 1 testId = [] # for img in os.listdir(TEST_DIR2): #kda 8alat # testId.append(img) sortedTest = [] for folder in os.listdir(TEST_DIR): sortedTest.append(folder) sortedTest.sort() for folder in sortedTest: folder_path = os.path.join(TEST_DIR, folder) for img in os.listdir(folder_path): testId.append(img)
class RNNGRU(object): """Process data for ingestion.""" def __init__( self, data, periods=288, batch_size=64, sequence_length=20, warmup_steps=50, epochs=20, display=False): """Instantiate the class. Args: data: Dict of values keyed by timestamp periods: Number of timestamp data points per vector batch_size: Size of batch sequence_length: Length of vectors for for each target warmup_steps: Returns: None """ # Initialize key variables self.periods = periods self.target_names = ['value'] self.warmup_steps = warmup_steps self.epochs = epochs self.batch_size = batch_size self.display = display ################################### # TensorFlow wizardry config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a total of half the GPU memory to be allocated config.gpu_options.per_process_gpu_memory_fraction = 0.95 # Crash with DeadlineExceeded instead of hanging forever when your # queues get full/empty config.operation_timeout_in_ms = 60000 # Create a session with the above options specified. backend.tensorflow_backend.set_session(tf.Session(config=config)) ################################### # Get data (x_data, y_data) = convert_data(data, periods, self.target_names) print('\n> Numpy Data Type: {}'.format(type(x_data))) print("> Numpy Data Shape: {}".format(x_data.shape)) print("> Numpy Data Row[0]: {}".format(x_data[0])) print('> Numpy Targets Type: {}'.format(type(y_data))) print("> Numpy Targets Shape: {}".format(y_data.shape)) ''' This is the number of observations (aka. data-points or samples) in the data-set: ''' num_data = len(x_data) ''' This is the fraction of the data-set that will be used for the training-set: ''' train_split = 0.9 ''' This is the number of observations in the training-set: ''' self.num_train = int(train_split * num_data) ''' This is the number of observations in the test-set: ''' num_test = num_data - self.num_train print('> Number of Samples: {}'.format(num_data)) print("> Number of Training Samples: {}".format(self.num_train)) print("> Number of Test Samples: {}".format(num_test)) # Create test and training data x_train = x_data[0:self.num_train] x_test = x_data[self.num_train:] self.y_train = y_data[0:self.num_train] self.y_test = y_data[self.num_train:] self.num_x_signals = x_data.shape[1] self.num_y_signals = y_data.shape[1] print("> Training Minimum Value:", np.min(x_train)) print("> Training Maximum Value:", np.max(x_train)) ''' steps_per_epoch is the number of batch iterations before a training epoch is considered finished. ''' self.steps_per_epoch = int(self.num_train / batch_size) + 1 print("> Epochs:", epochs) print("> Batch Size:", batch_size) print("> Steps:", self.steps_per_epoch) ''' Calculate the estimated memory footprint. ''' print("> Data size: {:.2f} Bytes".format(x_data.nbytes)) ''' if memory_footprint > 7: print('\n\n{}\n\n'.format( '> Estimated GPU memory usage too large. Use new parameters ' 'to reduce the footprint.')) sys.exit(0) ''' ''' The neural network works best on values roughly between -1 and 1, so we need to scale the data before it is being input to the neural network. We can use scikit-learn for this. We first create a scaler-object for the input-signals. Then we detect the range of values from the training-data and scale the training-data. ''' x_scaler = MinMaxScaler() self.x_train_scaled = x_scaler.fit_transform(x_train) print('> Scaled Training Minimum Value: {}'.format( np.min(self.x_train_scaled))) print('> Scaled Training Maximum Value: {}'.format( np.max(self.x_train_scaled))) self.x_test_scaled = x_scaler.transform(x_test) ''' The target-data comes from the same data-set as the input-signals, because it is the weather-data for one of the cities that is merely time-shifted. But the target-data could be from a different source with different value-ranges, so we create a separate scaler-object for the target-data. ''' self.y_scaler = MinMaxScaler() self.y_train_scaled = self.y_scaler.fit_transform(self.y_train) y_test_scaled = self.y_scaler.transform(self.y_test) # Data Generator ''' The data-set has now been prepared as 2-dimensional numpy arrays. The training-data has almost 300k observations, consisting of 20 input-signals and 3 output-signals. These are the array-shapes of the input and output data: ''' print('> Scaled Training Data Shape: {}'.format( self.x_train_scaled.shape)) print('> Scaled Training Targets Shape: {}'.format( self.y_train_scaled.shape)) # We then create the batch-generator. generator = self.batch_generator(batch_size, sequence_length) # Validation Set ''' The neural network trains quickly so we can easily run many training epochs. But then there is a risk of overfitting the model to the training-set so it does not generalize well to unseen data. We will therefore monitor the model's performance on the test-set after each epoch and only save the model's weights if the performance is improved on the test-set. The batch-generator randomly selects a batch of short sequences from the training-data and uses that during training. But for the validation-data we will instead run through the entire sequence from the test-set and measure the prediction accuracy on that entire sequence. ''' validation_data = (np.expand_dims(self.x_test_scaled, axis=0), np.expand_dims(y_test_scaled, axis=0)) # Create the Recurrent Neural Network self.model = Sequential() ''' We can now add a Gated Recurrent Unit (GRU) to the network. This will have 512 outputs for each time-step in the sequence. Note that because this is the first layer in the model, Keras needs to know the shape of its input, which is a batch of sequences of arbitrary length (indicated by None), where each observation has a number of input-signals (num_x_signals). ''' self.model.add(GRU( units=512, return_sequences=True, input_shape=(None, self.num_x_signals,))) ''' The GRU outputs a batch of sequences of 512 values. We want to predict 3 output-signals, so we add a fully-connected (or dense) layer which maps 512 values down to only 3 values. The output-signals in the data-set have been limited to be between 0 and 1 using a scaler-object. So we also limit the output of the neural network using the Sigmoid activation function, which squashes the output to be between 0 and 1.''' self.model.add(Dense(self.num_y_signals, activation='sigmoid')) ''' A problem with using the Sigmoid activation function, is that we can now only output values in the same range as the training-data. For example, if the training-data only has temperatures between -20 and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1. So if we limit the output of the neural network to be between 0 and 1 using the Sigmoid function, this can only be mapped back to temperature values between -20 and +30. We can use a linear activation function on the output instead. This allows for the output to take on arbitrary values. It might work with the standard initialization for a simple network architecture, but for more complicated network architectures e.g. with more layers, it might be necessary to initialize the weights with smaller values to avoid NaN values during training. You may need to experiment with this to get it working. ''' if False: # Maybe use lower init-ranges. # init = RandomUniform(minval=-0.05, maxval=0.05) init = RandomUniform(minval=-0.05, maxval=0.05) self.model.add(Dense( self.num_y_signals, activation='linear', kernel_initializer=init)) # Compile Model ''' This is the optimizer and the beginning learning-rate that we will use. We then compile the Keras model so it is ready for training. ''' optimizer = RMSprop(lr=1e-3) self.model.compile(loss=self.loss_mse_warmup, optimizer=optimizer) ''' This is a very small model with only two layers. The output shape of (None, None, 3) means that the model will output a batch with an arbitrary number of sequences, each of which has an arbitrary number of observations, and each observation has 3 signals. This corresponds to the 3 target signals we want to predict. ''' print('> Model Summary:\n') print(self.model.summary()) # Callback Functions ''' During training we want to save checkpoints and log the progress to TensorBoard so we create the appropriate callbacks for Keras. This is the callback for writing checkpoints during training. ''' path_checkpoint = '/tmp/23_checkpoint.keras' callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True) ''' This is the callback for stopping the optimization when performance worsens on the validation-set. ''' callback_early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1) ''' This is the callback for writing the TensorBoard log during training. ''' callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/', histogram_freq=0, write_graph=False) ''' This callback reduces the learning-rate for the optimizer if the validation-loss has not improved since the last epoch (as indicated by patience=0). The learning-rate will be reduced by multiplying it with the given factor. We set a start learning-rate of 1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4. We don't want the learning-rate to go any lower than this. ''' callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-4, patience=0, verbose=1) callbacks = [callback_early_stopping, callback_checkpoint, callback_tensorboard, callback_reduce_lr] # Train the Recurrent Neural Network '''We can now train the neural network. Note that a single "epoch" does not correspond to a single processing of the training-set, because of how the batch-generator randomly selects sub-sequences from the training-set. Instead we have selected steps_per_epoch so that one "epoch" is processed in a few minutes. With these settings, each "epoch" took about 2.5 minutes to process on a GTX 1070. After 14 "epochs" the optimization was stopped because the validation-loss had not decreased for 5 "epochs". This optimization took about 35 minutes to finish. Also note that the loss sometimes becomes NaN (not-a-number). This is often resolved by restarting and running the Notebook again. But it may also be caused by your neural network architecture, learning-rate, batch-size, sequence-length, etc. in which case you may have to modify those settings. ''' print('\n> Starting data training\n') try: self.model.fit_generator( generator=generator, epochs=self.epochs, steps_per_epoch=self.steps_per_epoch, validation_data=validation_data, callbacks=callbacks) except Exception as error: print('\n>{}\n'.format(error)) traceback.print_exc() sys.exit(0) # Load Checkpoint ''' Because we use early-stopping when training the model, it is possible that the model's performance has worsened on the test-set for several epochs before training was stopped. We therefore reload the last saved checkpoint, which should have the best performance on the test-set. ''' print('> Loading model weights') try: self.model.load_weights(path_checkpoint) except Exception as error: print('\n> Error trying to load checkpoint.\n\n{}'.format(error)) traceback.print_exc() sys.exit(0) # Performance on Test-Set ''' We can now evaluate the model's performance on the test-set. This function expects a batch of data, but we will just use one long time-series for the test-set, so we just expand the array-dimensionality to create a batch with that one sequence. ''' result = self.model.evaluate( x=np.expand_dims(self.x_test_scaled, axis=0), y=np.expand_dims(y_test_scaled, axis=0)) print('> Loss (test-set): {}'.format(result)) # If you have several metrics you can use this instead. if False: for res, metric in zip(result, self.model.metrics_names): print('{0}: {1:.3e}'.format(metric, res)) def batch_generator(self, batch_size, sequence_length): """Create generator function to create random batches of training-data. Args: batch_size: Size of batch sequence_length: Length of sequence Returns: (x_batch, y_batch) """ # Infinite loop. while True: # Allocate a new array for the batch of input-signals. x_shape = (batch_size, sequence_length, self.num_x_signals) x_batch = np.zeros(shape=x_shape, dtype=np.float16) # Allocate a new array for the batch of output-signals. y_shape = (batch_size, sequence_length, self.num_y_signals) y_batch = np.zeros(shape=y_shape, dtype=np.float16) # Fill the batch with random sequences of data. for i in range(batch_size): # Get a random start-index. # This points somewhere into the training-data. idx = np.random.randint(self.num_train - sequence_length) # Copy the sequences of data starting at this index. x_batch[i] = self.x_train_scaled[idx:idx+sequence_length] y_batch[i] = self.y_train_scaled[idx:idx+sequence_length] yield (x_batch, y_batch) def loss_mse_warmup(self, y_true, y_pred): """Calculate the Mean Squared Errror. Calculate the Mean Squared Error between y_true and y_pred, but ignore the beginning "warmup" part of the sequences. We will use Mean Squared Error (MSE) as the loss-function that will be minimized. This measures how closely the model's output matches the true output signals. However, at the beginning of a sequence, the model has only seen input-signals for a few time-steps, so its generated output may be very inaccurate. Using the loss-value for the early time-steps may cause the model to distort its later output. We therefore give the model a "warmup-period" of 50 time-steps where we don't use its accuracy in the loss-function, in hope of improving the accuracy for later time-steps Args: y_true: Desired output. y_pred: Model's output. Returns: loss_mean: Mean Squared Error """ warmup_steps = self.warmup_steps # The shape of both input tensors are: # [batch_size, sequence_length, num_y_signals]. # Ignore the "warmup" parts of the sequences # by taking slices of the tensors. y_true_slice = y_true[:, warmup_steps:, :] y_pred_slice = y_pred[:, warmup_steps:, :] # These sliced tensors both have this shape: # [batch_size, sequence_length - warmup_steps, num_y_signals] # Calculate the MSE loss for each value in these tensors. # This outputs a 3-rank tensor of the same shape. loss = tf.losses.mean_squared_error(labels=y_true_slice, predictions=y_pred_slice) # Keras may reduce this across the first axis (the batch) # but the semantics are unclear, so to be sure we use # the loss across the entire tensor, we reduce it to a # single scalar with the mean function. loss_mean = tf.reduce_mean(loss) return loss_mean def plot_comparison(self, start_idx, length=100, train=True): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. train: Boolean whether to use training- or test-set. Returns: None """ if train: # Use training-data. x_values = self.x_train_scaled y_true = self.y_train shim = 'Train' else: # Use test-data. x_values = self.x_test_scaled y_true = self.y_test shim = 'Test' # End-index for the sequences. end_idx = start_idx + length # Select the sequences from the given start-index and # of the given length. x_values = x_values[start_idx:end_idx] y_true = y_true[start_idx:end_idx] # Input-signals for the model. x_values = np.expand_dims(x_values, axis=0) # Use the model to predict the output-signals. y_pred = self.model.predict(x_values) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. y_pred_rescaled = self.y_scaler.inverse_transform(y_pred[0]) # For each output-signal. for signal in range(len(self.target_names)): # Create a filename filename = ( '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format( self.batch_size, self.epochs, self.num_train, signal, int(time.time()), shim) # Get the output-signal predicted by the model. signal_pred = y_pred_rescaled[:, signal] # Get the true output-signal from the data-set. signal_true = y_true[:, signal] # Make the plotting-canvas bigger. plt.figure(figsize=(15, 5)) # Plot and compare the two signals. plt.plot(signal_true, label='true') plt.plot(signal_pred, label='pred') # Plot grey box for warmup-period. _ = plt.axvspan( 0, self.warmup_steps, facecolor='black', alpha=0.15) # Plot labels etc. plt.ylabel(self.target_names[signal]) plt.legend() # Show and save the image if self.display is True: plt.savefig(filename, bbox_inches='tight') plt.show() else: plt.savefig(filename, bbox_inches='tight') print('> Saving file: {}'.format(filename))
class RNNGRU(object): """Support vector machine class.""" def __init__(self, batch_size=64, sequence_length=20, warmup_steps=50, epochs=20, display=False): """Instantiate the class. Args: batch_size: Size of batch sequence_length: Length of vectors for for each target save: Save charts if True Returns: None """ # Initialize key variables self.target_names = ['Temp', 'WindSpeed', 'Pressure'] self.warmup_steps = warmup_steps self.epochs = epochs self.batch_size = batch_size self.display = display # Get data x_data, y_data = self.data() print('\n> Numpy Data Type: {}'.format(type(x_data))) print("> Numpy Data Shape: {}".format(x_data.shape)) print("> Numpy Data Row[0]: {}".format(x_data[0])) print('> Numpy Targets Type: {}'.format(type(y_data))) print("> Numpy Targets Shape: {}".format(y_data.shape)) ''' This is the number of observations (aka. data-points or samples) in the data-set: ''' num_data = len(x_data) ''' This is the fraction of the data-set that will be used for the training-set: ''' train_split = 0.9 ''' This is the number of observations in the training-set: ''' self.num_train = int(train_split * num_data) ''' This is the number of observations in the test-set: ''' num_test = num_data - self.num_train print('> Number of Samples: {}'.format(num_data)) print("> Number of Training Samples: {}".format(self.num_train)) print("> Number of Test Samples: {}".format(num_test)) print("> Batch Size: {}".format(batch_size)) steps_per_epoch = int(self.num_train/batch_size) print("> Recommended Epoch Steps: {:.2f}".format(steps_per_epoch)) # Create test and training data x_train = x_data[0:self.num_train] x_test = x_data[self.num_train:] self.y_train = y_data[0:self.num_train] self.y_test = y_data[self.num_train:] self.num_x_signals = x_data.shape[1] self.num_y_signals = y_data.shape[1] print("> Training Minimum Value:", np.min(x_train)) print("> Training Maximum Value:", np.max(x_train)) ''' The neural network works best on values roughly between -1 and 1, so we need to scale the data before it is being input to the neural network. We can use scikit-learn for this. We first create a scaler-object for the input-signals. Then we detect the range of values from the training-data and scale the training-data. ''' x_scaler = MinMaxScaler() self.x_train_scaled = x_scaler.fit_transform(x_train) print('> Scaled Training Minimum Value: {}'.format( np.min(self.x_train_scaled))) print('> Scaled Training Maximum Value: {}'.format( np.max(self.x_train_scaled))) self.x_test_scaled = x_scaler.transform(x_test) ''' The target-data comes from the same data-set as the input-signals, because it is the weather-data for one of the cities that is merely time-shifted. But the target-data could be from a different source with different value-ranges, so we create a separate scaler-object for the target-data. ''' self.y_scaler = MinMaxScaler() self.y_train_scaled = self.y_scaler.fit_transform(self.y_train) y_test_scaled = self.y_scaler.transform(self.y_test) # Data Generator ''' The data-set has now been prepared as 2-dimensional numpy arrays. The training-data has almost 300k observations, consisting of 20 input-signals and 3 output-signals. These are the array-shapes of the input and output data: ''' print('> Scaled Training Data Shape: {}'.format( self.x_train_scaled.shape)) print('> Scaled Training Targets Shape: {}'.format( self.y_train_scaled.shape)) # We then create the batch-generator. generator = self.batch_generator(batch_size, sequence_length) # Validation Set ''' The neural network trains quickly so we can easily run many training epochs. But then there is a risk of overfitting the model to the training-set so it does not generalize well to unseen data. We will therefore monitor the model's performance on the test-set after each epoch and only save the model's weights if the performance is improved on the test-set. The batch-generator randomly selects a batch of short sequences from the training-data and uses that during training. But for the validation-data we will instead run through the entire sequence from the test-set and measure the prediction accuracy on that entire sequence. ''' validation_data = (np.expand_dims(self.x_test_scaled, axis=0), np.expand_dims(y_test_scaled, axis=0)) # Create the Recurrent Neural Network self.model = Sequential() ''' We can now add a Gated Recurrent Unit (GRU) to the network. This will have 512 outputs for each time-step in the sequence. Note that because this is the first layer in the model, Keras needs to know the shape of its input, which is a batch of sequences of arbitrary length (indicated by None), where each observation has a number of input-signals (num_x_signals). ''' self.model.add(GRU( units=512, return_sequences=True, input_shape=(None, self.num_x_signals,))) ''' The GRU outputs a batch of sequences of 512 values. We want to predict 3 output-signals, so we add a fully-connected (or dense) layer which maps 512 values down to only 3 values. The output-signals in the data-set have been limited to be between 0 and 1 using a scaler-object. So we also limit the output of the neural network using the Sigmoid activation function, which squashes the output to be between 0 and 1.''' self.model.add(Dense(self.num_y_signals, activation='sigmoid')) ''' A problem with using the Sigmoid activation function, is that we can now only output values in the same range as the training-data. For example, if the training-data only has temperatures between -20 and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1. So if we limit the output of the neural network to be between 0 and 1 using the Sigmoid function, this can only be mapped back to temperature values between -20 and +30. We can use a linear activation function on the output instead. This allows for the output to take on arbitrary values. It might work with the standard initialization for a simple network architecture, but for more complicated network architectures e.g. with more layers, it might be necessary to initialize the weights with smaller values to avoid NaN values during training. You may need to experiment with this to get it working. ''' if False: # Maybe use lower init-ranges. init = RandomUniform(minval=-0.05, maxval=0.05) self.model.add(Dense( self.num_y_signals, activation='linear', kernel_initializer=init)) # Compile Model ''' This is the optimizer and the beginning learning-rate that we will use. We then compile the Keras model so it is ready for training. ''' optimizer = RMSprop(lr=1e-3) self.model.compile(loss=self.loss_mse_warmup, optimizer=optimizer) ''' This is a very small model with only two layers. The output shape of (None, None, 3) means that the model will output a batch with an arbitrary number of sequences, each of which has an arbitrary number of observations, and each observation has 3 signals. This corresponds to the 3 target signals we want to predict. ''' print('> Model Summary:\n') print(self.model.summary()) # Callback Functions ''' During training we want to save checkpoints and log the progress to TensorBoard so we create the appropriate callbacks for Keras. This is the callback for writing checkpoints during training. ''' path_checkpoint = '/tmp/23_checkpoint.keras' callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True) ''' This is the callback for stopping the optimization when performance worsens on the validation-set. ''' callback_early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1) ''' This is the callback for writing the TensorBoard log during training. ''' callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/', histogram_freq=0, write_graph=False) ''' This callback reduces the learning-rate for the optimizer if the validation-loss has not improved since the last epoch (as indicated by patience=0). The learning-rate will be reduced by multiplying it with the given factor. We set a start learning-rate of 1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4. We don't want the learning-rate to go any lower than this. ''' callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-4, patience=0, verbose=1) callbacks = [callback_early_stopping, callback_checkpoint, callback_tensorboard, callback_reduce_lr] # Train the Recurrent Neural Network '''We can now train the neural network. Note that a single "epoch" does not correspond to a single processing of the training-set, because of how the batch-generator randomly selects sub-sequences from the training-set. Instead we have selected steps_per_epoch so that one "epoch" is processed in a few minutes. With these settings, each "epoch" took about 2.5 minutes to process on a GTX 1070. After 14 "epochs" the optimization was stopped because the validation-loss had not decreased for 5 "epochs". This optimization took about 35 minutes to finish. Also note that the loss sometimes becomes NaN (not-a-number). This is often resolved by restarting and running the Notebook again. But it may also be caused by your neural network architecture, learning-rate, batch-size, sequence-length, etc. in which case you may have to modify those settings. ''' self.model.fit_generator( generator=generator, epochs=self.epochs, steps_per_epoch=steps_per_epoch, validation_data=validation_data, callbacks=callbacks) # Load Checkpoint ''' Because we use early-stopping when training the model, it is possible that the model's performance has worsened on the test-set for several epochs before training was stopped. We therefore reload the last saved checkpoint, which should have the best performance on the test-set. ''' try: self.model.load_weights(path_checkpoint) except Exception as error: print('\n> Error trying to load checkpoint.\n\n{}'.format(error)) sys.exit(0) # Performance on Test-Set ''' We can now evaluate the model's performance on the test-set. This function expects a batch of data, but we will just use one long time-series for the test-set, so we just expand the array-dimensionality to create a batch with that one sequence. ''' result = self.model.evaluate( x=np.expand_dims(self.x_test_scaled, axis=0), y=np.expand_dims(y_test_scaled, axis=0)) print('> Loss (test-set): {}'.format(result)) # If you have several metrics you can use this instead. if False: for res, metric in zip(result, self.model.metrics_names): print('{0}: {1:.3e}'.format(metric, res)) def batch_generator(self, batch_size, sequence_length): """Generator function for creating random batches of training-data. Args: batch_size: Size of batch sequence_length: Length of sequence Returns: (x_batch, y_batch) """ # Infinite loop. while True: # Allocate a new array for the batch of input-signals. x_shape = (batch_size, sequence_length, self.num_x_signals) x_batch = np.zeros(shape=x_shape, dtype=np.float16) # Allocate a new array for the batch of output-signals. y_shape = (batch_size, sequence_length, self.num_y_signals) y_batch = np.zeros(shape=y_shape, dtype=np.float16) # Fill the batch with random sequences of data. for i in range(batch_size): # Get a random start-index. # This points somewhere into the training-data. idx = np.random.randint(self.num_train - sequence_length) # Copy the sequences of data starting at this index. x_batch[i] = self.x_train_scaled[idx:idx+sequence_length] y_batch[i] = self.y_train_scaled[idx:idx+sequence_length] yield (x_batch, y_batch) def plot_comparison(self, start_idx, length=100, train=True): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. train: Boolean whether to use training- or test-set. Returns: None """ if train: # Use training-data. x_values = self.x_train_scaled y_true = self.y_train shim = 'Train' else: # Use test-data. x_values = self.x_test_scaled y_true = self.y_test shim = 'Test' # End-index for the sequences. end_idx = start_idx + length # Select the sequences from the given start-index and # of the given length. x_values = x_values[start_idx:end_idx] y_true = y_true[start_idx:end_idx] # Input-signals for the model. x_values = np.expand_dims(x_values, axis=0) # Use the model to predict the output-signals. y_pred = self.model.predict(x_values) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. y_pred_rescaled = self.y_scaler.inverse_transform(y_pred[0]) # For each output-signal. for signal in range(len(self.target_names)): # Create a filename filename = ( '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format( self.batch_size, self.epochs, self.num_train, signal, int(time.time()), shim) # Get the output-signal predicted by the model. signal_pred = y_pred_rescaled[:, signal] # Get the true output-signal from the data-set. signal_true = y_true[:, signal] # Make the plotting-canvas bigger. plt.figure(figsize=(15, 5)) # Plot and compare the two signals. plt.plot(signal_true, label='true') plt.plot(signal_pred, label='pred') # Plot grey box for warmup-period. _ = plt.axvspan( 0, self.warmup_steps, facecolor='black', alpha=0.15) # Plot labels etc. plt.ylabel(self.target_names[signal]) plt.legend() # Show and save the image if self.display is True: plt.savefig(filename, bbox_inches='tight') plt.show() else: plt.savefig(filename, bbox_inches='tight') print('> Saving file: {}'.format(filename)) def data(self): """Get data to analyze. Args: None Returns: (x_data, y_data): X and Y values as numpy arrays """ # Download data weather.maybe_download_and_extract() # Import data into Pandas dataframe pandas_df = weather.load_resampled_data() print('\n> First Rows of Data:\n\n{}'.format(pandas_df.head(3))) # Print the cities cities = weather.cities print('\n> Cities: {}'.format(cities)) # Print dataframe shape print( '> Dataframe shape (Original): {}'.format(pandas_df.values.shape)) # The two signals that have missing data. (Columns with Nans) pandas_df.drop(('Esbjerg', 'Pressure'), axis=1, inplace=True) pandas_df.drop(('Roskilde', 'Pressure'), axis=1, inplace=True) # Print dataframe shape print('> Dataframe shape (New): {}'.format(pandas_df.values.shape)) # Verify that the columns have been dropped print( '\n> First Rows of Updated Data:\n\n{}'.format(pandas_df.head(1))) # Add Data ''' We can add some input-signals to the data that may help our model in making predictions. For example, given just a temperature of 10 degrees Celcius the model wouldn't know whether that temperature was measured during the day or the night, or during summer or winter. The model would have to infer this from the surrounding data-points which might not be very accurate for determining whether it's an abnormally warm winter, or an abnormally cold summer, or whether it's day or night. So having this information could make a big difference in how accurately the model can predict the next output. Although the data-set does contain the date and time information for each observation, it is only used in the index so as to order the data. We will therefore add separate input-signals to the data-set for the day-of-year (between 1 and 366) and the hour-of-day (between 0 and 23). ''' pandas_df['Various', 'Day'] = pandas_df.index.dayofyear pandas_df['Various', 'Hour'] = pandas_df.index.hour # Target Data for Prediction ''' We will try and predict the future weather-data for this city. ''' target_city = 'Odense' ''' We will try and predict these signals. ''' self.target_names = ['Temp', 'WindSpeed', 'Pressure'] ''' The following is the number of time-steps that we will shift the target-data. Our data-set is resampled to have an observation for each hour, so there are 24 observations for 24 hours. If we want to predict the weather 24 hours into the future, we shift the data 24 time-steps. If we want to predict the weather 7 days into the future, we shift the data 7 * 24 time-steps. ''' shift_days = 1 shift_steps = shift_days * 24 # Number of hours. # Create a new data-frame with the time-shifted data. ''' Note the negative time-shift! We want the future state targets to line up with the timestamp of the last value of each sample set. ''' df_targets = pandas_df[ target_city][self.target_names].shift(-shift_steps) ''' WARNING! You should double-check that you have shifted the data in the right direction! We want to predict the future, not the past! The shifted data-frame is confusing because Pandas keeps the original This is the first shift_steps + 5 rows of the original data-frame: ''' explanatory_hours = shift_steps + 5 print('\n> First Rows of Updated Data ({} hours):\n\n{}'.format( explanatory_hours, pandas_df[target_city][self.target_names].head(explanatory_hours))) ''' The following is the first 5 rows of the time-shifted data-frame. This should be identical to the last 5 rows shown above from the original data, except for the time-stamp. ''' print('\n> First Rows of Shifted Data - Target Labels ' '(Notice 1980 Dates):\n\n{}'.format(df_targets.head(5))) ''' The time-shifted data-frame has the same length as the original data-frame, but the last observations are NaN (not a number) because the data has been shifted backwards so we are trying to shift data that does not exist in the original data-frame. ''' print('\n> Last Rows of Shifted Data - Target Labels ' '(Notice 2018 Dates):\n\n{}'.format(df_targets.tail())) # NumPy Arrays ''' We now convert the Pandas data-frames to NumPy arrays that can be input to the neural network. We also remove the last part of the numpy arrays, because the target-data has NaN for the shifted period, and we only want to have valid data and we need the same array-shapes for the input- and output-data. These are the input-signals: ''' x_data = pandas_df.values[0:-shift_steps] y_data = df_targets.values[:-shift_steps] # Return return (x_data, y_data) def loss_mse_warmup(self, y_true, y_pred): """Calculate the Mean Squared Errror. Calculate the Mean Squared Error between y_true and y_pred, but ignore the beginning "warmup" part of the sequences. We will use Mean Squared Error (MSE) as the loss-function that will be minimized. This measures how closely the model's output matches the true output signals. However, at the beginning of a sequence, the model has only seen input-signals for a few time-steps, so its generated output may be very inaccurate. Using the loss-value for the early time-steps may cause the model to distort its later output. We therefore give the model a "warmup-period" of 50 time-steps where we don't use its accuracy in the loss-function, in hope of improving the accuracy for later time-steps Args: y_true: Desired output. y_pred: Model's output. Returns: loss_mean: Mean Squared Error """ warmup_steps = self.warmup_steps # The shape of both input tensors are: # [batch_size, sequence_length, num_y_signals]. # Ignore the "warmup" parts of the sequences # by taking slices of the tensors. y_true_slice = y_true[:, warmup_steps:, :] y_pred_slice = y_pred[:, warmup_steps:, :] # These sliced tensors both have this shape: # [batch_size, sequence_length - warmup_steps, num_y_signals] # Calculate the MSE loss for each value in these tensors. # This outputs a 3-rank tensor of the same shape. loss = tf.losses.mean_squared_error(labels=y_true_slice, predictions=y_pred_slice) # Keras may reduce this across the first axis (the batch) # but the semantics are unclear, so to be sure we use # the loss across the entire tensor, we reduce it to a # single scalar with the mean function. loss_mean = tf.reduce_mean(loss) return loss_mean