def predict(): # normalization scaler = MinMaxScaler() records['power'] = scaler.fit_transform(records['power']) saver = tf.train.Saver() with tf.Session() as sess: # restore model saver.restore(sess, model_path) test_data = get_cycle_time_batch_data(records, batch_size, cycle_timesteps) test_y_list = [] test_y_pre_list = [] test_all_loss = [] for batch in test_data: predict, loss = sess.run([y_pre, loss_func], feed_dict = {cycle_model.X_ : batch[0], y : batch[1]}) test_y_list.extend(batch[1]) test_y_pre_list.extend(predict) test_all_loss.append(loss) # display test_x = list(range(len(test_y_list))) # inverse normalization test_y_list = scaler.inverse_transform(test_y_list) test_y_pre_list = scaler.inverse_transform(test_y_pre_list) test_y_list = np.array(test_y_list) test_y_pre_list = np.array(test_y_pre_list) mse = np.mean( (test_y_list - test_y_pre_list) ** 2) print('---------------- Test Loss:', np.mean(test_all_loss), 'MSE:', mse) plt.plot(test_x, test_y_list, 'r', test_x, test_y_pre_list, 'b') plt.show()
def test_min_max_scaler_iris(): X = iris.data scaler = MinMaxScaler() # default params X_trans = scaler.fit_transform(X) assert_array_almost_equal(X_trans.min(axis=0), 0) assert_array_almost_equal(X_trans.min(axis=0), 0) assert_array_almost_equal(X_trans.max(axis=0), 1) X_trans_inv = scaler.inverse_transform(X_trans) assert_array_almost_equal(X, X_trans_inv) # not default params: min=1, max=2 scaler = MinMaxScaler(feature_range=(1, 2)) X_trans = scaler.fit_transform(X) assert_array_almost_equal(X_trans.min(axis=0), 1) assert_array_almost_equal(X_trans.max(axis=0), 2) X_trans_inv = scaler.inverse_transform(X_trans) assert_array_almost_equal(X, X_trans_inv) # min=-.5, max=.6 scaler = MinMaxScaler(feature_range=(-.5, .6)) X_trans = scaler.fit_transform(X) assert_array_almost_equal(X_trans.min(axis=0), -.5) assert_array_almost_equal(X_trans.max(axis=0), .6) X_trans_inv = scaler.inverse_transform(X_trans) assert_array_almost_equal(X, X_trans_inv) # raises on invalid range scaler = MinMaxScaler(feature_range=(2, 1)) assert_raises(ValueError, scaler.fit, X)
def sample_from_generator(history, nb_samples, latent_dim=12, valid_split=0.3, random_split=True, hidden_dims=None, **kwargs): scaler = MinMaxScaler() scaler.fit(history) scaled = scaler.transform(history) nb_train = history.shape[0] if not valid_split: nb_valid = 0 elif isinstance(valid_split, float): nb_valid = nb_train - int(np.floor(nb_train*valid_split)) else: nb_valid = valid_split if nb_valid > 0: if random_split: ind = np.arange(nb_train) np.random.shuffle(ind) x_valid = scaled[ind[-nb_valid:], :] x_train = scaled[ind[:-nb_valid], :] else: x_valid = scaled[-nb_valid:, :] x_train = scaled[:-nb_valid, :] else: x_valid = None x_train = scaled _, generator = build_model(latent_dim, x_train, x_valid=x_valid, hidden_dims=hidden_dims, **kwargs) normal_sample = np.random.standard_normal((nb_samples, latent_dim)) draws = generator.predict(normal_sample) return scaler.inverse_transform(draws)
def test_min_max_scaler_zero_variance_features(): """Check min max scaler on toy data with zero variance features""" X = [[0., 1., 0.5], [0., 1., -0.1], [0., 1., 1.1]] X_new = [[+0., 2., 0.5], [-1., 1., 0.0], [+0., 1., 1.5]] # default params scaler = MinMaxScaler() X_trans = scaler.fit_transform(X) X_expected_0_1 = [[0., 0., 0.5], [0., 0., 0.0], [0., 0., 1.0]] assert_array_almost_equal(X_trans, X_expected_0_1) X_trans_inv = scaler.inverse_transform(X_trans) assert_array_almost_equal(X, X_trans_inv) X_trans_new = scaler.transform(X_new) X_expected_0_1_new = [[+0., 1., 0.500], [-1., 0., 0.083], [+0., 0., 1.333]] assert_array_almost_equal(X_trans_new, X_expected_0_1_new, decimal=2) # not default params scaler = MinMaxScaler(feature_range=(1, 2)) X_trans = scaler.fit_transform(X) X_expected_1_2 = [[1., 1., 1.5], [1., 1., 1.0], [1., 1., 2.0]] assert_array_almost_equal(X_trans, X_expected_1_2)
def one_input_LSTM(dataset): # normalize the dataset scaler = MinMaxScaler(feature_range=(0, 1)) dataset = scaler.fit_transform(dataset) # split into train and test sets train_size = int(len(dataset) * 0.67) test_size = len(dataset) - train_size train, test = dataset[0:train_size,:], dataset[train_size:len(dataset),:] # reshape into X=t and Y=t+1 look_back = 1 trainX, trainY = create_dataset(train, look_back) testX, testY = create_dataset(test, look_back) # reshape input to be [samples, time steps, features] trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1])) # create and fit the LSTM network model = Sequential() model.add(LSTM(4, input_shape=(1, look_back))) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(trainX, trainY, epochs=20, batch_size=1, verbose=2) # make predictions trainPredict = model.predict(trainX) testPredict = model.predict(testX) # invert predictions trainPredict = scaler.inverse_transform(trainPredict) trainY = scaler.inverse_transform([trainY]) testPredict = scaler.inverse_transform(testPredict) testY = scaler.inverse_transform([testY]) # calculate root mean squared error trainScore = math.sqrt(mean_squared_error(trainY[0], trainPredict[:,0])) print('Train Score: %.2f RMSE' % (trainScore)) testScore = math.sqrt(mean_squared_error(testY[0], testPredict[:,0])) print('Test Score: %.2f RMSE' % (testScore)) # shift train predictions for plotting trainPredictPlot = np.empty_like(dataset) trainPredictPlot[:, :] = np.nan trainPredictPlot[look_back:len(trainPredict)+look_back, :] = trainPredict # shift test predictions for plotting testPredictPlot = np.empty_like(dataset) testPredictPlot[:, :] = np.nan testPredictPlot[len(trainPredict)+(look_back*2)+1:len(dataset)-1, :] = testPredict
class SerialDataScaler: def __init__(self, data): data = numpy.reshape(data, (len(data), 1)) data = data.astype("float32") self.scaler = MinMaxScaler(feature_range=(0, 1)) self.scaler.fit(data) def transform(self, X): #return X return self.scaler.transform(numpy.reshape(X, (len(X), 1))) def inverse_transform(self, x): return self.scaler.inverse_transform(x)
def get_net_prediction( train_data, train_truth, test_data, test_truth , hidden=(5,), weight_decay=0.0, dropout_prob=0.0 , learning_rate=None, epochs=25, verbose=False , iter_id=None ): container = NeuralNetContainer() container.learning_rate = learning_rate container.dropout_prob = dropout_prob container.weight_decay = weight_decay container.epochs = epochs container.hidden_layers = hidden container.verbose = verbose container.plot = get_should_plot() mms = MinMaxScaler(feature_range= (-1, 1)) # Scale output from -1 to 1. train_y = mms.fit_transform(train_truth[:,np.newaxis]) n_features = train_data.shape[1] collect_time_stats = get_is_time_stats() if collect_time_stats: start = time.time() # Find and return an effectively initialized network to start. container = _get_initial_net(container, n_features, train_data, train_y) # Train the network. if collect_time_stats: # Train a specific time, never terminating early. _train_net(container, train_data, train_y, override_epochs=TIMING_EPOCHS, is_check_train=False) else: # Normal training, enable all heuristics. _train_net(container, train_data, train_y) if collect_time_stats: end = time.time() print('Fitting took {} seconds'.format(end - start)) print(json.dumps({'seconds': end - start, 'hidden': container.hidden_layers})) # Unsupervised (test) dataset. predicted = _predict(container, test_data) predicted = mms.inverse_transform(predicted) return predicted.ravel()
def get_fast_nn_dom_prediction(train_data, train_truth, test_data, test_truth, hidden=(5,), weight_decay=0.0): # Convert data to individual alleles to capture dominance. train_data, test_data = tuple(map(_convert_to_individual_alleles, [train_data, test_data])) scaler = MinMaxScaler(feature_range = (-1, 1)) train_truth = scaler.fit_transform(train_truth) test_truth = scaler.transform(test_truth) net = _get_nn(train_data.shape[1], hidden) _train_nn(net, train_data, train_truth, weight_decay) out = [] for i in range(test_data.shape[0]): sample = test_data[i,:] res = net.run(sample) out.append(res) predicted = scaler.inverse_transform(np.array(out)) return predicted.ravel()
# model.load_weights('weights/bitcoin2012_2017_50_30_weights.hdf5') # model.compile(loss='mse', optimizer='adam') # scaler = MinMaxScaler(feature_range=(0, 1)) # scaler.fit(datas.reshape(-1)) # predicted_inverted = scaler.inverse_transform(predicted) # ground_true = scaler.inverse_transform(validation_next_price) # In[6]: predicted = model.predict(validation_datas) predicted_inverted = [] # In[7]: for i in range(original_datas.shape[1]): scaler.fit(original_datas[:,i].reshape(-1,1)) predicted_inverted.append(scaler.inverse_transform(predicted[:,:,i])) #get only the close data ground_true = ground_true[:,:,0].reshape(-1) output_times = output_times.reshape(-1) predicted_inverted = np.array(predicted_inverted)[:,:,0].reshape(-1) # In[8]: print output_times.shape, ground_true.shape plt.plot(output_times[-1000:],ground_true[-1000:]) plt.plot(output_times[-1000:],predicted_inverted[-1000:])
#we fit it on the training data and give it the chloride as predicted variable model = lm.fit(train_X, train_y) #make you predction on the test data (all the inputs expet for the chloride) predictions = lm.predict(test_X) ####Now we will mapp back to the original data set predictions = predictions.reshape((predictions.shape[0], 1)) # invert scaling for test data #take only the last number of features of the test data set #start should be the length of the variable "train" - the number of features "n_features" start=750 #end should be the length of the variable "train" end =900 ####we merge the predicted chloride with the rest of the data set inv_yhat = concatenate(( test_X[: , 750:900], predictions), axis=1) ##we invers the scaling inv_yhat = scaler.inverse_transform(inv_yhat) #we take the inversed scaling of the predicted chloride inv_yhat = inv_yhat[:,149] # invert scaling for real chloride test_y = test_y.reshape((len(test_y), 1)) #we merge the real chloride with the rest of the data set inv_y = concatenate(( test_X[: , 750:900],test_y), axis=1) # we invers the scaling inv_y = scaler.inverse_transform(inv_y) #we take the inversed scaling of the real chloride inv_y = inv_y[:,149] # calculate RMSE rmse = sqrt(mean_squared_error(inv_y, inv_yhat)) print('Test RMSE: %.3f' % rmse) #Calculate R squared from sklearn.metrics import r2_score
dataset_total = dataset_all[len(dataset_all) - (timesteps + testing_records):] inputs = dataset_total.iloc[:, 1:7].values #inputs = np.reshape(-1, 1) inputs = scale.transform(inputs) X_test = [] #y_test = [] for i in range(timesteps, inputs.shape[0]): X_test.append(inputs[i - timesteps:i, 0:6]) #y_test.append(inputs[i, 3:4]) y_scale = MinMaxScaler(feature_range=(0, 1)) y_training_set_scaled = y_scale.fit_transform(training_set[:, 3:4]) X_test = np.array(X_test) #X_test = np.reshape(X_test, (X_test.shape[0], timesteps, 6)) X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 6)) predicted_price = regressor.predict(X_test) predicted_price = y_scale.inverse_transform(predicted_price) real_stock_price = dataset_test.iloc[:, 3:4].values plt.plot(real_stock_price, color='red', label='Real Stock Price') plt.plot(predicted_price, color='blue', label='Predicted Stock Price') plt.title('Stock Price Prediction') plt.xlabel('Time') plt.ylabel('Stock Price') plt.legend() plt.show() #with tf.device('/gpu:0'):
model_JPM.add(LSTM(units=50)) model_JPM.add(Dropout(0.1)) model_JPM.add(Dense(units=1)) model_JPM.compile(optimizer ='adam', metrics = ['mse'], loss = 'mean_absolute_error') #fitting(epoch) model_JPM.fit(x_train, y_train, epochs = 200, batch_size = 62) x_test=[] y_test=[] for i in range(2001, len(JPM_stock)): x_test.append(newdata_scaled[i-30:i,0]) y_test.append(newdata_scaled[i,0]) x_test, y_test = np.array(x_test), np.array(y_test) x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1)) forecast_stock = model_JPM.predict(x_test) fore = sc.inverse_transform(np.array(forecast_stock).reshape(-1,1)) actual = sc.inverse_transform(np.array(y_test).reshape(-1,1)) plt.plot(actual, color='red', label='Actual price') plt.plot(fore, color='blue', label='forecasted price') plt.title('JPM stock prediction') plt.ylabel('Price') plt.xlabel('Time') plt.legend() plt.show()
look_back = 3 trainX, trainY = create_dataset(train, look_back) testX, testY = create_dataset(test, look_back) # reshape input to be [samples, time steps, features] trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1])) # create and fit the LSTM network model = Sequential() model.add(LSTM(4, input_dim=look_back)) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(trainX, trainY, nb_epoch=100, batch_size=1, verbose=2) # Estimate model performance trainScore = model.evaluate(trainX, trainY, verbose=0) trainScore = math.sqrt(trainScore) trainScore = scaler.inverse_transform(numpy.array([[trainScore]])) print('Train Score: %.2f RMSE' % (trainScore)) testScore = model.evaluate(testX, testY, verbose=0) testScore = math.sqrt(testScore) testScore = scaler.inverse_transform(numpy.array([[testScore]])) print('Test Score: %.2f RMSE' % (testScore)) # generate predictions for training trainPredict = model.predict(trainX) testPredict = model.predict(testX) # shift train predictions for plotting trainPredictPlot = numpy.empty_like(dataset) trainPredictPlot[:, :] = numpy.nan trainPredictPlot[look_back:len(trainPredict)+look_back, :] = trainPredict # shift test predictions for plotting testPredictPlot = numpy.empty_like(dataset) testPredictPlot[:, :] = numpy.nan
model.add(Dropout(0.5)) model.add(Conv1D( strides=2, filters=nb_features, kernel_size=2)) ''' model.load_weights( 'weights/bitcoin2015to2017_close_CNN_2_relu-44-0.00023.hdf5') model.compile(loss='mse', optimizer='adam') # In[336]: predicted = model.predict(validation_datas) predicted_inverted = [] # In[7]: for i in range(original_datas.shape[1]): scaler.fit(original_datas[:, i].reshape(-1, 1)) predicted_inverted.append(scaler.inverse_transform(predicted[:, :, i])) print(np.array(predicted_inverted).shape) #get only the close data ground_true = ground_true[:, :, 0].reshape(-1) ground_true_times = ground_true_times.reshape(-1) ground_true_times = pd.to_datetime(ground_true_times, unit='s') # since we are appending in the first dimension predicted_inverted = np.array(predicted_inverted)[0, :, :].reshape(-1) print(np.array(predicted_inverted).shape) validation_output_times = pd.to_datetime(validation_output_times.reshape(-1), unit='s') # In[337]: ground_true_df = pd.DataFrame() ground_true_df['times'] = ground_true_times
def create_dataset(dataset, look_back): dataX, dataY = [], [] for i in range(len(dataset) - look_back - 1): a = dataset[i:(i + look_back), :] dataX.append(a) dataY.append(dataset[i + look_back, :]) return np.array(dataX), np.array(dataY) # normalize dataset dataset = df.values dataset = dataset.astype('float32') # normalize the dataset scaler = MinMaxScaler(feature_range=(0, 1)) dataset = scaler.fit_transform(dataset) if (n_features != dataset.shape[1]): raise ValueError("Excel file format does not match with trained model.") test = dataset[35:, :] # reshape into X=t and Y=t+1 testX, testY = create_dataset(test, look_back) testPredict = model.predict(testX, batch_size=batch_size) normalizedTestPrediction = scaler.inverse_transform(testPredict) #normalizedTest = scaler.inverse_transform(test[3:]) print("Prediction for last 6 columns:\n", normalizedTestPrediction[:, 1:].astype(dtype=np.int32))
for i in xrange(0,numSteps): if (i%100==0): print float(i)/numSteps #batch_xs, batch_ys = GetSubset(trainDataX, trainDataY, batchsize, batchnum) batch_xs, batch_ys = GetRandomSubset(trainDataX, trainDataY, batchsize) batchnum += 1 if batchnum > (float(numPoints) / batchsize): batchnum = 0 sess.run(train, feed_dict={x: batch_xs, y_: batch_ys}) lossdata[i] = sess.run(loss, feed_dict={x: batch_xs, y_: batch_ys}) predTrain = sess.run(y, feed_dict={x: trainDataX}) predTest = sess.run(y, feed_dict={x: testDataX}) #predKaggleTest = sess.run(y, feed_dict={x: data}) predTrainPrice = scaler.inverse_transform(predTrain) predTestPrice = scaler.inverse_transform(predTest) #predKaggleTestPrice = scaler.inverse_transform(predKaggleTest) realPrice = scaler.inverse_transform(Y) """ i=1461 with open("hp_nnet_test.csv", "w") as f: f.write("Id,SalePrice\n") for x in predKaggleTestPrice: f.write(str(i)+","+str(x[0])+"\n") i+=1 """ f, axarr = plt.subplots(2) axarr[0].scatter(realPrice[:int(trainSplit*max(realPrice.shape))],predTrainPrice,c='Blue') axarr[0].scatter(realPrice[int(trainSplit*max(realPrice.shape)):],predTestPrice,c='Red')
model.add(Dense(units=1)) model.compile(optimizer='adam', loss='mean_squared_error') model.fit(xTrain, yTrain, epochs=100, batch_size=32) datasetTotal = pd.concat( (pricesDatasetTrain['adj_close'], pricesDatasetTest['adj_close']), axis=0) inputs = datasetTotal[len(datasetTotal) - len(pricesDatasetTest) - 40:].values inputs = inputs.reshape(-1, 1) inputs = min_max_scaler.transform(inputs) xTest = [] for i in range(40, len(pricesDatasetTest) + 40): xTest.append(inputs[i - 40:i, 0]) xTest = np.array(xTest) xTest = np.reshape(xTest, (xTest.shape[0], xTest.shape[1], 1)) predictions = model.predict(xTest) predictions = min_max_scaler.inverse_transform(predictions) plt.plot(testset, color='blue', label='Actual S&P500 Prices') plt.plot(predictions, color='green', label='LSTM Predictions') plt.title('S&P500 Predictions with Recurrent Neural Network') plt.xlabel('Time') plt.ylabel('Price') plt.legend() plt.show()
saver = tf.train.Saver() with tf.Session() as sesion: sesion.run(init) for iteracion in range(numero_iteraciones): lote_x, lote_y = lotes(entrenamiento_normalizado, tamaño_lote, numero_pasos) sesion.run(entrenamiento, feed_dict={x: lote_x, y: lote_y}) if iteracion % 100 == 0: error = funcion_error.eval(feed_dict={x: lote_x, y: lote_y}) print(iteracion, "\t Error", error) saver.save(sesion, "./modelo_series_temporales2") with tf.Session() as sesion: saver.restore(sesion, "./modelo_series_temporales2") entrenamiento_seed = list(entrenamiento_normalizado[-18:]) for iteracion in range(18): lote_x = np.array(entrenamiento_seed[-numero_pasos:]).reshape( 1, numero_pasos, 1) prediccion_y = sesion.run(salidas, feed_dict={x: lote_x}) entrenamiento_seed.append(prediccion_y[0, -1, 0]) resultados = normalizacion.inverse_transform( np.array(entrenamiento_seed[18:]).reshape(18, 1)) conjunto_pruebas['predicciones'] = resultados conjunto_pruebas conjunto_pruebas.plot()
class softArm: def __init__(self): # parameters self.name = 'softArm' self.enable_actions = (0, 1, 2, 3) self.action_bound = [[1900, 2500], [500, 1100], [500, 1100], [1900, 2500]] self.load_model() self.reset() def load_model(self): workbook = xlrd.open_workbook('Excel_Workbook220.xls') sheet1 = workbook.sheet_by_index(0) self.row_num = len(sheet1.col_values(0)) self.input_data = np.zeros([self.row_num, 4]) self.input_data[:, 0] = array(sheet1.col_values(0)) self.input_data[:, 1] = array(sheet1.col_values(1)) self.input_data[:, 2] = array(sheet1.col_values(2)) self.input_data[:, 3] = array(sheet1.col_values(3)) self.minMax0 = MinMaxScaler() self.input_data1 = self.minMax0.fit_transform(self.input_data) output_data = np.zeros([self.row_num, 12]) output_data[:, 0] = array(sheet1.col_values(4)) output_data[:, 1] = array(sheet1.col_values(5)) output_data[:, 2] = array(sheet1.col_values(6)) output_data[:, 3] = array(sheet1.col_values(7)) output_data[:, 4] = array(sheet1.col_values(8)) output_data[:, 5] = array(sheet1.col_values(9)) output_data[:, 6] = array(sheet1.col_values(10)) output_data[:, 7] = array(sheet1.col_values(11)) output_data[:, 8] = array(sheet1.col_values(12)) output_data[:, 9] = array(sheet1.col_values(13)) output_data[:, 10] = array(sheet1.col_values(14)) output_data[:, 11] = array(sheet1.col_values(15)) self.minMax2 = MinMaxScaler() self.output_data1 = self.minMax2.fit_transform(output_data) self.model = keras.Sequential() self.model.add( keras.layers.Dense(64, input_shape=(4, ), activation='tanh')) self.model.add(keras.layers.Dense(64, activation='tanh')) self.model.add(keras.layers.Dense(64, activation='tanh')) self.model.add(keras.layers.Dense(64, activation='tanh')) self.model.add(keras.layers.Dense(64, activation='tanh')) self.model.add(keras.layers.Dense(64, activation='tanh')) self.model.add(keras.layers.Dense(12, activation='tanh')) opt = keras.optimizers.Adam(lr=0.001) self.model.compile(loss='mse', optimizer=opt, metrics=['accuracy']) # self.model.load_weights('123.h') # self.model.fit(self.input_data1,self.output_data1,epochs=10000,verbose=2) # self.model.save_weights("123.h") self.model.load_weights('220.h') def draw(self): aa = self.minMax2.inverse_transform(self.current_pos) plt.plot(aa[0][0:6], -aa[0][6:12], '-') plt.hold(True) plt.plot(aa[0][0], -aa[0][6], 'o') plt.hold(True) aa = self.minMax2.inverse_transform(self.target) plt.plot(aa[0][0], -aa[0][6], '*') plt.title( str(self.motorA_pos) + ' ' + str(self.motorB_pos) + ' ' + str(self.motorC_pos) + ' ' + str(self.motorD_pos) + ' ' + str(self.reward)) plt.pause(0.001) plt.hold(False) def update(self, action): """ action: 0: 上节 move left 1: 上节 move right 2: 下节 move left 3: 下节 move right """ # update player position self.over_boundary = 0 if action == self.enable_actions[0]: #A move left if self.motorA_pos <= 1900: self.over_boundary = 1 elif self.motorB_pos == 500: if self.motorA_pos - self.step <= 1900: #判断可不可以执行 self.motorA_pos = 1900 else: self.motorA_pos -= self.step #25 else: if self.motorB_pos - self.step <= 500: #判断可不可以执行 self.motorB_pos = 500 else: self.motorB_pos -= self.step elif action == self.enable_actions[1]: #A move right if self.motorB_pos >= 1100: self.over_boundary = 1 elif self.motorA_pos >= 2500: #判断可不可以执行 if self.motorB_pos + self.step >= 1100: self.motorB_pos = 1100 else: self.motorB_pos += self.step else: #判断可不可以执行 if self.motorA_pos + self.step >= 2500: self.motorA_pos = 2500 else: self.motorA_pos += self.step elif action == self.enable_actions[2]: #B move left if self.motorC_pos >= 1100: self.over_boundary = 1 elif self.motorD_pos >= 2500: #判断可不可以执行 if self.motorC_pos + self.step >= 1100: self.motorC_pos = 1100 else: self.motorC_pos += self.step else: #判断可不可以执行 if self.motorD_pos + self.step >= 2500: self.motorD_pos = 2500 else: self.motorD_pos += self.step elif action == self.enable_actions[3]: #B move right if self.motorD_pos <= 1900: self.over_boundary = 1 elif self.motorC_pos <= 500: #判断可不可以执行 if self.motorD_pos - self.step <= 1900: self.motorD_pos = 1900 else: self.motorD_pos -= self.step else: #判断可不可以执行 if self.motorC_pos - self.step <= 500: self.motorC_pos = 500 else: self.motorC_pos -= self.step else: # do nothing print('?') if self.motorA_pos != 2500 and self.motorB_pos != 500: print('position error') if self.motorC_pos != 500 and self.motorD_pos != 2500: print('position error') # collision detection self.reward = 0 q = array([[ self.motorA_pos, self.motorB_pos, self.motorC_pos, self.motorD_pos ]]) input_ = self.minMax0.transform(q) self.current_pos = self.model.predict(input_) A_target = self.minMax2.inverse_transform(self.target) B_current_pos = self.minMax2.inverse_transform(self.current_pos) #-----------------last pos method--------------- c_last_pos = self.minMax2.inverse_transform(self.last_pos) #-------------------------------------------------- # self.distance = 0 # self.cur_distance = 0 self.last_distance = 0 self.terminal_dis = 0 # self.distance += abs(A[0][current_point] - B[0][current_point])# 奖励为距离的绝对值 #-----------------last pos method--------------------- self.terminal_dis = abs(A_target[0][0] - B_current_pos[0][0]) + abs(A_target[0][6] - B_current_pos[0][6]) self.last_distance = abs(A_target[0][0] - c_last_pos[0][0]) + abs(A_target[0][6] - c_last_pos[0][6]) # self.distance = self.last_distance - self.terminal_dis self.last_pos = self.current_pos #----------------------------------------------------------- self.run_step += 1 self.terminal = False if self.terminal_dis < 15: #15 # catch self.terminal = True self.reward = 2 else: if self.over_boundary == 1: self.reward = -1 elif self.terminal_dis < 20: self.reward = 1.2 - 0.005 * self.terminal_dis self.step = 5 elif self.terminal_dis < 50: self.reward = 1 - 0.005 * self.terminal_dis self.step = 10 elif self.terminal_dis < 90: self.reward = 0.5 - 0.002 * self.terminal_dis self.step = 25 elif self.terminal_dis < 150: self.reward = 0.1 - 0.001 * self.terminal_dis self.step = 50 else: self.reward = -0.001 * self.terminal_dis self.step = 75 if self.run_step > 150: self.terminal = True def observe(self): # self.draw() envir = [] target = np.array([self.target[0][0], self.target[0][6]]) current_pos = np.array( [self.current_pos[0][0], self.current_pos[0][6]]) envir.extend(target.flatten()) envir.extend(current_pos.flatten()) envir = np.array(envir) return envir, self.reward, self.terminal def execute_action(self, action): self.update(action) def reset(self): # reset player position start_pos = np.random.randint(0, self.row_num) self.motorA_pos = self.input_data[start_pos, 0] self.motorB_pos = self.input_data[start_pos, 1] self.motorC_pos = self.input_data[start_pos, 2] self.motorD_pos = self.input_data[start_pos, 3] end_pos = np.random.randint(0, self.row_num) # reset taget position c = self.input_data[end_pos, 0] d = self.input_data[end_pos, 1] e = self.input_data[end_pos, 2] f = self.input_data[end_pos, 3] q = array([[c, d, e, f]]) input_ = self.minMax0.transform(q) self.target = self.model.predict(input_) q = array([[ self.motorA_pos, self.motorB_pos, self.motorC_pos, self.motorD_pos ]]) input_ = self.minMax0.transform(q) self.current_pos = self.model.predict(input_) self.last_pos = self.current_pos # reset other variables self.reward = 0 self.run_step = 0 self.terminal = False self.distance = 0 self.step = 10 #步长 #function to get_pic from cv def get_from_cv(self): cap = cv2.VideoCapture(0) kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) for i in range(50): ret, frame = cap.read() while (1): # get a frame ret, frame = cap.read() # show a frame frame = frame[60:420, :630] # cv2.imshow("aaa",frame) gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) # cv2.imshow("aaa",gray) # 2-mode ret1, binary = cv2.threshold(gray, 23, 255, cv2.THRESH_BINARY) # cv2.imshow("aaa",binary) # 开运算 opened_pic = cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel) # cv2.imshow("aaa",opened_pic) #寻找边界 contours, boundary, ret2 = cv2.findContours( opened_pic, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cv2.drawContours(opened_pic, boundary, -1, (0, 0, 255), 3) # cv2.imshow("bbb",opened_pic) #计算坐标 x_min = np.zeros((len(boundary), 1)) x_max = np.zeros((len(boundary), 1)) y_max = np.zeros((len(boundary), 1)) y_min = np.zeros((len(boundary), 1)) color = (0, 255, 0) for i in range(len(boundary)): x_min[i] = boundary[i][0][0][0] x_max[i] = boundary[i][0][0][0] y_min[i] = boundary[i][0][0][1] y_max[i] = boundary[i][0][0][1] for j in range(len(boundary[i])): if boundary[i][j][0][0] > x_max[i]: x_max[i] = boundary[i][j][0][0] if boundary[i][j][0][0] < x_min[i]: x_min[i] = boundary[i][j][0][0] if boundary[i][j][0][1] > y_max[i]: y_max[i] = boundary[i][j][0][1] if boundary[i][j][0][1] < y_min[i]: y_min[i] = boundary[i][j][0][1] for i in range(len(boundary) - 1): i = i + 1 yuanxin_x = int((x_max[i] - x_min[i]) / 2 + x_min[i]) yuanxin_y = int((y_max[i] - y_min[i]) / 2 + y_min[i]) yuanxin = (yuanxin_x, yuanxin_y) cv2.circle(frame, yuanxin, 10, color) cv2.waitKey(50) cv2.imshow("get_characteristic", frame) cap.release()
mode='min', period=1) ts_model.fit(x=X_train, y=y_train, batch_size=16, epochs=20, verbose=1, callbacks=[save_best], validation_data=(X_val, y_val), shuffle=True) completeName = 'PRSA_data_Air_Pressure_MLP_weights.11-0.0001.hdf5' best_model = load_model( os.path.join(os.path.expanduser('~'), 'Documents', completeName)) preds = best_model.predict(X_val) pred_PRES = scaler.inverse_transform(preds) pred_PRES = np.squeeze(pred_PRES) from sklearn.metrics import r2_score r2 = r2_score(df_val['PRES'].loc[7:], pred_PRES) print('R-squared for the validation set:', round(r2, 4)) #Let's plot the first 50 actual and predicted values of air pressure. plt.figure(figsize=(5.5, 5.5)) plt.plot(range(50), df_val['PRES'].loc[7:56], linestyle='-', marker='*', color='r') plt.plot(range(50), pred_PRES[:50], linestyle='-', marker='.', color='b')
def lstm(data,neurons,look_back,ep,batch,ver,plot=False): # load the dataset dataset = data.values.reshape(-1,1) dataset = dataset.astype('float32') # normalize the dataset scaler = MinMaxScaler(feature_range=(0, 1)) dataset = scaler.fit_transform(dataset) # split into train and test sets train_size = int(len(dataset) * 0.67) test_size = len(dataset) - train_size train, test = dataset[0:train_size,:], dataset[train_size:len(dataset),:] # reshape into X=t and Y=t+1 trainX, trainY = create_dataset(train, look_back) testX, testY = create_dataset(test, look_back) # reshape input to be [samples, time steps, features] trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1])) # create and fit the LSTM network model = Sequential() model.add(LSTM(neurons, input_shape=(1, look_back))) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(trainX, trainY, epochs=ep, batch_size=batch, verbose=ver) # make predictions trainPredict = model.predict(trainX) testPredict = model.predict(testX) print('Obs: ', look_back) ev=explained_variance_score(testY,testPredict) mae=mean_absolute_error(testY,testPredict) r2=r2_score(testY, testPredict) corr=pearsonr(testY,testPredict.reshape(-1))[0] # invert predictions trainPredict = scaler.inverse_transform(trainPredict) trainY = scaler.inverse_transform([trainY]) testPredict = scaler.inverse_transform(testPredict) testY = scaler.inverse_transform([testY]) # shift train predictions for plotting trainPredictPlot = np.empty_like(dataset) trainPredictPlot[:, :] = np.nan trainPredictPlot[look_back:len(trainPredict)+look_back, :] = trainPredict # shift test predictions for plotting testPredictPlot = np.empty_like(dataset) testPredictPlot[:, :] = np.nan testPredictPlot[len(trainPredict)+(look_back*2)+1:len(dataset)-1, :] = testPredict if plot: # plot baseline and predictions plt.plot(scaler.inverse_transform(dataset)) plt.plot(trainPredictPlot) plt.plot(testPredictPlot) plt.show() return [ev,mae,r2,corr]
trainX, trainY = create_dataset(train, look_back, sentiment[0:train_size],sent=True) testX, testY = create_dataset(test, look_back, sentiment[train_size:len(scaled)], sent=True) trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1])) model = Sequential() model.add(LSTM(100, input_shape=(trainX.shape[1], trainX.shape[2]), return_sequences=True)) model.add(LSTM(100)) model.add(Dense(1)) model.compile(loss='mae', optimizer='adam') history = model.fit(trainX, trainY, epochs=300, batch_size=100, validation_data=(testX, testY), verbose=0, shuffle=False) yhat = model.predict(testX) yhat_inverse_sent = scaler.inverse_transform(yhat.reshape(-1, 1)) testY_inverse_sent = scaler.inverse_transform(testY.reshape(-1, 1)) rmse_sent = sqrt(mean_squared_error(testY_inverse_sent, yhat_inverse_sent)) print('Test RMSE: %.3f' % rmse_sent) import MySQLdb #Enter the values for you database connection dsn_database = "bitcoin" # e.g. "MySQLdbtest" dsn_hostname = "173.194.231.244" # e.g.: "mydbinstance.xyz.us-east-1.rds.amazonaws.com" dsn_port = 3306 # e.g. 3306 dsn_uid = "demo" # e.g. "user1" dsn_pwd = "qwerty@123" # e.g. "Password123" conn = MySQLdb.connect(host=dsn_hostname, port=dsn_port, user=dsn_uid, passwd=dsn_pwd, db=dsn_database)
regressor.add(Dense(1)) regressor.compile(optimizer="adam",loss = "mse") regressor.fit(x_train,y_train,epochs=100,batch_size=64) inputs = data[len(data)-len(test_data)-timestep:] inputs = inputs.values.reshape(-1,1) inputs = scaler.transform(inputs) x_test = [] for i in range(timestep,inputs.shape[0]): x_test.append(inputs[i-timestep:i,0]) x_test = np.array(x_test) x_test = x_test.reshape(x_test.shape[0],x_test.shape[1],1) predicted_data = regressor.predict(x_test) predicted_data = scaler.inverse_transform(predicted_data) data_test = np.array(test_data) data_test = data_test.reshape(len(data_test),1) #Visualizing results plt.figure(figsize = (8,4)) plt.plot(data_test,color = "r",label = "true result") plt.plot(predicted_data,color = "b",label = "predicted result") plt.legend(loc = "best") plt.xlabel("Time") plt.ylabel("Bitcoin Volume") plt.show()
X = sc.fit_transform(X) # Training the SOM from minisom import MiniSom som = MiniSom(x = 10, y = 10, input_len = 15, sigma = 1.0, learning_rate = 0.5) som.random_weights_init(X) som.train_random(data = X, num_iteration = 100) # Visualizing the results from pylab import bone, pcolor, colorbar, plot, show bone() pcolor(som.distance_map().T) colorbar() markers = ['o', 's'] colors = ['r', 'g'] for i, x in enumerate(X): w = som.winner(x) plot(w[0] + 0.5, w[1] + 0.5, markers[y[i]], markeredgecolor = colors[y[i]], markerfacecolor = 'None', markersize = 10, markeredgewidth = 2) show() # Finding the frauds mappings = som.win_map(X) frauds = np.concatenate((mappings[(8,1)], mappings[(6,8)]), axis = 0) frauds = sc.inverse_transform(frauds)
inputs_a = inputs_a.reshape(-1, 1) inputs_a = scaler.transform(inputs_a) inputs_v = g_dv[len(g_dv) - len(valid) - 20:].values inputs_v = inputs_v.reshape(-1, 1) inputs_v = scaler.transform(inputs_v) X_test_r = [] X_test_a = [] X_test_v = [] for i in range(20, inputs_r.shape[0]): X_test_r.append(inputs_r[i - 20:i, 0]) X_test_a.append(inputs_a[i - 20:i, 0]) X_test_v.append(inputs_v[i - 20:i, 0]) X_test_r = np.array(X_test_r) X_test_r = np.reshape(X_test_r, (X_test_r.shape[0], X_test_r.shape[1], 1)) growth_r = model.predict(X_test_r) growth_r = scaler.inverse_transform(growth_r) X_test_a = np.array(X_test_a) X_test_a = np.reshape(X_test_a, (X_test_a.shape[0], X_test_a.shape[1], 1)) growth_a = model.predict(X_test_a) growth_a = scaler.inverse_transform(growth_a) X_test_v = np.array(X_test_v) X_test_v = np.reshape(X_test_v, (X_test_v.shape[0], X_test_v.shape[1], 1)) growth_v = model.predict(X_test_v) growth_v = scaler.inverse_transform(growth_v) state_a = np.multiply(g_da > g_dr, g_da > g_dv) state_r = np.multiply(g_dr > g_da, g_dr > g_dv) state_v = np.multiply(g_dv > g_dr, g_dv > g_da) # dictionary for encoding and decoding seq. table = {0: 'A', 1: 'R', 2: 'V'} # initializing data set, one hot encoded seq. state = 1 * np.column_stack((state_a, state_r, state_v))
reframed.drop(reframed.columns[[7]], axis=1, inplace=True) values = reframed.values x_test = values[6:, :] x_test = x_test.reshape((x_test.shape[0],1,x_test.shape[1])) def aggValue(x_test, nVal): for i in range(x_test.shape[2]-1): x_test[0][0][i]=x_test[0][0][i+1] x_test[0][0][x_test.shape[2]-1]=nVal return x_test results=[] for i in range(7): parcial = model.predict(x_test) results.append(parcial[0]) x_test=aggValue(x_test, parcial[0]) adimen = [x for x in results] inverted = scaler.inverse_transform(adimen) prediccion1SemanaDic = pd.DataFrame(inverted) prediccion1SemanaDic.columns = ['Pronostico'] prediccion1SemanaDic.plot() diciembre2017 = df['2017-12-01':'2017-12-08'] plt.plot(diciembre2017.values, color='red', label='2017') plt.legend() plt.show()
# Get the final accuracy scores by running the "cost" operation on the training and test data sets final_training_cost = session.run(cost, feed_dict={ X: X_scaled_training, Y: Y_scaled_training }) final_testing_cost = session.run(cost, feed_dict={ X: X_scaled_testing, Y: Y_scaled_testing }) print("Final Training cost: {}".format(final_training_cost)) print("Final Testing cost: {}".format(final_testing_cost)) # Now that the neural network is trained, let's use it to make predictions for our test data. # Pass in the X testing data and run the "prediciton" operation Y_predicted_scaled = session.run(prediction, feed_dict={X: X_scaled_testing}) # Unscale the data back to it's original units (dollars) Y_predicted = Y_scaler.inverse_transform(Y_predicted_scaled) real_earnings = test_data_df['total_earnings'].values[0] predicted_earnings = Y_predicted[0][0] print("The actual earnings of Game #1 were ${}".format(real_earnings)) print("Our neural network predicted earnings of ${}".format( predicted_earnings))
for i in range(60,len(test_data)): x_test.append(test_data[i-60:i,0])##The independent testing variables or testing feature set 'X' (holds the stock price value for the past 60 days at position 'i-60') ##Convert x_test to a numpy array so we can use it to test and make predictions using the LSTM model we created x_test = np.array(x_test) ##A LSTM network expects the input to be 3-Dimensional in the form [samples, time steps, features]: ## samples is the number of data points (or rows/ records) we have, ## time steps is the number of time-dependent steps that are there in a single data point (60), ## features/indicators refers to the number of variables we have for the corresponding true value in Y, since we are only using one feature 'Close', ## the number of features/indicators will be one ##Reshape the data into the shape accepted by the LSTM x_test = np.reshape(x_test, (x_test.shape[0],x_test.shape[1],1)) ##Reshaping the variable to be 3-Dimensional in the form [samples, time steps, features] for the Neural Network ##Getting the models predicted price values predictions = model.predict(x_test) ##Getting the models predicted price values predictions = scaler.inverse_transform(predictions) ##Undo the scaling ##Get the root mean squared error (rmse) ##RMSE is a good measure of how accurately the model predicts the response ##(RMSE) is the standard deviation of the residuals (prediction errors). ##Lower values of RMSE indicate better fit. ##Usually you want to evaluate your model with other metrics as well to truly get an idea of how well your model performs. ##Calculate/Get the value of RMSE rmse=np.sqrt(np.mean((predictions- y_test)**2)) rmse ##Plot the data train = data[:training_data_len] ## A dataframe to get rows from index 0 to 1602 (1603 rows total) valid = data[training_data_len:] ## A dataframe to get rows from index 1603 to the end of the list (2004 - 1603 = 401 rows total) valid['Predictions'] = predictions ##Create a new column called 'Predictions' that will hold the predicted prices
test_inputs = apple_total[len(apple_total) - len(apple_testing_complete) - 60:].values test_inputs = test_inputs.reshape(-1, 1) test_inputs = scaler.transform(test_inputs) test_features = [] for i in range(60, 80): test_features.append(test_inputs[i - 60:i, 0]) test_features = np.array(test_features) test_features = np.reshape(test_features, (test_features.shape[0], test_features.shape[1], 1)) ## Making Predictions predictions = model.predict(test_features) predictions = scaler.inverse_transform(predictions) plt.figure(figsize=(10, 6)) plt.plot(apple_testing_processed, color='blue', label='Actual Apple Stock Price') plt.plot(predictions, color='red', label='Predicted Apple Stock Price') plt.title('Apple Stock Price Prediction') plt.xlabel('Date') plt.ylabel('Apple Stock Price') plt.legend() plt.show()
class PreprocessingDenga: def __init__(self, data_X, data_y, nan_fill_method='mean'): self.raw_data = data_X processing_data = self.raw_data.copy() processing_data.set_index('week_start_date', inplace=True) processing_data.drop('year', axis=1, inplace=True) # Drop redundant columns processing_data.drop('reanalysis_sat_precip_amt_mm', axis=1, inplace=True) processing_data.drop('reanalysis_specific_humidity_g_per_kg', axis=1, inplace=True) # Fill nan values self.values_to_nan_fill = None if nan_fill_method == 'mean': self.values_to_nan_fill = processing_data.mean() if self.values_to_nan_fill is not None: processing_data.fillna(self.values_to_nan_fill, inplace=True) # Convert city name to numerical, and week_of_year to circle self.train_city_index = processing_data.keys().get_loc('city') values = processing_data.values self.encoder = LabelBinarizer() city = self.encoder.fit_transform(values[:, self.train_city_index]) city_encoded = np.hstack( (city, 1 - city)) # convert 1 column to 2 column each for one city self.train_week_index = processing_data.keys().get_loc('weekofyear') week_values = values[:, self.train_week_index] self.encoder_of_weeks = CircleTransform(week_values) week_sin, week_cos = self.encoder_of_weeks.transform(week_values) week_sin_cos = np.column_stack((week_sin, week_cos)) values = np.delete(values, np.s_[self.train_city_index, self.train_week_index], axis=1) values = np.hstack((city_encoded, week_sin_cos, values)) values = values.astype('float32') # Get final count of features self.features_count = values.shape[ 1] # Warning: Shape may be dependent of transformations before, so check it. # normalize features self.scaler = MinMaxScaler(feature_range=(0, 1)) self.scaler.fit(values) self.scaled = self.scaler.transform( values) # TODO: Scaled to normal_version # data_y preprocessing: self.data_y = data_y['total_cases'] self.y_scaler = MinMaxScaler(feature_range=(0, 1)) self.y_scaler.fit(self.data_y.values.reshape(-1, 1)) def basic_X_preprocessing(self, given_data): """Prepare the same preprocessing process just like in init to new data.""" processing_data = given_data.copy() processing_data.set_index('week_start_date', inplace=True) processing_data.drop('year', axis=1, inplace=True) processing_data.drop('reanalysis_sat_precip_amt_mm', axis=1, inplace=True) processing_data.drop('reanalysis_specific_humidity_g_per_kg', axis=1, inplace=True) if self.values_to_nan_fill is not None: processing_data.fillna(self.values_to_nan_fill, inplace=True) # Transform of a city and a week attribute: city_index = processing_data.keys().get_loc('city') values = processing_data.values city = self.encoder.transform(values[:, city_index]) city_encoded = np.hstack((city, 1 - city)) week_index = processing_data.keys().get_loc('weekofyear') week_values = values[:, self.train_week_index] week_sin, week_cos = self.encoder_of_weeks.transform(week_values) week_sin_cos = np.column_stack((week_sin, week_cos)) values = np.delete(values, np.s_[city_index, week_index], axis=1) values = np.hstack((city_encoded, week_sin_cos, values)) scaled = self.scaler.transform(values) return scaled def get_truncated_data_X(self, data_X_prepared, truncated_backprop_length): data_X_truncated = PreprocessingSeries.prepare_for_truncated_backpropagation( data_X_prepared, truncated_backprop_length - 1, 0) return data_X_truncated def get_split_data_X_by_city(self, data_X): index_of_first_occurrence_next_city = self.get_city_split_index( data_X, self.train_city_index) city1 = data_X[:index_of_first_occurrence_next_city, :] city2 = data_X[index_of_first_occurrence_next_city:, :] return city1, city2 def get_split_and_truncated_data_y_by_city(self, data_y, truncated_backprop_length): index_of_first_occurrence_next_city = self.get_city_split_index(data_y) data_y = data_y['total_cases'] y_city_1 = data_y[:index_of_first_occurrence_next_city] y_city_2 = data_y[index_of_first_occurrence_next_city:] # Delete rows with not enough history y_city_1 = y_city_1[truncated_backprop_length - 1:] y_city_2 = y_city_2[truncated_backprop_length - 1:] return y_city_1, y_city_2 def get_normalized_y(self, given_data_y): data_y_norm = self.y_scaler.transform(given_data_y.reshape(-1, 1)) data_y_norm_and_shape = data_y_norm.reshape(-1) return data_y_norm_and_shape def get_inverse_y(self, given_y): y_result_not_shaped = self.y_scaler.inverse_transform(given_y) y_result = y_result_not_shaped.astype(int).reshape(-1) return y_result # TODO: Refactor def get_city_split_index(self, data, index=float('nan')): if math.isnan(index) and not isinstance(data, pd.DataFrame): raise AttributeError("Wrong arguments to split_data") if math.isnan(index): index = data.keys().get_loc('city') if isinstance(data, pd.DataFrame): data = data.values values_column = data[:, index] if values_column.dtype not in [ np.dtype(np.float16), np.dtype(np.float32), np.dtype(np.float64) ]: values_column = self.encoder.transform(values_column) index = np.argmax(values_column == 0) return index
loss = model.evaluate(test_x_scale, test_y_norm) print("loss (MSE):", loss) # R2 coefficient r2 = r2_score(test_y_norm, testPredict) print("R2:", r2) import math from sklearn.metrics import mean_squared_error from sklearn.metrics import mean_absolute_error # calculate root mean squared error trainScore = math.sqrt(mean_squared_error(train_y_norm, trainPredict)) print('Train Score: %.2f RMSE' % (trainScore * 100)) testScore = math.sqrt(mean_squared_error(test_y_norm, testPredict)) print('Test Score: %.2f RMSE' % (testScore * 100)) #calculate mean absolute percent error trainMAPE = mean_absolute_error(train_y, trainPredict) print('train MAPE: %.2f MAPE' % (trainMAPE * 100)) testMAPE = mean_absolute_error(test_y, testPredict) print('test MAPE: %.2f MAPE' % (testMAPE * 100)) # Get something which has as many features as dataset testPredict_extended = np.zeros((len(testPredict), 5)) # Put the predictions there testPredict_extended[:, 4] = testPredict[:, 0] # Inverse transform it and select the 5rd column. testPredict = scaler.inverse_transform(testPredict_extended)[:, 4] print('testPredict', testPredict) model.save("prediction.h5")
model.add(LSTM( 20, input_shape=(1, 1))) # (timestep, feature) # 층이 추가됨 (add) # 입력층, 첫번째 은닉층 model.add(Dense(1)) # 출력층 (하나) # 모델을 컴파일 (컴퓨터가 알아들을 수 있도록) # 오차함수, 최적화 방법 model.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy']) # metrics : 모델 수행 결과를 나타내게 설정 (과적합 문제 방지) model.fit(trainX, trainY, epochs=1000, batch_size=50, verbose=2) # 모델을 실제로 수행 # batch_size : 전체 데이터를 10개씩 사용 # verbose(로깅) 2 : epoch당 나오게 # 예측값 평가하기 (얼마나 정확한지) testPredict = model.predict(testX) testPredict = scaler.inverse_transform(testPredict) # testPredict : 예측 값 testY = scaler.inverse_transform(testY) # testY : 실제 값 testScore = math.sqrt(mean_squared_error( testY, testPredict)) # mean_squared_error : 평균 제곱근 오차 print('Train Score: %.2f RMSE' % testScore) # 예측 값과 실제 값 차이 출력 # 예측 데이터 출력 lastX = nptf[-1] lastX = np.reshape(lastX, (1, 1, 1)) lastY = model.predict(lastX) predict = scaler.inverse_transform(lastY) # 정규화 시킨 값을 역변환 print('Predict the Close value of final day: %d' % predict) # 데이터 입력 마지막 다음날 종가 예측 # 차트출력, 저장 plt.plot(testPredict)
def DayAheadMLP(endog, exog, date, lags=range(1, 169), hoursAhead=38, epochs=200, activation='relu', optimizer='adam', loss='mse', verbose=0): """ Trains a fresh MLP and returns a day-ahead forecast endog(pd.Series): Series (usually corresponding to a cluster) of hourly indexed load data exog(pd.DataFrame): DataFrame of covariates if applicable date(date-like obj): Hour after which to begin 38-hour-ahead forecast lags(List): List of lag features to generate (default=range(1,169)) hoursAhead(int): number of hours ahead to predict (default=38) epochs(int): number of epochs for training (default=200) activation(str): key for activation function (default='relu') optimizer(str): key for optimizer (default='adam') loss(str): key for loss function (default='mse') """ # force DataFrame dtype for y, copy X y = pd.DataFrame(endog) X = exog.copy(deep=True) testDate = pd.to_datetime(date) # scale y (0,1) on annual min and max, important assumption scaler = MinMaxScaler().fit(y) y = pd.DataFrame(data=scaler.transform(y), index=y.index, columns=y.columns) for i in lags: X['t-' + str(i)] = y.iloc[:, 0].shift(i) for j in range(1, 38): y['t+' + str(j)] = y.iloc[:, 0].shift(-j) # truncate on both sides to remove NaNs X.dropna(inplace=True) y = y.reindex(X.index, axis=0).dropna() X = X.reindex(y.index, axis=0).dropna() # train/test split, train range includes all available data up to two days prior to test date y_train = y.loc[y.index < testDate - pd.Timedelta(days=2)].values X_train = X.loc[X.index < testDate - pd.Timedelta(days=2)].values X_test = X.loc[X.index == testDate].values # set input and hidden layer dimensions inputDim = X_train.shape[1] hiddenDim = (inputDim - 38) // 2 + 38 # define model based on hyperparameters model = Sequential() model.add(Dense(hiddenDim, activation=activation, input_dim=inputDim)) model.add(Dense(38)) model.compile(optimizer=optimizer, loss=loss) # fit the model and make a prediction model.fit(X_train, y_train, epochs=epochs) y_pred = model.predict(X_test, verbose=0) # return result after reverse scaling return scaler.inverse_transform(y_pred).flatten()
class NeuralNetwork(BaseEstimator, RegressorMixin, object): _maxSteps = None _maxNonChangingSteps = None _learningRate = None _shrinkage = None _architecture = None _momentum = None _useDropout = None _alphaStandout = None _betaStandout = None _warmStart = None _batchSize = None _weights = None _inputDimension = None _outputDimension = None _step = None _lastDelta = None _percentageDropout = None _inputNormalizer = None _outputNormalizer = None def __init__(self, maxSteps=50, maxNonChangingSteps=5, learningRate=1e-6, shrinkage=0.9, architecture=[10], momentum=0.7, useDropout=False, alphaStandout=0, betaStandout=0.5, warmStart=False, startingWeights=None, batchSize = 1, step = 0, lastDelta = None, percentageDropout=1): self._maxSteps = maxSteps self._maxNonChangingSteps = maxNonChangingSteps self._learningRate = learningRate self._shrinkage = shrinkage self._architecture = architecture self._momentum = momentum self._useDropout = useDropout self._alphaStandout = alphaStandout self._betaStandout = betaStandout self._warmStart = warmStart self._weights = startingWeights self._batchSize = batchSize self._step = step self._lastDelta = None self._percentageDropout = percentageDropout def get_params(self, deep=True): params = {} params["maxSteps"] = self._maxSteps params["maxNonChangingSteps"] = self._maxNonChangingSteps params["learningRate"] = self._learningRate params["shrinkage"] = self._shrinkage params["architecture"] = self._architecture params["momentum"] = self._momentum params["useDropout"] = self._useDropout params["alphaStandout"] = self._alphaStandout params["betaStandout"] = self._betaStandout params["warmStart"] = self._warmStart params["batchSize"] = self._batchSize params["step"] = self._step params["lastDelta"] = self._lastDelta params["percentageDropout"] = self._percentageDropout return params def _initializeWeights(self, randomState=0): randomState = check_random_state(randomState) self._weights = [] for k in range(len(self.neuronsPerLayer())-1): self._weights.append(np.ones(shape=(self.neuronsPerLayer()[k]+1, self.neuronsPerLayer()[k+1]))) for k in range(len(self.neuronsPerLayer())-1): for i in range(len(self._weights[k])): for j in range(len(self._weights[k][i])): #Starting weights are set randomly, dependant on the number of inputs. Compare lecture 17, neuralnetworks slide 10. self._weights[k][i][j] = randomState.uniform(0, 1)/(self.neuronsPerLayer()[k+1])**0.5 #self._weights[k][i][j] = randomState.uniform(0, 1) def _batchify(self, X, batchSize, y=None): #first, set the batches. #A list of feature matrixes, with the ith column representing the ith example of said feature. index = 0 batchFeatures = [] #A list of matrices in the one of k coding scheme. if not y is None: batchTargets = [] while index < len(X): if batchSize != 0: numberExamples = min(batchSize, len(X) - index) else: numberExamples = len(X) batchFeatures.append(np.ones(shape=(self._inputDimension+1, numberExamples))) if (not y == None): batchTargets.append(np.zeros(shape=(self._outputDimension, numberExamples))) for i in range(numberExamples): for j in range(self._inputDimension): batchFeatures[-1][j, i] = X[index, j] #TODO in case of multiple dimensions, break glass. #Now, set the one out of k training scheme if (not y == None): for j in range(self._outputDimension): batchTargets[-1][j, i] = y[index, j] #batchTargets[-1][0, i] = y[index] index += 1 if not y == None: return batchFeatures, batchTargets else: return batchFeatures def neuronsPerLayer(self): neuronsPerLayer = [] neuronsPerLayer.append(self._inputDimension) neuronsPerLayer.extend(self._architecture) neuronsPerLayer.append(self._outputDimension) return neuronsPerLayer def set_params(self, **parameters): for parameter, value in parameters.items(): if (parameter == 'maxSteps'): self._maxSteps = value elif (parameter == 'maxNonChangingSteps'): self._maxNonChangingSteps = value elif (parameter == 'learningRate'): self._learningRate = value elif (parameter == 'shrinkage'): self._shrinkage = value elif (parameter == 'architecture'): self._architecture = value elif (parameter == 'momentum'): self._momentum = value elif (parameter == 'useDropout'): self._useDropout = value elif (parameter == 'alphaStandout'): self._alphaStandout = value elif (parameter == 'betaStandout'): self._betaStandout = value elif (parameter == 'warmStart'): self._warmStart = value elif (parameter == 'batchSize'): self._batchSize = value elif (parameter == 'step'): self._step = value elif (parameter == 'lastDelta'): self._lastDelta = value elif parameter == 'percentageDropout': self._percentageDropout = value return self def calcLayerOutputsBatch(self, batchFeatures, doDropout, randomState = 0): randomState = check_random_state(randomState) dropoutVectors = [] numExamples = batchFeatures.shape[1] for k in range(len(self.neuronsPerLayer())): if (k != len(self.neuronsPerLayer())-1): #if a bias neuron exists. dropoutVectors.append(np.ones((self.neuronsPerLayer()[k]+1, numExamples))) else: #else. dropoutVectors.append(np.ones((self.neuronsPerLayer()[k], numExamples))) outputsPerLayer = [] outputsPerLayer.append(batchFeatures) for k in range(0, len(self._weights)): #All the same except for the output layer. if (k == len(self._weights)-1): # Do not append the bias. #outputsPerLayer.append(np.maximum(np.matrix(np.dot(self._weights[k].transpose(), outputsPerLayer[k])), 0)) #outputsPerLayer.append(self.sigmoid(np.dot(self._weights[k].transpose(), outputsPerLayer[k]))) outputsPerLayer.append(self.sigmoid(np.dot(self._weights[k].transpose(), outputsPerLayer[k]))) else: #Do append the bias neuron. outputsPerLayer.append(np.ones((self.neuronsPerLayer()[k+1]+1, numExamples))) inputThisLayer = np.dot(self._weights[k].transpose(), outputsPerLayer[k]) #outputsPerLayer[k+1][:self.neuronsPerLayer()[k+1]] = np.maximum(inputThisLayer[:self.neuronsPerLayer()[k+1]], 0) #print(inputThisLayer) outputsPerLayer[k+1][:-1] = self.sigmoid(inputThisLayer) if (self._useDropout): dropoutNeuronNumber = int(self.neuronsPerLayer()[k+1]*self._percentageDropout) dropoutVectors[k+1][:dropoutNeuronNumber] = np.clip(self.sigmoidStandout(self._alphaStandout * inputThisLayer + self._betaStandout), 0, 1)[:dropoutNeuronNumber] #print(dropoutVectors[k+1]) if (doDropout): dropoutVectors[k+1] = np.ones((dropoutVectors[k+1].shape[0], dropoutVectors[k+1].shape[1])) * dropoutVectors[k+1] > np.random.rand(dropoutVectors[k+1].shape[0], dropoutVectors[k+1].shape[1]) #print(dropoutVectors[k+1]) outputsPerLayer[k+1] = np.multiply(outputsPerLayer[k+1], dropoutVectors[k+1]) #print(outputsPerLayer[-1]) if (doDropout): return outputsPerLayer, dropoutVectors else: return outputsPerLayer, dropoutVectors def _learnFromBatch(self, batchFeatures, batchTargets): outputsPerLayer, dropoutVectors = self.calcLayerOutputsBatch(batchFeatures, True) errorsPerLayer = [] for i in range(len(outputsPerLayer)-1): errorsPerLayer.append(np.zeros((outputsPerLayer[i].shape[0], len(batchTargets)))) #Set the error for the output layer. errorsPerLayer.append(batchTargets - outputsPerLayer[-1]) #now, it gets funny.: Calculate all of the errors. In both cases. dropout applies to the errorsPerLayer, too. A neuron that isn't 'active' will have no error. for k in range(len(self._weights)-1, -1, -1): if (k == len(self._weights)-1): errorsPerLayer[k] = np.dot(self._weights[k], errorsPerLayer[k+1]) else: errorsPerLayer[k] = np.dot(self._weights[k], errorsPerLayer[k+1][0:-1]) if (self._useDropout): errorsPerLayer[k] = np.multiply(errorsPerLayer[k], dropoutVectors[k]) #Calculate the deltaW. deltaW = [] for k in range(len(self._weights)): deltaW.append(np.zeros(shape=self._weights[k].shape)) for k in range(len(self._weights)-1, -1, -1): if (k == len(self._weights)-1): #derivative = 1./(np.exp(-outputsPerLayer[k+1])+1) #tmp = np.multiply(errorsPerLayer[k+1], derivative).transpose() tmp = np.multiply(np.multiply(errorsPerLayer[k+1], outputsPerLayer[k+1]), 1-outputsPerLayer[k+1]).transpose() else: #derivative = 1./(np.exp(-outputsPerLayer[k+1])+1) #tmp = np.multiply(errorsPerLayer[k+1], derivative)[0:-1].transpose() tmp = (np.multiply(np.multiply(errorsPerLayer[k+1], outputsPerLayer[k+1]), 1-outputsPerLayer[k+1]))[0:-1].transpose() #And again, a neuron which doesn't exist won't cause deltaWs. if (self._useDropout): deltaW[k] = np.dot(np.multiply(outputsPerLayer[k], dropoutVectors[k]), tmp) else: deltaW[k] = np.dot(outputsPerLayer[k], tmp) #print(deltaW) #raw_input() return deltaW def fit(self, X, y): X = np.matrix(X) y = np.matrix(y) self._outputNormalizer = MinMaxScaler() self._inputNormalizer = MinMaxScaler() self._outputNormalizer = self._outputNormalizer.fit(y) self._inputNormalizer = self._inputNormalizer.fit(X) self._inputDimension = X.shape[1] self._outputDimension = y.shape[1]#For now, hardcoded to 1-dimensional regression problems. if (not self._warmStart or self._weights == None): self._initializeWeights() self._lastDelta = None batchFeatures, batchTargets = self._batchify(np.matrix(self._inputNormalizer.transform(X)), self._batchSize, np.matrix(self._outputNormalizer.transform(y))) #do for each step until the maximum steps: for i in range(self._maxSteps): reducedLearningRate = self._learningRate * self._shrinkage ** self._step for j in range(len(batchFeatures)): deltaW = self._learnFromBatch(batchFeatures[j], batchTargets[j]) if (self._lastDelta == None): self._lastDelta = deltaW for k in range(len(self._weights)): self._lastDelta[k] = ((1-self._momentum) * deltaW[k] + self._momentum * self._lastDelta[k]) self._weights[k] = self._weights[k] + reducedLearningRate * self._lastDelta[k] #self._positifyWeights() self._step += 1 #print(step) return self def predict(self, X, debug=False): X = np.matrix(X) batchFeatures = self._batchify(self._inputNormalizer.transform(X), self._batchSize) batchResults = np.zeros((X.shape[0], self._outputDimension)) dropoutResults = [] for k in range(len(self.neuronsPerLayer())): if (k != len(self.neuronsPerLayer())-1): #if a bias neuron exists. dropoutResults.append(np.zeros((self.neuronsPerLayer()[k]+1, 1))) else: #else. dropoutResults.append(np.zeros((self.neuronsPerLayer()[k], 1))) begin = 0 end = batchFeatures[0].shape[1] for i in range(len(batchFeatures)): outputsLast, dropoutFeatures = self.calcLayerOutputsBatch(batchFeatures[i], False) outputsLast = outputsLast[-1] batchResults[begin:end, :] = outputsLast.transpose() begin = end end = end + batchFeatures[i].shape[1] #for featureList in batchFeatures: # outputsLast, dropoutFeatures = self.calcLayerOutputsBatch(featureList, False) # outputsLast = outputsLast[-1] # batchResults.extend(list(np.array(outputsLast).reshape(-1,))) # for i in range(len(dropoutFeatures)): # summed = np.matrix(np.sum(dropoutFeatures[i], 1)).transpose() # dropoutResults[i] += summed batchResults = np.matrix(batchResults) if (debug): return self._outputNormalizer.inverse_transform(batchResults), dropoutResults else: return self._outputNormalizer.inverse_transform(batchResults) def sigmoid(self, X): #return 1 / (1 + np.exp(-X)) return 0.5 * (X/(1+abs(X))+1) def sigmoidStandout(self, X): #return 1 / (1 + np.exp(-X)) sigmoidResult = 0.5 * (X/(1+abs(X))+1) #return 4*(sigmoidResult * (1-sigmoidResult)) return sigmoidResult
saver.save(sess, model_path) # display if False: x = list(range(len(y_list))) plt.subplot(211) plt.plot(x, y_list, 'r', x, y_pre_list, 'b') plt.subplot(212) plt.plot(list(range(len(all_loss))), all_loss, 'g') plt.show() # predict if False: test_data = get_both_batch_data(test_records, batch_size, series_timesteps, cycle_timesteps) test_y_list = [] test_y_pre_list = [] test_all_loss = [] for batch in test_data: predict, loss = sess.run([y_pre, loss_func], feed_dict = {series_model.X_ : batch[0], cycle_model.X_ : batch[1], y : batch[2]}) test_y_list.extend(batch[2]) test_y_pre_list.extend(predict) test_all_loss.append(loss) # display test_x = list(range(len(test_y_list))) print('---------------- Test Loss:', np.mean(test_all_loss)) plt.plot(test_x, scaler.inverse_transform(test_y_list), 'r', test_x, scaler.inverse_transform(test_y_pre_list), 'b') plt.show()
axis=1) x_1 yhat[0] = model.predict(x_1) ### predict the next X steps for i in range(1, steps): x_n = np.append(np.append(yhat[i - 1], zsy[i - 1, :]).reshape(1, 1, features + 1), x_1[-1, 0:(timesteps - 1), :].reshape(1, (timesteps - 1), features + 1), axis=1) yhat[i] = model.predict(x_n) x_1 = x_n yhat = scaler.inverse_transform(yhat.reshape(steps, 1)) trainPredictPlot = numpy.zeros(shape=(len(train) + steps, 1)) trainPredictPlot[:, :] = numpy.nan trainPredictPlot[:len(train), :] = train.reshape(len(train), 1) trainPredictPlot[len(train):len(train) + steps, :] = test.reshape(len(test), 1) testPredictPlot = numpy.zeros(shape=(len(train) + steps, 1)) testPredictPlot[:, :] = numpy.nan testPredictPlot[len(train):len(train) + steps, :] = yhat # yhat0=yhat plt.plot(trainPredictPlot) plt.plot(testPredictPlot) plt.show()
class RNNGRU(object): """Process data for ingestion. Roughly based on: https://github.com/Hvass-Labs/TensorFlow-Tutorials/blob/master/23_Time-Series-Prediction.ipynb """ def __init__( self, _data, batch_size=64, epochs=20, sequence_length=20, warmup_steps=50, dropout=0, layers=1, patience=10, units=512, display=False, binary=False): """Instantiate the class. Args: data: Tuple of (x_data, y_data, target_names) batch_size: Size of batch sequence_length: Length of vectors for for each target warmup_steps: display: Show charts of results if True binary: Process data for predicting boolean up / down movement vs actual values if True Returns: None """ # Initialize key variables self._warmup_steps = warmup_steps self._binary = binary self._display = display self._data = _data self._gpus = len(general.get_available_gpus()) # Set key file locations path_prefix = '/tmp/keras-{}'.format(int(time.time())) self._path_checkpoint = '{}.checkpoint'.format(path_prefix) self._path_model_weights = '{}.weights.h5'.format(path_prefix) self._path_model_parameters = '{}.model'.format(path_prefix) # Initialize parameters self.hyperparameters = { 'units': units, 'dropout': dropout, 'layers': int(abs(layers)), 'sequence_length': sequence_length, 'patience': patience, 'batch_size': batch_size * self._gpus, 'epochs': epochs } # Delete any stale checkpoint file if os.path.exists(self._path_checkpoint) is True: os.remove(self._path_checkpoint) ################################### # TensorFlow wizardry config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a total of half the GPU memory to be allocated config.gpu_options.per_process_gpu_memory_fraction = 0.8 # Crash with DeadlineExceeded instead of hanging forever when your # queues get full/empty config.operation_timeout_in_ms = 60000 # Create a session with the above options specified. backend.tensorflow_backend.set_session(tf.Session(config=config)) ################################### # Get data self._y_current = self._data.values() # Create test and training arrays for VALIDATION and EVALUATION (x_train, x_validation, _x_test, self._y_train, self._y_validation, self._y_test) = self._data.train_validation_test_split() (self.training_rows, self._training_vector_count) = x_train.shape (self.test_rows, _) = _x_test.shape (_, self._training_class_count) = self._y_train.shape ''' The neural network works best on values roughly between -1 and 1, so we need to scale the data before it is being input to the neural network. We can use scikit-learn for this. We first create a scaler-object for the input-signals. Then we detect the range of values from the training-data and scale the training-data. From StackOverflow: To center the data (make it have zero mean and unit standard error), you subtract the mean and then divide the result by the standard deviation. x'=x−μσ You do that on the training set of data. But then you have to apply the same transformation to your testing set (e.g. in cross-validation), or to newly obtained examples before forecast. But you have to use the same two parameters μ and σ (values) that you used for centering the training set. Hence, every sklearn's transform's fit() just calculates the parameters (e.g. μ and σ in case of StandardScaler) and saves them as an internal objects state. Afterwards, you can call its transform() method to apply the transformation to a particular set of examples. fit_transform() joins these two steps and is used for the initial fitting of parameters on the training set x, but it also returns a transformed x'. Internally, it just calls first fit() and then transform() on the same data. ''' self._x_scaler = MinMaxScaler() _ = self._x_scaler.fit_transform(self._data.vectors()) self._x_train_scaled = self._x_scaler.transform(x_train) self._x_validation_scaled = self._x_scaler.transform(x_validation) self._x_test_scaled = self._x_scaler.transform(_x_test) '''print(np.amin(self._x_train_scaled), np.amax(self._x_train_scaled)) print(np.amin(self._x_train_scaled), np.amax(self._x_train_scaled)) print(np.amin(self._x_test_scaled), np.amax(self._x_test_scaled)) print('\n', _x_test, '\n') print('\n', self._x_test_scaled, '\n')''' ''' The target-data comes from the same data-set as the input-signals, because it is the weather-data for one of the cities that is merely time-shifted. But the target-data could be from a different source with different value-ranges, so we create a separate scaler-object for the target-data. ''' self._y_scaler = MinMaxScaler() _ = self._y_scaler.fit_transform(self._data.classes()) self._y_train_scaled = self._y_scaler.transform(self._y_train) self._y_validation_scaled = self._y_scaler.transform( self._y_validation) self._y_test_scaled = self._y_scaler.transform(self._y_test) '''print(np.amin(self._y_train_scaled), np.amax(self._y_train_scaled)) print(np.amin(self._y_train_scaled), np.amax(self._y_train_scaled)) print(np.amin(self._y_test_scaled), np.amax(self._y_test_scaled)) sys.exit()''' # Print stuff print('\n> Numpy Data Type: {}'.format(type(x_train))) print("> Numpy Data Shape: {}".format(x_train.shape)) print("> Numpy Data Row[0]: {}".format(x_train[0])) print("> Numpy Data Row[Last]: {}".format(x_train[-1])) print('> Numpy Targets Type: {}'.format(type(self._y_train))) print("> Numpy Vector Feature Type: {}".format(type(x_train[0][0]))) print("> Numpy Targets Shape: {}".format(self._y_train.shape)) print('> Number of Samples: {}'.format(self._y_current.shape[0])) print('> Number of Training Samples: {}'.format(x_train.shape[0])) print('> Number of Training Classes: {}'.format( self._training_class_count)) print('> Number of Test Samples: {}'.format(self.test_rows)) print("> Training Minimum Value:", np.min(x_train)) print("> Training Maximum Value:", np.max(x_train)) print('> Number X signals: {}'.format(self._training_vector_count)) print('> Number Y signals: {}'.format(self._training_class_count)) # Print epoch related data print('> Epochs:', self.hyperparameters['epochs']) print('> Batch Size:', self.hyperparameters['batch_size']) # Display estimated memory footprint of training data. print("> Data size: {:.2f} Bytes".format(x_train.nbytes)) print('> Scaled Training Minimum Value: {}'.format( np.min(self._x_train_scaled))) print('> Scaled Training Maximum Value: {}'.format( np.max(self._x_train_scaled))) ''' The data-set has now been prepared as 2-dimensional numpy arrays. The training-data has almost 300k observations, consisting of 20 input-signals and 3 output-signals. These are the array-shapes of the input and output data: ''' print('> Scaled Training Data Shape: {}'.format( self._x_train_scaled.shape)) print('> Scaled Training Targets Shape: {}'.format( self._y_train_scaled.shape)) def model(self, params=None): """Create the Recurrent Neural Network. Args: None Returns: _model: RNN model """ # Initialize key variables if params is None: _hyperparameters = self.hyperparameters else: params['batch_size'] = params['batch_size'] * self._gpus _hyperparameters = params '''print(_hyperparameters['batch_size']) print(self.hyperparameters['batch_size']) sys.exit(0)''' # Calculate the steps per epoch epoch_steps = int( self.training_rows / _hyperparameters['batch_size']) + 1 ''' Instantiate the base model (or "template" model). We recommend doing this with under a CPU device scope, so that the model's weights are hosted on CPU memory. Otherwise they may end up hosted on a GPU, which would complicate weight sharing. NOTE: multi_gpu_model values will be way off if you don't do this. ''' with tf.device('/cpu:0'): serial_model = Sequential() ''' We can now add a Gated Recurrent Unit (GRU) to the network. This will have 512 outputs for each time-step in the sequence. Note that because this is the first layer in the model, Keras needs to know the shape of its input, which is a batch of sequences of arbitrary length (indicated by None), where each observation has a number of input-signals (num_x_signals). ''' serial_model.add(GRU( _hyperparameters['units'], stateful=True, batch_size=_hyperparameters['batch_size'], return_sequences=True, recurrent_dropout=_hyperparameters['dropout'], input_shape=(None, self._training_vector_count,))) for _ in range(1, _hyperparameters['layers']): serial_model.add(GRU( _hyperparameters['units'], stateful=True, batch_size=_hyperparameters['batch_size'], recurrent_dropout=_hyperparameters['dropout'], return_sequences=True)) ''' The GRU outputs a batch from keras_contrib.layers.advanced_activations of sequences of 512 values. We want to predict 3 output-signals, so we add a fully-connected (or dense) layer which maps 512 values down to only 3 values. The output-signals in the data-set have been limited to be between 0 and 1 using a scaler-object. So we also limit the output of the neural network using the Sigmoid activation function, which squashes the output to be between 0 and 1. ''' if False: serial_model.add(Dense( self._training_class_count, activation='sigmoid')) ''' A problem with using the Sigmoid activation function, is that we can now only output values in the same range as the training-data. For example, if the training-data only has values between -20 and +30, then the scaler-object will map -20 to 0 and +30 to 1. So if we limit the output of the neural network to be between 0 and 1 using the Sigmoid function, this can only be mapped back to values between -20 and +30. We can use a linear activation function on the output instead. This allows for the output to take on arbitrary values. It might work with the standard initialization for a simple network architecture, but for more complicated network architectures e.g. with more layers, it might be necessary to initialize the weights with smaller values to avoid NaN values during training. You may need to experiment with this to get it working. ''' if True: # Maybe use lower init-ranges. init = RandomUniform(minval=-0.05, maxval=0.05) serial_model.add(Dense( self._training_class_count, activation='linear', kernel_initializer=init)) '''print(inspect.getmembers(_model, predicate=inspect.ismethod)) print('\n\n----------------------\n\n')''' # Apply multi-GPU logic. try: # We have to wrap multi_gpu_model this way to get callbacks to work if False: parallel_model = ModelMGPU( serial_model, cpu_relocation=True, gpus=self._gpus) #_model = _model if True: parallel_model = multi_gpu_model( serial_model, cpu_relocation=True, gpus=self._gpus) # parallel_model = serial_model if False: parallel_model = serial_model print('> Training using multiple GPUs...') except ValueError: parallel_model = serial_model print('> Training using single GPU or CPU...') # Compile Model ''' This is the optimizer and the beginning learning-rate that we will use. We then compile the Keras model so it is ready for training. ''' optimizer = RMSprop(lr=1e-3) # optimizer = Adam(lr=1e-3) if self._binary is True: parallel_model.compile( loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) else: parallel_model.compile( loss=self._loss_mse_warmup, optimizer=optimizer, metrics=['accuracy']) ''' This is a very small model with only two layers. The output shape of (None, None, 3) means that the model will output a batch with an arbitrary number of sequences, each of which has an arbitrary number of observations, and each observation has 3 signals. This corresponds to the 3 target signals we want to predict. ''' print('\n> Model Summary (Serial):\n') print(serial_model.summary()) print('\n> Model Summary (Parallel):\n') print(parallel_model.summary()) '''print(_hyperparameters['batch_size']) sys.exit(0)''' # Create the batch-generator. generator = self._batch_generator( _hyperparameters['batch_size'], _hyperparameters['sequence_length']) # Validation Set ''' The neural network trains quickly so we can easily run many training epochs. But then there is a risk of overfitting the model to the training-set so it does not generalize well to unseen data. We will therefore monitor the model's performance on the test-set after each epoch and only save the model's weights if the performance is improved on the test-set. The batch-generator randomly selects a batch of short sequences from the training-data and uses that during training. But for the validation-data we will instead run through the entire sequence from the test-set and measure the prediction accuracy on that entire sequence. ''' validation_data = (np.expand_dims(self._x_validation_scaled, axis=0), np.expand_dims(self._y_validation_scaled, axis=0)) # Callback Functions ''' During training we want to save checkpoints and log the progress to TensorBoard so we create the appropriate callbacks for Keras. This is the callback for writing checkpoints during training. ''' callback_checkpoint = ModelCheckpoint( filepath=self._path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True) ''' This is the callback for stopping the optimization when performance worsens on the validation-set. ''' callback_early_stopping = EarlyStopping( monitor='val_loss', patience=_hyperparameters['patience'], verbose=1) ''' This is the callback for writing the TensorBoard log during training. ''' callback_tensorboard = TensorBoard( log_dir='/tmp/23_logs/', histogram_freq=0, write_graph=False) ''' This callback reduces the learning-rate for the optimizer if the validation-loss has not improved since the last epoch (as indicated by patience=0). The learning-rate will be reduced by multiplying it with the given factor. We set a start learning-rate of 1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4. We don't want the learning-rate to go any lower than this. ''' callback_reduce_lr = ReduceLROnPlateau( monitor='val_loss', factor=0.1, min_lr=1e-4, patience=0, verbose=1) callbacks = [ callback_early_stopping, callback_checkpoint, callback_tensorboard, callback_reduce_lr] # Train the Recurrent Neural Network '''We can now train the neural network. Note that a single "epoch" does not correspond to a single processing of the training-set, because of how the batch-generator randomly selects sub-sequences from the training-set. Instead we have selected steps_per_epoch so that one "epoch" is processed in a few minutes. With these settings, each "epoch" took about 2.5 minutes to process on a GTX 1070. After 14 "epochs" the optimization was stopped because the validation-loss had not decreased for 5 "epochs". This optimization took about 35 minutes to finish. Also note that the loss sometimes becomes NaN (not-a-number). This is often resolved by restarting and running the Notebook again. But it may also be caused by your neural network architecture, learning-rate, batch-size, sequence-length, etc. in which case you may have to modify those settings. ''' print('\n> Parameters for training\n') pprint(_hyperparameters) print('\n> Starting data training\n') history = parallel_model.fit_generator( generator=generator, epochs=_hyperparameters['epochs'], steps_per_epoch=epoch_steps, use_multiprocessing=True, validation_data=validation_data, callbacks=callbacks) '''v_steps = (self.test_rows // _hyperparameters['batch_size']) - 1 parallel_model.fit( self._x_train_scaled, self._y_train_scaled, epochs=_hyperparameters['epochs'], steps_per_epoch=epoch_steps, validation_data=validation_data, validation_steps=v_steps, callbacks=callbacks)''' # Save model '''self.save(serial_model) serial_model = self.load_model()''' # Determine whether the weights of the parallel and serial models match if general.weights_match(serial_model, parallel_model) is True: print('> Weights match (Parallel / Serial)') else: print('> Weights different (Parallel / Serial)') # sys.exit(0) '''print("Ploting History") plt.plot(history.history['loss'], label='Parallel Training Loss') plt.plot(history.history['val_loss'], label='Parallel Validation Loss') # plt.plot(parallel_model.history['loss'], label='Parallel Training Loss') # plt.plot(parallel_model.history['val_loss'], label='Parallel Validation Loss') plt.legend() plt.show() sys.exit(0)''' # Return return serial_model def save(self, _model): """Save the Recurrent Neural Network model. Args: None Returns: _model: RNN model """ # Serialize model to JSON model_yaml = _model.to_yaml() with open(self._path_model_parameters, 'w') as yaml_file: yaml_file.write(model_yaml) # Serialize weights to HDF5 _model.save_weights(self._path_model_weights) print('> Saved model to disk') def load_model(self): """Load the Recurrent Neural Network model from disk. Args: None Returns: _model: RNN model """ # Load json and create model print('> Loading model from disk') with open(self._path_model_parameters, 'r') as yaml_file: loaded_model_yaml = yaml_file.read() _model = model_from_yaml(loaded_model_yaml) # Load weights into new model _model.load_weights(self._path_model_weights) print('> Finished loading model from disk') # Return return _model def evaluate(self, _model): """Evaluate the model. Args: _model: Model to evaluate Returns: None """ # Load Checkpoint ''' Because we use early-stopping when training the model, it is possible that the model's performance has worsened on the test-set for several epochs before training was stopped. We therefore reload the last saved checkpoint, which should have the best performance on the test-set. ''' print('> Loading model weights') if os.path.exists(self._path_checkpoint): _model.load_weights(self._path_checkpoint) '''optimizer = RMSprop(lr=1e-3) if self._binary is True: _model.compile( loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) else: _model.compile( loss=self._loss_mse_warmup, optimizer=optimizer, metrics=['accuracy'])''' # Performance on Test-Set. ''' We can now evaluate the model's performance on the validation-set. This function expects a batch of data, but we will just use one long time-series for the test-set, so we just expand the array-dimensionality to create a batch with that one sequence. ''' if self._binary is False: x_scaled = self._x_test_scaled y_scaled = self._y_test_scaled else: # Get the filtered vectors and classes (filtered_vectors, filtered_classes) = self._data.stochastic_vectors_classes() # Scale and then evaluate x_scaled = self._x_scaler.transform(filtered_vectors) y_scaled = self._y_scaler.transform(filtered_classes) # Evaluate the MSE accuracy result = _model.evaluate( x=np.expand_dims(x_scaled, axis=0), y=np.expand_dims(y_scaled, axis=0)) # If you have several metrics you can use this instead. print('> Metrics (test-set):') for _value, _metric in zip(result, _model.metrics_names): print('\t{}: {:.10f}'.format(_metric, _value)) if self._binary is True: # Input-signals for the model. x_values = np.expand_dims(x_scaled, axis=0) # Get the predictions predictions_scaled = _model.predict_classes(x_values, verbose=1) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. predictions = self._y_scaler.inverse_transform( predictions_scaled[0]) # Print meaningful human accuracy values print( '> Human accuracy {:.3f} %' ''.format(general.binary_accuracy( predictions, filtered_classes) * 100)) def objective(self, params=None): """Optimize the Recurrent Neural Network. Args: None Returns: _model: RNN model """ model = deepcopy(self.model(params=params)) if bool(self._binary) is False: scaled_vectors = self._x_test_scaled test_classes = self._y_test else: # Get the filtered vectors and classes (filtered_vectors, test_classes) = self._data.stochastic_vectors_classes() # Scale and then evaluate scaled_vectors = self._x_scaler.transform(filtered_vectors) # Input-signals for the model. x_values = np.expand_dims(scaled_vectors, axis=0) # Get the predictions predictions_scaled = model.predict(x_values, verbose=1) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. predictions = self._y_scaler.inverse_transform( predictions_scaled[0]) # Get the error value accuracy = mean_absolute_error(test_classes, predictions) # Free object memory del model gc.collect() # Print meaningful human accuracy values if self._binary is True: # Print predictions and actuals: print( '> Human accuracy {:.5f} %' ''.format(general.binary_accuracy( predictions, test_classes) * 100)) # Return return { 'loss': (accuracy * -1), 'status': STATUS_OK, 'estimated_accuracy': accuracy, 'hyperparameters': params} def cleanup(self): """Release memory and delete checkpoint files. Args: None Returns: None """ # Delete os.remove(self._path_checkpoint) def stationary(self): """Evaluate wether the timeseries is stationary. non-stationary timeseries are probably random walks and not suitable for forecasting. Args: None Returns: state: True if stationary """ # Initialize key variables state = False values = [] # statistical test result = adfuller(self._y_current) adf = result[0] print('> Stationarity Test:') print(' ADF Statistic: {:.3f}'.format(adf)) print(' p-value: {:.3f}'.format(result[1])) print(' Critical Values:') for key, value in result[4].items(): print('\t{}: {:.3f}'.format(key, value)) values.append(value) # Return if adf < min(values): state = True print(' Stationarity: {}'.format(state)) return state def _batch_generator(self, batch_size, sequence_length): """Create generator function to create random batches of training-data. Args: batch_size: Size of batch sequence_length: Length of sequence Returns: (x_batch, y_batch) """ # Infinite loop. while True: # Allocate a new array for the batch of input-signals. x_shape = ( batch_size, sequence_length, self._training_vector_count) x_batch = np.zeros(shape=x_shape, dtype=np.float16) # Allocate a new array for the batch of output-signals. y_shape = (batch_size, sequence_length, self._training_class_count) y_batch = np.zeros(shape=y_shape, dtype=np.float16) # Fill the batch with random sequences of data. for i in range(batch_size): # Get a random start-index. # This points somewhere into the training-data. idx = np.random.randint( self.training_rows - sequence_length) # Copy the sequences of data starting at this index. x_batch[i] = self._x_train_scaled[idx:idx+sequence_length] y_batch[i] = self._y_train_scaled[idx:idx+sequence_length] yield (x_batch, y_batch) def _loss_mse_warmup(self, y_true, y_pred): """Calculate the Mean Squared Errror. Calculate the Mean Squared Error between y_true and y_pred, but ignore the beginning "warmup" part of the sequences. We will use Mean Squared Error (MSE) as the loss-function that will be minimized. This measures how closely the model's output matches the true output signals. However, at the beginning of a sequence, the model has only seen input-signals for a few time-steps, so its generated output may be very inaccurate. Using the loss-value for the early time-steps may cause the model to distort its later output. We therefore give the model a "warmup-period" of 50 time-steps where we don't use its accuracy in the loss-function, in hope of improving the accuracy for later time-steps Args: y_true: Desired output. y_pred: Model's output. Returns: loss_mean: Mean Squared Error """ warmup_steps = self._warmup_steps # The shape of both input tensors are: # [batch_size, sequence_length, num_y_signals]. # Ignore the "warmup" parts of the sequences # by taking slices of the tensors. y_true_slice = y_true[:, warmup_steps:, :] y_pred_slice = y_pred[:, warmup_steps:, :] # These sliced tensors both have this shape: # [batch_size, sequence_length - warmup_steps, num_y_signals] # Calculate the MSE loss for each value in these tensors. # This outputs a 3-rank tensor of the same shape. loss = tf.losses.mean_squared_error(labels=y_true_slice, predictions=y_pred_slice) # Keras may reduce this across the first axis (the batch) # but the semantics are unclear, so to be sure we use # the loss across the entire tensor, we reduce it to a # single scalar with the mean function. loss_mean = tf.reduce_mean(loss) return loss_mean def plot_train(self, model, start_idx, length=100): """Plot the predicted and true output-signals. Args: model: Training model start_idx: Start-index for the time-series. length: Sequence-length to process and plot. Returns: None """ # Plot self._plot_comparison(model, start_idx, length=length, train=True) def plot_test(self, model, start_idx, length=100): """Plot the predicted and true output-signals. Args: model: Training model start_idx: Start-index for the time-series. length: Sequence-length to process and plot. Returns: None """ # Plot self._plot_comparison(model, start_idx, length=length, train=False) def _plot_comparison(self, model, start_idx, length=100, train=True): """Plot the predicted and true output-signals. Args: model: Training model start_idx: Start-index for the time-series. length: Sequence-length to process and plot. train: Boolean whether to use training- or test-set. Returns: None """ # Initialize key variables datetimes = {} num_train = self.training_rows # Don't plot if we are looking at binary classes if bool(self._binary) is True: print('> Will not plot charts for binary class values.') return # End-index for the sequences. end_idx = start_idx + length # Variables for date formatting days = mdates.DayLocator() # Every day months = mdates.MonthLocator() # Every month months_format = mdates.DateFormatter('%b %Y') days_format = mdates.DateFormatter('%d') # Assign other variables dependent on the type of data we are plotting if train is True: # Use training-data. x_values = self._x_train_scaled[start_idx:end_idx] y_true = self._y_train[start_idx:end_idx] shim = 'Train' # Datetimes to use for training datetimes[shim] = self._data.datetime()[ :num_train][start_idx:end_idx] else: # Scale the data x_test_scaled = self._x_scaler.transform( self._data.vectors_test_all()) # Use test-data. x_values = x_test_scaled[start_idx:end_idx] y_true = self._y_test[start_idx:end_idx] shim = 'Test' # Datetimes to use for testing datetimes[shim] = self._data.datetime()[ -self.test_rows-1:][start_idx:end_idx] # Input-signals for the model. x_values = np.expand_dims(x_values, axis=0) # Use the model to predict the output-signals. y_pred = model.predict(x_values) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. y_pred_rescaled = self._y_scaler.inverse_transform(y_pred[0]) # For each output-signal. for signal in range(len(self._data.labels())): # Assign other variables dependent on the type of data plot if train is True: # Only get current values that are a part of the training data current = self._y_current[:num_train][start_idx:end_idx] # The number of datetimes for the 'actual' plot must match # that of current values datetimes['actual'] = self._data.datetime()[ :num_train][start_idx:end_idx] else: # Only get current values that are a part of the test data. current = self._y_current[ -self.test_rows:][start_idx:] # The number of datetimes for the 'actual' plot must match # that of current values datetimes['actual'] = self._data.datetime()[ -self.test_rows:][start_idx:] # Create a filename filename = ( '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format( self.hyperparameters['batch_size'], self.hyperparameters['epochs'], num_train, signal, int(time.time()), shim) # Get the output-signal predicted by the model. signal_pred = y_pred_rescaled[:, signal] # Get the true output-signal from the data-set. signal_true = y_true[:, signal] # Create a new chart (fig, axis) = plt.subplots(figsize=(15, 5)) # Plot and compare the two signals. axis.plot( datetimes[shim][:len(signal_true)], signal_true, label='Current +{}'.format(self._data.labels()[signal])) axis.plot( datetimes[shim][:len(signal_pred)], signal_pred, label='Prediction') axis.plot(datetimes['actual'], current, label='Current') # Set plot labels and titles axis.set_title('{1}ing Forecast ({0} Future Intervals)'.format( self._data.labels()[signal], shim)) axis.set_ylabel('Values') axis.legend( bbox_to_anchor=(1.04, 0.5), loc='center left', borderaxespad=0) # Add gridlines and ticks ax = plt.gca() ax.grid(True) # Add major gridlines ax.xaxis.grid(which='major', color='black', alpha=0.2) ax.yaxis.grid(which='major', color='black', alpha=0.2) # Add minor ticks (They must be turned on first) ax.minorticks_on() ax.xaxis.grid(which='minor', color='black', alpha=0.1) ax.yaxis.grid(which='minor', color='black', alpha=0.1) # Format the tick labels ax.xaxis.set_major_locator(months) ax.xaxis.set_major_formatter(months_format) ax.xaxis.set_minor_locator(days) # Remove tick marks ax.tick_params(axis='both', which='both', length=0) # Print day numbers on xaxis for Test data only if train is False: ax.xaxis.set_minor_formatter(days_format) plt.setp(ax.xaxis.get_minorticklabels(), rotation=90) # Rotates and right aligns the x labels, and moves the bottom of # the axes up to make room for them fig.autofmt_xdate() # Plot grey box for warmup-period if we are working with training # data and the start is within the warmup-period if (0 < start_idx < self._warmup_steps): if train is True: plt.axvspan( datetimes[shim][start_idx], datetimes[shim][self._warmup_steps], facecolor='black', alpha=0.15) # Show and save the image if self._display is True: fig.savefig(filename, bbox_inches='tight') plt.show() else: fig.savefig(filename, bbox_inches='tight') print('> Saving file: {}'.format(filename)) # Close figure plt.close(fig=fig)
X_final = np.zeros_like(X_test) print(X_final.shape) X_train = X_train.reshape((X_train.shape[0], X_train.shape[1], 1)) X_test = X_test.reshape((X_test.shape[0], X_test.shape[1], 1)) # predict the model.In case of volume use high no of epochs ~500-800 to achieve the shape of the graph. no_of_models = 5 for k in range(no_of_models): history = model.fit(X_train, y_train, epochs=20, validation_split=0.015, shuffle=False) Xt = model.predict(X_test) score = model.evaluate(X_test, y_test, verbose=1) Xt = scl.inverse_transform(Xt.reshape(-1, look_forward)) # print(Xt.shape) # print(Xt) X_final += Xt Xt = X_final / no_of_models y_test = scl.inverse_transform(y_test.reshape(-1, look_forward)) model.save('open_lstm.h5') plt.figure(figsize=(18, 9)) plt.subplot(1, 2, 1) plt.plot(y_test, color='red', label='actual') plt.plot(Xt, color='blue', label='predict') plt.legend() error = abs((y_test - Xt) / y_test) for i in range(len(Xt)):
dataset = df.values dataset = dataset.astype('float32') # normalize the dataset scaler = MinMaxScaler(feature_range=(0, 1)) dataset = scaler.fit_transform(dataset) #prepare the X and Y label X,y = create_dataset(dataset, 1) print X.shape, 'XXXXX', y.shape #Take 80% of data as the training sample and 20% as testing sample trainX, testX, trainY, testY = train_test_split(X, y, test_size=0.20, shuffle=False) print scaler.inverse_transform(trainX[:10]) print len(testY), 'testY length' # reshape input to be [samples, time steps, features] trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) print trainX[:3],trainX.shape, 'ttttttttttX' testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1])) # create and fit the LSTM network model = Sequential() model.add(LSTM(4, input_shape=(1, 1))) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') history=model.fit(trainX, trainY, nb_epoch=5, batch_size=1, validation_data=(testX, testY), verbose=2) plt.plot(history.history['loss'], label='train')
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1)) X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1)) # LSTM model = Sequential() model.add(LSTM(32, input_dim=1)) #look_back)) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(X_train, y_train, nb_epoch=100, batch_size=5, verbose=2) train_pred = model.predict(X_train) test_pred = model.predict(X_test) # scale back train_pred = scaler.inverse_transform(train_pred) y_train = scaler.inverse_transform(y_train) test_pred = scaler.inverse_transform(test_pred) y_test = scaler.inverse_transform(y_test) # shift predictions for plotting train_pred_plot = np.empty_like(dataset) train_pred_plot[:,:] = np.nan train_pred_plot[look_back:len(train_pred)+look_back,:] = train_pred test_pred_plot = np.empty_like(dataset) test_pred_plot[:,:] = np.nan test_pred_plot[len(train_pred)+(look_back*2)+1:len(dataset)-1,:] = test_pred f = plt.figure() plt.plot(scaler.inverse_transform(dataset), color='b', lw=2.0, label='S&P 500')
# reshape input to be [samples, time steps, features] trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1])) # create and fit the LSTM network model = Sequential() model.add(LSTM(6, input_dim=look_back)) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(trainX, trainY, nb_epoch=100, batch_size=1, verbose=2) # Estimate model performance trainScore = model.evaluate(trainX, trainY, verbose=0) print('Train Score: ', scaler.inverse_transform(numpy.array([[trainScore]]))) testScore = model.evaluate(testX, testY, verbose=0) print('Test Score: ', scaler.inverse_transform(numpy.array([[testScore]]))) # generate predictions for training trainPredict = model.predict(trainX) testPredict = model.predict(testX) #print(scaler.inverse_transform(testPredict)) # shift train predictions for plotting trainPredictPlot = numpy.empty_like(dataset) trainPredictPlot[:, :] = numpy.nan trainPredictPlot[look_back:len(trainPredict)+look_back, :] = trainPredict # shift test predictions for plotting
#training set plus testset dataset_total = pd.concat((prices_dataset_train['adj_close'],prices_dataset_test['adj_close']), axis=0) #vertical axis=0 horizontal axis=1 #all inputs for test set inputs = dataset_total[len(dataset_total)-len(prices_dataset_test)-40:].values inputs = inputs.reshape(-1,1) #neural net trained on the scaled values we have to min-max normalize the inputs #it is already fitted so we can use transform directly inputs = min_max_scaler.transform(inputs) X_test = [] for i in range(40,len(prices_dataset_test)+40): X_test.append(inputs[i-40:i,0]) X_test = np.array(X_test) X_test = np.reshape(X_test, (X_test.shape[0],X_test.shape[1],1)) predictions = model.predict(X_test) #inverse the predicitons because we applied normalization but we want to compare with the original prices predictions = min_max_scaler.inverse_transform(predictions) #plotting the results plt.plot(testset, color='blue', label='Actual S&P500 Prices') plt.plot(predictions, color='green', label='LSTM Predictions') plt.title('S&P500 Predictions with Reccurent Neural Network') plt.xlabel('Time') plt.ylabel('Price') plt.legend() plt.show()
prediction. The default sigmoid activation function is used for the LSTM blocks. The network is trained for 100 epochs and a batch size of 1 is used.""" # create and fit the LSTM network model = Sequential() model.add(LSTM(4, input_dim=look_back)) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(trainX, trainY, nb_epoch=100, batch_size=1, verbose=2) # make predictions trainPredict = model.predict(trainX) testPredict = model.predict(testX) # invert predictions trainPredict = scaler.inverse_transform(trainPredict) trainY = scaler.inverse_transform([trainY]) testPredict = scaler.inverse_transform(testPredict) testY = scaler.inverse_transform([testY]) # calculate root mean squared error trainScore = math.sqrt(mean_squared_error(trainY[0], trainPredict[:,0])) print(trainY[0]) print(trainPredict[:,0]) print('Train Score: %.2f RMSE' % (trainScore)) testScore = math.sqrt(mean_squared_error(testY[0], testPredict[:,0])) print(testY[0]) print(testPredict[:,0]) print('Test Score: %.2f RMSE' % (testScore)) # shift train predictions for plotting trainPredictPlot = numpy.empty_like(dataset)
file.close() loss = history.history['loss'] val_loss = history.history['val_loss'] plt.plot(loss, label='Training Loss') plt.plot(val_loss, label='Validation Loss') plt.title('Training and Validation Loss') plt.legend() plt.show() ################## predict ###################### # 测试集输入模型进行预测 predicted_stock_price = model.predict(x_test) # 对预测数据还原---从(0,1)反归一化到原始范围 predicted_stock_price = sc.inverse_transform(predicted_stock_price) # 对真实数据还原---从(0,1)反归一化到原始范围 real_stock_price = sc.inverse_transform(test_set[60:]) # 画出真实数据和预测数据的对比曲线 plt.plot(real_stock_price, color='red', label='MaoTai Stock Price') plt.plot(predicted_stock_price, color='blue', label='Predicted MaoTai Stock Price') plt.title('MaoTai Stock Price Prediction') plt.xlabel('Time') plt.ylabel('MaoTai Stock Price') plt.legend() plt.show() ##########evaluate############## # calculate MSE 均方误差 ---> E[(预测值-真实值)^2] (预测值减真实值求平方后求均值)
scaler = MinMaxScaler(feature_range=(0, 1)) # 正则化函数 MAPE_list = [0 for i in range(120)] for i in range(120): train_data_norm = scaler.fit_transform(pd.DataFrame(data_list[i][1:145])) train_data_norm = np.array(train_data_norm).tolist() train_data_norm = list(chain.from_iterable(train_data_norm)) # del(train_data_norm[0]) X, y = split_sequence(train_data_norm, 24) n_features = 1 X = X.reshape((X.shape[0], X.shape[1], n_features)) # reshape input to be [samples, time steps, features] #trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) model.fit(X, y, epochs=50, batch_size=1, verbose=2) pred_val = [] pred_data = train_data_norm[120:145] for i in range(24): # 预测一天 pred_X = array(pred_data).reshape((1, 24, 1)) pred = model.predict(pred_X)[:, 0].tolist()[0] # print(pred) pred_val.append(pred) del (pred_data[0]) pred_data.append(pred) pred_val = scaler.inverse_transform(pd.DataFrame(pred_val)) pred_val = np.array(pred_val).tolist() pred_val = list(chain.from_iterable(pred_val)) ## 计算MAPE MAPE_list[i] = MAPE(pred_val, data_list[i][145:169]) print(MAPE_list[i])
model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') history=model.fit(trainX, trainY, nb_epoch=5, batch_size=1, validation_data=(testX, testY), verbose=2) # plot the loss vs val_loss plt.plot(history.history['loss'], label='train') plt.plot(history.history['val_loss'], label='test') plt.legend() plt.show() # predict the bitcoin price in train_predict and test_predict train_predict = model.predict(trainX) test_predict = model.predict(testX) futurePredict = model.predict(np.asarray([[test_predict[-1]]])) futurePredict = scaler.inverse_transform(futurePredict) # invert predictions train_predict = scaler.inverse_transform(train_predict) trainY = scaler.inverse_transform(trainY) test_predict = scaler.inverse_transform(test_predict) testY = scaler.inverse_transform(testY) print("Price for last 10 days: ") print(test_predict[-10:]) print("Bitcoin price for tomorrow: ", futurePredict) print ('test prdict length', len(test_predict)) # calculate root mean squared error trainScore = math.sqrt(mean_squared_error(trainY[:,0], train_predict[:,0]))
#LSTM creation model = Sequential() model.add(LSTM(10, input_shape=(1, timesteps))) model.add(Dense(units=1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(trainX, trainY, epochs=50, batch_size=1) #Prediction trainPredict = model.predict(trainX) testPredict = model.predict(testX) #İnvert scaling trainPredict = scaler.inverse_transform(trainPredict) testPredict = scaler.inverse_transform(testPredict) trainY = scaler.inverse_transform([trainY]) testY = scaler.inverse_transform([testY]) #Error hesabı trainScore = math.sqrt(mean_squared_error(trainY[0], trainPredict[:, 0])) print('Train score:%.2f RMSE' % (trainScore)) testScore = math.sqrt(mean_squared_error(testY[0], testPredict[:, 0])) print('Test score:%.2f RMSE' % (testScore)) #Shifting and Plotting plt.plot(scaler.inverse_transform(dataset)) trainPredictPlot = np.empty_like(dataset) trainPredictPlot[:, :] = np.nan
def main(): # fix random seed for reproducibility numpy.random.seed(7) # Get CLI arguments parser = argparse.ArgumentParser() parser.add_argument( '--gpus', help='Number of GPUs to use.', type=int, default=1) args = parser.parse_args() gpus = args.gpus # load the dataset dataframe = DataFrame( [0.00000, 5.99000, 11.92016, 17.73121, 23.36510, 28.76553, 33.87855, 38.65306, 43.04137, 46.99961, 50.48826, 53.47244, 55.92235, 57.81349, 59.12698, 59.84970, 59.97442, 59.49989, 58.43086, 56.77801, 54.55785, 51.79256, 48.50978, 44.74231, 40.52779, 35.90833, 30.93008, 25.64279, 20.09929, 14.35496, 8.46720, 2.49484, -3.50245, -9.46474, -15.33247, -21.04699, -26.55123, -31.79017, -36.71147, -41.26597, -45.40815, -49.09663, -52.29455, -54.96996, -57.09612, -58.65181, -59.62146, -59.99540, -59.76988, -58.94716, -57.53546, -55.54888, -53.00728, -49.93605, -46.36587, -42.33242, -37.87600, -33.04113, -27.87613, -22.43260, -16.76493, -10.92975, -4.98536, 1.00883, 6.99295, 12.90720, 18.69248, 24.29100, 29.64680, 34.70639, 39.41920, 43.73814, 47.62007, 51.02620, 53.92249, 56.28000, 58.07518, 59.29009, 59.91260, 59.93648, 59.36149, 58.19339, 56.44383, 54.13031, 51.27593, 47.90923, 44.06383, 39.77815, 35.09503, 30.06125, 24.72711, 19.14590, 13.37339, 7.46727, 1.48653, -4.50907, -10.45961, -16.30564, -21.98875, -27.45215, -32.64127, -37.50424, -41.99248, -46.06115, -49.66959, -52.78175, -55.36653, -57.39810, -58.85617, -59.72618, -59.99941, -59.67316, -58.75066, -57.24115, -55.15971, -52.52713, -49.36972, -45.71902, -41.61151, -37.08823, -32.19438, -26.97885, -21.49376, -15.79391, -9.93625, -3.97931, 2.01738, 7.99392, 13.89059, 19.64847, 25.21002, 30.51969, 35.52441, 40.17419, 44.42255, 48.22707, 51.54971, 54.35728, 56.62174, 58.32045, 59.43644, 59.95856, 59.88160, 59.20632, 57.93947, 56.09370, 53.68747, 50.74481, 47.29512, 43.37288, 39.01727, 34.27181, 29.18392, 23.80443, 18.18710, 12.38805, 6.46522, 0.47779, -5.51441, -11.45151]) dataset = dataframe.values dataset = dataset.astype('float32') # normalize the dataset scaler = MinMaxScaler(feature_range=(0, 1)) dataset = scaler.fit_transform(dataset) # split into train and test sets train_size = int(len(dataset) * 0.67) test_size = len(dataset) - train_size train, test = dataset[0:train_size, :], dataset[train_size:len(dataset), :] # reshape into X=t and Y=t+1 look_back = 1 trainX, trainY = create_dataset(train, look_back) testX, testY = create_dataset(test, look_back) # reshape input to be [samples, time steps, features] trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1])) # Create layers for model x_tensor = Input(shape=(1, look_back)) layer_1 = LSTM(4)(x_tensor) y_tensor = Dense(1)(layer_1) # Create and fit the LSTM network with tf.device('/cpu:0'): serial_model = Model(inputs=x_tensor, outputs=y_tensor) # Modify model for GPUs if necessary if gpus == 1: parallel_model = serial_model else: parallel_model = multi_gpu_model( serial_model, cpu_relocation=True, gpus=gpus) parallel_model.compile( loss='mean_squared_error', optimizer='adam') parallel_model.fit( trainX, trainY, epochs=100, batch_size=int(dataset.size * gpus / 20), verbose=2) # make predictions if gpus == 1: trainPredict = parallel_model.predict(trainX) testPredict = parallel_model.predict(testX) else: trainPredict = serial_model.predict(trainX) testPredict = serial_model.predict(testX) # invert predictions trainPredict = scaler.inverse_transform(trainPredict) trainY = scaler.inverse_transform([trainY]) testPredict = scaler.inverse_transform(testPredict) testY = scaler.inverse_transform([testY]) # calculate root mean squared error trainScore = math.sqrt(mean_squared_error(trainY[0], trainPredict[:, 0])) print('Train Score: %.2f RMSE' % (trainScore)) testScore = math.sqrt(mean_squared_error(testY[0], testPredict[:, 0])) print('Test Score: %.2f RMSE' % (testScore)) # shift train predictions for plotting trainPredictPlot = numpy.empty_like(dataset) trainPredictPlot[:, :] = numpy.nan trainPredictPlot[look_back:len(trainPredict)+look_back, :] = trainPredict # shift test predictions for plotting testPredictPlot = numpy.empty_like(dataset) testPredictPlot[:, :] = numpy.nan testPredictPlot[ len(trainPredict)+(look_back*2)+1:len(dataset)-1, :] = testPredict # plot baseline and predictions plt.plot(scaler.inverse_transform(dataset), label='Complete Data') plt.plot(trainPredictPlot, label='Training Data') plt.plot(testPredictPlot, label='Prediction Data') plt.legend(loc='upper left') plt.title('Using {} GPUs'.format(gpus)) plt.show()
history = model.fit(train_X, train_y, epochs=50, batch_size=20, validation_data=(test_X, test_y), verbose=2, shuffle=False) # plot training and test loss curve pyplot.plot(history.history['loss'], label='train') pyplot.plot(history.history['val_loss'], label='test') pyplot.legend() pyplot.show() # make a prediction yhat = model.predict(test_X) test_X = test_X.reshape((test_X.shape[0], test_X.shape[2])) # invert scaling for forecast inv_yhat = np.concatenate((yhat, test_X[:, 1:]), axis=1) inv_yhat = scaler.inverse_transform(inv_yhat) inv_yhat = inv_yhat[:, 0] # invert scaling for actual test_y = test_y.reshape((len(test_y), 1)) inv_y = np.concatenate((test_y, test_X[:, 1:]), axis=1) inv_y = scaler.inverse_transform(inv_y) inv_y = inv_y[:, 0] # calculate RMSE rmse = sqrt(mean_squared_error(inv_y, inv_yhat)) print('Test RMSE: %.3f' % rmse) # model.save('baseline_model')
# Training is now complete! # Get the final accuracy scores by running the "cost" operation on the training and test data sets final_training_cost = session.run(cost, feed_dict={X: X_scaled_training, Y: Y_scaled_training}) final_testing_cost = session.run(cost, feed_dict={X: X_scaled_testing, Y: Y_scaled_testing}) print("Final Training cost: {}".format(final_training_cost)) print("Final Testing cost: {}".format(final_testing_cost)) # Now that the neural network is trained, let's use it to make predictions for our test data. # Pass in the X testing data and run the "prediciton" operation Y_predicted_scaled = session.run(prediction, feed_dict={X: X_scaled_testing}) # Unscale the data back to it's original units (dollars) Y_predicted = Y_scaler.inverse_transform(Y_predicted_scaled) real_earnings = test_data_df['total_earnings'].values[0] predicted_earnings = Y_predicted[0][0] print("The actual earnings of Game #1 were ${}".format(real_earnings)) print("Our neural network predicted earnings of ${}".format(predicted_earnings)) model_builder = tf.saved_model.builder.SavedModelBuilder("exported_model") inputs = { 'input': tf.saved_model.utils.build_tensor_info(X) } outputs = { 'earnings': tf.saved_model.utils.build_tensor_info(prediction) }
# Part 3 - Making the predictions and visualising the results # Getting the real stock price of 2018 feb dataset_test = pd.read_csv('enscotest.csv') real_stock_price = dataset_test.iloc[:, 1:2].values # Getting the predicted stock price of 2018 feb dataset_total = pd.concat((dataset_train['open'], dataset_test['open']), axis = 0) inputs = dataset_total[len(dataset_total) - len(dataset_test) - 60:].values inputs = inputs.reshape(-1,1) inputs = sc.transform(inputs) X_test = [] for i in range(60, 80): X_test.append(inputs[i-60:i, 0]) X_test = np.array(X_test) X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1)) predicted_stock_price = regressor.predict(X_test) predicted_stock_price = sc.inverse_transform(predicted_stock_price) # Visualising the results plt.plot(real_stock_price, color = 'red', label = 'Real Ensco Stock Price') plt.plot(predicted_stock_price, color = 'blue', label = 'Predicted Ensco Stock Price') plt.title('Ensco Stock Price Prediction') plt.xlabel('Time') plt.ylabel('Ensco Stock Price') plt.legend() plt.show()
# verify import matplotlib.pyplot as plt plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('loss func') plt.ylabel('loss/val_loss') plt.xlabel('epochs') plt.legend(['train', 'test'], loc='upper right') plt.show() scores = model.evaluate(tX_test, ty_test) print('\n%s: %.3f' % (model.metrics_names, scores)) # Step 4: test y_pred = model.predict(tX_test) y_pred = y_scaler.inverse_transform(y_pred) print('y_pred: \n', y_pred[:5]) import pandas as pd dfy = pd.DataFrame({'Truth class': y_test[...,0], 'Pred class': y_pred[...,0]}) long_dfy = pd.melt(dfy, value_vars=['Truth class', 'Pred class']) from plotnine import * (ggplot(long_dfy, aes(x='value', color='variable', fill='variable')) + geom_density(alpha=0.5) + theme_bw())
class Train: def __init__(self, f, maps): self.data = pandas.DataFrame(read_csv(f, header=0)) self.scaler = None self.map_condition = maps def series2supervise(self, data, n_in=24, n_out=6, is_drop=True): data = pandas.DataFrame(data) cols, names = list(), list() # add input patterns for i in range(n_in, 0, -1): cols.append(data.shift(i)) # add output patterns for i in range(0, n_out): cols.append(data.shift(-i)) # concat cols to array res = concat(cols, axis=1) if is_drop: res.dropna(inplace=True) return res def data_prepro(self): dt = self.data.sort_values([self.data.columns[1], self.data.columns[2]], ascending=True) v = dt.values v[:, 0] = [self.map_condition[x.lower()] for x in v[:, 0]] v = v.astype('float32') reframed = self.series2supervise(v) reframed.columns = range(6*30) drop = [] for i in range(30): if i < 24: drop += [i*6+1, i*6+2] else: drop += [i*6+1, i*6+2, i*6+3, i*6+4, i*6+5] print(reframed.values[10, :]) reframed.drop(reframed.columns[drop], axis=1, inplace=True) print(reframed.shape) print(reframed.values[10, :]) self.scaler = MinMaxScaler(feature_range=(0, 1)) reframed = pandas.DataFrame(self.scaler.fit_transform(reframed)) return reframed def model(self): reframed = self.data_prepro() values = reframed.values n_train = 365*24 n_test = 30*24 train = values[:-n_test, :] test = values[-n_test:, :] train_X, train_y = train[:, :-6], train[:, -6:] test_X, test_y = test[:, :-6], test[:, -6:] print(train.shape) # reshape train_X = train_X.reshape(train_X.shape[0], 1, train_X.shape[1]) test_X = test_X.reshape(test_X.shape[0], 1, test_X.shape[1]) # build network model = Sequential() model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2]))) model.add(Dense(6)) model.compile(loss='mae', optimizer='adam') # fit fit = model.fit(train_X, train_y, epochs=1, batch_size=72, validation_data=(test_X, test_y), verbose=2, shuffle=False) print(" -------train loss-------") # predict res_y = model.predict(test_X) # get true value test_X = test_X.reshape(test_X.shape[0], test_X.shape[2]) y = concatenate((test_X[:, :], res_y), axis=1) y = self.scaler.inverse_transform(y) y = y[:, -6:] # evaluate RMSE ground_y = test_y.reshape((len(test_y), 6)) ground_y = concatenate((test_X[:, :], ground_y), axis=1) ground_y = self.scaler.inverse_transform(ground_y) ground_y = ground_y[:, -6:] print(" -------PREDICTED Condition-------") # print(y) #print(ground_y) print(" -------ERROR DELTA------") #print(y-ground_y) rmse = sqrt(mean_squared_error(y, ground_y)) print(" -------RMSE------") print(rmse) return model, self.scaler
# Fitting the RNN to the Training set regressor.fit(X_train, y_train, batch_size = 32, epochs = 200) # Part 3 - Making the predictions and visualising the results # Getting the real stock price of 2017 test_set = pd.read_csv('Google_Stock_Price_Test.csv') real_stock_price = test_set.iloc[:,1:2].values # Getting the predicted stock price of 2017 inputs = real_stock_price inputs = sc.transform(inputs) inputs = np.reshape(inputs, (20, 1, 1)) predicted_stock_price = regressor.predict(inputs) predicted_stock_price = sc.inverse_transform(predicted_stock_price) # Visualising the results plt.plot(real_stock_price, color = 'red', label = 'Real Google Stock Price') plt.plot(predicted_stock_price, color = 'blue', label = 'Predicted Google Stock Price') plt.title('Google Stock Price Prediction') plt.xlabel('Time') plt.ylabel('Google Stock Price') plt.legend() plt.show() # Homework # Getting the real stock price of 2012 - 2016 real_stock_price_train = pd.read_csv('Google_Stock_Price_Train.csv') real_stock_price_train = real_stock_price_train.iloc[:,1:2].values
def trainTestAndPredict(startDateForPrediction, dateToPredict, tick, dim_window): fix.pdr_override() start = "2005-01-01" end = str(date.today()) sc = MinMaxScaler(feature_range=(0, 1)) alldata = ist.get_data(tick, start_date=start, end_date=end) process = DataProcessing("stock_prices.csv", 0.9) process.generate_test(30) process.generate_train(30) X_train = sc.fit_transform(process.X_train) Y_train = sc.fit_transform(process.Y_train) X_test = sc.fit_transform(process.X_test) Y_test = sc.fit_transform(process.Y_test) model = Sequential() model.add(Dense(60, activation=tf.nn.selu)) model.add(Dropout(0.2)) model.add(Dense(40, activation=tf.nn.selu)) model.add(Dropout(0.2)) model.add(Dense(1, activation=tf.nn.selu)) model.compile(optimizer="Adam", loss="mean_squared_error", metrics=['mean_absolute_error', 'mean_squared_error']) history = model.fit(X_train, Y_train, epochs=100, validation_split=0.2, batch_size=30) print(model.summary()) loss, mae, mse = model.evaluate(X_test, Y_test) a = 'Adam MLP.png' b = 'Adam MLP' # stampa di errori MAE e MSE def plot_history(storia): hist = pd.DataFrame(storia.history) hist['epoch'] = storia.epoch plt.figure() plt.title("Mean Absolute Error " + str(b)) plt.xlabel('Epoch') plt.ylabel('Mean Abs Error') plt.plot(hist['epoch'], hist['mean_absolute_error'], label='Train Error') plt.plot(hist['epoch'], hist['val_mean_absolute_error'], label='Val Error') plt.legend() plt.savefig('MAE' + str(a)) plt.figure() plt.title("Mean Squared Error " + str(b)) plt.xlabel('Epoch') plt.ylabel('Mean Square Error') plt.plot(hist['epoch'], hist['mean_squared_error'], label='Train Error') plt.plot(hist['epoch'], hist['val_mean_squared_error'], label='Val Error') plt.legend() fig = plt.gcf() plt.show() plt.draw() fig.savefig('MSE' + str(a)) plot_history(history) # stampa punti nello spazio con valori predizioni vs valori reali test_predictions = sc.inverse_transform(model.predict(X_test)).flatten() Y_test = sc.inverse_transform(Y_test) plt.figure() plt.title("True Value VS Predicted Value " + str(b)) plt.scatter(Y_test, test_predictions) plt.xlabel('True Values') plt.ylabel('Predictions') plt.axis('equal') plt.axis('square') _ = plt.plot([-100, 100], [-100, 100]) fig = plt.gcf() plt.show() plt.draw() fig.savefig('TrueVSPredicted' + str(a)) # stampa conteggio errori plt.figure() plt.title("Error Distribution " + str(b)) error = test_predictions - Y_test plt.hist(error, bins=25) plt.xlabel("Prediction Error") _ = plt.ylabel("Count") fig = plt.gcf() plt.show() plt.draw() fig.savefig('PredictionError' + str(a)) Y_train = sc.inverse_transform(Y_train) # stampa valore predetto nel test/ valore reale + storia plt.figure() plt.title("True vs Predicted Test " + str(b)) plt.plot(test_predictions, label="Predicted") plt.plot(Y_test, label="Real") plt.xlabel('Date') plt.ylabel('Price') plt.legend(loc='upper left') fig = plt.gcf() plt.show() plt.draw() fig.savefig('History' + str(a)) return 0
class RNNGRU(object): """Process data for ingestion.""" def __init__( self, data, periods=288, batch_size=64, sequence_length=20, warmup_steps=50, epochs=20, display=False): """Instantiate the class. Args: data: Dict of values keyed by timestamp periods: Number of timestamp data points per vector batch_size: Size of batch sequence_length: Length of vectors for for each target warmup_steps: Returns: None """ # Initialize key variables self.periods = periods self.target_names = ['value'] self.warmup_steps = warmup_steps self.epochs = epochs self.batch_size = batch_size self.display = display ################################### # TensorFlow wizardry config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a total of half the GPU memory to be allocated config.gpu_options.per_process_gpu_memory_fraction = 0.95 # Crash with DeadlineExceeded instead of hanging forever when your # queues get full/empty config.operation_timeout_in_ms = 60000 # Create a session with the above options specified. backend.tensorflow_backend.set_session(tf.Session(config=config)) ################################### # Get data (x_data, y_data) = convert_data(data, periods, self.target_names) print('\n> Numpy Data Type: {}'.format(type(x_data))) print("> Numpy Data Shape: {}".format(x_data.shape)) print("> Numpy Data Row[0]: {}".format(x_data[0])) print('> Numpy Targets Type: {}'.format(type(y_data))) print("> Numpy Targets Shape: {}".format(y_data.shape)) ''' This is the number of observations (aka. data-points or samples) in the data-set: ''' num_data = len(x_data) ''' This is the fraction of the data-set that will be used for the training-set: ''' train_split = 0.9 ''' This is the number of observations in the training-set: ''' self.num_train = int(train_split * num_data) ''' This is the number of observations in the test-set: ''' num_test = num_data - self.num_train print('> Number of Samples: {}'.format(num_data)) print("> Number of Training Samples: {}".format(self.num_train)) print("> Number of Test Samples: {}".format(num_test)) # Create test and training data x_train = x_data[0:self.num_train] x_test = x_data[self.num_train:] self.y_train = y_data[0:self.num_train] self.y_test = y_data[self.num_train:] self.num_x_signals = x_data.shape[1] self.num_y_signals = y_data.shape[1] print("> Training Minimum Value:", np.min(x_train)) print("> Training Maximum Value:", np.max(x_train)) ''' steps_per_epoch is the number of batch iterations before a training epoch is considered finished. ''' self.steps_per_epoch = int(self.num_train / batch_size) + 1 print("> Epochs:", epochs) print("> Batch Size:", batch_size) print("> Steps:", self.steps_per_epoch) ''' Calculate the estimated memory footprint. ''' print("> Data size: {:.2f} Bytes".format(x_data.nbytes)) ''' if memory_footprint > 7: print('\n\n{}\n\n'.format( '> Estimated GPU memory usage too large. Use new parameters ' 'to reduce the footprint.')) sys.exit(0) ''' ''' The neural network works best on values roughly between -1 and 1, so we need to scale the data before it is being input to the neural network. We can use scikit-learn for this. We first create a scaler-object for the input-signals. Then we detect the range of values from the training-data and scale the training-data. ''' x_scaler = MinMaxScaler() self.x_train_scaled = x_scaler.fit_transform(x_train) print('> Scaled Training Minimum Value: {}'.format( np.min(self.x_train_scaled))) print('> Scaled Training Maximum Value: {}'.format( np.max(self.x_train_scaled))) self.x_test_scaled = x_scaler.transform(x_test) ''' The target-data comes from the same data-set as the input-signals, because it is the weather-data for one of the cities that is merely time-shifted. But the target-data could be from a different source with different value-ranges, so we create a separate scaler-object for the target-data. ''' self.y_scaler = MinMaxScaler() self.y_train_scaled = self.y_scaler.fit_transform(self.y_train) y_test_scaled = self.y_scaler.transform(self.y_test) # Data Generator ''' The data-set has now been prepared as 2-dimensional numpy arrays. The training-data has almost 300k observations, consisting of 20 input-signals and 3 output-signals. These are the array-shapes of the input and output data: ''' print('> Scaled Training Data Shape: {}'.format( self.x_train_scaled.shape)) print('> Scaled Training Targets Shape: {}'.format( self.y_train_scaled.shape)) # We then create the batch-generator. generator = self.batch_generator(batch_size, sequence_length) # Validation Set ''' The neural network trains quickly so we can easily run many training epochs. But then there is a risk of overfitting the model to the training-set so it does not generalize well to unseen data. We will therefore monitor the model's performance on the test-set after each epoch and only save the model's weights if the performance is improved on the test-set. The batch-generator randomly selects a batch of short sequences from the training-data and uses that during training. But for the validation-data we will instead run through the entire sequence from the test-set and measure the prediction accuracy on that entire sequence. ''' validation_data = (np.expand_dims(self.x_test_scaled, axis=0), np.expand_dims(y_test_scaled, axis=0)) # Create the Recurrent Neural Network self.model = Sequential() ''' We can now add a Gated Recurrent Unit (GRU) to the network. This will have 512 outputs for each time-step in the sequence. Note that because this is the first layer in the model, Keras needs to know the shape of its input, which is a batch of sequences of arbitrary length (indicated by None), where each observation has a number of input-signals (num_x_signals). ''' self.model.add(GRU( units=512, return_sequences=True, input_shape=(None, self.num_x_signals,))) ''' The GRU outputs a batch of sequences of 512 values. We want to predict 3 output-signals, so we add a fully-connected (or dense) layer which maps 512 values down to only 3 values. The output-signals in the data-set have been limited to be between 0 and 1 using a scaler-object. So we also limit the output of the neural network using the Sigmoid activation function, which squashes the output to be between 0 and 1.''' self.model.add(Dense(self.num_y_signals, activation='sigmoid')) ''' A problem with using the Sigmoid activation function, is that we can now only output values in the same range as the training-data. For example, if the training-data only has temperatures between -20 and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1. So if we limit the output of the neural network to be between 0 and 1 using the Sigmoid function, this can only be mapped back to temperature values between -20 and +30. We can use a linear activation function on the output instead. This allows for the output to take on arbitrary values. It might work with the standard initialization for a simple network architecture, but for more complicated network architectures e.g. with more layers, it might be necessary to initialize the weights with smaller values to avoid NaN values during training. You may need to experiment with this to get it working. ''' if False: # Maybe use lower init-ranges. # init = RandomUniform(minval=-0.05, maxval=0.05) init = RandomUniform(minval=-0.05, maxval=0.05) self.model.add(Dense( self.num_y_signals, activation='linear', kernel_initializer=init)) # Compile Model ''' This is the optimizer and the beginning learning-rate that we will use. We then compile the Keras model so it is ready for training. ''' optimizer = RMSprop(lr=1e-3) self.model.compile(loss=self.loss_mse_warmup, optimizer=optimizer) ''' This is a very small model with only two layers. The output shape of (None, None, 3) means that the model will output a batch with an arbitrary number of sequences, each of which has an arbitrary number of observations, and each observation has 3 signals. This corresponds to the 3 target signals we want to predict. ''' print('> Model Summary:\n') print(self.model.summary()) # Callback Functions ''' During training we want to save checkpoints and log the progress to TensorBoard so we create the appropriate callbacks for Keras. This is the callback for writing checkpoints during training. ''' path_checkpoint = '/tmp/23_checkpoint.keras' callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True) ''' This is the callback for stopping the optimization when performance worsens on the validation-set. ''' callback_early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1) ''' This is the callback for writing the TensorBoard log during training. ''' callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/', histogram_freq=0, write_graph=False) ''' This callback reduces the learning-rate for the optimizer if the validation-loss has not improved since the last epoch (as indicated by patience=0). The learning-rate will be reduced by multiplying it with the given factor. We set a start learning-rate of 1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4. We don't want the learning-rate to go any lower than this. ''' callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-4, patience=0, verbose=1) callbacks = [callback_early_stopping, callback_checkpoint, callback_tensorboard, callback_reduce_lr] # Train the Recurrent Neural Network '''We can now train the neural network. Note that a single "epoch" does not correspond to a single processing of the training-set, because of how the batch-generator randomly selects sub-sequences from the training-set. Instead we have selected steps_per_epoch so that one "epoch" is processed in a few minutes. With these settings, each "epoch" took about 2.5 minutes to process on a GTX 1070. After 14 "epochs" the optimization was stopped because the validation-loss had not decreased for 5 "epochs". This optimization took about 35 minutes to finish. Also note that the loss sometimes becomes NaN (not-a-number). This is often resolved by restarting and running the Notebook again. But it may also be caused by your neural network architecture, learning-rate, batch-size, sequence-length, etc. in which case you may have to modify those settings. ''' print('\n> Starting data training\n') try: self.model.fit_generator( generator=generator, epochs=self.epochs, steps_per_epoch=self.steps_per_epoch, validation_data=validation_data, callbacks=callbacks) except Exception as error: print('\n>{}\n'.format(error)) traceback.print_exc() sys.exit(0) # Load Checkpoint ''' Because we use early-stopping when training the model, it is possible that the model's performance has worsened on the test-set for several epochs before training was stopped. We therefore reload the last saved checkpoint, which should have the best performance on the test-set. ''' print('> Loading model weights') try: self.model.load_weights(path_checkpoint) except Exception as error: print('\n> Error trying to load checkpoint.\n\n{}'.format(error)) traceback.print_exc() sys.exit(0) # Performance on Test-Set ''' We can now evaluate the model's performance on the test-set. This function expects a batch of data, but we will just use one long time-series for the test-set, so we just expand the array-dimensionality to create a batch with that one sequence. ''' result = self.model.evaluate( x=np.expand_dims(self.x_test_scaled, axis=0), y=np.expand_dims(y_test_scaled, axis=0)) print('> Loss (test-set): {}'.format(result)) # If you have several metrics you can use this instead. if False: for res, metric in zip(result, self.model.metrics_names): print('{0}: {1:.3e}'.format(metric, res)) def batch_generator(self, batch_size, sequence_length): """Create generator function to create random batches of training-data. Args: batch_size: Size of batch sequence_length: Length of sequence Returns: (x_batch, y_batch) """ # Infinite loop. while True: # Allocate a new array for the batch of input-signals. x_shape = (batch_size, sequence_length, self.num_x_signals) x_batch = np.zeros(shape=x_shape, dtype=np.float16) # Allocate a new array for the batch of output-signals. y_shape = (batch_size, sequence_length, self.num_y_signals) y_batch = np.zeros(shape=y_shape, dtype=np.float16) # Fill the batch with random sequences of data. for i in range(batch_size): # Get a random start-index. # This points somewhere into the training-data. idx = np.random.randint(self.num_train - sequence_length) # Copy the sequences of data starting at this index. x_batch[i] = self.x_train_scaled[idx:idx+sequence_length] y_batch[i] = self.y_train_scaled[idx:idx+sequence_length] yield (x_batch, y_batch) def loss_mse_warmup(self, y_true, y_pred): """Calculate the Mean Squared Errror. Calculate the Mean Squared Error between y_true and y_pred, but ignore the beginning "warmup" part of the sequences. We will use Mean Squared Error (MSE) as the loss-function that will be minimized. This measures how closely the model's output matches the true output signals. However, at the beginning of a sequence, the model has only seen input-signals for a few time-steps, so its generated output may be very inaccurate. Using the loss-value for the early time-steps may cause the model to distort its later output. We therefore give the model a "warmup-period" of 50 time-steps where we don't use its accuracy in the loss-function, in hope of improving the accuracy for later time-steps Args: y_true: Desired output. y_pred: Model's output. Returns: loss_mean: Mean Squared Error """ warmup_steps = self.warmup_steps # The shape of both input tensors are: # [batch_size, sequence_length, num_y_signals]. # Ignore the "warmup" parts of the sequences # by taking slices of the tensors. y_true_slice = y_true[:, warmup_steps:, :] y_pred_slice = y_pred[:, warmup_steps:, :] # These sliced tensors both have this shape: # [batch_size, sequence_length - warmup_steps, num_y_signals] # Calculate the MSE loss for each value in these tensors. # This outputs a 3-rank tensor of the same shape. loss = tf.losses.mean_squared_error(labels=y_true_slice, predictions=y_pred_slice) # Keras may reduce this across the first axis (the batch) # but the semantics are unclear, so to be sure we use # the loss across the entire tensor, we reduce it to a # single scalar with the mean function. loss_mean = tf.reduce_mean(loss) return loss_mean def plot_comparison(self, start_idx, length=100, train=True): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. train: Boolean whether to use training- or test-set. Returns: None """ if train: # Use training-data. x_values = self.x_train_scaled y_true = self.y_train shim = 'Train' else: # Use test-data. x_values = self.x_test_scaled y_true = self.y_test shim = 'Test' # End-index for the sequences. end_idx = start_idx + length # Select the sequences from the given start-index and # of the given length. x_values = x_values[start_idx:end_idx] y_true = y_true[start_idx:end_idx] # Input-signals for the model. x_values = np.expand_dims(x_values, axis=0) # Use the model to predict the output-signals. y_pred = self.model.predict(x_values) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. y_pred_rescaled = self.y_scaler.inverse_transform(y_pred[0]) # For each output-signal. for signal in range(len(self.target_names)): # Create a filename filename = ( '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format( self.batch_size, self.epochs, self.num_train, signal, int(time.time()), shim) # Get the output-signal predicted by the model. signal_pred = y_pred_rescaled[:, signal] # Get the true output-signal from the data-set. signal_true = y_true[:, signal] # Make the plotting-canvas bigger. plt.figure(figsize=(15, 5)) # Plot and compare the two signals. plt.plot(signal_true, label='true') plt.plot(signal_pred, label='pred') # Plot grey box for warmup-period. _ = plt.axvspan( 0, self.warmup_steps, facecolor='black', alpha=0.15) # Plot labels etc. plt.ylabel(self.target_names[signal]) plt.legend() # Show and save the image if self.display is True: plt.savefig(filename, bbox_inches='tight') plt.show() else: plt.savefig(filename, bbox_inches='tight') print('> Saving file: {}'.format(filename))
testX, testY = create_dataset(test, look_back) # reshape input to be [samples, time steps, features] trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1])) # simple lstm network learning model = Sequential() model.add(LSTM(16, input_shape=(1, look_back))) model.add(Dense(1, activation='linear')) model.compile(loss='mean_squared_error', optimizer=Adam(lr=0.05)) #lr 학습률 model.fit(trainX, trainY, epochs=500, batch_size=1, verbose=2) # make prediction trainPredict = model.predict(trainX) trainPredict = scaler.inverse_transform(trainPredict) testPredict = model.predict(testX) testPredict = scaler.inverse_transform(testPredict) testY = scaler.inverse_transform(testY) testScore = math.sqrt(mean_squared_error(testY, testPredict)) print('Train Score: %.2f RMSE' % testScore) # plot fig = plt.figure(facecolor='white') plt.plot(testY, label='test data') plt.plot(testPredict, label='Predict') plt.legend() plt.show() #save plot to 'png'file
class RNNGRU(DataGRU): """Process data for ingestion.""" def __init__( self, filename, lookahead_periods, batch_size=64, epochs=20, sequence_length=20, warmup_steps=50, dropout=0, layers=1, patience=10, units=512, display=False): """Instantiate the class. Args: data: Tuple of (x_data, y_data, target_names) batch_size: Size of batch sequence_length: Length of vectors for for each target warmup_steps: Returns: None """ # Setup inheritance DataGRU.__init__(self, filename, lookahead_periods) # Initialize key variables self._warmup_steps = warmup_steps self._epochs = epochs self._batch_size = batch_size self._patience = patience self._sequence_length = sequence_length self._units = units self._dropout = dropout self._layers = int(abs(layers)) self.display = display self._path_checkpoint = '/tmp/checkpoint.keras' # Delete any stale checkpoint file if os.path.exists(self._path_checkpoint) is True: os.remove(self._path_checkpoint) ################################### # TensorFlow wizardry config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a total of half the GPU memory to be allocated config.gpu_options.per_process_gpu_memory_fraction = 0.8 # Crash with DeadlineExceeded instead of hanging forever when your # queues get full/empty config.operation_timeout_in_ms = 60000 # Create a session with the above options specified. backend.tensorflow_backend.set_session(tf.Session(config=config)) ################################### # Get data self._y_current = self.close() # Create test and training arrays for VALIDATION and EVALUATION (x_train, xv_test, self._y_train, self._yv_test) = self.train_test_split(.1) (self.training_rows, self._training_vector_count) = x_train.shape (self.test_rows, _) = xv_test.shape (_, self._training_class_count) = self._y_train.shape # Define the number of steps per epoch self._epoch_steps = int(self.training_rows / self._batch_size) + 1 # Print stuff print('\n> Numpy Data Type: {}'.format(type(x_train))) print("> Numpy Data Shape: {}".format(x_train.shape)) print("> Numpy Data Row[0]: {}".format(x_train[0])) print("> Numpy Data Row[Last]: {}".format(x_train[-1])) print('> Numpy Targets Type: {}'.format(type(self._y_train))) print("> Numpy Targets Shape: {}".format(self._y_train.shape)) print('> Number of Samples: {}'.format(self._y_current.shape[0])) print('> Number of Training Samples: {}'.format(x_train.shape[0])) print('> Number of Training Classes: {}'.format( self._training_class_count)) print('> Number of Test Samples: {}'.format(self.test_rows)) print("> Training Minimum Value:", np.min(x_train)) print("> Training Maximum Value:", np.max(x_train)) print('> Number X signals: {}'.format(self._training_vector_count)) print('> Number Y signals: {}'.format(self._training_class_count)) # Print epoch related data print('> Epochs:', self._epochs) print('> Batch Size:', self._batch_size) print('> Steps:', self._epoch_steps) # Display estimated memory footprint of training data. print("> Data size: {:.2f} Bytes".format(x_train.nbytes)) ''' The neural network works best on values roughly between -1 and 1, so we need to scale the data before it is being input to the neural network. We can use scikit-learn for this. We first create a scaler-object for the input-signals. Then we detect the range of values from the training-data and scale the training-data. ''' self._x_scaler = MinMaxScaler() self._x_train_scaled = self._x_scaler.fit_transform(x_train) print('> Scaled Training Minimum Value: {}'.format( np.min(self._x_train_scaled))) print('> Scaled Training Maximum Value: {}'.format( np.max(self._x_train_scaled))) self._xv_test_scaled = self._x_scaler.transform(xv_test) ''' The target-data comes from the same data-set as the input-signals, because it is the weather-data for one of the cities that is merely time-shifted. But the target-data could be from a different source with different value-ranges, so we create a separate scaler-object for the target-data. ''' self._y_scaler = MinMaxScaler() self._y_train_scaled = self._y_scaler.fit_transform(self._y_train) self._yv_test_scaled = self._y_scaler.transform(self._yv_test) # Data Generator ''' The data-set has now been prepared as 2-dimensional numpy arrays. The training-data has almost 300k observations, consisting of 20 input-signals and 3 output-signals. These are the array-shapes of the input and output data: ''' print('> Scaled Training Data Shape: {}'.format( self._x_train_scaled.shape)) print('> Scaled Training Targets Shape: {}'.format( self._y_train_scaled.shape)) # Create RNN model self.model = self._model() def _model(self): """Create the Recurrent Neural Network. Args: None Returns: _model: RNN model """ # Create the model object _model = Sequential() ''' We can now add a Gated Recurrent Unit (GRU) to the network. This will have 512 outputs for each time-step in the sequence. Note that because this is the first layer in the model, Keras needs to know the shape of its input, which is a batch of sequences of arbitrary length (indicated by None), where each observation has a number of input-signals (num_x_signals). ''' _model.add(GRU( units=self._units, return_sequences=True, recurrent_dropout=self._dropout, input_shape=(None, self._training_vector_count,))) for _ in range(0, self._layers): _model.add(GRU( units=self._units, recurrent_dropout=self._dropout, return_sequences=True)) ''' The GRU outputs a batch of sequences of 512 values. We want to predict 3 output-signals, so we add a fully-connected (or dense) layer which maps 512 values down to only 3 values. The output-signals in the data-set have been limited to be between 0 and 1 using a scaler-object. So we also limit the output of the neural network using the Sigmoid activation function, which squashes the output to be between 0 and 1.''' _model.add( Dense(self._training_class_count, activation='sigmoid')) ''' A problem with using the Sigmoid activation function, is that we can now only output values in the same range as the training-data. For example, if the training-data only has temperatures between -20 and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1. So if we limit the output of the neural network to be between 0 and 1 using the Sigmoid function, this can only be mapped back to temperature values between -20 and +30. We can use a linear activation function on the output instead. This allows for the output to take on arbitrary values. It might work with the standard initialization for a simple network architecture, but for more complicated network architectures e.g. with more layers, it might be necessary to initialize the weights with smaller values to avoid NaN values during training. You may need to experiment with this to get it working. ''' if False: # Maybe use lower init-ranges. # init = RandomUniform(minval=-0.05, maxval=0.05) init = RandomUniform(minval=-0.05, maxval=0.05) _model.add(Dense( self._training_class_count, activation='linear', kernel_initializer=init)) # Compile Model ''' This is the optimizer and the beginning learning-rate that we will use. We then compile the Keras model so it is ready for training. ''' optimizer = RMSprop(lr=1e-3) _model.compile( loss=self._loss_mse_warmup, optimizer=optimizer, metrics=['accuracy']) ''' This is a very small model with only two layers. The output shape of (None, None, 3) means that the model will output a batch with an arbitrary number of sequences, each of which has an arbitrary number of observations, and each observation has 3 signals. This corresponds to the 3 target signals we want to predict. ''' print('> Model Summary:\n') print(_model.summary()) # Return return _model def test(self): """Test the Recurrent Neural Network. Args: None Returns: _model: RNN model """ # Initialize key variables patience = self._patience sequence_length = self._sequence_length # Create the batch-generator. generator = self._batch_generator( self._batch_size, sequence_length) # Validation Set ''' The neural network trains quickly so we can easily run many training epochs. But then there is a risk of overfitting the model to the training-set so it does not generalize well to unseen data. We will therefore monitor the model's performance on the test-set after each epoch and only save the model's weights if the performance is improved on the test-set. The batch-generator randomly selects a batch of short sequences from the training-data and uses that during training. But for the validation-data we will instead run through the entire sequence from the test-set and measure the prediction accuracy on that entire sequence. ''' validation_data = (np.expand_dims(self._xv_test_scaled, axis=0), np.expand_dims(self._yv_test_scaled, axis=0)) # Callback Functions ''' During training we want to save checkpoints and log the progress to TensorBoard so we create the appropriate callbacks for Keras. This is the callback for writing checkpoints during training. ''' callback_checkpoint = ModelCheckpoint(filepath=self._path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True) ''' This is the callback for stopping the optimization when performance worsens on the validation-set. ''' callback_early_stopping = EarlyStopping(monitor='val_loss', patience=patience, verbose=1) ''' This is the callback for writing the TensorBoard log during training. ''' callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/', histogram_freq=0, write_graph=False) ''' This callback reduces the learning-rate for the optimizer if the validation-loss has not improved since the last epoch (as indicated by patience=0). The learning-rate will be reduced by multiplying it with the given factor. We set a start learning-rate of 1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4. We don't want the learning-rate to go any lower than this. ''' callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, min_lr=1e-4, patience=0, verbose=1) callbacks = [callback_early_stopping, callback_checkpoint, callback_tensorboard, callback_reduce_lr] # Train the Recurrent Neural Network '''We can now train the neural network. Note that a single "epoch" does not correspond to a single processing of the training-set, because of how the batch-generator randomly selects sub-sequences from the training-set. Instead we have selected steps_per_epoch so that one "epoch" is processed in a few minutes. With these settings, each "epoch" took about 2.5 minutes to process on a GTX 1070. After 14 "epochs" the optimization was stopped because the validation-loss had not decreased for 5 "epochs". This optimization took about 35 minutes to finish. Also note that the loss sometimes becomes NaN (not-a-number). This is often resolved by restarting and running the Notebook again. But it may also be caused by your neural network architecture, learning-rate, batch-size, sequence-length, etc. in which case you may have to modify those settings. ''' print('\n> Starting data training\n') self.model.fit_generator( generator=generator, epochs=self._epochs, steps_per_epoch=self._epoch_steps, validation_data=validation_data, callbacks=callbacks) # Load Checkpoint ''' Because we use early-stopping when training the model, it is possible that the model's performance has worsened on the test-set for several epochs before training was stopped. We therefore reload the last saved checkpoint, which should have the best performance on the test-set. ''' print('> Loading model weights') if os.path.exists(self._path_checkpoint): self.model.load_weights(self._path_checkpoint) # Performance on Test-Set ''' We can now evaluate the model's performance on the test-set. This function expects a batch of data, but we will just use one long time-series for the test-set, so we just expand the array-dimensionality to create a batch with that one sequence. ''' result = self.model.evaluate( x=np.expand_dims(self._xv_test_scaled, axis=0), y=np.expand_dims(self._yv_test_scaled, axis=0)) print('> Loss (test-set): {}'.format(result)) # If you have several metrics you can use this instead. if False: for res, metric in zip(result, self.model.metrics_names): print('{0}: {1:.3e}'.format(metric, res)) def _batch_generator(self, batch_size, sequence_length): """Create generator function to create random batches of training-data. Args: batch_size: Size of batch sequence_length: Length of sequence Returns: (x_batch, y_batch) """ # Infinite loop. while True: # Allocate a new array for the batch of input-signals. x_shape = ( batch_size, sequence_length, self._training_vector_count) x_batch = np.zeros(shape=x_shape, dtype=np.float16) # Allocate a new array for the batch of output-signals. y_shape = (batch_size, sequence_length, self._training_class_count) y_batch = np.zeros(shape=y_shape, dtype=np.float16) # Fill the batch with random sequences of data. for i in range(batch_size): # Get a random start-index. # This points somewhere into the training-data. idx = np.random.randint( self.training_rows - sequence_length) # Copy the sequences of data starting at this index. x_batch[i] = self._x_train_scaled[idx:idx+sequence_length] y_batch[i] = self._y_train_scaled[idx:idx+sequence_length] yield (x_batch, y_batch) def _loss_mse_warmup(self, y_true, y_pred): """Calculate the Mean Squared Errror. Calculate the Mean Squared Error between y_true and y_pred, but ignore the beginning "warmup" part of the sequences. We will use Mean Squared Error (MSE) as the loss-function that will be minimized. This measures how closely the model's output matches the true output signals. However, at the beginning of a sequence, the model has only seen input-signals for a few time-steps, so its generated output may be very inaccurate. Using the loss-value for the early time-steps may cause the model to distort its later output. We therefore give the model a "warmup-period" of 50 time-steps where we don't use its accuracy in the loss-function, in hope of improving the accuracy for later time-steps Args: y_true: Desired output. y_pred: Model's output. Returns: loss_mean: Mean Squared Error """ warmup_steps = self._warmup_steps # The shape of both input tensors are: # [batch_size, sequence_length, num_y_signals]. # Ignore the "warmup" parts of the sequences # by taking slices of the tensors. y_true_slice = y_true[:, warmup_steps:, :] y_pred_slice = y_pred[:, warmup_steps:, :] # These sliced tensors both have this shape: # [batch_size, sequence_length - warmup_steps, num_y_signals] # Calculate the MSE loss for each value in these tensors. # This outputs a 3-rank tensor of the same shape. loss = tf.losses.mean_squared_error(labels=y_true_slice, predictions=y_pred_slice) # Keras may reduce this across the first axis (the batch) # but the semantics are unclear, so to be sure we use # the loss across the entire tensor, we reduce it to a # single scalar with the mean function. loss_mean = tf.reduce_mean(loss) return loss_mean def plot_train(self, start_idx, length=100): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. Returns: None """ # Plot self._plot_comparison(start_idx, length=length, train=True) def plot_test(self, start_idx, length=100): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. Returns: None """ # Plot self._plot_comparison(start_idx, length=length, train=False) def _plot_comparison(self, start_idx, length=100, train=True): """Plot the predicted and true output-signals. Args: start_idx: Start-index for the time-series. length: Sequence-length to process and plot. train: Boolean whether to use training- or test-set. Returns: None """ # Initialize key variables datetimes = {} num_train = self.training_rows # End-index for the sequences. end_idx = start_idx + length # Variables for date formatting days = mdates.DayLocator() # Every day months = mdates.MonthLocator() # Every month months_format = mdates.DateFormatter('%b %Y') days_format = mdates.DateFormatter('%d') # Assign other variables dependent on the type of data we are plotting if train is True: # Use training-data. x_values = self._x_train_scaled[start_idx:end_idx] y_true = self._y_train[start_idx:end_idx] shim = 'Train' # Datetimes to use for training datetimes[shim] = self.datetime()[ :num_train][start_idx:end_idx] else: # Scale the data x_test_scaled = self._x_scaler.transform( self.vectors_test_all()) # Use test-data. x_values = x_test_scaled[start_idx:end_idx] y_true = self._yv_test[start_idx:end_idx] shim = 'Test' # Datetimes to use for testing datetimes[shim] = self.datetime()[ num_train:][start_idx:end_idx] # Input-signals for the model. x_values = np.expand_dims(x_values, axis=0) # Use the model to predict the output-signals. y_pred = self.model.predict(x_values) # The output of the model is between 0 and 1. # Do an inverse map to get it back to the scale # of the original data-set. y_pred_rescaled = self._y_scaler.inverse_transform(y_pred[0]) # For each output-signal. for signal in range(len(self.labels())): # Assign other variables dependent on the type of data plot if train is True: # Only get current values that are a part of the training data current = self._y_current[:num_train][start_idx:end_idx] # The number of datetimes for the 'actual' plot must match # that of current values datetimes['actual'] = self.datetime()[ :num_train][start_idx:end_idx] else: # Only get current values that are a part of the test data. current = self._y_current[ num_train:][start_idx:] # The number of datetimes for the 'actual' plot must match # that of current values datetimes['actual'] = self.datetime()[ num_train:][start_idx:] # Create a filename filename = ( '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format( self._batch_size, self._epochs, num_train, signal, int(time.time()), shim) # Get the output-signal predicted by the model. signal_pred = y_pred_rescaled[:, signal] # Get the true output-signal from the data-set. signal_true = y_true[:, signal] # Create a new chart (fig, axis) = plt.subplots(figsize=(15, 5)) # Plot and compare the two signals. axis.plot( datetimes[shim][:len(signal_true)], signal_true, label='Current +{}'.format(self.labels()[signal])) axis.plot( datetimes[shim][:len(signal_pred)], signal_pred, label='Prediction') axis.plot(datetimes['actual'], current, label='Current') # Set plot labels and titles axis.set_title('{1}ing Forecast ({0} Future Intervals)'.format( self.labels()[signal], shim)) axis.set_ylabel('Values') axis.legend( bbox_to_anchor=(1.04, 0.5), loc='center left', borderaxespad=0) # Add gridlines and ticks ax = plt.gca() ax.grid(True) # Add major gridlines ax.xaxis.grid(which='major', color='black', alpha=0.2) ax.yaxis.grid(which='major', color='black', alpha=0.2) # Add minor ticks (They must be turned on first) ax.minorticks_on() ax.xaxis.grid(which='minor', color='black', alpha=0.1) ax.yaxis.grid(which='minor', color='black', alpha=0.1) # Format the tick labels ax.xaxis.set_major_locator(months) ax.xaxis.set_major_formatter(months_format) ax.xaxis.set_minor_locator(days) # Remove tick marks ax.tick_params(axis='both', which='both', length=0) # Print day numbers on xaxis for Test data only if train is False: ax.xaxis.set_minor_formatter(days_format) plt.setp(ax.xaxis.get_minorticklabels(), rotation=90) # Rotates and right aligns the x labels, and moves the bottom of # the axes up to make room for them fig.autofmt_xdate() # Plot grey box for warmup-period if we are working with training # data and the start is within the warmup-period if (0 < start_idx < self._warmup_steps): if train is True: plt.axvspan( datetimes[shim][start_idx], datetimes[shim][self._warmup_steps], facecolor='black', alpha=0.15) # Show and save the image if self.display is True: fig.savefig(filename, bbox_inches='tight') plt.show() else: fig.savefig(filename, bbox_inches='tight') print('> Saving file: {}'.format(filename)) # Close figure plt.close(fig=fig)
# create and fit the LSTM network model = Sequential() model.add(LSTM(4, input_shape=(1, look_back))) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(trainX, trainY, epochs=6000, batch_size=1, verbose=2) # make predictions trainPredict = model.predict(trainX) testPredict = model.predict(testX) showPredict = model.predict(showX) # invert predictions, needs change for different look_back # for train data tmp3 = numpy.concatenate((trainPredict, trainPredict, trainPredict), axis=1) trainPredict = scaler.inverse_transform(tmp3) trainPredict = trainPredict[:, 0] tmp4 = numpy.concatenate(([trainY], [trainY], [trainY]), axis=0) tmp4 = tmp4.transpose() trainY = scaler.inverse_transform(tmp4) trainY = trainY[:, 0] # for test data tmp5 = numpy.concatenate((testPredict, testPredict, testPredict), axis=1) testPredict = scaler.inverse_transform(tmp5) testPredict = testPredict[:, 0] tmp6 = numpy.concatenate(([testY], [testY], [testY]), axis=0) tmp6 = tmp6.transpose() testY = scaler.inverse_transform(tmp6)
def predict_new(self, input): model = self.train_model() assert len(input) == 5 and type(input) == list scaler = MinMaxScaler(feature_range=(0, 1)) scaler.fit(self.data) inp = scaler.transform([input]) print(scaler.inverse_transform(model.predict(numpy.array(inp).reshape(1, 1, 5))))