def evaluate(Y_test, predictions, Y_test_inv_scaled, predictions_inv_scaled):
rmse = (mean_squared_error(Y_test, predictions)**0.5)
print('\nNormalized RMSE: %.3f' % (rmse))
nrmse = ((mean_squared_error(Y_test, predictions)**0.5)) / np.mean(Y_test)
print('Normalized NRMSE: %.3f' % (nrmse))
mae = mean_absolute_error(Y_test, predictions)
print('Normalized MAE: %.3f' % (mae))
mape = compute_mape(Y_test, predictions)
print('Normalized MAPE: %.3f' % (mape))
correlation = np.corrcoef(Y_test.T, predictions.T)
print("Normalized Correlation: %.3f" % (correlation[0, 1]))
r2 = r2_score(Y_test, predictions)
print("Normalized r^2: %.3f" % (r2))
normalized_metrics = [rmse, nrmse, mae, mape, correlation[0, 1], r2]
#evaluating the model on the inverse-normalized dataset
rmse = (mean_squared_error(Y_test_inv_scaled, predictions_inv_scaled)**0.5)
print('\nInverse-Normalized Outsample RMSE: %.3f' % (rmse))
nrmse = ((mean_squared_error(Y_test_inv_scaled, predictions_inv_scaled)**
0.5)) / np.mean(Y_test)
print('Normalized NRMSE: %.3f' % (nrmse))
mae = mean_absolute_error(Y_test_inv_scaled, predictions_inv_scaled)
print('Normalized MAE: %.3f' % (mae))
mape = compute_mape(Y_test_inv_scaled, predictions_inv_scaled)
print('Inverse-Normalized Outsample MAPE: %.3f' % (mape))
correlation = np.corrcoef(Y_test_inv_scaled.T, predictions_inv_scaled.T)
print("Inverse-Normalized Outsample Correlation: %.3f" %
(correlation[0, 1]))
r2 = r2_score(Y_test_inv_scaled, predictions_inv_scaled)
print("Inverse-Normalized Outsample r^2: %.3f" % (r2))
inv_normalized_metrics = [rmse, nrmse, mae, mape, correlation[0, 1], r2]
return normalized_metrics, inv_normalized_metrics
X_train, Y_train, X_test, Y_test = ds.dataset_reshape(dataset, future_gap,
split)
#kNN model
model = knn.knn(5)
#fitting the training data
model.train(X_train, Y_train)
#predictions
predictions = model.query(X_test, normalize=False, addDiff=False)
#evaluating the model on the normalized dataset
rmse = calculate_rmse(predictions, Y_test)
print('Normalized Test RMSE: %.3f' % (rmse))
mape = ds.compute_mape(Y_test, predictions)
print('Normalized Outsample MAPE: %.3f' % (mape))
correlation = np.corrcoef(predictions, Y_test)
print("Normalized Correlation: %.3f" % (correlation[0, 1]))
r2 = r2_score(predictions, Y_test)
print("Normalized Outsample r^2: %.3f" % (r2))
#evaluating the model on the inverse-normalized dataset
predictions = predictions.reshape((predictions.shape[0], 1))
Y_test = Y_test.reshape((Y_test.shape[0], 1))
predictions_inv_scaled = scaler.inverse_transform(predictions)
Y_test_inv_scaled = scaler.inverse_transform(Y_test)
rmse = calculate_rmse(predictions_inv_scaled, Y_test_inv_scaled)
print('\nInverse-Normalized Outsample RMSE: %.3f' % (rmse))
def main(internal_eval=False):
#building the dataset
print("> building the dataset...")
stock_symbol = '^GSPC'
start_date = '1950-01-01'
end_date = '2017-12-31'
window = 2
dataframe, scaler = ffnn.bulid_new_TIs_dataset(stock_symbol, start_date,
end_date, window)
#reshaping the dataset for FFNN
print("\n> reshaping the dataset for FFNN...")
dataset = dataframe.values
future_gap = 1 #1 trading day
split = 0.8 #80% of the dataset
X_train, Y_train, X_test, Y_test = ffnn.ffnn_dataset_reshape(
dataset, future_gap, split)
#building the FFNN model
print("\n> building the FFNN model...")
features = X_train.shape[1]
neurons = [256, 256, 16, 1]
drop_out = 0.3
verbose = 1
model = ffnn.build_model(features, neurons, drop_out)
#fitting the training data
print("\n> fitting the training data...")
early_stopping_callback = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=50,
verbose=verbose,
mode='auto')
batch_size = 4096
epochs = 200
validation_split = 0.1
_ = ffnn.model_fit(model, X_train, Y_train, batch_size, epochs,
validation_split, verbose, [early_stopping_callback])
#internal evaluation
if internal_eval:
print("\n> internal evaluation...")
_, _ = ffnn.evaluate_model(model, X_train, Y_train, X_test, Y_test,
verbose)
#predictions
predictions = model.predict(X_test)
predictions = predictions.reshape((predictions.shape[0], 1))
Y_test = Y_test.reshape((Y_test.shape[0], 1))
#evaluating the model on the normalized dataset
rmse = (mean_squared_error(predictions, Y_test)**0.5)
print('\nNormalized Test RMSE: %.3f' % (rmse))
mape = compute_mape(Y_test, predictions)
print('Normalized Outsample MAPE: %.3f' % (mape))
correlation = np.corrcoef(predictions.T, Y_test.T)
print("Normalized Correlation: %.3f" % (correlation[0, 1]))
r2 = r2_score(predictions, Y_test)
print("Normalized Outsample r^2: %.3f" % (r2))
#evaluating the model on the inverse-normalized dataset
predictions_inv_scaled = scaler.inverse_transform(predictions)
Y_test_inv_scaled = scaler.inverse_transform(Y_test)
rmse = (mean_squared_error(predictions_inv_scaled, Y_test_inv_scaled)**0.5)
print('\nInverse-Normalized Outsample RMSE: %.3f' % (rmse))
mape = compute_mape(Y_test_inv_scaled, predictions_inv_scaled)
print('Inverse-Normalized Outsample MAPE: %.3f' % (mape))
correlation = np.corrcoef(predictions_inv_scaled.T, Y_test_inv_scaled.T)
print("Inverse-Normalized Outsample Correlation: %.3f" %
(correlation[0, 1]))
r2 = r2_score(predictions_inv_scaled, Y_test_inv_scaled)
print("Inverse-Normalized Outsample r^2: %.3f" % (r2))
#plotting the results
print("\n> plotting the results...")
_, ax2 = plt.subplots()
'''ax1.plot(history.history['loss'], label='Training')
ax1.plot(history.history['val_loss'], label='Validation')
ax1.set_xlabel('Epoch #')
ax1.set_ylabel('Loss')
ax1.legend(loc='best')
ax1.grid(True)
'''
ax2.plot(range(len(predictions_inv_scaled)),
predictions_inv_scaled,
label='Prediction')
ax2.plot(range(len(Y_test_inv_scaled)), Y_test_inv_scaled, label='Actual')
ax2.set_xlabel('Trading Day')
ax2.set_ylabel('Price')
ax2.legend(loc='best')
ax2.grid(True)
plt.show()