def naive_persistence_model_sunspots(new_dir=os.getcwd()): """ Main function """ # %% # IMPORTS os.chdir(new_dir) # code repository sub-package imports from artificial_neural_networks.code.utils.download_monthly_sunspots import \ download_monthly_sunspots from artificial_neural_networks.code.utils.generic_utils import series_to_supervised, \ affine_transformation from artificial_neural_networks.code.utils.vis_utils import regression_figs # %% # SETTINGS parser = argparse.ArgumentParser() # General settings parser.add_argument('--verbose', type=int, default=1) parser.add_argument('--reproducible', type=bool, default=True) parser.add_argument('--seed', type=int, default=0) parser.add_argument('--plot', type=bool, default=False) # Settings for preprocessing and hyperparameters parser.add_argument('--look_back', type=int, default=3) parser.add_argument('--scaling_factor', type=float, default=(1 / 780)) parser.add_argument('--translation', type=float, default=0) args = parser.parse_args() if args.verbose > 0: print(args) # For reproducibility if args.reproducible: os.environ['PYTHONHASHSEED'] = '0' np.random.seed(args.seed) rn.seed(args.seed) # %% # Load the Monthly sunspots dataset sunspots_path = download_monthly_sunspots() sunspots = np.genfromtxt(fname=sunspots_path, dtype=np.float32, delimiter=",", skip_header=1, usecols=1) # %% # Train-Test split L_series = len(sunspots) split_ratio = 2 / 3 # between zero and one n_split = int(L_series * split_ratio) look_back = args.look_back train = np.concatenate((np.zeros(look_back), sunspots[:n_split])) test = sunspots[n_split - look_back:] train_x, train_y = series_to_supervised(train, look_back) test_x, test_y = series_to_supervised(test, look_back) # %% # PREPROCESSING STEP scaling_factor = args.scaling_factor translation = args.translation # Apply preprocessing train_x_ = affine_transformation(train_x, scaling_factor, translation) test_x_ = affine_transformation(test_x, scaling_factor, translation) # %% # TRAINING PHASE def model_predict(x): """ Predict using the Naive Persistence Model """ last = x.shape[1] - 1 y_pred = [] y_pred.append(x[0][last]) for i in range(x.shape[0] - 1): y_pred.append(x[i][last]) return np.array(y_pred) # %% # TESTING PHASE # Predict preprocessed values train_y_pred_ = model_predict(train_x_) test_y_pred_ = model_predict(test_x_) # Remove preprocessing train_y_pred = affine_transformation(train_y_pred_, scaling_factor, translation, inverse=True) test_y_pred = affine_transformation(test_y_pred_, scaling_factor, translation, inverse=True) train_rmse = sqrt(mean_squared_error(train_y, train_y_pred)) train_mae = mean_absolute_error(train_y, train_y_pred) train_r2 = r2_score(train_y, train_y_pred) test_rmse = sqrt(mean_squared_error(test_y, test_y_pred)) test_mae = mean_absolute_error(test_y, test_y_pred) test_r2 = r2_score(test_y, test_y_pred) if args.verbose > 0: print('Train RMSE: %.4f ' % (train_rmse)) print('Train MAE: %.4f ' % (train_mae)) print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2)) print('Train R_squared: %.4f ' % (train_r2)) print('') print('Test RMSE: %.4f ' % (test_rmse)) print('Test MAE: %.4f ' % (test_mae)) print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2)) print('Test R_squared: %.4f ' % (test_r2)) # %% # Data Visualization if args.plot: regression_figs(train_y=train_y, train_y_pred=train_y_pred, test_y=test_y, test_y_pred=test_y_pred)
def lstm_dense_sunspots(new_dir=os.getcwd()): """ Main function """ # %% # IMPORTS os.chdir(new_dir) # code repository sub-package imports from artificial_neural_networks.code.utils.download_monthly_sunspots import \ download_monthly_sunspots from artificial_neural_networks.code.utils.generic_utils import none_or_int, none_or_float, \ save_regress_model, series_to_supervised, affine_transformation from artificial_neural_networks.code.utils.vis_utils import regression_figs # %% # SETTINGS parser = argparse.ArgumentParser() # General settings parser.add_argument('--verbose', type=int, default=1) parser.add_argument('--reproducible', type=bool, default=True) parser.add_argument('--seed', type=int, default=0) parser.add_argument('--time_training', type=bool, default=True) parser.add_argument('--plot', type=bool, default=True) # Settings for preprocessing and hyperparameters parser.add_argument('--look_back', type=int, default=10) parser.add_argument('--scaling_factor', type=float, default=(1 / 780)) parser.add_argument('--translation', type=float, default=0) parser.add_argument('--layer_size', type=int, default=4) parser.add_argument('--n_epochs', type=int, default=13) parser.add_argument('--batch_size', type=none_or_int, default=1) parser.add_argument('--optimizer', type=str, default='Adam') parser.add_argument('--lrearning_rate', type=float, default=1e-3) parser.add_argument('--epsilon', type=none_or_float, default=None) parser.add_argument('--diff', type=int, default=126) # Settings for saving the model parser.add_argument('--save_architecture', type=bool, default=True) parser.add_argument('--save_last_weights', type=bool, default=True) parser.add_argument('--save_last_model', type=bool, default=True) parser.add_argument('--save_models', type=bool, default=False) parser.add_argument('--save_weights_only', type=bool, default=False) parser.add_argument('--save_best', type=bool, default=True) args = parser.parse_args() if args.verbose > 0: print(args) # For reproducibility if (args.reproducible): os.environ['PYTHONHASHSEED'] = '0' np.random.seed(args.seed) rn.seed(args.seed) tf.set_random_seed(args.seed) sess = tf.Session(graph=tf.get_default_graph()) K.set_session(sess) # print(hash("keras")) # %% # Load the Monthly sunspots dataset sunspots_path = download_monthly_sunspots() sunspots = np.genfromtxt(fname=sunspots_path, dtype=np.float32, delimiter=",", skip_header=1, usecols=1) # %% # Train-Test split L_series = len(sunspots) split_ratio = 2 / 3 # between zero and one n_split = int(L_series * split_ratio) look_back = args.look_back diff = args.diff train = sunspots[:n_split] test = sunspots[n_split - diff - look_back:] train_y = train test_y = test[diff + look_back:] # Apply diferencing for Seasonality Adjustment to make the it stationary d_train = train[diff:] - train[:-diff] d_test = test[diff:] - test[:-diff] train_first = train[look_back:look_back + diff] test_first = test[look_back:look_back + diff] d_train_x, d_train_y = series_to_supervised(d_train, look_back) d_test_x, d_test_y = series_to_supervised(d_test, look_back) # %% # PREPROCESSING STEP scaling_factor = args.scaling_factor translation = args.translation n_train = d_train_x.shape[0] # number of training examples/samples n_test = d_test_x.shape[0] # number of test examples/samples n_in = d_train_x.shape[1] # number of features / dimensions n_out = 1 # number of classes/labels # Reshape training and test sets d_train_x = d_train_x.reshape(n_train, n_in, 1) d_test_x = d_test_x.reshape(n_test, n_in, 1) # Apply preprocessing d_train_x_ = affine_transformation(d_train_x, scaling_factor, translation) d_train_y_ = affine_transformation(d_train_y, scaling_factor, translation) d_test_x_ = affine_transformation(d_test_x, scaling_factor, translation) d_test_y_ = affine_transformation(d_test_y, scaling_factor, translation) # %% # Model hyperparameters and ANN Architecture x = Input(shape=(n_in, 1)) # input layer h = x h = LSTM(units=args.layer_size)(h) # hidden layer out = Dense(units=n_out, activation=None)(h) # output layer model = Model(inputs=x, outputs=out) if (args.verbose > 0): model.summary() def root_mean_squared_error(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) loss_function = root_mean_squared_error metrics = ['mean_absolute_error', 'mean_absolute_percentage_error'] lr = args.lrearning_rate epsilon = args.epsilon optimizer_selection = { 'Adadelta': optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0), 'Adagrad': optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0), 'Adam': optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0, amsgrad=False), 'Adamax': optimizers.Adamax(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0), 'Nadam': optimizers.Nadam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, schedule_decay=0.004), 'RMSprop': optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0), 'SGD': optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False) } optimizer = optimizer_selection[args.optimizer] model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics) # %% # Save trained models for every epoch models_path = r'artificial_neural_networks/trained_models/' model_name = 'sunspots_lstm_dense_des' weights_path = models_path + model_name + '_weights' model_path = models_path + model_name + '_model' file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}' if (args.save_weights_only): file_path = weights_path else: file_path = model_path file_path += file_suffix monitor = 'val_loss' if args.save_models: checkpoint = ModelCheckpoint(file_path + '.h5', monitor=monitor, verbose=args.verbose, save_best_only=args.save_best, mode='auto', save_weights_only=args.save_weights_only) callbacks = [checkpoint] else: callbacks = [] # %% # TRAINING PHASE if args.time_training: start = timer() model.fit(x=d_train_x_, y=d_train_y_, validation_data=(d_test_x_, d_test_y_), batch_size=args.batch_size, epochs=args.n_epochs, verbose=args.verbose, callbacks=callbacks) if args.time_training: end = timer() duration = end - start print('Total time for training (in seconds):') print(duration) # %% # TESTING PHASE # Predict values (with preprocessing and differencing) d_train_y_pred_ = model.predict(d_train_x_)[:, 0] d_test_y_pred_ = model.predict(d_test_x_)[:, 0] # Remove preprocessing d_train_y_pred = affine_transformation(d_train_y_pred_, scaling_factor, translation, inverse=True) d_test_y_pred = affine_transformation(d_test_y_pred_, scaling_factor, translation, inverse=True) def remove_diff(d_y, y_first): """ Remove differencing """ y = np.insert(d_y, 0, y_first) d = y_first.shape[0] n_y = y.shape[0] n_zeros = d - (n_y % d) d_y_pad = np.insert(np.zeros(n_zeros), 0, y) n_rows = len(d_y_pad) // d n_cols = d d_y_reshaped = d_y_pad.reshape(n_rows, n_cols) y_pad = d_y_reshaped.cumsum(axis=0).reshape(n_rows * n_cols) y = y_pad[:-n_zeros] y[:d] = np.zeros(d) return y # Remove differencing train_y_pred = np.concatenate( (np.zeros(look_back), remove_diff(d_train_y_pred, train_first))) test_y_pred = remove_diff(d_test_y_pred, test_first)[diff:] train_rmse = sqrt(mean_squared_error(train_y, train_y_pred)) train_mae = mean_absolute_error(train_y, train_y_pred) train_r2 = r2_score(train_y, train_y_pred) test_rmse = sqrt(mean_squared_error(test_y, test_y_pred)) test_mae = mean_absolute_error(test_y, test_y_pred) test_r2 = r2_score(test_y, test_y_pred) if (args.verbose > 0): print('Train RMSE: %.4f ' % (train_rmse)) print('Train MAE: %.4f ' % (train_mae)) print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2)) print('Train R_squared: %.4f ' % (train_r2)) print('') print('Test RMSE: %.4f ' % (test_rmse)) print('Test MAE: %.4f ' % (test_mae)) print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2)) print('Test R_squared: %.4f ' % (test_r2)) # %% # Data Visualization if (args.plot): regression_figs(train_y=train_y, train_y_pred=train_y_pred, test_y=test_y, test_y_pred=test_y_pred) # %% # Save the architecture and the lastly trained model save_regress_model(model, models_path, model_name, weights_path, model_path, file_suffix, test_rmse, test_mae, args) # %% return model
def bi_lstm_dropout_sunspots(args): """ Main function """ # %% # IMPORTS # code repository sub-package imports from artificial_neural_networks.code.utils.download_monthly_sunspots import \ download_monthly_sunspots from artificial_neural_networks.code.utils.generic_utils import save_regress_model, \ series_to_supervised, affine_transformation # %% if args.verbose > 0: print(args) # For reproducibility if args.reproducible: os.environ['PYTHONHASHSEED'] = '0' np.random.seed(args.seed) rn.seed(args.seed) tf.set_random_seed(args.seed) sess = tf.Session(graph=tf.get_default_graph()) K.set_session(sess) # print(hash("keras")) # %% # Load the Monthly sunspots dataset sunspots_path = download_monthly_sunspots() sunspots = np.genfromtxt(fname=sunspots_path, dtype=np.float32, delimiter=",", skip_header=1, usecols=1) # %% # Train-Test split L_series = len(sunspots) split_ratio = 2 / 3 # between zero and one n_split = int(L_series * split_ratio) look_back = args.look_back train = sunspots[:n_split] test = sunspots[n_split - look_back:] train_x, train_y = series_to_supervised(train, look_back) test_x, test_y = series_to_supervised(test, look_back) # %% # PREPROCESSING STEP scaling_factor = args.scaling_factor translation = args.translation n_train = train_x.shape[0] # number of training examples/samples n_test = test_x.shape[0] # number of test examples/samples n_in = train_x.shape[1] # number of features / dimensions n_out = train_y.shape[1] # number of steps ahead to be predicted # Reshape training and test sets train_x = train_x.reshape(n_train, n_in, 1) test_x = test_x.reshape(n_test, n_in, 1) # Apply preprocessing train_x_ = affine_transformation(train_x, scaling_factor, translation) train_y_ = affine_transformation(train_y, scaling_factor, translation) test_x_ = affine_transformation(test_x, scaling_factor, translation) test_y_ = affine_transformation(test_y, scaling_factor, translation) # %% # Model hyperparameters and ANN Architecture x = Input(shape=(n_in, 1)) # input layer h = Dropout(rate=args.dropout_rate_input)(x) h = Bidirectional(LSTM(units=args.layer_size))(h) # hidden layer h = Dropout(rate=args.dropout_rate_hidden)(h) out = Dense(units=n_out, activation=None)(h) # output layer model = Model(inputs=x, outputs=out) if args.verbose > 0: model.summary() def root_mean_squared_error(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) loss_function = root_mean_squared_error metrics = ['mean_absolute_error', 'mean_absolute_percentage_error'] lr = args.lrearning_rate epsilon = args.epsilon optimizer_selection = { 'Adadelta': optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0), 'Adagrad': optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0), 'Adam': optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0, amsgrad=False), 'Adamax': optimizers.Adamax(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0), 'Nadam': optimizers.Nadam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, schedule_decay=0.004), 'RMSprop': optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0), 'SGD': optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False) } optimizer = optimizer_selection[args.optimizer] model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics) # %% # Save trained models for every epoch models_path = r'artificial_neural_networks/trained_models/' model_name = 'sunspots_bi_lstm_dropout' weights_path = models_path + model_name + '_weights' model_path = models_path + model_name + '_model' file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}' if args.save_weights_only: file_path = weights_path else: file_path = model_path file_path += file_suffix monitor = 'val_loss' if args.save_models: checkpoint = ModelCheckpoint(file_path + '.h5', monitor=monitor, verbose=args.verbose, save_best_only=args.save_best, mode='auto', save_weights_only=args.save_weights_only) callbacks = [checkpoint] else: callbacks = [] # %% # TRAINING PHASE if args.time_training: start = timer() model.fit(x=train_x_, y=train_y_, validation_data=(test_x_, test_y_), batch_size=args.batch_size, epochs=args.n_epochs, verbose=args.verbose, callbacks=callbacks) if args.time_training: end = timer() duration = end - start print('Total time for training (in seconds):') print(duration) # %% # TESTING PHASE # Predict preprocessed values train_y_pred_ = model.predict(train_x_)[:, 0] test_y_pred_ = model.predict(test_x_)[:, 0] # Remove preprocessing train_y_pred = affine_transformation(train_y_pred_, scaling_factor, translation, inverse=True) test_y_pred = affine_transformation(test_y_pred_, scaling_factor, translation, inverse=True) train_rmse = sqrt(mean_squared_error(train_y, train_y_pred)) train_mae = mean_absolute_error(train_y, train_y_pred) train_r2 = r2_score(train_y, train_y_pred) test_rmse = sqrt(mean_squared_error(test_y, test_y_pred)) test_mae = mean_absolute_error(test_y, test_y_pred) test_r2 = r2_score(test_y, test_y_pred) if args.verbose > 0: print('Train RMSE: %.4f ' % (train_rmse)) print('Train MAE: %.4f ' % (train_mae)) print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2)) print('Train R_squared: %.4f ' % (train_r2)) print('') print('Test RMSE: %.4f ' % (test_rmse)) print('Test MAE: %.4f ' % (test_mae)) print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2)) print('Test R_squared: %.4f ' % (test_r2)) # %% # Data Visualization if args.plot: plt.figure() plt.plot(train_y) plt.plot(train_y_pred) plt.title('Time Series of the training set') plt.show() plt.figure() plt.plot(test_y) plt.plot(test_y_pred) plt.title('Time Series of the test set') plt.show() train_errors = train_y - train_y_pred plt.figure() plt.hist(train_errors, bins='auto') plt.title('Histogram of training errors') plt.show() test_errors = test_y - test_y_pred plt.figure() plt.hist(test_errors, bins='auto') plt.title('Histogram of test errors') plt.show() plt.figure() plt.scatter(x=train_y, y=train_y_pred, edgecolors=(0, 0, 0)) plt.plot([train_y.min(), train_y.max()], [train_y.min(), train_y.max()], 'k--', lw=4) plt.title('Predicted vs Actual for training set') plt.show() plt.figure() plt.scatter(x=test_y, y=test_y_pred, edgecolors=(0, 0, 0)) plt.plot([test_y.min(), test_y.max()], [test_y.min(), test_y.max()], 'k--', lw=4) plt.title('Predicted vs Actual for test set') plt.show() plt.figure() plt.scatter(x=train_y_pred, y=train_errors, edgecolors=(0, 0, 0)) plt.plot([train_y.min(), train_y.max()], [0, 0], 'k--', lw=4) plt.title('Residuals vs Predicted for training set') plt.show() plt.figure() plt.scatter(x=test_y_pred, y=test_errors, edgecolors=(0, 0, 0)) plt.plot([test_y.min(), test_y.max()], [0, 0], 'k--', lw=4) plt.title('Residuals vs Predicted for test set') plt.show() # %% # Save the architecture and the lastly trained model save_regress_model(model, models_path, model_name, weights_path, model_path, file_suffix, test_rmse, test_mae, args) # %% return model
def lstm_dense_sunspots(args): """ Main function """ # %% # IMPORTS # code repository sub-package imports from artificial_neural_networks.code.utils.download_monthly_sunspots import \ download_monthly_sunspots from artificial_neural_networks.code.utils.generic_utils import save_regress_model, \ series_to_supervised, affine_transformation from artificial_neural_networks.code.utils.vis_utils import regression_figs # %% if args.verbose > 0: print(args) # For reproducibility if args.reproducible: os.environ['PYTHONHASHSEED'] = '0' np.random.seed(args.seed) rn.seed(args.seed) tf.set_random_seed(args.seed) sess = tf.Session(graph=tf.get_default_graph()) K.set_session(sess) # print(hash("keras")) # %% # Load the Monthly sunspots dataset sunspots_path = download_monthly_sunspots() sunspots = np.genfromtxt(fname=sunspots_path, dtype=np.float32, delimiter=",", skip_header=1, usecols=1) # %% # Train-Test split L_series = len(sunspots) split_ratio = 2 / 3 # between zero and one n_split = int(L_series * split_ratio) look_back = args.look_back steps_ahead = args.steps_ahead train = sunspots[:n_split + (steps_ahead - 1)] test = sunspots[n_split - look_back:] train_x, train_y = series_to_supervised(train, look_back, steps_ahead) test_x, test_y = series_to_supervised(test, look_back, steps_ahead) train_y_series = train[look_back:train.shape[0] - (steps_ahead - 1)] test_y_series = test[look_back:] # %% # PREPROCESSING STEP scaling_factor = args.scaling_factor translation = args.translation n_train = train_x.shape[0] # number of training examples/samples n_test = test_x.shape[0] # number of test examples/samples n_in = train_x.shape[1] # number of features / dimensions n_out = train_y.shape[1] # number of steps ahead to be predicted # Reshape training and test sets train_x = train_x.reshape(n_train, n_in, 1) test_x = test_x.reshape(n_test, n_in, 1) # Apply preprocessing train_x_ = affine_transformation(train_x, scaling_factor, translation) train_y_ = affine_transformation(train_y, scaling_factor, translation) test_x_ = affine_transformation(test_x, scaling_factor, translation) test_y_ = affine_transformation(test_y, scaling_factor, translation) train_y_series_ = affine_transformation(train_y_series, scaling_factor, translation) test_y_series_ = affine_transformation(test_y_series, scaling_factor, translation) # %% # Model hyperparameters and ANN Architecture stateful = args.stateful if stateful: x = Input(shape=(n_in, 1), batch_shape=(1, n_in, 1)) # input layer else: x = Input(shape=(n_in, 1)) # input layer h = x h = LSTM(units=args.layer_size, stateful=stateful)(h) # hidden layer out = Dense(units=n_out, activation=None)(h) # output layer model = Model(inputs=x, outputs=out) if args.verbose > 0: model.summary() def root_mean_squared_error(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) loss_function = root_mean_squared_error metrics = ['mean_absolute_error', 'mean_absolute_percentage_error'] lr = args.lrearning_rate epsilon = args.epsilon optimizer_selection = { 'Adadelta': optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0), 'Adagrad': optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0), 'Adam': optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0, amsgrad=False), 'Adamax': optimizers.Adamax(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0), 'Nadam': optimizers.Nadam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, schedule_decay=0.004), 'RMSprop': optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0), 'SGD': optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False) } optimizer = optimizer_selection[args.optimizer] model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics) # %% # Save trained models for every epoch models_path = r'artificial_neural_networks/trained_models/' model_name = 'sunspots_lstm_dense' weights_path = models_path + model_name + '_weights' model_path = models_path + model_name + '_model' file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}' if args.save_weights_only: file_path = weights_path else: file_path = model_path file_path += file_suffix monitor = 'val_loss' if args.save_models: checkpoint = ModelCheckpoint(file_path + '.h5', monitor=monitor, verbose=args.verbose, save_best_only=args.save_best, mode='auto', save_weights_only=args.save_weights_only) callbacks = [checkpoint] else: callbacks = [] # %% # TRAINING PHASE """ if stateful: shuffle = False else: shuffle = True """ if args.time_training: start = timer() for i in range(0, args.n_epochs): if args.verbose > 0: print('Epoch: {0}/{1}'.format(i + 1, args.n_epochs)) model.fit(x=train_x_, y=train_y_, validation_data=(test_x_, test_y_), batch_size=args.batch_size, epochs=1, verbose=args.verbose, callbacks=callbacks, shuffle=True) if stateful: model.reset_states() if args.time_training: end = timer() duration = end - start print('Total time for training (in seconds):') print(duration) # %% def model_predict(x_, y_): """ Predict using the LSTM Model (Multi-step ahead Forecasting) """ n_y_ = y_.shape[0] y_pred = np.zeros(n_y_) if args.recursive: # Recursive Strategy if args.verbose > 0: print('Following Recursive Strategy ...') n_x_ = x_.shape[0] n_iter = int(np.floor(n_x_ / steps_ahead)) L_last_window = n_x_ % steps_ahead first = 0 # Multi-step ahead Forecasting of all the full windows for i in range(0, n_iter): """ if args.verbose > 0: print('Completed: {0}/{1}'.format(i + 1, n_iter + 1)) """ pred_start = i * steps_ahead pred_end = pred_start + steps_ahead # first time step of each window (no recursion possible) j = pred_start k = j - pred_start # (always zero and unused) x_dyn = np.copy(x_[j:j + 1]) # use actual values only y_dyn = model.predict(x_dyn)[:, first] y_pred[j:j + 1] = y_dyn # remaining time steps of each window (with recursion) for j in range(pred_start + 1, pred_end): k = j - pred_start x_dyn = np.copy(x_[j:j + 1]) # use actual values (if possible) x_start = np.max([0, look_back - k]) y_start = np.max([0, k - look_back]) + pred_start # y_start = np.max([pred_start, j - look_back]) x_dyn[0, x_start:look_back, 0] = np.copy(y_pred[y_start:j]) # use pred. values y_dyn = model.predict(x_dyn)[:, first] # y_after = np.max([0, y_dyn]) + 0.015 * np.random.randn() y_pred[j:j + 1] = np.max([0, y_dyn]) # y_pred[j:j + 1] = y_dyn # Multi-step ahead Forecasting of the last window if L_last_window > 0: """ if args.verbose > 0: print('Completed: {0}/{1}'.format(n_iter + 1, n_iter + 1)) """ pred_start = n_x_ - L_last_window pred_end = n_y_ # first time step of the last window (no recursion possible) j = pred_start k = j - pred_start # (always zero and unused) x_dyn = np.copy(x_[j:j + 1]) # use actual values only y_dyn = model.predict(x_dyn)[:, first] y_pred[j:j + 1] = y_dyn # remaining time steps of the last window (with recursion) for j in range(pred_start + 1, pred_end): k = j - pred_start x_dyn = np.roll(x_dyn, -1) # use act. values (if possible) x_start = np.max([0, look_back - k]) y_start = np.max([0, k - look_back]) + pred_start # y_start = np.max([pred_start, j - look_back]) x_dyn[0, x_start:look_back, 0] = np.copy(y_pred[y_start:j]) # use pred. values y_dyn = model.predict(x_dyn)[:, first] # y_after = np.max([0, y_dyn]) + 0.015 * np.random.randn() y_pred[j:j + 1] = np.max([0, y_dyn]) # y_pred[j:j + 1] = y_dyn """ # One-step ahead Forecasting n_x_ = x_.shape[0] for i in range(0, n_x_): x_dyn = x_[i:i+1] y_dyn = model.predict(x_dyn)[0, 0] y_pred[i] = y_dyn for i in range(n_x_, n_y): x_dyn[0, :, 0] = y_[i - look_back:i] y_dyn = model.predict(x_dyn)[0, 0] y_pred[i] = y_dyn """ else: # Multiple Ouptput Strategy if args.verbose > 0: print('Following Multiple Ouptput Strategy ...') n_iter = int(np.floor(n_y_ / steps_ahead)) L_last_window = n_y_ % steps_ahead # Multi-step ahead Forecasting of all the full windows for i in range(0, n_iter): pred_start = i * steps_ahead pred_end = pred_start + steps_ahead x_dyn = x_[pred_start:pred_start + 1] y_dyn = model.predict(x_dyn)[0] y_pred[pred_start:pred_end] = y_dyn # Multi-step ahead Forecasting of the last window if L_last_window > 0: pred_start = n_y_ - L_last_window pred_end = n_y_ x_dyn[0, :, 0] = y_[pred_end - look_back:pred_end] y_dyn = model.predict(x_dyn)[0] y_pred[pred_start:pred_end] = y_dyn[:L_last_window] return y_pred # %% # TESTING PHASE # Predict preprocessed values train_y_sum = [0] test_y_sum = [0] reps = 1 for i in range(reps): train_y_pred_ = model_predict(train_x_, train_y_series_) test_y_pred_ = model_predict(test_x_, test_y_series_) train_y_sum = np.sum([train_y_sum, train_y_pred_], axis=0) test_y_sum = np.sum([test_y_sum, test_y_pred_], axis=0) train_y_pred_ = train_y_sum / reps test_y_pred_ = test_y_sum / reps # Remove preprocessing train_y_pred = affine_transformation(train_y_pred_, scaling_factor, translation, inverse=True) test_y_pred = affine_transformation(test_y_pred_, scaling_factor, translation, inverse=True) train_rmse = sqrt(mean_squared_error(train_y_series, train_y_pred)) train_mae = mean_absolute_error(train_y_series, train_y_pred) train_r2 = r2_score(train_y_series, train_y_pred) test_rmse = sqrt(mean_squared_error(test_y_series, test_y_pred)) test_mae = mean_absolute_error(test_y_series, test_y_pred) test_r2 = r2_score(test_y_series, test_y_pred) if args.verbose > 0: print('Train RMSE: %.4f ' % (train_rmse)) print('Train MAE: %.4f ' % (train_mae)) print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2)) print('Train R_squared: %.4f ' % (train_r2)) print('') print('Test RMSE: %.4f ' % (test_rmse)) print('Test MAE: %.4f ' % (test_mae)) print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2)) print('Test R_squared: %.4f ' % (test_r2)) # %% # Data Visualization if args.plot: regression_figs(train_y=train_y_series, train_y_pred=train_y_pred, test_y=test_y_series, test_y_pred=test_y_pred) # %% # Save the architecture and the lastly trained model save_regress_model(model, models_path, model_name, weights_path, model_path, file_suffix, test_rmse, test_mae, args) # %% return model