def lstm_dense_sunspots(args):
"""
Main function
"""
# %%
# IMPORTS
# code repository sub-package imports
from artificial_neural_networks.utils.download_monthly_sunspots import \
download_monthly_sunspots
from artificial_neural_networks.utils.generic_utils import save_regress_model, \
series_to_supervised, affine_transformation
from artificial_neural_networks.utils.vis_utils import regression_figs
# %%
if args.verbose > 0:
print(args)
# For reproducibility
if args.reproducible:
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(args.seed)
rn.seed(args.seed)
tf.set_random_seed(args.seed)
sess = tf.Session(graph=tf.get_default_graph())
K.set_session(sess)
# print(hash("keras"))
# %%
# Load the Monthly sunspots dataset
sunspots_path = download_monthly_sunspots()
sunspots = np.genfromtxt(
fname=sunspots_path, dtype=np.float32, delimiter=",", skip_header=1, usecols=1)
# %%
# Train-Test split
L_series = len(sunspots)
split_ratio = 2 / 3 # between zero and one
n_split = int(L_series * split_ratio)
look_back = args.look_back
train = sunspots[:n_split]
test = sunspots[n_split - look_back:]
train_x, train_y = series_to_supervised(train, look_back)
test_x, test_y = series_to_supervised(test, look_back)
# %%
# PREPROCESSING STEP
scaling_factor = args.scaling_factor
translation = args.translation
n_train = train_x.shape[0] # number of training examples/samples
n_test = test_x.shape[0] # number of test examples/samples
n_in = train_x.shape[1] # number of features / dimensions
n_out = train_y.shape[1] # number of steps ahead to be predicted
# Reshape training and test sets
train_x = train_x.reshape(n_train, n_in, 1)
test_x = test_x.reshape(n_test, n_in, 1)
# Apply preprocessing
train_x_ = affine_transformation(train_x, scaling_factor, translation)
train_y_ = affine_transformation(train_y, scaling_factor, translation)
test_x_ = affine_transformation(test_x, scaling_factor, translation)
test_y_ = affine_transformation(test_y, scaling_factor, translation)
# %%
# Model hyperparameters and ANN Architecture
x = Input(shape=(n_in, 1)) # input layer
h = x
h = LSTM(units=args.layer_size)(h) # hidden layer
out = Dense(units=n_out, activation=None)(h) # output layer
model = Model(inputs=x, outputs=out)
if args.verbose > 0:
model.summary()
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))
loss_function = root_mean_squared_error
metrics = ['mean_absolute_error', 'mean_absolute_percentage_error']
lr = args.lrearning_rate
epsilon = args.epsilon
optimizer_selection = {
'Adadelta':
optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0),
'Adagrad':
optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0),
'Adam':
optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0, amsgrad=False),
'Adamax':
optimizers.Adamax(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0),
'Nadam':
optimizers.Nadam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, schedule_decay=0.004),
'RMSprop':
optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0),
'SGD':
optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False)
}
optimizer = optimizer_selection[args.optimizer]
model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics)
# %%
# Save trained models for every epoch
models_path = r'artificial_neural_networks/trained_models/'
model_name = 'sunspots_lstm_dense'
weights_path = models_path + model_name + '_weights'
model_path = models_path + model_name + '_model'
file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}'
if args.save_weights_only:
file_path = weights_path
else:
file_path = model_path
file_path += file_suffix
monitor = 'val_loss'
if args.save_models:
checkpoint = ModelCheckpoint(
file_path + '.h5',
monitor=monitor,
verbose=args.verbose,
save_best_only=args.save_best,
mode='auto',
save_weights_only=args.save_weights_only)
callbacks = [checkpoint]
else:
callbacks = []
# %%
# TRAINING PHASE
if args.time_training:
start = timer()
model.fit(
x=train_x_,
y=train_y_,
validation_data=(test_x_, test_y_),
batch_size=args.batch_size,
epochs=args.n_epochs,
verbose=args.verbose,
callbacks=callbacks)
if args.time_training:
end = timer()
duration = end - start
print('Total time for training (in seconds):')
print(duration)
# %%
# TESTING PHASE
# Predict preprocessed values
train_y_pred_ = model.predict(train_x_)[:, 0]
test_y_pred_ = model.predict(test_x_)[:, 0]
# Remove preprocessing
train_y_pred = affine_transformation(train_y_pred_, scaling_factor, translation, inverse=True)
test_y_pred = affine_transformation(test_y_pred_, scaling_factor, translation, inverse=True)
train_rmse = sqrt(mean_squared_error(train_y, train_y_pred))
train_mae = mean_absolute_error(train_y, train_y_pred)
train_r2 = r2_score(train_y, train_y_pred)
test_rmse = sqrt(mean_squared_error(test_y, test_y_pred))
test_mae = mean_absolute_error(test_y, test_y_pred)
test_r2 = r2_score(test_y, test_y_pred)
if args.verbose > 0:
print('Train RMSE: %.4f ' % (train_rmse))
print('Train MAE: %.4f ' % (train_mae))
print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2))
print('Train R_squared: %.4f ' % (train_r2))
print('')
print('Test RMSE: %.4f ' % (test_rmse))
print('Test MAE: %.4f ' % (test_mae))
print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2))
print('Test R_squared: %.4f ' % (test_r2))
# %%
# Data Visualization
if args.plot:
regression_figs(train_y=train_y, train_y_pred=train_y_pred,
test_y=test_y, test_y_pred=test_y_pred)
# %%
# Save the architecture and the lastly trained model
save_regress_model(model, models_path, model_name, weights_path, model_path, file_suffix,
test_rmse, test_mae, args)
# %%
return model
def lstm_dense_sunspots(args):
"""
Main function
"""
# %%
# IMPORTS
# code repository sub-package imports
from artificial_neural_networks.utils.download_monthly_sunspots import \
download_monthly_sunspots
from artificial_neural_networks.utils.generic_utils import save_regress_model, \
series_to_supervised, affine_transformation
from artificial_neural_networks.utils.vis_utils import regression_figs
# %%
if args.verbose > 0:
print(args)
# For reproducibility
if args.reproducible:
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(args.seed)
rn.seed(args.seed)
tf.set_random_seed(args.seed)
sess = tf.Session(graph=tf.get_default_graph())
K.set_session(sess)
# print(hash("keras"))
# %%
# Load the Monthly sunspots dataset
sunspots_path = download_monthly_sunspots()
sunspots = np.genfromtxt(fname=sunspots_path,
dtype=np.float32,
delimiter=",",
skip_header=1,
usecols=1)
# %%
# Train-Test split
L_series = len(sunspots)
split_ratio = 2 / 3 # between zero and one
n_split = int(L_series * split_ratio)
look_back = args.look_back
steps_ahead = args.steps_ahead
train = sunspots[:n_split + (steps_ahead - 1)]
test = sunspots[n_split - look_back:]
train_x, train_y = series_to_supervised(train, look_back, steps_ahead)
test_x, test_y = series_to_supervised(test, look_back, steps_ahead)
train_y_series = train[look_back:train.shape[0] - (steps_ahead - 1)]
test_y_series = test[look_back:]
# %%
# PREPROCESSING STEP
scaling_factor = args.scaling_factor
translation = args.translation
n_train = train_x.shape[0] # number of training examples/samples
n_test = test_x.shape[0] # number of test examples/samples
n_in = train_x.shape[1] # number of features / dimensions
n_out = train_y.shape[1] # number of steps ahead to be predicted
# Reshape training and test sets
train_x = train_x.reshape(n_train, n_in, 1)
test_x = test_x.reshape(n_test, n_in, 1)
# Apply preprocessing
train_x_ = affine_transformation(train_x, scaling_factor, translation)
train_y_ = affine_transformation(train_y, scaling_factor, translation)
test_x_ = affine_transformation(test_x, scaling_factor, translation)
test_y_ = affine_transformation(test_y, scaling_factor, translation)
train_y_series_ = affine_transformation(train_y_series, scaling_factor,
translation)
test_y_series_ = affine_transformation(test_y_series, scaling_factor,
translation)
# %%
# Model hyperparameters and ANN Architecture
stateful = args.stateful
if stateful:
x = Input(shape=(n_in, 1), batch_shape=(1, n_in, 1)) # input layer
else:
x = Input(shape=(n_in, 1)) # input layer
h = x
h = LSTM(units=args.layer_size, stateful=stateful)(h) # hidden layer
out = Dense(units=n_out, activation=None)(h) # output layer
model = Model(inputs=x, outputs=out)
if args.verbose > 0:
model.summary()
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))
loss_function = root_mean_squared_error
metrics = ['mean_absolute_error', 'mean_absolute_percentage_error']
lr = args.lrearning_rate
epsilon = args.epsilon
optimizer_selection = {
'Adadelta':
optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0),
'Adagrad':
optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0),
'Adam':
optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
decay=0.0,
amsgrad=False),
'Adamax':
optimizers.Adamax(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
decay=0.0),
'Nadam':
optimizers.Nadam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
schedule_decay=0.004),
'RMSprop':
optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0),
'SGD':
optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False)
}
optimizer = optimizer_selection[args.optimizer]
model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics)
# %%
# Save trained models for every epoch
models_path = r'artificial_neural_networks/trained_models/'
model_name = 'sunspots_lstm_dense'
weights_path = models_path + model_name + '_weights'
model_path = models_path + model_name + '_model'
file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}'
if args.save_weights_only:
file_path = weights_path
else:
file_path = model_path
file_path += file_suffix
monitor = 'val_loss'
if args.save_models:
checkpoint = ModelCheckpoint(file_path + '.h5',
monitor=monitor,
verbose=args.verbose,
save_best_only=args.save_best,
mode='auto',
save_weights_only=args.save_weights_only)
callbacks = [checkpoint]
else:
callbacks = []
# %%
# TRAINING PHASE
"""
if stateful:
shuffle = False
else:
shuffle = True
"""
if args.time_training:
start = timer()
for i in range(0, args.n_epochs):
if args.verbose > 0:
print('Epoch: {0}/{1}'.format(i + 1, args.n_epochs))
model.fit(x=train_x_,
y=train_y_,
validation_data=(test_x_, test_y_),
batch_size=args.batch_size,
epochs=1,
verbose=args.verbose,
callbacks=callbacks,
shuffle=True)
if stateful:
model.reset_states()
if args.time_training:
end = timer()
duration = end - start
print('Total time for training (in seconds):')
print(duration)
# %%
def model_predict(x_, y_):
"""
Predict using the LSTM Model (Multi-step ahead Forecasting)
"""
n_y_ = y_.shape[0]
y_pred = np.zeros(n_y_)
if args.recursive: # Recursive Strategy
if args.verbose > 0:
print('Following Recursive Strategy ...')
n_x_ = x_.shape[0]
n_iter = int(np.floor(n_x_ / steps_ahead))
L_last_window = n_x_ % steps_ahead
first = 0
# Multi-step ahead Forecasting of all the full windows
for i in range(0, n_iter):
""" if args.verbose > 0:
print('Completed: {0}/{1}'.format(i + 1, n_iter + 1)) """
pred_start = i * steps_ahead
pred_end = pred_start + steps_ahead
# first time step of each window (no recursion possible)
j = pred_start
k = j - pred_start # (always zero and unused)
x_dyn = np.copy(x_[j:j + 1]) # use actual values only
y_dyn = model.predict(x_dyn)[:, first]
y_pred[j:j + 1] = y_dyn
# remaining time steps of each window (with recursion)
for j in range(pred_start + 1, pred_end):
k = j - pred_start
x_dyn = np.copy(x_[j:j +
1]) # use actual values (if possible)
x_start = np.max([0, look_back - k])
y_start = np.max([0, k - look_back]) + pred_start
# y_start = np.max([pred_start, j - look_back])
x_dyn[0, x_start:look_back,
0] = np.copy(y_pred[y_start:j]) # use pred. values
y_dyn = model.predict(x_dyn)[:, first]
# y_after = np.max([0, y_dyn]) + 0.015 * np.random.randn()
y_pred[j:j + 1] = np.max([0, y_dyn])
# y_pred[j:j + 1] = y_dyn
# Multi-step ahead Forecasting of the last window
if L_last_window > 0:
""" if args.verbose > 0:
print('Completed: {0}/{1}'.format(n_iter + 1, n_iter + 1)) """
pred_start = n_x_ - L_last_window
pred_end = n_y_
# first time step of the last window (no recursion possible)
j = pred_start
k = j - pred_start # (always zero and unused)
x_dyn = np.copy(x_[j:j + 1]) # use actual values only
y_dyn = model.predict(x_dyn)[:, first]
y_pred[j:j + 1] = y_dyn
# remaining time steps of the last window (with recursion)
for j in range(pred_start + 1, pred_end):
k = j - pred_start
x_dyn = np.roll(x_dyn, -1) # use act. values (if possible)
x_start = np.max([0, look_back - k])
y_start = np.max([0, k - look_back]) + pred_start
# y_start = np.max([pred_start, j - look_back])
x_dyn[0, x_start:look_back,
0] = np.copy(y_pred[y_start:j]) # use pred. values
y_dyn = model.predict(x_dyn)[:, first]
# y_after = np.max([0, y_dyn]) + 0.015 * np.random.randn()
y_pred[j:j + 1] = np.max([0, y_dyn])
# y_pred[j:j + 1] = y_dyn
"""
# One-step ahead Forecasting
n_x_ = x_.shape[0]
for i in range(0, n_x_):
x_dyn = x_[i:i+1]
y_dyn = model.predict(x_dyn)[0, 0]
y_pred[i] = y_dyn
for i in range(n_x_, n_y):
x_dyn[0, :, 0] = y_[i - look_back:i]
y_dyn = model.predict(x_dyn)[0, 0]
y_pred[i] = y_dyn
"""
else: # Multiple Ouptput Strategy
if args.verbose > 0:
print('Following Multiple Ouptput Strategy ...')
n_iter = int(np.floor(n_y_ / steps_ahead))
L_last_window = n_y_ % steps_ahead
# Multi-step ahead Forecasting of all the full windows
for i in range(0, n_iter):
pred_start = i * steps_ahead
pred_end = pred_start + steps_ahead
x_dyn = x_[pred_start:pred_start + 1]
y_dyn = model.predict(x_dyn)[0]
y_pred[pred_start:pred_end] = y_dyn
# Multi-step ahead Forecasting of the last window
if L_last_window > 0:
pred_start = n_y_ - L_last_window
pred_end = n_y_
x_dyn[0, :, 0] = y_[pred_end - look_back:pred_end]
y_dyn = model.predict(x_dyn)[0]
y_pred[pred_start:pred_end] = y_dyn[:L_last_window]
return y_pred
# %%
# TESTING PHASE
# Predict preprocessed values
train_y_sum = [0]
test_y_sum = [0]
reps = 1
for i in range(reps):
train_y_pred_ = model_predict(train_x_, train_y_series_)
test_y_pred_ = model_predict(test_x_, test_y_series_)
train_y_sum = np.sum([train_y_sum, train_y_pred_], axis=0)
test_y_sum = np.sum([test_y_sum, test_y_pred_], axis=0)
train_y_pred_ = train_y_sum / reps
test_y_pred_ = test_y_sum / reps
# Remove preprocessing
train_y_pred = affine_transformation(train_y_pred_,
scaling_factor,
translation,
inverse=True)
test_y_pred = affine_transformation(test_y_pred_,
scaling_factor,
translation,
inverse=True)
train_rmse = sqrt(mean_squared_error(train_y_series, train_y_pred))
train_mae = mean_absolute_error(train_y_series, train_y_pred)
train_r2 = r2_score(train_y_series, train_y_pred)
test_rmse = sqrt(mean_squared_error(test_y_series, test_y_pred))
test_mae = mean_absolute_error(test_y_series, test_y_pred)
test_r2 = r2_score(test_y_series, test_y_pred)
if args.verbose > 0:
print('Train RMSE: %.4f ' % (train_rmse))
print('Train MAE: %.4f ' % (train_mae))
print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2))
print('Train R_squared: %.4f ' % (train_r2))
print('')
print('Test RMSE: %.4f ' % (test_rmse))
print('Test MAE: %.4f ' % (test_mae))
print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2))
print('Test R_squared: %.4f ' % (test_r2))
# %%
# Data Visualization
if args.plot:
regression_figs(train_y=train_y_series,
train_y_pred=train_y_pred,
test_y=test_y_series,
test_y_pred=test_y_pred)
# %%
# Save the architecture and the lastly trained model
save_regress_model(model, models_path, model_name, weights_path,
model_path, file_suffix, test_rmse, test_mae, args)
# %%
return model
def naive_persistence_model_sunspots(args):
"""
Main function
"""
# %%
# IMPORTS
# code repository sub-package imports
from artificial_neural_networks.utils.download_monthly_sunspots import \
download_monthly_sunspots
from artificial_neural_networks.utils.generic_utils import series_to_supervised, \
affine_transformation
from artificial_neural_networks.utils.vis_utils import regression_figs
# %%
if args.verbose > 0:
print(args)
# For reproducibility
if args.reproducible:
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(args.seed)
rn.seed(args.seed)
# %%
# Load the Monthly sunspots dataset
sunspots_path = download_monthly_sunspots()
sunspots = np.genfromtxt(fname=sunspots_path,
dtype=np.float32,
delimiter=",",
skip_header=1,
usecols=1)
# %%
# Train-Test split
L_series = len(sunspots)
split_ratio = 2 / 3 # between zero and one
n_split = int(L_series * split_ratio)
look_back = args.look_back
train = np.concatenate((np.zeros(look_back), sunspots[:n_split]))
test = sunspots[n_split - look_back:]
train_x, train_y = series_to_supervised(train, look_back)
test_x, test_y = series_to_supervised(test, look_back)
# %%
# PREPROCESSING STEP
scaling_factor = args.scaling_factor
translation = args.translation
# Apply preprocessing
train_x_ = affine_transformation(train_x, scaling_factor, translation)
test_x_ = affine_transformation(test_x, scaling_factor, translation)
# %%
# TRAINING PHASE
def model_predict(x):
"""
Predict using the Naive Persistence Model
"""
last = x.shape[1] - 1
y_pred = []
y_pred.append(x[0][last])
for i in range(x.shape[0] - 1):
y_pred.append(x[i][last])
return np.array(y_pred)
# %%
# TESTING PHASE
# Predict preprocessed values
train_y_pred_ = model_predict(train_x_)
test_y_pred_ = model_predict(test_x_)
# Remove preprocessing
train_y_pred = affine_transformation(train_y_pred_,
scaling_factor,
translation,
inverse=True)
test_y_pred = affine_transformation(test_y_pred_,
scaling_factor,
translation,
inverse=True)
train_rmse = sqrt(mean_squared_error(train_y, train_y_pred))
train_mae = mean_absolute_error(train_y, train_y_pred)
train_r2 = r2_score(train_y, train_y_pred)
test_rmse = sqrt(mean_squared_error(test_y, test_y_pred))
test_mae = mean_absolute_error(test_y, test_y_pred)
test_r2 = r2_score(test_y, test_y_pred)
if args.verbose > 0:
print('Train RMSE: %.4f ' % (train_rmse))
print('Train MAE: %.4f ' % (train_mae))
print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2))
print('Train R_squared: %.4f ' % (train_r2))
print('')
print('Test RMSE: %.4f ' % (test_rmse))
print('Test MAE: %.4f ' % (test_mae))
print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2))
print('Test R_squared: %.4f ' % (test_r2))
# %%
# Data Visualization
if args.plot:
regression_figs(train_y=train_y,
train_y_pred=train_y_pred,
test_y=test_y,
test_y_pred=test_y_pred)