def inverse_transform(X, scaler, trend=None):
X = X.astype(np.float32)
X = scaler.inverse_transform(X)
try:
X += trend
except TypeError as e:
logger.warn(str(e))
except Exception as e:
logger.warn(
'General error (not a TypeError) while adding back time series trend. \n {}'
.format(str(e)))
return X
def inverse_transform(X, scaler, trend=None):
"""
:param X: the data
:param scaler: the scaler that have been used for transforming X
:param trend: the trebd values that has been removed from X. None if no detrending has been used.
It has to be the same dim. as X.
:return:
X with trend adds back and de-standardized
"""
X = X.astype(np.float32)
X = scaler.inverse_transform(X)
try:
X += trend
except TypeError as e:
logger.warn(str(e))
except Exception as e:
logger.warn(
'General error (not a TypeError) while adding back time series trend. \n {}'
.format(str(e)))
return X
def load_data(fill_nan=None,
preprocessing=True,
detrend=False,
exogenous_vars=False,
train_len=365 * 3 * SAMPLES_PER_DAY,
test_len=365 * SAMPLES_PER_DAY,
valid_len=0,
split_type='simple',
is_train=False,
use_prebuilt=True):
"""
Create a split of the data according to the given dimensions for each set.
:param fill_nan: string that identifies how NaN values should be filled. Options are:
-bfill: fill NaN value at index i with value at index i-1
-ffill: fill NaN value at index i with value at index i+1
-mean: fill NaN value at index i with the mean value over all dataset at the same hour,minute
-median: fill NaN value at index i with the median value over all dataset at the same hour,minute
-drop: drop all rows with missing values
:param preprocessing: if True, standardize features using standard scaler
:param detrend: if True, use train weekly statistics to detrend the time series.
(WORKS ONLY FOR split_type=simple or split_type=default when is_train=False)
:param exogenous_vars: if True, add exogenous features to the input data
(temperatures + date/time feature + holiday feature)
:param train_len: length of the train dataset
:param test_len: length of the test set
:param valid_len: length of the validation set
:param split_type: 'simple', 'multi' or 'default'.
- 'simple': See dts.utils.split.simple_split
- 'multi': See dts.utils.split.multiple_split
- 'default': Uses 'simple' split for train-test, then divides training using the 'multi' approach.
:param use_prebuilt: if True, load already splitted data files from disk
:return: a dict having the following (key, value) pairs:
- train = training dataset, np.array of shape()
- test = test dataset, np.array of shape()
- scaler = the scaler used to preprocess the data
- trend = None or the values that has to be added back after prediction if pdetrending has been used.
"""
dataset = dict(
train=None,
test=None,
scaler=None,
trend=[None, None],
)
if valid_len == 0:
valid_len = int(0.1 * train_len)
if split_type == 'simple':
train_test_split = lambda x: simple_split(
x, train_len=None, valid_len=0, test_len=test_len)
train_valid_split = lambda x: simple_split(
train_test_split(x)[0],
train_len=len(train_test_split(x)[0]) - valid_len,
valid_len=0,
test_len=valid_len)
elif split_type == 'multi':
train_test_split = lambda x: multiple_splits(
x, train_len=train_len + valid_len, valid_len=0, test_len=test_len)
train_valid_split = lambda x: [
x[0][:, :train_len, :], None, x[0][:, train_len:, :]
]
elif split_type == 'default':
train_test_split = lambda x: simple_split(
x, train_len=None, valid_len=0, test_len=int(0.1 * df.shape[0]))
train_valid_split = lambda x: multiple_splits(
train_test_split(x)[0],
train_len=5 * 31 * SAMPLES_PER_DAY,
valid_len=0,
test_len=31 * SAMPLES_PER_DAY)
else:
raise ValueError('{} is not a valid split type.'.format(split_type))
if not use_prebuilt:
logger.info(
'Fetching and preprocessing data. This will take a while...')
try:
df = load_dataset()
except FileNotFoundError:
logger.info(
'The dataset seems to be unavailable on your disk at {}. \n'
'Downloading...'.format(
os.path.join(config['data'], 'gefcom2014.csv')))
download()
df = load_dataset()
if detrend:
if split_type == 'default' and not is_train:
df, trend_values = apply_detrend(
df, df.shape[0] - 365 * SAMPLES_PER_DAY)
trend_values = train_test_split(
np.expand_dims(trend_values, -1))[::2]
elif split_type == 'simple' and is_train:
df, trend_values = apply_detrend(df, train_len)
trend_values = train_valid_split(
np.expand_dims(trend_values, -1))[::2]
elif split_type == 'simple':
df, trend_values = apply_detrend(df, train_len + valid_len)
trend_values = train_test_split(
np.expand_dims(trend_values, -1))[::2]
else:
raise ValueError(
'Detrend cannot be applied with this type of split.')
dataset['trend'] = trend_values
X = df[TARGET].values[:-1] # load values
X = np.expand_dims(X, axis=-1)
if preprocessing:
# init scaler using only information for training
scaler, _ = transform(X[:train_len])
# actual preprocess
_, X = transform(X, scaler)
else:
scaler = None
if exogenous_vars:
# init scaler using only temperature information for training
X_temp, X_ex = add_exogenous_variables(df, one_hot=True)
scaler_temp, _ = transform(X_temp[:train_len],
scaler_type='minmax')
_, X_temp = transform(X_temp, scaler_temp)
X = np.concatenate([X, X_temp, X_ex],
axis=1) # Load @ t-1, Datetime @ t, Temp @ t
if is_train:
data = train_valid_split(X)
else:
data = train_test_split(X)
dataset['scaler'] = scaler
dataset['train'] = np.array(data[0], dtype=np.float32)
dataset['test'] = np.array(data[2], dtype=np.float32)
return dataset
else:
logger.info('Fetching preprocessed data from disk...')
try:
return load_prebuilt_data(split_type=split_type,
exogenous_vars=exogenous_vars,
detrend=detrend,
is_train=is_train,
dataset_name=NAME)
except FileNotFoundError:
logger.warn(
'An already preprocessed version of the data do not exists on disk. '
'The train/test data will be created now.')
return load_data(fill_nan,
preprocessing,
detrend,
exogenous_vars,
train_len,
test_len,
valid_len,
split_type,
is_train,
use_prebuilt=False)
def main(_run):
################################
# Load Experiment's paramaters #
################################
params = vars(args)
logger.info(params)
################################
# Load Dataset #
################################
dataset_name = params['dataset']
if dataset_name == 'gefcom':
dataset = gefcom2014
else:
dataset = uci_single_households
data = dataset.load_data(fill_nan='median',
preprocessing=True,
split_type='simple',
is_train=params['train'],
detrend=params['detrend'],
exogenous_vars=params['exogenous'],
use_prebuilt=True)
scaler, train, test, trend = data['scaler'], data['train'], data[
'test'], data['trend']
if not params['detrend']:
trend = None
if params['recursive_forecast']:
horizon = 1
else:
horizon = params['output_sequence_length']
X_train, y_train = get_rnn_inputs(
train,
window_size=params['input_sequence_length'],
horizon=horizon,
shuffle=True,
multivariate_output=params['exogenous'])
################################
# Build & Train Model #
################################
if params['ffnn_type'] == 'simple':
ffnn = SimpleNet
else:
ffnn = ResNet
ffnn = ffnn(layers=params['layers'],
kernel_initializer='glorot_normal',
kernel_regularizer=l2(params['l2_reg']),
bias_regularizer=l2(params['l2_reg']),
use_bias=False,
recursive_forecast=params['recursive_forecast'])
if params['exogenous']:
exog_var_train = y_train[:, :, 1:] # [n_samples, 1, n_features]
y_train = y_train[:, :, 0] # [n_samples, 1]
conditions_shape = (exog_var_train.shape[1], exog_var_train.shape[-1])
X_test, y_test = get_rnn_inputs(
test,
window_size=params['input_sequence_length'],
horizon=params['output_sequence_length'],
shuffle=False,
multivariate_output=True)
exog_var_test = y_test[:, :, 1:] # [n_samples, 1, n_features]
y_test = y_test[:, :, 0] # [n_samples, 1]
else:
X_test, y_test = get_rnn_inputs(
test,
window_size=params['input_sequence_length'],
horizon=params['output_sequence_length'],
shuffle=False)
exog_var_train = None
exog_var_test = None
conditions_shape = None
# IMPORTANT: Remember to pass the trend values through the same ops as the inputs values
if params['detrend']:
X_trend_test, y_trend_test = get_rnn_inputs(
trend[1],
window_size=params['input_sequence_length'],
horizon=params['output_sequence_length'],
shuffle=False)
trend = y_trend_test
model = ffnn.build_model(input_shape=(X_train.shape[1], X_train.shape[-1]),
horizon=params['output_sequence_length'],
conditions_shape=conditions_shape)
if params['load']:
logger.info("Loading model's weights from disk using {}".format(
params['load']))
model.load_weights(params['load'])
optimizer = Adam(params['learning_rate'])
model.compile(optimizer=optimizer, loss=['mse'], metrics=metrics)
callbacks = [EarlyStopping(patience=50, monitor='val_loss')]
if params['exogenous']:
history = model.fit([X_train, exog_var_train],
y_train,
validation_split=0.1,
batch_size=params['batch_size'],
epochs=params['epochs'],
callbacks=callbacks,
verbose=2)
else:
history = model.fit(X_train,
y_train,
validation_split=0.1,
batch_size=params['batch_size'],
epochs=params['epochs'],
callbacks=callbacks,
verbose=2)
################################
# Save weights #
################################
model_filepath = os.path.join(
config['weights'], '{}_{}_{}'.format(params['ffnn_type'],
params['dataset'], time.time()))
model.save_weights(model_filepath)
logger.info("Model's weights saved at {}".format(model_filepath))
#################################
# Evaluate on Validation & Test #
#################################
fn_inverse_val = lambda x: dataset.inverse_transform(
x, scaler=scaler, trend=None)
fn_inverse_test = lambda x: dataset.inverse_transform(
x, scaler=scaler, trend=trend)
fn_plot = lambda x: plot(x, dataset.SAMPLES_PER_DAY, save_at=None)
if params['recursive_forecast']:
val_scores = []
txt = "When FFNN is trained in Recursive mode training and inference are different. Specifically, training is "\
"a 1 step forecasting problem and inference is multi step forecasting problem. Thus, "\
"validation results will not be provided as they are not comparable with test results"
logger.warn(txt)
_run.info['extra'] = txt
else:
# has to add this filter because of unexpected behaviour of history.validation_data when using resent.
validation_data = list(
filter(lambda x: isinstance(x, np.ndarray),
history.validation_data))
val_scores = ffnn.evaluate(validation_data[:-1],
fn_inverse=fn_inverse_val)
if params['exogenous']:
test_scores = ffnn.evaluate([[X_test, exog_var_test], y_test],
fn_inverse=fn_inverse_test,
fn_plot=fn_plot)
else:
test_scores = ffnn.evaluate([X_test, y_test],
fn_inverse=fn_inverse_test,
fn_plot=fn_plot)
metrics_names = [
m.__name__ if not isinstance(m, str) else m for m in model.metrics
]
return dict(zip(metrics_names, val_scores)), \
dict(zip(metrics_names, test_scores)), \
model_filepath
def main(_run):
################################
# Load Experiment's paramaters #
################################
params = vars(args)
print(params)
################################
# Load Dataset #
################################
dataset_name = params['dataset']
if dataset_name == 'gefcom':
dataset = gefcom2014
else:
dataset = uci_single_households
data = dataset.load_data(fill_nan='median',
preprocessing=True,
split_type='simple',
is_train=params['train'],
detrend=params['detrend'],
exogenous_vars=params['exogenous'],
use_prebuilt=True)
scaler, train, test, trend = data['scaler'], data['train'], data['test'], data['trend']
if not params['detrend']:
trend = None
if params['MIMO']:
X_train, y_train = get_rnn_inputs(train,
window_size=params['input_sequence_length'],
horizon=params['output_sequence_length'],
shuffle=True,
multivariate_output=True)
else:
X_train, y_train = get_rnn_inputs(train,
window_size=params['input_sequence_length'],
horizon=1,
shuffle=True,
multivariate_output=True)
if params['exogenous']:
exog_var_train = y_train[:, :, 1:]
y_train = y_train[:, :, 0]
exogenous_shape = (exog_var_train.shape[1], exog_var_train.shape[-1])
if params['MIMO']:
X_train = [X_train, exog_var_train]
X_test, y_test = get_rnn_inputs(test,
window_size=params['input_sequence_length'],
horizon=params['output_sequence_length'],
shuffle=False,
multivariate_output=True)
exog_var_test = y_test[:, :, 1:] # [n_samples, 1, n_features]
y_test = y_test[:, :, 0] # [n_samples, 1]
else:
y_train = y_train[:, :, 0]
X_test, y_test = get_rnn_inputs(test,
window_size=params['input_sequence_length'],
horizon=params['output_sequence_length'],
shuffle=False)
exog_var_train = None
exog_var_test = None
exogenous_shape = None
# IMPORTANT: Remember to pass the trend values through the same ops as the inputs values
if params['detrend']:
X_trend_test, y_trend_test = get_rnn_inputs(trend[1],
window_size=params['input_sequence_length'],
horizon=params['output_sequence_length'],
shuffle=False)
trend = y_trend_test
################################
# Build & Train Model #
################################
cell_params = dict(units=params['units'],
activation='tanh',
dropout=params['dropout'],
kernel_regularizer=l2(params['l2']),
recurrent_regularizer=l2(params['l2']),
kernel_initializer='lecun_uniform',
recurrent_initializer='lecun_uniform')
if params['MIMO']:
rnn = RecurrentNN_MIMO(cell_type=params['cell'],
layers=params['layers'],
cell_params=cell_params)
if params['exogenous']:
model = rnn.build_model(input_shape=(params['input_sequence_length'], X_train[0].shape[-1]),
horizon=params['output_sequence_length'],
exogenous_shape=exogenous_shape)
else:
model = rnn.build_model(input_shape=(params['input_sequence_length'], X_train.shape[-1]),
horizon=params['output_sequence_length'])
else:
rnn = RecurrentNN_Rec(cell_type=params['cell'],
layers=params['layers'],
cell_params=cell_params)
model = rnn.build_model(input_shape=(params['input_sequence_length'], X_train.shape[-1]),
horizon=params['output_sequence_length'])
model.compile(optimizer=Adam(params['learning_rate']), loss='mse', metrics=metrics)
callbacks = [EarlyStopping(patience=100, monitor='val_loss'),
# LambdaCallback(on_epoch_end=lambda _, logs: f_log_metrics(logs=logs))
]
if params['load']:
logger.info("Loading model's weights from disk using {}".format(params['load']))
model.load_weights(params['load'])
history = model.fit(X_train, y_train,
validation_split=0.1,
batch_size = params['batch_size'],
# steps_per_epoch= train.shape[0] // params['batch_size'],
epochs=params['epochs'],
callbacks=callbacks,
verbose=2)
################################
# Save weights #
################################
model_filepath = os.path.join(config['weights'],'{}_{}_{}'.format(
params['cell'], params['dataset'], time.time()))
model.save_weights(model_filepath)
logger.info("Model's weights saved at {}".format(model_filepath))
#################################
# Evaluate on Validation & Test #
#################################
fn_inverse_val = lambda x: dataset.inverse_transform(x, scaler=scaler, trend=None)
fn_inverse_test = lambda x: dataset.inverse_transform(x, scaler=scaler, trend=trend)
fn_plot = lambda x: plot(x, dataset.SAMPLES_PER_DAY, save_at=None)
if params['MIMO']:
val_scores = rnn.evaluate(history.validation_data[:-1], fn_inverse=fn_inverse_val)
else:
val_scores = []
txt = "When RNN is trained in Recursive mode training and inference are different. Specifically, training is "\
"a 1 step forecasting problem and inference is multi step forecasting problem. Thus, "\
"validation results will not be provided as they are not comparable with test results"
logger.warn(txt)
_run.info['extra'] = txt
if params['exogenous']:
test_scores = rnn.evaluate([[X_test, exog_var_test], y_test], fn_inverse=fn_inverse_test, fn_plot=fn_plot)
else:
test_scores = rnn.evaluate([X_test, y_test], fn_inverse=fn_inverse_test, fn_plot=fn_plot)
metrics_names = [m.__name__ if not isinstance(m, str) else m for m in model.metrics]
return dict(zip(metrics_names, val_scores)), \
dict(zip(metrics_names, test_scores)), \
model_filepath