def train(self, train: pd.DataFrame, cross_val_call: bool = False) -> dict:
"""
Train model
:param train: train set
:param cross_val_call: called to perform cross validation
:return dictionary with cross validated scores (if specified)
"""
TrainHelper.init_pytorch_seeds()
cross_val_score_dict = {}
if cross_val_call:
cross_val_score_dict_ts, self.model = self.get_cross_val_score(
train=train)
cross_val_score_dict_shuf, self.model = self.get_cross_val_score(
train=train, normal_cv=True)
cross_val_score_dict = {
**cross_val_score_dict_ts,
**cross_val_score_dict_shuf
}
# create train and validation set
train_loader, x_valid, y_valid = self.create_train_valid_sequence_sets(
train=train)
# run optim loop
self.run_pytorch_optim_loop(train_loader=train_loader,
x_valid=x_valid,
y_valid=y_valid,
model=self.model,
checkpoint_name='lstm_train')
return cross_val_score_dict
def insample(self, train: pd.DataFrame) -> pd.DataFrame:
"""
Deliver insample predictions
:param train: train set
:return: DataFrame with insample predictions
"""
TrainHelper.init_pytorch_seeds()
self.model.eval()
# predict on cpu
self.model.to(torch.device("cpu"))
# scale
x_train_scaled = self.x_scaler.transform(
train.drop(self.target_column, axis=1))
y_train_scaled = self.y_scaler.transform(
train[self.target_column].values.reshape(-1, 1))
# create sequences
x_seq_train, _ = self.create_sequences(
data=np.hstack((x_train_scaled, y_train_scaled)))
x_train = torch.tensor(x_seq_train.astype(np.float32))
# predict and transform back
y_insample = self.y_scaler.inverse_transform(
self.model(x_train).data.numpy())
# insert dummy values for train samples before first full sequence
y_insample = np.insert(y_insample, 0, self.seq_length * [-9999])
insample = pd.DataFrame(data=y_insample,
index=train.index,
columns=['Insample'])
return insample
def create_train_valid_sequence_sets(self, train: pd.DataFrame) -> tuple:
"""
Create train and validation sequenced set respective train loader with sequenced batches
:param train: train data to use
:return: DataLoader with batches of sequenced train data as well as sequenced validation data
"""
TrainHelper.init_pytorch_seeds()
# scale input data
x_train_scaled = self.x_scaler.fit_transform(
train.drop(self.target_column, axis=1))
y_train_scaled = self.y_scaler.fit_transform(
train[self.target_column].values.reshape(-1, 1))
# create sequences
x_seq_train, y_seq_train = self.create_sequences(
data=np.hstack((x_train_scaled, y_train_scaled)))
# split into train and validation set
valid_size = 0.2
split_ind = int(x_seq_train.shape[0] * (1 - valid_size))
x_train = torch.tensor(x_seq_train[:split_ind, :, :].astype(
np.float32))
x_valid = torch.tensor(x_seq_train[split_ind:, :, :].astype(
np.float32))
y_train = torch.tensor(y_seq_train[:split_ind].reshape(-1, 1).astype(
np.float32))
y_valid = torch.tensor(y_seq_train[split_ind:].reshape(-1, 1).astype(
np.float32))
train_loader = torch.utils.data.DataLoader(
dataset=torch.utils.data.TensorDataset(x_train, y_train),
batch_size=self.batch_size,
shuffle=False,
drop_last=False)
return train_loader, x_valid, y_valid
def __init__(self,
n_feature: int,
n_hidden: int,
num_hidden_layer: int,
n_output: int = 1,
dropout_rate: float = 0.0):
"""
:param n_feature: number of features for ANN input
:param n_hidden: number of hidden neurons (first hidden layer)
:param num_hidden_layer: number of hidden layers
:param n_output: number of outputs
:param dropout_rate: probability of element being zeroed in dropout layer
"""
super(ANN, self).__init__()
TrainHelper.init_pytorch_seeds()
self.hidden_layer = nn.ModuleList()
hidden_in = n_feature
hidden_out = n_hidden
for layer_num in range(num_hidden_layer):
self.hidden_layer.append(
nn.Linear(in_features=hidden_in, out_features=hidden_out))
hidden_in = hidden_out
hidden_out = int(hidden_in / 2)
self.output_layer = nn.Linear(in_features=hidden_in,
out_features=n_output)
self.dropout = nn.Dropout(p=dropout_rate)
def create_train_valid_sets(self, train: pd.DataFrame) -> tuple:
"""
Create train and validation set respective train loader with batches
:param train: train dataset
:return: DataLoader with batches of train data as well as validation data
"""
TrainHelper.init_pytorch_seeds()
# create train and validation set
valid_size = 0.2
split_ind = int(train.shape[0] * (1 - valid_size))
train_data = train.iloc[:split_ind]
valid_data = train.iloc[split_ind:]
# scale input data
x_train = self.x_scaler.fit_transform(
train_data.drop(self.target_column, axis=1))
x_valid = self.x_scaler.transform(
valid_data.drop(self.target_column, axis=1))
# create train ready data
x_train = torch.tensor(x_train.astype(np.float32))
x_valid = torch.tensor(x_valid.astype(np.float32))
y_train = torch.tensor(data=train_data[
self.target_column].values.reshape(-1, 1).astype(np.float32))
y_valid = torch.tensor(data=valid_data[
self.target_column].values.reshape(-1, 1).astype(np.float32))
train_loader = torch.utils.data.DataLoader(
dataset=torch.utils.data.TensorDataset(x_train, y_train),
batch_size=self.batch_size,
shuffle=False,
drop_last=False,
worker_init_fn=np.random.seed(0))
return train_loader, x_valid, y_valid
def predict(self, test: pd.DataFrame, train: pd.DataFrame) -> pd.DataFrame:
"""
Deliver, if specified one step ahead, out-of-sample predictions
:param test: test set
:param train: train set
:return: DataFrame with predictions, upper and lower confidence level
"""
TrainHelper.init_pytorch_seeds()
x_test = torch.tensor(data=self.x_scaler.transform(
test.drop(self.target_column, axis=1)).astype(np.float32))
if self.one_step_ahead:
train_manip = train.copy()
predict_lst = []
# deep copy model as predict function should not change class model
model = copy.deepcopy(self.model)
for i in range(0, test.shape[0]):
model.eval()
# predict on cpu
model.to(torch.device("cpu"))
fc = model(x=x_test[i].view(1, -1)).item()
train_manip = train_manip.append(test.iloc[[i]])
self.update(train=train_manip, model=model)
predict_lst.append(fc)
predict = np.array(predict_lst).flatten()
else:
self.model.eval()
# predict on cpu
self.model.to(torch.device("cpu"))
predict = self.model(x=x_test).data.numpy().flatten()
predictions = pd.DataFrame({'Prediction': predict}, index=test.index)
return predictions
def run_pytorch_optim_loop(self, train_loader, x_valid, y_valid, model, checkpoint_name: str = 'train'):
"""
Optimization of hyperparameters
:param train_loader: DataLoader with sequenced train batches
:param x_valid: sequenced validation data
:param y_valid: validation labels
:param model: model to optimize
:param checkpoint_name: save name for best checkpoints
:return:
"""
TrainHelper.init_pytorch_seeds()
# name for checkpoint for temporary storing during optimization with early stopping
# detailed timestamp to prevent interference with parallel running jobs using same directory
checkpoint_name += '_' + datetime.datetime.now().strftime("%d-%b-%Y_%H-%M-%S-%f")
min_valid_loss = 99999999
epochs_wo_improvement_threshold = 0
epochs_wo_improvement_total = 0
# instantiate new optimizer to ensure independence of previous runs
optimizer = torch.optim.Adam(params=model.parameters(), lr=self.learning_rate)
# get device and shift model and data to it
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)
x_valid, y_valid = x_valid.to(device), y_valid.to(device)
for e in range(self.epochs):
model.train()
for (batch_x, batch_y) in train_loader:
TrainHelper.init_pytorch_seeds()
# copy data to device
batch_x, batch_y = batch_x.to(device), batch_y.to(device)
# gradients are summed up so they need to be zeroed for new run
optimizer.zero_grad()
y_pred = model(batch_x)
loss_train = self.loss(y_pred, batch_y)
loss_train.backward()
optimizer.step()
model.eval()
y_pred_valid = model(x_valid)
loss_valid = self.loss(y_pred_valid, y_valid).item()
if loss_valid < min_valid_loss:
min_valid_loss = loss_valid
epochs_wo_improvement_threshold = 0
epochs_wo_improvement_total = 0
torch.save(model.state_dict(), 'Checkpoints/checkpoint_' + checkpoint_name + '.pt')
elif (loss_valid - min_valid_loss) > self.min_val_loss_improvement:
# Early Stopping with thresholds for counter incrementing and max_epochs
epochs_wo_improvement_threshold += 1
if epochs_wo_improvement_threshold > self.max_epochs_wo_improvement:
print('Early Stopping after epoch ' + str(e))
break
elif loss_valid >= min_valid_loss:
# Early stopping if there is no improvement with a higher threshold
epochs_wo_improvement_total += 1
if epochs_wo_improvement_total > 2 * self.max_epochs_wo_improvement:
print('Early Stopping after epoch ' + str(e))
break
if e % 100 == 0:
print('Epoch ' + str(e) + ': valid loss = ' + str(loss_valid)
+ ', min_valid_loss = ' + str(min_valid_loss))
model.load_state_dict(state_dict=torch.load('Checkpoints/checkpoint_' + checkpoint_name + '.pt'))
os.remove('Checkpoints/checkpoint_' + checkpoint_name + '.pt')
def __init__(self,
target_column: str,
seasonal_periods: int,
one_step_ahead: bool,
n_feature: int,
lstm_hidden_dim: int = 10,
lstm_num_layers: int = 1,
seq_length: int = 7,
n_output: int = 1,
dropout_rate: float = 0.0,
epochs: int = 5000,
batch_size: int = 16,
learning_rate: float = 1e-3,
loss=nn.MSELoss(),
min_val_loss_improvement: float = 1000,
max_epochs_wo_improvement: int = 100):
"""
:param target_column: target_column for prediction
:param seasonal_periods: period of seasonality
:param one_step_ahead: perform one step ahead prediction
:param n_feature: number of features for ANN input
:param lstm_hidden_dim: dimensionality of hidden layer
:param lstm_num_layers: depth of lstm network
:param seq_length: sequence length for input of lstm network
:param n_output: number of outputs
:param dropout_rate: probability of element being zeroed in dropout layer
:param epochs: number of epochs
:param batch_size: size of a batch
:param learning_rate: learning rate for optimizer
:param loss: loss function to use
:param min_val_loss_improvement: deviation validation loss to min_val_loss for being counted for early stopping
:param max_epochs_wo_improvement: maximum number of epochs without improvement before early stopping
"""
super().__init__(target_column=target_column,
seasonal_periods=seasonal_periods,
name='LSTM',
one_step_ahead=one_step_ahead)
TrainHelper.init_pytorch_seeds()
self.model = LSTM(n_feature=n_feature,
lstm_hidden_dim=lstm_hidden_dim,
lstm_num_layers=lstm_num_layers,
n_output=n_output,
dropout_rate=dropout_rate)
self.seq_length = seq_length
self.optimizer = 'adam'
self.learning_rate = learning_rate
self.loss = loss
self.x_scaler = sklearn.preprocessing.StandardScaler()
self.y_scaler = sklearn.preprocessing.StandardScaler()
self.batch_size = batch_size
self.epochs = epochs
self.min_val_loss_improvement = min_val_loss_improvement
self.max_epochs_wo_improvement = max_epochs_wo_improvement
def forward(self, x):
"""
Feedforward path
:param x: data to process
:return: prediction value
"""
TrainHelper.init_pytorch_seeds()
for layer in self.hidden_layer:
x = F.relu(layer(x))
x = self.dropout(x)
out = self.output_layer(x)
return out
def forward(self, x):
"""
Feedforward path
:param x: data to process
:return: prediction value
"""
TrainHelper.init_pytorch_seeds()
# input (batch x seq_length x input_size) (batch_first is set True)
lstm_out, (hn, cn) = self.lstm(x.view(x.shape[0], x.shape[1], -1))
# only take last output of sequence
out = self.dropout(lstm_out[:, -1, :])
out = self.output_layer(out)
return out
def update(self, train: pd.DataFrame, model):
"""
Update existing model due to new samples
:param train: train set with new samples
:param model: model to update
"""
TrainHelper.init_pytorch_seeds()
train_loader, x_valid, y_valid = self.create_train_valid_sequence_sets(
train=train)
self.run_pytorch_optim_loop(train_loader=train_loader,
x_valid=x_valid,
y_valid=y_valid,
model=model,
checkpoint_name='lstm_update')
def predict(self, test: pd.DataFrame, train: pd.DataFrame) -> pd.DataFrame:
"""
Deliver, if specified one step ahead, out-of-sample predictions
:param test: test set
:param train: train set
:return: DataFrame with predictions, upper and lower confidence level
"""
TrainHelper.init_pytorch_seeds()
x_train_scaled = self.x_scaler.transform(
train.drop(self.target_column, axis=1))
y_train_scaled = self.y_scaler.transform(
train[self.target_column].values.reshape(-1, 1))
x_test_scaled = self.x_scaler.transform(
test.drop(self.target_column, axis=1))
y_test_scaled = self.y_scaler.transform(
test[self.target_column].values.reshape((-1, 1)))
# add last elements of train to complete first test sequence
x_test_full = np.vstack(
(x_train_scaled[-self.seq_length:], x_test_scaled))
y_test_full = np.vstack(
(y_train_scaled[-self.seq_length:], y_test_scaled))
# create test sequences
x_seq_test, _ = self.create_sequences(data=np.hstack((x_test_full,
y_test_full)))
if self.one_step_ahead:
predict_lst = []
train_manip = train.copy()
# deep copy model as predict function should not change class model
model = copy.deepcopy(self.model)
for i in range(0, test.shape[0]):
test_seq = x_seq_test[i].reshape(1, self.seq_length, -1)
model.eval()
# predict on cpu
model.to(torch.device("cpu"))
fc = self.y_scaler.inverse_transform(
model(x=torch.tensor(test_seq.astype(
np.float32))).data.numpy())
train_manip = train_manip.append(test.iloc[[i]])
self.update(train=train, model=model)
predict_lst.append(fc)
predict = np.array(predict_lst).flatten()
else:
# predict on cpu
self.model.to(torch.device("cpu"))
self.model.eval()
predict = self.y_scaler.inverse_transform(
self.model(x=torch.tensor(x_seq_test.astype(
np.float32))).data.numpy()).flatten()
predictions = pd.DataFrame({'Prediction': predict}, index=test.index)
return predictions
def insample(self, train: pd.DataFrame) -> pd.DataFrame:
"""
Deliver insample predictions
:param train: train set
:return: DataFrame with insample predictions
"""
TrainHelper.init_pytorch_seeds()
self.model.eval()
# predict on cpu
self.model.to(torch.device("cpu"))
x_train = torch.tensor(data=self.x_scaler.transform(
train.drop(self.target_column, axis=1)).astype(np.float32))
insample = pd.DataFrame(data=self.model(x=x_train).data.numpy(),
index=train.index,
columns=['Insample'])
return insample
def evaluate(self, train: pd.DataFrame, test: pd.DataFrame) -> dict:
"""
Evaluate model against all implemented evaluation metrics and baseline methods.
Deliver dictionary with evaluation metrics.
:param train: train set
:param test: test set
:return: dictionary with evaluation metrics of model and all baseline methods
"""
TrainHelper.init_pytorch_seeds()
insample_rw, prediction_rw = SimpleBaselines.RandomWalk(one_step_ahead=self.one_step_ahead)\
.get_insample_prediction(train=train, test=test, target_column=self.target_column)
insample_seasrw, prediction_seasrw = SimpleBaselines.RandomWalk(one_step_ahead=self.one_step_ahead)\
.get_insample_prediction(train=train, test=test, target_column=self.target_column,
seasonal_periods=self.seasonal_periods)
insample_ha, prediction_ha = SimpleBaselines.HistoricalAverage(one_step_ahead=self.one_step_ahead)\
.get_insample_prediction(train=train, test=test, target_column=self.target_column)
insample_model = self.insample(train=train)
prediction_model = self.predict(test=test, train=train)
rmse_train_rw, mape_train_rw, smape_train_rw = EvaluationHelper.get_all_eval_vals(
actual=train[self.target_column], prediction=insample_rw['Insample'])
rmse_test_rw, mape_test_rw, smape_test_rw = EvaluationHelper.get_all_eval_vals(
actual=test[self.target_column], prediction=prediction_rw['Prediction'])
rmse_train_seasrw, mape_train_seasrw, smape_train_seasrw = EvaluationHelper.get_all_eval_vals(
actual=train[self.target_column], prediction=insample_seasrw['Insample'])
rmse_test_seasrw, mape_test_seasrw, smape_test_seasrw = EvaluationHelper.get_all_eval_vals(
actual=test[self.target_column], prediction=prediction_seasrw['Prediction'])
rmse_train_ha, mape_train_ha, smape_train_ha = EvaluationHelper.get_all_eval_vals(
actual=train[self.target_column], prediction=insample_ha['Insample'])
rmse_test_ha, mape_test_ha, smape_test_ha = EvaluationHelper.get_all_eval_vals(
actual=test[self.target_column], prediction=prediction_ha['Prediction'])
rmse_train_model, mape_train_model, smape_train_model = EvaluationHelper.get_all_eval_vals(
actual=train[self.target_column], prediction=insample_model['Insample'])
rmse_test_model, mape_test_model, smape_test_model = EvaluationHelper.get_all_eval_vals(
actual=test[self.target_column], prediction=prediction_model['Prediction'])
return {'RMSE_Train_RW': rmse_train_rw, 'MAPE_Train_RW': mape_train_rw, 'sMAPE_Train_RW': smape_train_rw,
'RMSE_Test_RW': rmse_test_rw, 'MAPE_Test_RW': mape_test_rw, 'sMAPE_Test_RW': smape_test_rw,
'RMSE_Train_seasRW': rmse_train_seasrw, 'MAPE_Train_seasRW': mape_train_seasrw,
'sMAPE_Train_seasRW': smape_train_seasrw,
'RMSE_Test_seasRW': rmse_test_seasrw, 'MAPE_Test_seasRW': mape_test_seasrw,
'sMAPE_Test_seasRW': smape_test_seasrw,
'RMSE_Train_HA': rmse_train_ha, 'MAPE_Train_HA': mape_train_ha, 'sMAPE_Train_HA': smape_train_ha,
'RMSE_Test_HA': rmse_test_ha, 'MAPE_Test_HA': mape_test_ha, 'sMAPE_Test_HA': smape_test_ha,
'RMSE_Train': rmse_train_model, 'MAPE_Train': mape_train_model, 'sMAPE_Train': smape_train_model,
'RMSE_Test': rmse_test_model, 'MAPE_Test': mape_test_model, 'sMAPE_Test': smape_test_model
}
def __init__(self,
n_feature: int,
lstm_hidden_dim: int,
lstm_num_layers: int = 1,
n_output: int = 1,
dropout_rate: float = 0.0):
"""
:param n_feature: number of features for ANN input
:param lstm_hidden_dim: dimensionality of hidden layer
:param lstm_num_layers: depth of lstm network
:param n_output: number of outputs
:param dropout_rate: probability of element being zeroed in dropout layer
"""
super(LSTM, self).__init__()
TrainHelper.init_pytorch_seeds()
self.lstm = nn.LSTM(input_size=n_feature,
hidden_size=lstm_hidden_dim,
num_layers=lstm_num_layers,
batch_first=True,
dropout=dropout_rate)
self.dropout = nn.Dropout(p=dropout_rate)
self.output_layer = nn.Linear(in_features=lstm_hidden_dim,
out_features=n_output)