def get_cross_val_score(self, train: pd.DataFrame, normal_cv: bool = False) -> tuple:
"""
Deliver cross validated evaluation scores
:param train: train set
:param normal_cv: specify whether normal cv can be performed
:return: dictionary with mean and std of cross validated evaluation scores
"""
# backup model so train on full dataset afterwards is independet of cv training
backup_model = copy.deepcopy(self.model)
if train.shape[0] < 80:
print('Train set too small for Cross Validation')
return {}, backup_model
train = train.copy()
rmse_lst, mape_lst, smape_lst = [], [], []
prefix = 'ts_'
splitter = sklearn.model_selection.TimeSeriesSplit(n_splits=3)
if normal_cv:
splitter = sklearn.model_selection.ShuffleSplit(n_splits=5, test_size=0.2, random_state=0)
prefix = 'shuf_'
for train_index, test_index in splitter.split(train):
cv_train, cv_test = train.loc[train.index[train_index]], train.loc[train.index[test_index]]
# ES Model with seasonality is only working if n_samples is bigger than seasonality
# noinspection PyUnresolvedReferences
if self.name == 'ExponentialSmoothing' and self.seasonal is not None:
if cv_train.shape[0] <= self.seasonal_periods:
print('CV train set too small for seasonality')
continue
self.model = copy.deepcopy(backup_model)
try:
self.train(train=cv_train)
predictions = self.predict(test=cv_test, train=cv_train)
rmse_test, mape_test, smape_test = EvaluationHelper.get_all_eval_vals(
actual=cv_test[self.target_column], prediction=predictions['Prediction'])
rmse_lst.append(rmse_test)
mape_lst.append(mape_test)
smape_lst.append(smape_test)
except Exception as exc:
print(exc)
continue
rmse_mean, mape_mean, smape_mean = \
np.mean(np.asarray(rmse_lst)), np.mean(np.asarray(mape_lst)), np.mean(np.asarray(smape_lst))
rmse_std, mape_std, smape_std = \
np.std(np.asarray(rmse_lst)), np.std(np.asarray(mape_lst)), np.std(np.asarray(smape_lst))
cv_dict = {prefix + 'cv_rmse_mean': rmse_mean, prefix + 'cv_rmse_std': rmse_std,
prefix + 'cv_mape_mean': mape_mean, prefix + 'cv_mape_std': mape_std,
prefix + 'cv_smape_mean': smape_mean, prefix + 'cv_smape_std': smape_std}
for cv_number in range(len(rmse_lst)):
cv_dict[prefix + 'cv_rmse_' + str(cv_number)] = rmse_lst[cv_number]
cv_dict[prefix + 'cv_mape_' + str(cv_number)] = mape_lst[cv_number]
cv_dict[prefix + 'cv_smape_' + str(cv_number)] = smape_lst[cv_number]
return cv_dict, backup_model
def evaluate(self, train: pd.DataFrame, test: pd.DataFrame) -> dict:
"""
Evaluate model against all implemented evaluation metrics and baseline methods.
Deliver dictionary with evaluation metrics.
:param train: train set
:param test: test set
:return: dictionary with evaluation metrics of model and all baseline methods
"""
TrainHelper.init_pytorch_seeds()
insample_rw, prediction_rw = SimpleBaselines.RandomWalk(one_step_ahead=self.one_step_ahead)\
.get_insample_prediction(train=train, test=test, target_column=self.target_column)
insample_seasrw, prediction_seasrw = SimpleBaselines.RandomWalk(one_step_ahead=self.one_step_ahead)\
.get_insample_prediction(train=train, test=test, target_column=self.target_column,
seasonal_periods=self.seasonal_periods)
insample_ha, prediction_ha = SimpleBaselines.HistoricalAverage(one_step_ahead=self.one_step_ahead)\
.get_insample_prediction(train=train, test=test, target_column=self.target_column)
insample_model = self.insample(train=train)
prediction_model = self.predict(test=test, train=train)
rmse_train_rw, mape_train_rw, smape_train_rw = EvaluationHelper.get_all_eval_vals(
actual=train[self.target_column], prediction=insample_rw['Insample'])
rmse_test_rw, mape_test_rw, smape_test_rw = EvaluationHelper.get_all_eval_vals(
actual=test[self.target_column], prediction=prediction_rw['Prediction'])
rmse_train_seasrw, mape_train_seasrw, smape_train_seasrw = EvaluationHelper.get_all_eval_vals(
actual=train[self.target_column], prediction=insample_seasrw['Insample'])
rmse_test_seasrw, mape_test_seasrw, smape_test_seasrw = EvaluationHelper.get_all_eval_vals(
actual=test[self.target_column], prediction=prediction_seasrw['Prediction'])
rmse_train_ha, mape_train_ha, smape_train_ha = EvaluationHelper.get_all_eval_vals(
actual=train[self.target_column], prediction=insample_ha['Insample'])
rmse_test_ha, mape_test_ha, smape_test_ha = EvaluationHelper.get_all_eval_vals(
actual=test[self.target_column], prediction=prediction_ha['Prediction'])
rmse_train_model, mape_train_model, smape_train_model = EvaluationHelper.get_all_eval_vals(
actual=train[self.target_column], prediction=insample_model['Insample'])
rmse_test_model, mape_test_model, smape_test_model = EvaluationHelper.get_all_eval_vals(
actual=test[self.target_column], prediction=prediction_model['Prediction'])
return {'RMSE_Train_RW': rmse_train_rw, 'MAPE_Train_RW': mape_train_rw, 'sMAPE_Train_RW': smape_train_rw,
'RMSE_Test_RW': rmse_test_rw, 'MAPE_Test_RW': mape_test_rw, 'sMAPE_Test_RW': smape_test_rw,
'RMSE_Train_seasRW': rmse_train_seasrw, 'MAPE_Train_seasRW': mape_train_seasrw,
'sMAPE_Train_seasRW': smape_train_seasrw,
'RMSE_Test_seasRW': rmse_test_seasrw, 'MAPE_Test_seasRW': mape_test_seasrw,
'sMAPE_Test_seasRW': smape_test_seasrw,
'RMSE_Train_HA': rmse_train_ha, 'MAPE_Train_HA': mape_train_ha, 'sMAPE_Train_HA': smape_train_ha,
'RMSE_Test_HA': rmse_test_ha, 'MAPE_Test_HA': mape_test_ha, 'sMAPE_Test_HA': smape_test_ha,
'RMSE_Train': rmse_train_model, 'MAPE_Train': mape_train_model, 'sMAPE_Train': smape_train_model,
'RMSE_Test': rmse_test_model, 'MAPE_Test': mape_test_model, 'sMAPE_Test': smape_test_model
}