def groe_owa(ts: TimeSeries, model: ForecastingModel, n1: int, m: int, p: int,
fq: int):
"""
Backtesting.
Compute the OWA score iteratively on ´p´ timepoints following an expanding window mode.
Parameters
---------
ts
TimeSeries on which backtesting will be done.
model
model to backtest.
n1
minimum number of datapoints to take during training
m
step size
p
number of steps to iterate over
fq
Frequency of the time series.
Returns
-------
Sum of all OWA errors
"""
n = len(ts)
errors = []
for i in range(p):
if n1 + i * m == n:
break
ni = n1 + i * m
npred = n - ni
train = ts[:ni]
if npred >= 3:
test = ts[ni:]
else:
test = TimeSeries(ts.pd_series()[ni:], freq=ts.freq_str())
forecast_naive2 = naive2_groe(train, npred, fq)
error_sape_n2 = mase_m4(train, test, forecast_naive2)
error_ase_n2 = smape_m4(test, forecast_naive2)
model.fit(train)
forecast = model.predict(npred)
try:
error_sape = mase_m4(train, test, forecast)
error_ase = smape_m4(test, forecast)
owa = 0.5 * (error_sape / error_sape_n2) + 0.5 * (error_ase /
error_ase_n2)
errors.append(np.sum(owa))
except (ZeroDivisionError, ValueError):
errors.append(0)
errors = np.sum(errors)
return errors
def fallback(mase_all, smape_all, train, test, m):
naive = NaiveSeasonal(K=m)
naive.fit(train)
forecast = naive.predict(len(test))
mase_all.append(np.vstack([
mase_m4(train, test, forecast, m=m),
]))
smape_all.append(np.vstack([
smape_m4(test, forecast),
]))
holt.fit(train_des)
damp.fit(train_des)
forecast_naiveSeason = naiveSeason.predict(len(test))
forecast_naiveDrift = naive.predict(len(test))
forecast_naive = forecast_naiveSeason + forecast_naiveDrift - train.last_value(
)
forecast_naive2 = naive2.predict(len(test)) * seasonOut
forecast_ses = ses.predict(len(test)) * seasonOut
forecast_holt = holt.predict(len(test)) * seasonOut
forecast_damp = damp.predict(len(test)) * seasonOut
forecast_comb = ((forecast_ses + forecast_holt + forecast_damp) /
3)
mase_all.append(
np.vstack([
mase_m4(train, test, forecast_naiveSeason, m=m),
mase_m4(train, test, forecast_naive, m=m),
mase_m4(train, test, forecast_naive2, m=m),
mase_m4(train, test, forecast_ses, m=m),
mase_m4(train, test, forecast_holt, m=m),
mase_m4(train, test, forecast_damp, m=m),
mase_m4(train, test, forecast_comb, m=m),
]))
smape_all.append(
np.vstack([
smape_m4(test, forecast_naiveSeason),
smape_m4(test, forecast_naive),
smape_m4(test, forecast_naive2),
smape_m4(test, forecast_ses),
smape_m4(test, forecast_holt),
smape_m4(test, forecast_damp),
season_diff.append(
np.abs(train_des.values() - r_train_des) / np.abs(r_train_des))
train_des = TimeSeries.from_times_and_values(
train.time_index(), r_train_des)
seasonOut = TimeSeries.from_times_and_values(
test.time_index(), r_seasonOut)
naive2 = NaiveSeasonal(K=1)
naive2.fit(train_des)
forecast_naive2 = naive2.predict(len(test)) * seasonOut
mase_all.append(
np.vstack([
mase_m4(train, test, forecast_naive2, m=m),
]))
smape_all.append(np.vstack([
smape_m4(test, forecast_naive2),
]))
rel_error = np.mean(np.hstack(season_diff))
pkl.dump([mase_all, smape_all, season_diff],
open("rnaive2_" + cat + ".pkl", "wb"))
print(
np.round(np.nanmean(np.stack(mase_all), axis=(
0, 2)), 3) == baseline[cat][1])
print(
np.round(np.nanmean(np.stack(smape_all), axis=(
0, 2)), 3) == baseline[cat][0])
print("MASE; Naive2: {:.3f}".format(
*tuple(np.nanmean(np.stack(mase_all), axis=(0, 2)))))
def groe_owa(ts: TimeSeries, model: ForecastingModel, fq: int, n1: int, m: int,
p: int) -> float:
"""
Implementation of Generalized Rolling Origin Evaluation using OWA score.
The concept is to cross-validate a model on a time series with rolling origin, using OWA score from M4 competition.
Parameters
-----------
ts
The time series object to use to cross-validate
model
The Darts model to evaluate
fq
Period of the seasonality of the time series
n1
First origin to use for the cross-validation
m
Stride used for rolling the origin
p
number of stride to operate
Returns
-------
Float
sum of OWA score for all different origins
If there is an error with one of the origin, return 0.
"""
# todo: Implement generalized version from R
n = len(ts)
errors = []
for i in range(p):
# if origin is further than end timestamp, end function
if n1 + i * m >= n:
break
ni = n1 + i * m
npred = n - ni
train = ts[:ni]
test = ts[ni:]
forecast_naive2 = rNaive2(train.values(), npred, fq)
forecast_naive2 = TimeSeries.from_times_and_values(
test.time_index(), forecast_naive2)
try:
error_ase_n2 = mase_m4(train, test, forecast_naive2)
error_sape_n2 = smape_m4(test, forecast_naive2)
except ValueError:
errors.append(0)
continue
try:
model.fit(train)
forecast = model.predict(npred)
except RRuntimeError:
errors.append(0)
continue
try:
error_ase = mase_m4(train, test, forecast)
error_sape = smape_m4(test, forecast)
OWA = 0.5 * (error_sape / error_sape_n2) + 0.5 * (error_ase /
error_ase_n2)
errors.append(np.sum(OWA))
except ValueError:
errors.append(0)
errors = np.sum(errors)
return errors
continue
Score = 1 / np.array(criterion)
pesos = Score / Score.sum()
groe_ensemble = 0
for prediction, weight in zip(model_predictions, pesos):
groe_ensemble = prediction * weight + groe_ensemble
if (groe_ensemble.univariate_values() < 0).any():
indices = test.time_index()[
groe_ensemble.univariate_values() < 0]
groe_ensemble = groe_ensemble.update(indices,
np.zeros(len(indices)))
mase_all.append(
np.vstack([
mase_m4(train, test, groe_ensemble, m=m),
]))
smape_all.append(np.vstack([
smape_m4(test, groe_ensemble),
]))
print("MASE GROE: {:.3f}".format(
np.nanmean(np.stack(mase_all), axis=(0, 2))))
print("sMAPE GROE: {:.3f}".format(
np.nanmean(np.stack(smape_all), axis=(0, 2))))
print("OWA GROE: {:.3f}".format(
owa_m4(freq, np.nanmean(np.stack(smape_all), axis=(0, 2)),
np.nanmean(np.stack(mase_all), axis=(0, 2)))))
pickle.dump(mase_all, open("groeR_mase_" + freq + ".pkl", "wb"))
pickle.dump(smape_all, open("groeR_smape_" + freq + ".pkl", "wb"))
groe_ensemble = prediction * weight + groe_ensemble
# BO3 ensembling
score = np.argsort(Score)[::-1][:3]
pesos2 = Score[score] / Score[score].sum()
bo3_ensemble = 0
bo3_mean = 0
for i, model in enumerate(score):
bo3_ensemble = model_predictions[model] * pesos2[i] + bo3_ensemble
bo3_mean = model_predictions[model] / len(score) + bo3_mean
# compute score
m = info_dataset.Frequency[freq[0] + '1']
mase_all.append(np.vstack([
mase_m4(train, test, models_des_predictions[0], m=m),
mase_m4(train, test, ensemble_pred, m=m),
mase_m4(train, test, mean_pred, m=m),
mase_m4(train, test, groe_ensemble, m=m),
mase_m4(train, test, bo3_ensemble, m=m),
mase_m4(train, test, bo3_mean, m=m),
]))
smape_all.append(np.vstack([
smape_m4(test, models_des_predictions[0]),
smape_m4(test, ensemble_pred),
smape_m4(test, mean_pred),
smape_m4(test, groe_ensemble),
smape_m4(test, bo3_ensemble),
smape_m4(test, bo3_mean),
]))
pkl.dump(mase_all, open("ensembling_mase_"+freq+".pkl", "wb"))
m = int(info_dataset.Frequency[cat[0] + "1"])
for train, test in _build_tqdm_iterator(zip(ts_train, ts_test),
verbose=True):
try:
try:
fft = FFT.gridsearch(
{
'nr_freqs_to_keep': [5, 7, 10, 15, 25, 50],
'trend': ['poly', 'exp'],
'trend_poly_degree': [0, 1, 2, 3],
'required_matches': [None]
},
train[:-2 * len(test)],
val_target_series=train[-2 * len(test):],
metric=lambda x, y: np.mean(
mase_m4(train[-2 * len(test)], x, y, m=m)))
except ValueError:
fft = FFT.gridsearch(
{
'nr_freqs_to_keep': [5, 7, 10, 15, 25, 50],
'trend': ['poly', 'exp'],
'trend_poly_degree': [0, 1, 2, 3],
'required_matches': [None]
},
train[:-len(test)],
val_target_series=train[-len(test):],
metric=lambda x, y: np.mean(
mase_m4(train[-len(test)], x, y, m=m)))
fft.fit(train)
forecast_fft = fft.predict(len(test))
mase_all.append(
derivate = np.diff(train.univariate_values(), n=1)
jump = derivate.max() / (train.max().max() - train.min().min())
try:
if jump <= 0.5:
prophet.fit(train)
else:
prophet.fit(
train.drop_before(
train.time_index[np.argmax(derivate) + 1]))
except ValueError as e:
raise e
forecast_prophet = prophet.predict(len(test))
m = info_dataset.Frequency[cat[0] + "1"]
mase_all.append(
np.vstack([
mase_m4(train, test, forecast_prophet, m=m),
]))
smape_all.append(
np.vstack([
smape_m4(test, forecast_prophet),
]))
except Exception as e:
print(e)
break
pkl.dump(mase_all, open("prophet_mase_" + cat + ".pkl", "wb"))
pkl.dump(smape_all, open("prophet_smape_" + cat + ".pkl", "wb"))
print("MASE; Prophet: {}".format(
*tuple(np.nanmean(np.stack(mase_all), axis=(0, 2)))))
print("sMAPE; Prophet: {}".format(
*tuple(np.nanmean(np.stack(smape_all), axis=(0, 2)))))
print(
_, season = extract_trend_and_seasonality(
train, m, model=SeasonalityMode.MULTIPLICATIVE)
train_des = remove_from_series(
train, season, model=SeasonalityMode.MULTIPLICATIVE)
seasonOut = season[-m:].shift(m)
seasonOut = seasonOut.append_values(seasonOut.values())
seasonOut = seasonOut[:len(test)]
forecast_theta = train_theta(train_des, seasonOut, len(test))
forecast_fourtheta = train_4theta(train, len(test))
forecast_thetaBC = train_theta_boxcox(train_des, seasonOut,
len(test))
mase_all.append(
np.vstack([
mase_m4(train, test, forecast_theta, m=m),
mase_m4(train, test, forecast_fourtheta, m=m),
mase_m4(train, test, forecast_thetaBC, m=m),
]))
smape_all.append(
np.vstack([
smape_m4(test, forecast_theta),
smape_m4(test, forecast_fourtheta),
smape_m4(test, forecast_thetaBC),
]))
pkl.dump(mase_all, open("theta_mase_" + cat + ".pkl", "wb"))
pkl.dump(smape_all, open("theta_smape_" + cat + ".pkl", "wb"))
print("MASE; Theta: {:.3f}, 4theta: {:.3f},"
" Theta-BoxCox: {:.3f}".format(
*tuple(np.nanmean(np.stack(smape_all), axis=(0, 2)))))
print("sMAPE; Theta: {:.3f}, 4theta: {:.3f},"