def groe_owa(ts: TimeSeries, model: ForecastingModel, n1: int, m: int, p: int, fq: int): """ Backtesting. Compute the OWA score iteratively on ´p´ timepoints following an expanding window mode. Parameters --------- ts TimeSeries on which backtesting will be done. model model to backtest. n1 minimum number of datapoints to take during training m step size p number of steps to iterate over fq Frequency of the time series. Returns ------- Sum of all OWA errors """ n = len(ts) errors = [] for i in range(p): if n1 + i * m == n: break ni = n1 + i * m npred = n - ni train = ts[:ni] if npred >= 3: test = ts[ni:] else: test = TimeSeries(ts.pd_series()[ni:], freq=ts.freq_str()) forecast_naive2 = naive2_groe(train, npred, fq) error_sape_n2 = mase_m4(train, test, forecast_naive2) error_ase_n2 = smape_m4(test, forecast_naive2) model.fit(train) forecast = model.predict(npred) try: error_sape = mase_m4(train, test, forecast) error_ase = smape_m4(test, forecast) owa = 0.5 * (error_sape / error_sape_n2) + 0.5 * (error_ase / error_ase_n2) errors.append(np.sum(owa)) except (ZeroDivisionError, ValueError): errors.append(0) errors = np.sum(errors) return errors
def fallback(mase_all, smape_all, train, test, m): naive = NaiveSeasonal(K=m) naive.fit(train) forecast = naive.predict(len(test)) mase_all.append(np.vstack([ mase_m4(train, test, forecast, m=m), ])) smape_all.append(np.vstack([ smape_m4(test, forecast), ]))
forecast_comb = ((forecast_ses + forecast_holt + forecast_damp) / 3) mase_all.append( np.vstack([ mase_m4(train, test, forecast_naiveSeason, m=m), mase_m4(train, test, forecast_naive, m=m), mase_m4(train, test, forecast_naive2, m=m), mase_m4(train, test, forecast_ses, m=m), mase_m4(train, test, forecast_holt, m=m), mase_m4(train, test, forecast_damp, m=m), mase_m4(train, test, forecast_comb, m=m), ])) smape_all.append( np.vstack([ smape_m4(test, forecast_naiveSeason), smape_m4(test, forecast_naive), smape_m4(test, forecast_naive2), smape_m4(test, forecast_ses), smape_m4(test, forecast_holt), smape_m4(test, forecast_damp), smape_m4(test, forecast_comb), ])) pkl.dump(mase_all, open("baseline_mase_" + cat + ".pkl", "wb")) pkl.dump(smape_all, open("baseline_smape_" + cat + ".pkl", "wb")) print( "MASE; sNaive: {:.3f}, sNaive+Drift: {:.3f}, Naive2: {:.3f}, SES: {:.3f}, Holt: {:.3f}, " "Damped: {:.3f}, Comb: {:.3f}".format( *tuple(np.nanmean(np.stack(mase_all), axis=(0, 2))))) print( "sMAPE; sNaive: {:.3f}, sNaive+Drift: {:.3f}, Naive2: {:.3f}, SES: {:.3f}, Holt: {:.3f}, "
train_des = TimeSeries.from_times_and_values( train.time_index(), r_train_des) seasonOut = TimeSeries.from_times_and_values( test.time_index(), r_seasonOut) naive2 = NaiveSeasonal(K=1) naive2.fit(train_des) forecast_naive2 = naive2.predict(len(test)) * seasonOut mase_all.append( np.vstack([ mase_m4(train, test, forecast_naive2, m=m), ])) smape_all.append(np.vstack([ smape_m4(test, forecast_naive2), ])) rel_error = np.mean(np.hstack(season_diff)) pkl.dump([mase_all, smape_all, season_diff], open("rnaive2_" + cat + ".pkl", "wb")) print( np.round(np.nanmean(np.stack(mase_all), axis=( 0, 2)), 3) == baseline[cat][1]) print( np.round(np.nanmean(np.stack(smape_all), axis=( 0, 2)), 3) == baseline[cat][0]) print("MASE; Naive2: {:.3f}".format( *tuple(np.nanmean(np.stack(mase_all), axis=(0, 2))))) print("sMAPE; Naive2: {:.3f}".format( *tuple(np.nanmean(np.stack(smape_all), axis=(0, 2))))) print(
def groe_owa(ts: TimeSeries, model: ForecastingModel, fq: int, n1: int, m: int, p: int) -> float: """ Implementation of Generalized Rolling Origin Evaluation using OWA score. The concept is to cross-validate a model on a time series with rolling origin, using OWA score from M4 competition. Parameters ----------- ts The time series object to use to cross-validate model The Darts model to evaluate fq Period of the seasonality of the time series n1 First origin to use for the cross-validation m Stride used for rolling the origin p number of stride to operate Returns ------- Float sum of OWA score for all different origins If there is an error with one of the origin, return 0. """ # todo: Implement generalized version from R n = len(ts) errors = [] for i in range(p): # if origin is further than end timestamp, end function if n1 + i * m >= n: break ni = n1 + i * m npred = n - ni train = ts[:ni] test = ts[ni:] forecast_naive2 = rNaive2(train.values(), npred, fq) forecast_naive2 = TimeSeries.from_times_and_values( test.time_index(), forecast_naive2) try: error_ase_n2 = mase_m4(train, test, forecast_naive2) error_sape_n2 = smape_m4(test, forecast_naive2) except ValueError: errors.append(0) continue try: model.fit(train) forecast = model.predict(npred) except RRuntimeError: errors.append(0) continue try: error_ase = mase_m4(train, test, forecast) error_sape = smape_m4(test, forecast) OWA = 0.5 * (error_sape / error_sape_n2) + 0.5 * (error_ase / error_ase_n2) errors.append(np.sum(OWA)) except ValueError: errors.append(0) errors = np.sum(errors) return errors
continue Score = 1 / np.array(criterion) pesos = Score / Score.sum() groe_ensemble = 0 for prediction, weight in zip(model_predictions, pesos): groe_ensemble = prediction * weight + groe_ensemble if (groe_ensemble.univariate_values() < 0).any(): indices = test.time_index()[ groe_ensemble.univariate_values() < 0] groe_ensemble = groe_ensemble.update(indices, np.zeros(len(indices))) mase_all.append( np.vstack([ mase_m4(train, test, groe_ensemble, m=m), ])) smape_all.append(np.vstack([ smape_m4(test, groe_ensemble), ])) print("MASE GROE: {:.3f}".format( np.nanmean(np.stack(mase_all), axis=(0, 2)))) print("sMAPE GROE: {:.3f}".format( np.nanmean(np.stack(smape_all), axis=(0, 2)))) print("OWA GROE: {:.3f}".format( owa_m4(freq, np.nanmean(np.stack(smape_all), axis=(0, 2)), np.nanmean(np.stack(mase_all), axis=(0, 2))))) pickle.dump(mase_all, open("groeR_mase_" + freq + ".pkl", "wb")) pickle.dump(smape_all, open("groeR_smape_" + freq + ".pkl", "wb"))
for i, model in enumerate(score): bo3_ensemble = model_predictions[model] * pesos2[i] + bo3_ensemble bo3_mean = model_predictions[model] / len(score) + bo3_mean # compute score m = info_dataset.Frequency[freq[0] + '1'] mase_all.append(np.vstack([ mase_m4(train, test, models_des_predictions[0], m=m), mase_m4(train, test, ensemble_pred, m=m), mase_m4(train, test, mean_pred, m=m), mase_m4(train, test, groe_ensemble, m=m), mase_m4(train, test, bo3_ensemble, m=m), mase_m4(train, test, bo3_mean, m=m), ])) smape_all.append(np.vstack([ smape_m4(test, models_des_predictions[0]), smape_m4(test, ensemble_pred), smape_m4(test, mean_pred), smape_m4(test, groe_ensemble), smape_m4(test, bo3_ensemble), smape_m4(test, bo3_mean), ])) pkl.dump(mase_all, open("ensembling_mase_"+freq+".pkl", "wb")) pkl.dump(smape_all, open("ensembling_smape_"+freq+".pkl", "wb")) print("MASE; Naive2: {:.3f}, Linear Regression: {:.3f}, Mean ensembling: {:.3f}, GROE ensembling: {:.3f}, " "BO3 ensembling: {:.3f}, BO3 Mean: {:.3f}".format(*tuple(np.nanmean(np.stack(mase_all), axis=(0, 2))))) print("sMAPE; Naive2: {:.3f}, Linear Regression: {:.3f}, Mean ensembling: {:.3f}, GROE ensembling: {:.3f}, " "BO3 ensembling: {:.3f}, BO3 Mean: {:.3f}".format(*tuple(np.nanmean(np.stack(smape_all), axis=(0, 2))))) print("OWA: ", owa_m4(freq, np.nanmean(np.stack(smape_all), axis=(0, 2)), np.nanmean(np.stack(mase_all), axis=(0, 2))))
'trend': ['poly', 'exp'], 'trend_poly_degree': [0, 1, 2, 3], 'required_matches': [None] }, train[:-len(test)], val_target_series=train[-len(test):], metric=lambda x, y: np.mean( mase_m4(train[-len(test)], x, y, m=m))) fft.fit(train) forecast_fft = fft.predict(len(test)) mase_all.append( np.vstack([ mase_m4(train, test, forecast_fft, m=m), ])) smape_all.append(np.vstack([ smape_m4(test, forecast_fft), ])) except Exception as e: print(e) pkl.dump(mase_all, open("FFT_mase_" + cat + ".pkl", "wb")) pkl.dump(smape_all, open("FFT_smape_" + cat + ".pkl", "wb")) break pkl.dump(mase_all, open("FFT_mase_" + cat + "2.pkl", "wb")) pkl.dump(smape_all, open("FFT_smape_" + cat + "2.pkl", "wb")) print("MASE; fft: {}".format( *tuple(np.nanmean(np.stack(mase_all), axis=(0, 2))))) print("sMAPE; fft: {}".format( *tuple(np.nanmean(np.stack(smape_all), axis=(0, 2))))) print( "OWA: ", owa_m4(cat, np.nanmean(np.stack(smape_all), axis=(0, 2)),
prophet.fit(train) else: prophet.fit( train.drop_before( train.time_index[np.argmax(derivate) + 1])) except ValueError as e: raise e forecast_prophet = prophet.predict(len(test)) m = info_dataset.Frequency[cat[0] + "1"] mase_all.append( np.vstack([ mase_m4(train, test, forecast_prophet, m=m), ])) smape_all.append( np.vstack([ smape_m4(test, forecast_prophet), ])) except Exception as e: print(e) break pkl.dump(mase_all, open("prophet_mase_" + cat + ".pkl", "wb")) pkl.dump(smape_all, open("prophet_smape_" + cat + ".pkl", "wb")) print("MASE; Prophet: {}".format( *tuple(np.nanmean(np.stack(mase_all), axis=(0, 2))))) print("sMAPE; Prophet: {}".format( *tuple(np.nanmean(np.stack(smape_all), axis=(0, 2))))) print( "OWA: ", owa_m4(cat, np.nanmean(np.stack(smape_all), axis=(0, 2)), np.nanmean(np.stack(mase_all), axis=(0, 2))))
seasonOut = seasonOut[:len(test)] forecast_theta = train_theta(train_des, seasonOut, len(test)) forecast_fourtheta = train_4theta(train, len(test)) forecast_thetaBC = train_theta_boxcox(train_des, seasonOut, len(test)) mase_all.append( np.vstack([ mase_m4(train, test, forecast_theta, m=m), mase_m4(train, test, forecast_fourtheta, m=m), mase_m4(train, test, forecast_thetaBC, m=m), ])) smape_all.append( np.vstack([ smape_m4(test, forecast_theta), smape_m4(test, forecast_fourtheta), smape_m4(test, forecast_thetaBC), ])) pkl.dump(mase_all, open("theta_mase_" + cat + ".pkl", "wb")) pkl.dump(smape_all, open("theta_smape_" + cat + ".pkl", "wb")) print("MASE; Theta: {:.3f}, 4theta: {:.3f}," " Theta-BoxCox: {:.3f}".format( *tuple(np.nanmean(np.stack(smape_all), axis=(0, 2))))) print("sMAPE; Theta: {:.3f}, 4theta: {:.3f}," " Theta-BoxCox: {:.3f}".format( *tuple(np.nanmean(np.stack(mase_all), axis=(0, 2))))) print( "OWA: ", owa_m4(cat, np.nanmean(np.stack(mase_all), axis=(0, 2)), np.nanmean(np.stack(smape_all), axis=(0, 2))))