plot_acf(airpassengers_train, ax7) ax3.set_title('ACF of seasonal series') plot_pacf(airpassengers_train, ax8) ax4.set_title('PACF of seasonal series') plt.show() # %% sarimax = SARIMAX(airpassengers_train, order=(3,1,1), seasonal_order=(0,1,0,12)).fit() sarimax.summary() # %% sarimax.plot_diagnostics(figsize=(16, 8)) plt.show() # %% sarimax_forecast = sarimax.get_forecast(24) sarimax_forecast_conf_int = sarimax_forecast.conf_int() # %% plt.plot(airpassengers_train, label='train') plt.plot(airpassengers_test, label='test') plt.plot(sarimax_forecast.predicted_mean, label='forecast') plt.fill_between(sarimax_forecast_conf_int.index,
s = 'GDPC1' # real gdp, seasonally adjusted df = alf(s, log=1, diff=1, start=19591201, freq='Q').loc[:20191231].dropna() df.index = pd.DatetimeIndex(df.index.astype(str), freq='infer') df_train = df[df.index <= '2017-12-31'] df_test = df[df.index > '2017-12-31'] lags = ar_select_order(df_train, maxlag=13, ic='bic', old_names=False).ar_lags print('(BIC) lags= ', len(lags), ':', lags) # AR and SARIMAX ## AR(p) is simplest time-model, can nest in SARIMAX(p,d,q,s) with ## moving average MA(q), integration order I(d), seasonality S(s), exogenous X from statsmodels.tsa.statespace.sarimax import SARIMAX adf = alf(s, log=1, freq='Q').loc[19591201:20171231] adf.index = pd.DatetimeIndex(adf.index.astype(str), freq='infer') arima = SARIMAX(adf, order=(2, 1, 0), trend='c').fit() fig = arima.plot_diagnostics(figsize=(10, 6)) plt.tight_layout(pad=2) plt.savefig(os.path.join(imgdir, 'ar.jpg')) plt.show() arima.summary() # Forecasting ## One-step ahead predictions model = AutoReg(df_train, lags=lags, old_names=False).fit() print(model.summary()) # Observations to predict are from the test split from sklearn.metrics import mean_squared_error all_dates = AutoReg(df, lags=lags, old_names=False) df_pred = all_dates.predict(model.params, start=df_train.index[-1]).shift(1).iloc[1:]
#___________________________ #ARIMA(1, 1, 1) SARIMAX(1, 0, 1, 52) model = SARIMAX(series, order=(1, 1, 1), seasonal_order=(1, 1, 1, 52), trend='n', enforce_stationarity=False, enforce_invertibility=False).fit() print("________________________") print("MODEL SUMMARY") print(model.summary().tables[1]) # Nice way to check residuals follow a Gaussian distribution model.plot_diagnostics(figsize=(15, 12)) plt.show() train_pred = model.predict() train_pred_cpy = train_pred.copy() print(train_pred_cpy) print(type(train_pred_cpy)) print(type(series)) cdf_index = a_organic[0:train_size].index #print(cdf_index) #print(type(cdf_index)) #________________________________________________ #Comparing the FIT with the trained data #________________________________________________
parameters = product(p, q, P, Q) parameters_list = list(parameters) print(len(parameters_list)) ## Uncomment to find the optimization parameters: result_df = optimize_SARIMA(parameters_list, 1, 1, m, dft_f['cnt_smooth']) print('\nSarima Optimization\n', result_df) best_param = result_df['(p,q)x(P,Q)'][0] (p_best, q_best, P_best, Q_best) = best_param # best_model = SARIMAX(dft_f['cnt_smooth'], order=(1, 1, 1), seasonal_order=(0, 1, 1, m)).fit(dis=-1) best_model = SARIMAX(dft_f['cnt_smooth'], order=(p_best, 1, q_best), seasonal_order=(P_best, 1, Q_best, m)).fit(dis=-1) print(best_model.summary()) best_model.plot_diagnostics(figsize=(12,8)) plt.suptitle(f'Diagnostic Best model') plt.savefig(f'{baseSave}/diagnostic_plot_station_{s}.png', dpi=150) plt.clf() ''' END: Compute Sarima optimization ''' end_endto = args.forecast_upto.date() + timedelta(days=-1) dft_all = pd.read_csv(f'{baseSave}/smoothed_to_compare_s_{s}.csv', header=[0], index_col=[0], sep=';', parse_dates=True) dft_all_upto = dft_all.loc[selection[s]['stop']:end_endto.strftime('%Y-%m-%d')] dft_f_from = dft_f.loc[selection[s]['start']:] pred_uc = best_model.get_forecast(steps=pd.to_datetime(args.forecast_upto.date().strftime('%Y-%m-%d'))) pred_ci = pred_uc.conf_int() ax = dft_f_from['cnt_smooth'].plot(label='Observed Smoothed Data', c='r', figsize=(12, 8)) dft_all_upto['cnt_smooth'].plot(label='Observed "Forecasted" Data', c='b', figsize=(12, 8))
try: mod = sm.tsa.statespace.SARIMAX(y, order=param, seasonal_order=param_seasonal, enforce_stationarity=False, enforce_invertibility=False) results = mod.fit() # st.write('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic)) except: continue sarimax = SARIMAX(y, order=(0,1,1), seasonal_order=(0,1,1,12)).fit() st.write('Model Summary') sarimax.summary() st.pyplot(sarimax.plot_diagnostics(figsize=(20, 10))) residuals =pd.Series(sarimax.resid) def check_residuals(series): fig = plt.figure(figsize=(20, 10)) gs = fig.add_gridspec(2,2) ax1 = fig.add_subplot(gs[0, :]) ax1.plot(series) ax1.set_title('residuals') ax2 = fig.add_subplot(gs[1,0]) plot_acf(series, ax=ax2, title='ACF') ax3 = fig.add_subplot(gs[1,1]) sns.kdeplot(series, ax=ax3) ax3.set_title('density')
enforce_invertability=False) results = sarima_model.fit() print("ARIMA{}x{} - AIC:{:.2f}".format(param, param_seasonal, results.aic)) except: continue # plug in results with lowest AIC score sarima_model = SARIMAX(y, order=(1,1,1), seasonal_order=(0,1,1,12)) sarima_model = sarima_model.fit(disp=False) # summary table of SARIMA print("SARIMA summary table:") print(sarima_model.summary().tables[1]) # show plot diagnostics sarima_model.plot_diagnostics(figsize=(15,12)) plt.show() # Show predictions using one-step forecast pred = sarima_model.get_prediction(start=pd.to_datetime('1998-01-01'), dynamic=False) pred_ci = pred.conf_int() ax = y['1990':].plot(label='observed') pred.predicted_mean.plot(ax=ax, label='One step ahead forecast', alpha=0.7) ax.fill_between(pred_ci.index, pred_ci.iloc[:, 0], pred_ci.iloc[:, 1], color='k', alpha=0.2) ax.set_xlabel('Date') ax.set_ylabel('CO2 Levels') plt.legend() plt.show()
def projectexample_modelling(series, model_name, parameters): """ Function that performs the following plots shape of the series the first items :params series: univariate time series :type series: dataframe :return: - error (int): variable with error code """ # modelling error = 0 try: print("{} time series modelling".format('-' * 20)) print("{} {} model".format('-' * 20, model_name)) if model_name=='SARIMAX': p = parameters[0] d = parameters[1] q = parameters[2] P = parameters[3] D = parameters[4] Q = parameters[5] S = parameters[6] t = parameters[7] print("{} fitting model".format('-' * 20)) # fit the model model = SARIMAX(series.values, trend = t, order = (p, d, q), seasonal_order = (P, D, Q, S), enforce_stationarity = False, enforce_invertibility = False).fit() # Model summary print("{} Model summary".format('-' * 20)) print(model.summary().tables[1]) # Model diagnostic print("{} Model diagnostic".format('-' * 20)) fig = model.plot_diagnostics(figsize=(20, 12)) fig.savefig(os.path.join(os.getcwd(), 'figures\\diagnostic_{}.png'.format(model_name))) fig.show() except Exception as exception_msg: print('{} (!) Error in projectexample_modelling: '.format('-' * 20) + str(exception_msg)) error = 1 model = [] return model, error # Metrics print("{} Metrics".format('-' * 20)) try: # Regression metrics y_fitted = model.predict() R2 = round(r2_score(series, y_fitted), 3) MAE = round(mean_absolute_error(series, y_fitted), 3) RMSE = round(np.sqrt(mean_squared_error(series, y_fitted)), 3) print("{} R2: {}".format('-' * 20, R2)) print("{} MAE: {}".format('-' * 20, MAE)) print("{} RMSE: {}".format('-' * 20, RMSE)) except Exception as exception_msg: print('{} (!) Error in projectexample_modelling (metrics): '.format('-' * 20) + str(exception_msg)) error = 2 return model, error return model, error
def mod_sarima(train, test, dependent_var_col, trend, p, d, q, P, D, Q, S, is_log, outpath, name, xreg, plot_regressors, mle_regression=True, time_varying_regression=False, periodicity='daily'): """ This function trains and tests the SARIMA model. for this two dataframes must be given, train and test. trend, pdq and PDQS, are the statsmodels.SARIMAX variables. :param train (Pandas Dataframe): train data :param test (Pandas Dataframe): test data :param ts_col (int): column of the objective variable :param trend (str): Parameter controlling the deterministic trend polynomial A(t) :param p (int): Autorregresive parameter :param d (int): Differencing parameter :param q (int): Differencing Moving Average parameter :param P (int): Seasonal Autorregresive parameter :param D (int): Seasonal Differencing parameter :param Q (int): Seasonal Differencing Moving Average parameter :param S (int): Lags for the seasonal :param is_log (bool): true if the series is in logarithm. defaults to False. :param outpath (str): path where the results will be stored :param name (str): name to use when saving the files returned by the model :xreg(list): list of strings with names of columns in the test/train datasets to be used as regressors :plot_regressors: whether the regressors should be plotted in the function :return: mae_error (float): Mean Absolute Error rmse_error (float): root mean squared error res_df (Pandas Dataframe): Dataframe with all data and the prediction in the Forecast column. mod (statsmodel object): Model object. """ print( 'Modelling \n', name, ' Forecast - SARIMAX ' + '(' + str(p) + ',' + str(d) + ',' + str(q) + ')' + 'S' + '(' + str(P) + ',' + str(D) + ',' + str(Q) + ')' + str(S)) # path definition if name not in os.listdir(outpath): os.mkdir(outpath + name) print('creating output folder in: \n', outpath + name) report_output_path = str(outpath) + str(name) + '/' # fit the model if len(xreg) == 0: mod = SARIMAX(train[dependent_var_col], trend=trend, order=(p, d, q), seasonal_order=(P, D, Q, S), time_varying_regression=time_varying_regression, mle_regression=mle_regression).fit() else: mod = SARIMAX(train[dependent_var_col], trend=trend, order=(p, d, q), seasonal_order=(P, D, Q, S), exog=train[xreg], enforce_stationarity=False, time_varying_regression=time_varying_regression, mle_regression=mle_regression).fit() # plot diagnostics plt.figure() plt.title('Plot diagnostics for' + dependent_var_col + ' Forecast - SARIMA ' + '(' + str(p) + ',' + str(d) + ',' + str(q) + ')' + 'S' + '(' + str(P) + ',' + str(D) + ',' + str(Q) + ')' + str(S)) mod.plot_diagnostics(figsize=(15, 9), lags=40) plt.savefig(report_output_path + 'diagnostics_' + name + '.png') # predict with the model # I know this seems like a lot, but to be able to support broken time series in the forecast you need to reset the indexes test_aux = test.copy(deep=True) # TODO: remove this parameter test_aux[xreg] = np.exp(test_aux[xreg]) test_aux[xreg] = test_aux[xreg] * 0.9 test_aux[xreg] = np.log(test_aux[xreg]) test_aux.reset_index(drop=True, inplace=True) train_aux = train.copy(deep=True) train_aux.reset_index(drop=True, inplace=True) # get the predictions with the model if len(xreg) == 0: predictions = mod.predict(train_aux.index.max() + 1, end=train_aux.index.max() + 1 + test_aux.index.max()) conf_intervals = mod.get_prediction( train_aux.index.max() + 1, end=train_aux.index.max() + 1 + test_aux.index.max()).conf_int(alpha=0.5) else: predictions = mod.predict(train_aux.index.max() + 1, end=train_aux.index.max() + 1 + test_aux.index.max(), exog=test_aux[xreg]) conf_intervals = mod.get_prediction( train_aux.index.max() + 1, end=train_aux.index.max() + 1 + test_aux.index.max(), exog=test_aux[xreg]).conf_int(alpha=0.5) predictions.index = test.index conf_intervals.index = test.index # the confidence interval is trimmed for extreme values so they don't overextort after missing dates and doing the inverse log transf (exp) conf_intervals = pd.DataFrame(conf_intervals) # conf_intervals[(conf_intervals['lower log_revenue_emi'] < conf_intervals['lower log_revenue_emi'].quantile(q=0.01)) | ( # conf_intervals['upper log_revenue_emi'] > conf_intervals['upper log_revenue_emi'].quantile(q=0.99))] = np.nan conf_intervals.index = conf_intervals.index.date conf_intervals.index = conf_intervals.index.map(str) # assign the predictions to the test dataframe to be used later in the plotting test['Forecast'] = predictions train['Forecast'] = mod.fittedvalues # add the columns that are in the regressors to the dataframe that will be used and get a dataframe to plot (train aux) columns = [dependent_var_col, 'Forecast'] columns.append(xreg) columns = list(flatten(columns)) train_aux = train[columns] test_aux = test[columns] test_aux = pd.merge(test_aux, conf_intervals, left_index=True, right_index=True) # transform the data back from logarithm if the series is in that scale if is_log is True: res_df = pd.concat([train_aux, test_aux]) res_df['Forecast'] = np.exp(res_df['Forecast']) res_df[dependent_var_col] = np.exp(res_df[dependent_var_col]) mae_error = mean_absolute_error(np.exp(test[dependent_var_col]), np.exp(predictions)) rmse_error = np.sqrt( mean_squared_error(np.exp(test[dependent_var_col]), np.exp(predictions))) mape = mean_absolute_percentage_error(np.exp(test[dependent_var_col]), np.exp(predictions)) preds = np.exp(predictions) else: res_df = pd.concat([train_aux, test_aux]) mae_error = mean_absolute_error(test[dependent_var_col], predictions) rmse_error = np.sqrt( mean_squared_error(test[dependent_var_col], predictions)) mape = mean_absolute_percentage_error(test[dependent_var_col], predictions) preds = predictions # Create a text box for the iteration results textstr = 'MAE:' + str(round(mae_error, 0)) + '\n' + 'MAPE:' + str( round(mape, 2)) aux_res_df = res_df.tail(365) # only plot the 6 months aux_res_df.index = pd.to_datetime(aux_res_df.index) if str(periodicity).upper() is 'daily': aux_res_df = aux_res_df.reindex(pd.date_range(aux_res_df.index.min(), aux_res_df.index.max()), fill_value=np.nan) # Upper and lower confidence intervals lower = aux_res_df[str('lower ' + str(dependent_var_col))] upper = aux_res_df[str('upper ' + str(dependent_var_col))] if is_log is True: lower = np.exp(lower) upper = np.exp(upper) # plot the figure with the prediction fig, ax = plt.subplots(figsize=(15, 10)) plt.subplots_adjust(right=0.85, left=0.05, bottom=0.1) ax2 = ax.twinx() ax.plot(aux_res_df["Forecast"], color='darkred', label='Forecast') ax.plot(aux_res_df[dependent_var_col], color='darkblue', label='Real') if plot_regressors is True: for i in xreg: ax2.plot(aux_res_df[i], color='grey', alpha=0.4, label=str(i)) ax.plot(lower, color='darkgreen', label='Lower', alpha=0.5) ax.plot(upper, color='darkgreen', label='Upper', alpha=0.5) ax.fill_between(upper.dropna().index, upper.dropna(), lower.dropna(), facecolor='darkgreen', alpha=0.2, interpolate=False) ax.axvline(x=pd.to_datetime(test.index.min(), format='%Y-%m-%d'), color='grey', linestyle='--') ax.xaxis.set_major_locator(mticker.MultipleLocator(30)) plt.gcf().autofmt_xdate() # generate a text box props = dict(boxstyle='round', facecolor='white') # place a text box in upper left in axes coords ax.text(0.05, 0.95, textstr, transform=ax.transAxes, fontsize=14, verticalalignment='top', bbox=props) ax.legend(title='Forecast Legend', bbox_to_anchor=(1.05, 1), loc='upper left') ax2.legend(title='Regressors', bbox_to_anchor=(1.05, 0.7), loc='center left') plt.savefig(report_output_path + 'Forecast_' + name + '_' + str( datetime.strftime(pd.to_datetime(test.index.min()), format='%Y-%m-%d')) + '.png') plt.title('SARIMAX Forecast of ' + name) plt.show() plt.close('all') # plotting the results in plotly fig = go.Figure() fig.add_trace( go.Scatter(x=res_df.index, y=res_df[dependent_var_col], mode='lines', name='Real')) fig.add_trace( go.Scatter(x=res_df.index, y=res_df['Forecast'], mode='lines+markers', name='Fitted - Forecasted')) fig.add_shape( dict(type="line", x0=test.index.min(), y0=res_df[dependent_var_col].min(), x1=test.index.min(), y1=res_df[dependent_var_col].max(), line=dict(color="grey", width=1))) fig.update_xaxes(rangeslider_visible=True) fig.update_layout(title=dependent_var_col + ' Forecast - SARIMA ' + '(' + str(p) + ',' + str(d) + ',' + str(q) + ')' + 'S' + '(' + str(P) + ',' + str(D) + ',' + str(Q) + ')' + str(S), xaxis_title=dependent_var_col, yaxis_title='Date', font=dict(family="Century gothic", size=18, color="darkgrey")) fig.write_html(report_output_path + name + '_forecast_SARIMA.html') plt.close('all') print('MAE', mae_error) print('RMSE', rmse_error) print('MAPE', mape) print(mod.summary()) return mae_error, rmse_error, mape, name, preds, conf_intervals