plot_acf(airpassengers_train, ax7)
ax3.set_title('ACF of seasonal series')
plot_pacf(airpassengers_train, ax8)
ax4.set_title('PACF of seasonal series')
plt.show()
# %%
sarimax = SARIMAX(airpassengers_train, order=(3,1,1), seasonal_order=(0,1,0,12)).fit()
sarimax.summary()
# %%
sarimax.plot_diagnostics(figsize=(16, 8))
plt.show()
# %%
sarimax_forecast = sarimax.get_forecast(24)
sarimax_forecast_conf_int = sarimax_forecast.conf_int()
# %%
plt.plot(airpassengers_train, label='train')
plt.plot(airpassengers_test, label='test')
plt.plot(sarimax_forecast.predicted_mean, label='forecast')
plt.fill_between(sarimax_forecast_conf_int.index,
s = 'GDPC1' # real gdp, seasonally adjusted
df = alf(s, log=1, diff=1, start=19591201, freq='Q').loc[:20191231].dropna()
df.index = pd.DatetimeIndex(df.index.astype(str), freq='infer')
df_train = df[df.index <= '2017-12-31']
df_test = df[df.index > '2017-12-31']
lags = ar_select_order(df_train, maxlag=13, ic='bic', old_names=False).ar_lags
print('(BIC) lags= ', len(lags), ':', lags)
# AR and SARIMAX
## AR(p) is simplest time-model, can nest in SARIMAX(p,d,q,s) with
## moving average MA(q), integration order I(d), seasonality S(s), exogenous X
from statsmodels.tsa.statespace.sarimax import SARIMAX
adf = alf(s, log=1, freq='Q').loc[19591201:20171231]
adf.index = pd.DatetimeIndex(adf.index.astype(str), freq='infer')
arima = SARIMAX(adf, order=(2, 1, 0), trend='c').fit()
fig = arima.plot_diagnostics(figsize=(10, 6))
plt.tight_layout(pad=2)
plt.savefig(os.path.join(imgdir, 'ar.jpg'))
plt.show()
arima.summary()
# Forecasting
## One-step ahead predictions
model = AutoReg(df_train, lags=lags, old_names=False).fit()
print(model.summary())
# Observations to predict are from the test split
from sklearn.metrics import mean_squared_error
all_dates = AutoReg(df, lags=lags, old_names=False)
df_pred = all_dates.predict(model.params,
start=df_train.index[-1]).shift(1).iloc[1:]
#___________________________
#ARIMA(1, 1, 1) SARIMAX(1, 0, 1, 52)
model = SARIMAX(series,
order=(1, 1, 1),
seasonal_order=(1, 1, 1, 52),
trend='n',
enforce_stationarity=False,
enforce_invertibility=False).fit()
print("________________________")
print("MODEL SUMMARY")
print(model.summary().tables[1])
# Nice way to check residuals follow a Gaussian distribution
model.plot_diagnostics(figsize=(15, 12))
plt.show()
train_pred = model.predict()
train_pred_cpy = train_pred.copy()
print(train_pred_cpy)
print(type(train_pred_cpy))
print(type(series))
cdf_index = a_organic[0:train_size].index
#print(cdf_index)
#print(type(cdf_index))
#________________________________________________
#Comparing the FIT with the trained data
#________________________________________________
parameters = product(p, q, P, Q)
parameters_list = list(parameters)
print(len(parameters_list))
## Uncomment to find the optimization parameters:
result_df = optimize_SARIMA(parameters_list, 1, 1, m, dft_f['cnt_smooth'])
print('\nSarima Optimization\n', result_df)
best_param = result_df['(p,q)x(P,Q)'][0]
(p_best, q_best, P_best, Q_best) = best_param
# best_model = SARIMAX(dft_f['cnt_smooth'], order=(1, 1, 1), seasonal_order=(0, 1, 1, m)).fit(dis=-1)
best_model = SARIMAX(dft_f['cnt_smooth'], order=(p_best, 1, q_best), seasonal_order=(P_best, 1, Q_best, m)).fit(dis=-1)
print(best_model.summary())
best_model.plot_diagnostics(figsize=(12,8))
plt.suptitle(f'Diagnostic Best model')
plt.savefig(f'{baseSave}/diagnostic_plot_station_{s}.png', dpi=150)
plt.clf()
'''
END: Compute Sarima optimization
'''
end_endto = args.forecast_upto.date() + timedelta(days=-1)
dft_all = pd.read_csv(f'{baseSave}/smoothed_to_compare_s_{s}.csv', header=[0], index_col=[0], sep=';', parse_dates=True)
dft_all_upto = dft_all.loc[selection[s]['stop']:end_endto.strftime('%Y-%m-%d')]
dft_f_from = dft_f.loc[selection[s]['start']:]
pred_uc = best_model.get_forecast(steps=pd.to_datetime(args.forecast_upto.date().strftime('%Y-%m-%d')))
pred_ci = pred_uc.conf_int()
ax = dft_f_from['cnt_smooth'].plot(label='Observed Smoothed Data', c='r', figsize=(12, 8))
dft_all_upto['cnt_smooth'].plot(label='Observed "Forecasted" Data', c='b', figsize=(12, 8))
try:
mod = sm.tsa.statespace.SARIMAX(y,
order=param,
seasonal_order=param_seasonal,
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
# st.write('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic))
except:
continue
sarimax = SARIMAX(y, order=(0,1,1), seasonal_order=(0,1,1,12)).fit()
st.write('Model Summary')
sarimax.summary()
st.pyplot(sarimax.plot_diagnostics(figsize=(20, 10)))
residuals =pd.Series(sarimax.resid)
def check_residuals(series):
fig = plt.figure(figsize=(20, 10))
gs = fig.add_gridspec(2,2)
ax1 = fig.add_subplot(gs[0, :])
ax1.plot(series)
ax1.set_title('residuals')
ax2 = fig.add_subplot(gs[1,0])
plot_acf(series, ax=ax2, title='ACF')
ax3 = fig.add_subplot(gs[1,1])
sns.kdeplot(series, ax=ax3)
ax3.set_title('density')
enforce_invertability=False)
results = sarima_model.fit()
print("ARIMA{}x{} - AIC:{:.2f}".format(param, param_seasonal, results.aic))
except:
continue
# plug in results with lowest AIC score
sarima_model = SARIMAX(y, order=(1,1,1), seasonal_order=(0,1,1,12))
sarima_model = sarima_model.fit(disp=False)
# summary table of SARIMA
print("SARIMA summary table:")
print(sarima_model.summary().tables[1])
# show plot diagnostics
sarima_model.plot_diagnostics(figsize=(15,12))
plt.show()
# Show predictions using one-step forecast
pred = sarima_model.get_prediction(start=pd.to_datetime('1998-01-01'), dynamic=False)
pred_ci = pred.conf_int()
ax = y['1990':].plot(label='observed')
pred.predicted_mean.plot(ax=ax, label='One step ahead forecast', alpha=0.7)
ax.fill_between(pred_ci.index,
pred_ci.iloc[:, 0],
pred_ci.iloc[:, 1], color='k', alpha=0.2)
ax.set_xlabel('Date')
ax.set_ylabel('CO2 Levels')
plt.legend()
plt.show()
def projectexample_modelling(series, model_name, parameters):
""" Function that performs the following plots
shape of the series
the first items
:params series: univariate time series
:type series: dataframe
:return:
- error (int): variable with error code
"""
# modelling
error = 0
try:
print("{} time series modelling".format('-' * 20))
print("{} {} model".format('-' * 20, model_name))
if model_name=='SARIMAX':
p = parameters[0]
d = parameters[1]
q = parameters[2]
P = parameters[3]
D = parameters[4]
Q = parameters[5]
S = parameters[6]
t = parameters[7]
print("{} fitting model".format('-' * 20))
# fit the model
model = SARIMAX(series.values,
trend = t,
order = (p, d, q),
seasonal_order = (P, D, Q, S),
enforce_stationarity = False,
enforce_invertibility = False).fit()
# Model summary
print("{} Model summary".format('-' * 20))
print(model.summary().tables[1])
# Model diagnostic
print("{} Model diagnostic".format('-' * 20))
fig = model.plot_diagnostics(figsize=(20, 12))
fig.savefig(os.path.join(os.getcwd(), 'figures\\diagnostic_{}.png'.format(model_name)))
fig.show()
except Exception as exception_msg:
print('{} (!) Error in projectexample_modelling: '.format('-' * 20) + str(exception_msg))
error = 1
model = []
return model, error
# Metrics
print("{} Metrics".format('-' * 20))
try:
# Regression metrics
y_fitted = model.predict()
R2 = round(r2_score(series, y_fitted), 3)
MAE = round(mean_absolute_error(series, y_fitted), 3)
RMSE = round(np.sqrt(mean_squared_error(series, y_fitted)), 3)
print("{} R2: {}".format('-' * 20, R2))
print("{} MAE: {}".format('-' * 20, MAE))
print("{} RMSE: {}".format('-' * 20, RMSE))
except Exception as exception_msg:
print('{} (!) Error in projectexample_modelling (metrics): '.format('-' * 20) + str(exception_msg))
error = 2
return model, error
return model, error
def mod_sarima(train,
test,
dependent_var_col,
trend,
p,
d,
q,
P,
D,
Q,
S,
is_log,
outpath,
name,
xreg,
plot_regressors,
mle_regression=True,
time_varying_regression=False,
periodicity='daily'):
"""
This function trains and tests the SARIMA model. for this two dataframes must be given, train and test.
trend, pdq and PDQS, are the statsmodels.SARIMAX variables.
:param train (Pandas Dataframe): train data
:param test (Pandas Dataframe): test data
:param ts_col (int): column of the objective variable
:param trend (str): Parameter controlling the deterministic trend polynomial A(t)
:param p (int): Autorregresive parameter
:param d (int): Differencing parameter
:param q (int): Differencing Moving Average parameter
:param P (int): Seasonal Autorregresive parameter
:param D (int): Seasonal Differencing parameter
:param Q (int): Seasonal Differencing Moving Average parameter
:param S (int): Lags for the seasonal
:param is_log (bool): true if the series is in logarithm. defaults to False.
:param outpath (str): path where the results will be stored
:param name (str): name to use when saving the files returned by the model
:xreg(list): list of strings with names of columns in the test/train datasets to be used as regressors
:plot_regressors: whether the regressors should be plotted in the function
:return: mae_error (float): Mean Absolute Error
rmse_error (float): root mean squared error
res_df (Pandas Dataframe): Dataframe with all data and the prediction in the Forecast column.
mod (statsmodel object): Model object.
"""
print(
'Modelling \n', name,
' Forecast - SARIMAX ' + '(' + str(p) + ',' + str(d) + ',' + str(q) +
')' + 'S' + '(' + str(P) + ',' + str(D) + ',' + str(Q) + ')' + str(S))
# path definition
if name not in os.listdir(outpath):
os.mkdir(outpath + name)
print('creating output folder in: \n', outpath + name)
report_output_path = str(outpath) + str(name) + '/'
# fit the model
if len(xreg) == 0:
mod = SARIMAX(train[dependent_var_col],
trend=trend,
order=(p, d, q),
seasonal_order=(P, D, Q, S),
time_varying_regression=time_varying_regression,
mle_regression=mle_regression).fit()
else:
mod = SARIMAX(train[dependent_var_col],
trend=trend,
order=(p, d, q),
seasonal_order=(P, D, Q, S),
exog=train[xreg],
enforce_stationarity=False,
time_varying_regression=time_varying_regression,
mle_regression=mle_regression).fit()
# plot diagnostics
plt.figure()
plt.title('Plot diagnostics for' + dependent_var_col +
' Forecast - SARIMA ' + '(' + str(p) + ',' + str(d) + ',' +
str(q) + ')' + 'S' + '(' + str(P) + ',' + str(D) + ',' + str(Q) +
')' + str(S))
mod.plot_diagnostics(figsize=(15, 9), lags=40)
plt.savefig(report_output_path + 'diagnostics_' + name + '.png')
# predict with the model
# I know this seems like a lot, but to be able to support broken time series in the forecast you need to reset the indexes
test_aux = test.copy(deep=True)
# TODO: remove this parameter
test_aux[xreg] = np.exp(test_aux[xreg])
test_aux[xreg] = test_aux[xreg] * 0.9
test_aux[xreg] = np.log(test_aux[xreg])
test_aux.reset_index(drop=True, inplace=True)
train_aux = train.copy(deep=True)
train_aux.reset_index(drop=True, inplace=True)
# get the predictions with the model
if len(xreg) == 0:
predictions = mod.predict(train_aux.index.max() + 1,
end=train_aux.index.max() + 1 +
test_aux.index.max())
conf_intervals = mod.get_prediction(
train_aux.index.max() + 1,
end=train_aux.index.max() + 1 +
test_aux.index.max()).conf_int(alpha=0.5)
else:
predictions = mod.predict(train_aux.index.max() + 1,
end=train_aux.index.max() + 1 +
test_aux.index.max(),
exog=test_aux[xreg])
conf_intervals = mod.get_prediction(
train_aux.index.max() + 1,
end=train_aux.index.max() + 1 + test_aux.index.max(),
exog=test_aux[xreg]).conf_int(alpha=0.5)
predictions.index = test.index
conf_intervals.index = test.index
# the confidence interval is trimmed for extreme values so they don't overextort after missing dates and doing the inverse log transf (exp)
conf_intervals = pd.DataFrame(conf_intervals)
# conf_intervals[(conf_intervals['lower log_revenue_emi'] < conf_intervals['lower log_revenue_emi'].quantile(q=0.01)) | (
# conf_intervals['upper log_revenue_emi'] > conf_intervals['upper log_revenue_emi'].quantile(q=0.99))] = np.nan
conf_intervals.index = conf_intervals.index.date
conf_intervals.index = conf_intervals.index.map(str)
# assign the predictions to the test dataframe to be used later in the plotting
test['Forecast'] = predictions
train['Forecast'] = mod.fittedvalues
# add the columns that are in the regressors to the dataframe that will be used and get a dataframe to plot (train aux)
columns = [dependent_var_col, 'Forecast']
columns.append(xreg)
columns = list(flatten(columns))
train_aux = train[columns]
test_aux = test[columns]
test_aux = pd.merge(test_aux,
conf_intervals,
left_index=True,
right_index=True)
# transform the data back from logarithm if the series is in that scale
if is_log is True:
res_df = pd.concat([train_aux, test_aux])
res_df['Forecast'] = np.exp(res_df['Forecast'])
res_df[dependent_var_col] = np.exp(res_df[dependent_var_col])
mae_error = mean_absolute_error(np.exp(test[dependent_var_col]),
np.exp(predictions))
rmse_error = np.sqrt(
mean_squared_error(np.exp(test[dependent_var_col]),
np.exp(predictions)))
mape = mean_absolute_percentage_error(np.exp(test[dependent_var_col]),
np.exp(predictions))
preds = np.exp(predictions)
else:
res_df = pd.concat([train_aux, test_aux])
mae_error = mean_absolute_error(test[dependent_var_col], predictions)
rmse_error = np.sqrt(
mean_squared_error(test[dependent_var_col], predictions))
mape = mean_absolute_percentage_error(test[dependent_var_col],
predictions)
preds = predictions
# Create a text box for the iteration results
textstr = 'MAE:' + str(round(mae_error, 0)) + '\n' + 'MAPE:' + str(
round(mape, 2))
aux_res_df = res_df.tail(365) # only plot the 6 months
aux_res_df.index = pd.to_datetime(aux_res_df.index)
if str(periodicity).upper() is 'daily':
aux_res_df = aux_res_df.reindex(pd.date_range(aux_res_df.index.min(),
aux_res_df.index.max()),
fill_value=np.nan)
# Upper and lower confidence intervals
lower = aux_res_df[str('lower ' + str(dependent_var_col))]
upper = aux_res_df[str('upper ' + str(dependent_var_col))]
if is_log is True:
lower = np.exp(lower)
upper = np.exp(upper)
# plot the figure with the prediction
fig, ax = plt.subplots(figsize=(15, 10))
plt.subplots_adjust(right=0.85, left=0.05, bottom=0.1)
ax2 = ax.twinx()
ax.plot(aux_res_df["Forecast"], color='darkred', label='Forecast')
ax.plot(aux_res_df[dependent_var_col], color='darkblue', label='Real')
if plot_regressors is True:
for i in xreg:
ax2.plot(aux_res_df[i], color='grey', alpha=0.4, label=str(i))
ax.plot(lower, color='darkgreen', label='Lower', alpha=0.5)
ax.plot(upper, color='darkgreen', label='Upper', alpha=0.5)
ax.fill_between(upper.dropna().index,
upper.dropna(),
lower.dropna(),
facecolor='darkgreen',
alpha=0.2,
interpolate=False)
ax.axvline(x=pd.to_datetime(test.index.min(), format='%Y-%m-%d'),
color='grey',
linestyle='--')
ax.xaxis.set_major_locator(mticker.MultipleLocator(30))
plt.gcf().autofmt_xdate()
# generate a text box
props = dict(boxstyle='round', facecolor='white')
# place a text box in upper left in axes coords
ax.text(0.05,
0.95,
textstr,
transform=ax.transAxes,
fontsize=14,
verticalalignment='top',
bbox=props)
ax.legend(title='Forecast Legend',
bbox_to_anchor=(1.05, 1),
loc='upper left')
ax2.legend(title='Regressors',
bbox_to_anchor=(1.05, 0.7),
loc='center left')
plt.savefig(report_output_path + 'Forecast_' + name + '_' + str(
datetime.strftime(pd.to_datetime(test.index.min()), format='%Y-%m-%d'))
+ '.png')
plt.title('SARIMAX Forecast of ' + name)
plt.show()
plt.close('all')
# plotting the results in plotly
fig = go.Figure()
fig.add_trace(
go.Scatter(x=res_df.index,
y=res_df[dependent_var_col],
mode='lines',
name='Real'))
fig.add_trace(
go.Scatter(x=res_df.index,
y=res_df['Forecast'],
mode='lines+markers',
name='Fitted - Forecasted'))
fig.add_shape(
dict(type="line",
x0=test.index.min(),
y0=res_df[dependent_var_col].min(),
x1=test.index.min(),
y1=res_df[dependent_var_col].max(),
line=dict(color="grey", width=1)))
fig.update_xaxes(rangeslider_visible=True)
fig.update_layout(title=dependent_var_col + ' Forecast - SARIMA ' + '(' +
str(p) + ',' + str(d) + ',' + str(q) + ')' + 'S' + '(' +
str(P) + ',' + str(D) + ',' + str(Q) + ')' + str(S),
xaxis_title=dependent_var_col,
yaxis_title='Date',
font=dict(family="Century gothic",
size=18,
color="darkgrey"))
fig.write_html(report_output_path + name + '_forecast_SARIMA.html')
plt.close('all')
print('MAE', mae_error)
print('RMSE', rmse_error)
print('MAPE', mape)
print(mod.summary())
return mae_error, rmse_error, mape, name, preds, conf_intervals
