def test_autoreg_append_deterministic(append_data):
y = append_data.y
y_oos = append_data.y_oos
y_both = append_data.y_both
x = append_data.x
x_oos = append_data.x_oos
x_both = append_data.x_both
terms = [TimeTrend(constant=True, order=1), Seasonality(12)]
dp = DeterministicProcess(y.index, additional_terms=terms)
res = AutoReg(y, lags=3, trend="n", deterministic=dp).fit()
res_append = res.append(y_oos, refit=True)
res_direct = AutoReg(y_both,
lags=3,
trend="n",
deterministic=dp.apply(y_both.index)).fit()
assert_allclose(res_append.params, res_direct.params)
res_np = AutoReg(np.asarray(y), lags=3, trend="n", deterministic=dp).fit()
res_append_np = res_np.append(np.asarray(y_oos))
assert_allclose(res_np.params, res_append_np.params)
res = AutoReg(y, exog=x, lags=3, trend="n", deterministic=dp).fit()
res_append = res.append(y_oos, exog=x_oos, refit=True)
res_direct = AutoReg(
y_both,
exog=x_both,
lags=3,
trend="n",
deterministic=dp.apply(y_both.index),
).fit()
assert_allclose(res_append.params, res_direct.params)
def test_range_error():
idx = pd.Index([0, 1, 1, 2, 3, 5, 8, 13])
dp = DeterministicProcess(
idx, constant=True, order=2, seasonal=True, period=2
)
with pytest.raises(TypeError, match="The index in the deterministic"):
dp.range(0, 12)
def check(self, dp, X):
from statsmodels.tsa.deterministic import (CalendarFourier,
DeterministicProcess)
y = load_average_sales()['2017']
fourier = CalendarFourier(freq='M', order=4)
dp = DeterministicProcess(
index=y.index,
constant=True,
order=1,
seasonal=True,
additional_terms=[fourier],
drop=True,
)
X_true = dp.in_sample()
import pandas as pd
assert all(
dp._index == y.index
), f"`index` argument to `DeterministicProcess` should be `y.index`. You gave {dp._index}."
assert dp._constant, f"`constant` argument to `DeterministicProcess` should be `True`. You gave {dp._constant}."
assert dp._order == 1, f"`order` argument to `DeterministicProcess` should be `1`. You gave {dp._order}."
assert dp._seasonal, f"`seasonal` argument to `DeterministicProcess` should be `True`. You gave {dp._seasonal}."
assert len(
dp._additional_terms
) == 1, f"`additional_terms` argument to `DeterministicProcess` should be `[fourier]`. You gave {dp._additional_terms}."
assert isinstance(
dp._additional_terms[0], CalendarFourier
), f"`additional_terms` argument to `DeterministicProcess` should be `[fourier]`. You gave {dp._additional_terms}."
assert dp._additional_terms[
0]._order == 4, f"`order` argument to `CalendarFourier` should be `4`. You gave {dp._additional_terms[0]._order}."
assert isinstance(
dp._additional_terms[0]._freq, pd.offsets.MonthEnd
), f"`freq` argument to `CalendarFourier` should be `'M'`."
assert dp._drop, f"`additional_terms` argument to `DeterministicProcess` should be `True`. You gave {dp._drop}."
assert_equal(X, X_true, 'X')
def test_drop():
index = pd.RangeIndex(0, 200)
dummy = DummyTerm()
str(dummy)
assert dummy != TimeTrend()
dp = DeterministicProcess(index, additional_terms=[dummy], drop=True)
in_samp = dp.in_sample()
assert in_samp.shape == (200, 4)
oos = dp.out_of_sample(37)
assert oos.shape == (37, 4)
assert list(oos.columns) == list(in_samp.columns)
valid = ("const", "trend", "dummy", "normal")
for valid_col in valid:
assert sum([1 for col in oos if valid_col in col]) == 1
class Q3(EqualityCheckProblem): # Create trend feature
from statsmodels.tsa.deterministic import DeterministicProcess
y = load_average_sales()
dp = DeterministicProcess(index=y.index, order=3)
X = dp.in_sample()
X_fore = dp.out_of_sample(steps=90)
_vars = ['X', 'X_fore']
_expected = [X, X_fore]
_hint = """Your answer should look like:
```python
from statsmodels.tsa.deterministic import DeterministicProcess
y = average_sales.copy()
dp = DeterministicProcess(index=____, order=____)
X = dp.in_sample()
X_fore = dp.out_of_sample(steps=____)
```
"""
_solution = CS("""
from statsmodels.tsa.deterministic import DeterministicProcess
y = average_sales.copy()
dp = DeterministicProcess(index=y.index, order=3)
X = dp.in_sample()
X_fore = dp.out_of_sample(steps=90)
""")
def test_deterministic_process(time_index, constant, order, seasonal, fourier,
period, drop):
if seasonal and fourier:
return
dp = DeterministicProcess(
time_index,
constant=constant,
order=order,
seasonal=seasonal,
fourier=fourier,
period=period,
drop=drop,
)
terms = dp.in_sample()
pd.testing.assert_index_equal(terms.index, time_index)
terms = dp.out_of_sample(23)
assert isinstance(terms, pd.DataFrame)
def test_range_casting():
idx = np.arange(120)
dp = DeterministicProcess(
idx, constant=True, order=1, seasonal=True, period=12
)
idx = pd.RangeIndex(0, 120)
dp2 = DeterministicProcess(
idx, constant=True, order=1, seasonal=True, period=12
)
pd.testing.assert_frame_equal(dp.in_sample(), dp2.in_sample())
pd.testing.assert_frame_equal(dp.range(100, 150), dp2.range(100, 150))
def test_drop_two_consants(time_index):
tt = TimeTrend(constant=True, order=1)
dp = DeterministicProcess(
time_index, constant=True, additional_terms=[tt], drop=True
)
assert dp.in_sample().shape[1] == 2
dp2 = DeterministicProcess(time_index, additional_terms=[tt], drop=True)
pd.testing.assert_frame_equal(dp.in_sample(), dp2.in_sample())
def test_deterministic(reset_randomstate):
y = pd.Series(np.random.normal(size=200))
terms = [TimeTrend(constant=True, order=1), Seasonality(12)]
dp = DeterministicProcess(y.index, additional_terms=terms)
m = AutoReg(y, trend="n", seasonal=False, lags=2, deterministic=dp)
res = m.fit()
m2 = AutoReg(y, trend="ct", seasonal=True, lags=2, period=12)
res2 = m2.fit()
assert_almost_equal(np.asarray(res.params), np.asarray(res2.params))
with pytest.warns(RuntimeWarning, match="When using deterministic, trend"):
AutoReg(y, trend="ct", seasonal=False, lags=2, deterministic=dp)
def test_additional_terms(time_index):
add_terms = [TimeTrend(True, order=1)]
dp = DeterministicProcess(time_index, additional_terms=add_terms)
dp2 = DeterministicProcess(time_index, constant=True, order=1)
pd.testing.assert_frame_equal(dp.in_sample(), dp2.in_sample())
with pytest.raises(ValueError,
match="One or more terms in additional_terms"):
DeterministicProcess(time_index,
additional_terms=add_terms + add_terms)
with pytest.raises(ValueError,
match="One or more terms in additional_terms"):
DeterministicProcess(time_index,
constant=True,
order=1,
additional_terms=add_terms)
def test_non_unit_range():
idx = pd.RangeIndex(0, 700, 7)
dp = DeterministicProcess(idx, constant=True)
with pytest.raises(ValueError, match="The step of the index is not 1"):
dp.range(11, 900)
class Q4(EqualityCheckProblem): # Create time series features
import pandas as pd
from sklearn.linear_model import LinearRegression
from statsmodels.tsa.deterministic import (CalendarFourier,
DeterministicProcess)
family_sales = load_family_sales()
y = family_sales.loc[:,
('sales',
'SCHOOL AND OFFICE SUPPLIES')].rename("Supply Sales")
onpromotion = family_sales.loc[:, (
'onpromotion', 'SCHOOL AND OFFICE SUPPLIES')].rename("onpromotion")
fourier = CalendarFourier(freq='M', order=4)
dp = DeterministicProcess(
constant=True,
index=y.index,
order=1,
seasonal=True,
drop=True,
additional_terms=[fourier],
)
X_time = dp.in_sample()
X_time['NewYearsDay'] = (X_time.index.dayofyear == 1)
model = LinearRegression(fit_intercept=False)
model.fit(X_time, y)
y_deseason = y - model.predict(X_time)
y_deseason.name = 'sales_deseasoned'
X_lags = make_lags(y_deseason, lags=1)
X_promo = pd.concat([
make_lags(onpromotion, lags=1),
onpromotion,
make_leads(onpromotion, leads=1),
],
axis=1)
_vars = ['X_lags', 'X_promo']
_expected = [X_lags, X_promo]
_hint = """Your solution should look like:
```python
X_lags = make_lags(y_deseason, lags=____)
X_promo = pd.concat([
make_lags(onpromotion, lags=____),
onpromotion,
make_leads(onpromotion, leads=____),
], axis=1)
X = pd.concat([X_time, X_lags, X_promo], axis=1).dropna()
y, X = y.align(X, join='inner')
```
"""
_solution = CS("""
X_lags = make_lags(y_deseason, lags=1)
X_promo = pd.concat([
make_lags(onpromotion, lags=1),
onpromotion,
make_leads(onpromotion, leads=1),
], axis=1)
X = pd.concat([X_time, X_lags, X_promo], axis=1).dropna()
y, X = y.align(X, join='inner')
""")
# `DeterministicProcess`. These can include a constant, a time trend of any
# order, and either a seasonal or a Fourier component.
#
# The process requires an index, which is the index of the full-sample (or
# in-sample).
#
# First, we initialize a deterministic process with a constant, a linear
# time trend, and a 5-period seasonal term. The `in_sample` method returns
# the full set of values that match the index.
from statsmodels.tsa.deterministic import DeterministicProcess
index = pd.RangeIndex(0, 100)
det_proc = DeterministicProcess(index,
constant=True,
order=1,
seasonal=True,
period=5)
det_proc.in_sample()
# The `out_of_sample` returns the next `steps` values after the end of the
# in-sample.
det_proc.out_of_sample(15)
# `range(start, stop)` can also be used to produce the deterministic terms
# over any range including in- and out-of-sample.
#
# ### Notes
#
# * When the index is a pandas `DatetimeIndex` or a `PeriodIndex`, then
def test_deterministic_process_errors(time_index):
with pytest.raises(ValueError, match="seasonal and fourier"):
DeterministicProcess(time_index, seasonal=True, fourier=2, period=5)
with pytest.raises(TypeError, match="All additional terms"):
DeterministicProcess(time_index, seasonal=True, additional_terms=[1])
# includes them as exogenous regressors. from statsmodels.tsa.api import SARIMAX sarimax_mod = SARIMAX(ind_prod, order=((1, 5, 12, 13), 0, 0), trend="c") sarimax_res = sarimax_mod.fit() print(sarimax_res.summary()) sarimax_params = sarimax_res.params.iloc[:-1].copy() sarimax_params.index = res_glob.params.index params = pd.concat([res_glob.params, sarimax_params], axis=1, sort=False) params.columns = ["AutoReg", "SARIMAX"] params # ## Custom Deterministic Processes # # The `deterministic` parameter allows a custom `DeterministicProcess` to # be used. This allows for more complex deterministic terms to be # constructed, for example one that includes seasonal components with two # periods, or, as the next example shows, one that uses a Fourier series # rather than seasonal dummies. from statsmodels.tsa.deterministic import DeterministicProcess dp = DeterministicProcess(housing.index, constant=True, period=12, fourier=2) mod = AutoReg(housing, 2, trend="n", seasonal=False, deterministic=dp) res = mod.fit() print(res.summary()) fig = res.plot_predict(720, 840)
def test_range_index_basic():
idx = pd.date_range("2000-1-1", freq="M", periods=120)
dp = DeterministicProcess(idx, constant=True, order=1, seasonal=True)
dp.range("2001-1-1", "2008-1-1")
dp.range("2001-1-1", "2015-1-1")
dp.range("2013-1-1", "2008-1-1")
dp.range(0, 100)
dp.range(100, 150)
dp.range(130, 150)
with pytest.raises(ValueError):
dp.range("1990-1-1", "2010-1-1")
idx = pd.period_range("2000-1-1", freq="M", periods=120)
dp = DeterministicProcess(idx, constant=True, order=1, seasonal=True)
dp.range("2001-1-1", "2008-1-1")
dp.range("2001-1-1", "2015-1-1")
dp.range("2013-1-1", "2008-1-1")
with pytest.raises(ValueError, match="start must be non-negative"):
dp.range(-7, 200)
dp.range(0, 100)
dp.range(100, 150)
dp.range(130, 150)
idx = pd.RangeIndex(0, 120)
dp = DeterministicProcess(idx,
constant=True,
order=1,
seasonal=True,
period=12)
dp.range(0, 100)
dp.range(100, 150)
dp.range(120, 150)
dp.range(130, 150)
with pytest.raises(ValueError):
dp.range(-10, 0)
def test_determintic_term_equiv(index):
base = DeterministicProcess(pd.RangeIndex(0, 200), constant=True, order=2)
dp = DeterministicProcess(index, constant=True, order=2)
np.testing.assert_array_equal(base.in_sample(), dp.in_sample())
np.testing.assert_array_equal(base.out_of_sample(37), dp.out_of_sample(37))
np.testing.assert_array_equal(base.range(200, 237), dp.range(200, 237))
np.testing.assert_array_equal(base.range(50, 150), dp.range(50, 150))
np.testing.assert_array_equal(base.range(50, 250), dp.range(50, 250))