def model_local_level(endog=None, params=None, direct=False):
if endog is None:
y1 = 10.2394
endog = np.r_[y1, [1] * 9]
if params is None:
params = [1.993, 8.253]
sigma2_y, sigma2_mu = params
if direct:
mod = None
# Construct the basic representation
ssm = KalmanSmoother(k_endog=1, k_states=1, k_posdef=1)
ssm.bind(endog)
init = Initialization(ssm.k_states, initialization_type='diffuse')
ssm.initialize(init)
# ssm.filter_univariate = True # should not be required
# Fill in the system matrices for a local level model
ssm['design', :] = 1
ssm['obs_cov', :] = sigma2_y
ssm['transition', :] = 1
ssm['selection', :] = 1
ssm['state_cov', :] = sigma2_mu
else:
mod = UnobservedComponents(endog, 'llevel')
mod.update(params)
ssm = mod.ssm
ssm.initialize(Initialization(ssm.k_states, 'diffuse'))
return mod, ssm
def model_local_linear_trend(endog=None, params=None, direct=False):
if endog is None:
y1 = 10.2394
y2 = 4.2039
y3 = 6.123123
endog = np.r_[y1, y2, y3, [1] * 7]
if params is None:
params = [1.993, 8.253, 2.334]
sigma2_y, sigma2_mu, sigma2_beta = params
if direct:
mod = None
# Construct the basic representation
ssm = KalmanSmoother(k_endog=1, k_states=2, k_posdef=2)
ssm.bind(endog)
init = Initialization(ssm.k_states, initialization_type='diffuse')
ssm.initialize(init)
# ssm.filter_univariate = True # should not be required
# Fill in the system matrices for a local level model
ssm['design', 0, 0] = 1
ssm['obs_cov', 0, 0] = sigma2_y
ssm['transition'] = np.array([[1, 1], [0, 1]])
ssm['selection'] = np.eye(2)
ssm['state_cov'] = np.diag([sigma2_mu, sigma2_beta])
else:
mod = UnobservedComponents(endog, 'lltrend')
mod.update(params)
ssm = mod.ssm
ssm.initialize(Initialization(ssm.k_states, 'diffuse'))
return mod, ssm
def test_forecast_exog():
# Test forecasting with various shapes of `exog`
nobs = 100
endog = np.ones(nobs) * 2.0
exog = np.ones(nobs)
mod = UnobservedComponents(endog, 'irregular', exog=exog)
res = mod.smooth([1.0, 2.0])
# 1-step-ahead, valid
exog_fcast_scalar = 1.
exog_fcast_1dim = np.ones(1)
exog_fcast_2dim = np.ones((1, 1))
assert_allclose(res.forecast(1, exog=exog_fcast_scalar), 2.)
assert_allclose(res.forecast(1, exog=exog_fcast_1dim), 2.)
assert_allclose(res.forecast(1, exog=exog_fcast_2dim), 2.)
# h-steps-ahead, valid
h = 10
exog_fcast_1dim = np.ones(h)
exog_fcast_2dim = np.ones((h, 1))
assert_allclose(res.forecast(h, exog=exog_fcast_1dim), 2.)
assert_allclose(res.forecast(h, exog=exog_fcast_2dim), 2.)
# h-steps-ahead, invalid
assert_raises(ValueError, res.forecast, h, exog=1.)
assert_raises(ValueError, res.forecast, h, exog=[1, 2])
assert_raises(ValueError, res.forecast, h, exog=np.ones((h, 2)))
def test_mle_reg():
endog = np.arange(100)*1.0
exog = endog*2
# Make the fit not-quite-perfect
endog[::2] += 0.01
endog[1::2] -= 0.01
with warnings.catch_warnings(record=True) as w:
mod1 = UnobservedComponents(endog, irregular=True, exog=exog, mle_regression=False)
res1 = mod1.fit(disp=-1)
mod2 = UnobservedComponents(endog, irregular=True, exog=exog, mle_regression=True)
res2 = mod2.fit(disp=-1)
assert_allclose(res1.regression_coefficients.filtered[0, -1], 0.5, atol=1e-5)
assert_allclose(res2.params[1], 0.5, atol=1e-5)
def test_specifications():
# Clear warnings
structural.__warningregistry__ = {}
endog = [1, 2]
# Test that when nothing specified, a warning is issued and the model that
# is fit is one with irregular=True and nothing else.
with warnings.catch_warnings(record=True) as w:
mod = UnobservedComponents(endog)
message = ("Specified model does not contain a stochastic element;"
" irregular component added.")
assert_equal(str(w[0].message), message)
assert_equal(mod.trend_specification, 'irregular')
# Test an invalid string trend specification
with pytest.raises(ValueError):
UnobservedComponents(endog, 'invalid spec')
# Test that if a trend component is specified without a level component,
# a warning is issued and a deterministic level component is added
with warnings.catch_warnings(record=True) as w:
mod = UnobservedComponents(endog, trend=True, irregular=True)
message = ("Trend component specified without level component;"
" deterministic level component added.")
assert_equal(str(w[0].message), message)
assert_equal(mod.trend_specification, 'deterministic trend')
# Test that if a string specification is provided, a warning is issued if
# the boolean attributes are also specified
trend_attributes = [
'irregular', 'trend', 'stochastic_level', 'stochastic_trend'
]
for attribute in trend_attributes:
with warnings.catch_warnings(record=True) as w:
kwargs = {attribute: True}
mod = UnobservedComponents(endog, 'deterministic trend', **kwargs)
message = ("Value of `%s` may be overridden when the trend"
" component is specified using a model string." %
attribute)
assert_equal(str(w[0].message), message)
# Test that a seasonal with period less than two is invalid
with pytest.raises(ValueError):
UnobservedComponents(endog, seasonal=1)
def get_model(y, x, **kwargs):
"""
return the current model
:param y: array
:param x: array
:param kwargs: model parameter
:return: UnonbservedComponents model object
"""
return UnobservedComponents(y, exog=x, **kwargs)
def test_forecast():
endog = np.arange(50) + 10
exog = np.arange(50)
mod = UnobservedComponents(endog, exog=exog, level='dconstant', seasonal=4)
res = mod.smooth([1e-15, 0, 1])
actual = res.forecast(10, exog=np.arange(50, 60)[:, np.newaxis])
desired = np.arange(50, 60) + 10
assert_allclose(actual, desired)
def test_custom_model_input_validation(rand_data, pre_int_period,
post_int_period):
with pytest.raises(ValueError) as excinfo:
ci = CausalImpact(rand_data,
pre_int_period,
post_int_period,
model='test')
assert str(
excinfo.value) == 'Input model must be of type UnobservedComponents.'
ucm = UnobservedComponents(rand_data.iloc[:101, 0],
level='llevel',
exog=rand_data.iloc[:101, 1:])
ucm.level = False
with pytest.raises(ValueError) as excinfo:
ci = CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=ucm)
assert str(excinfo.value) == 'Model must have level attribute set.'
ucm = UnobservedComponents(rand_data.iloc[:101, 0],
level='llevel',
exog=rand_data.iloc[:101, 1:])
ucm.exog = None
with pytest.raises(ValueError) as excinfo:
ci = CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=ucm)
assert str(excinfo.value) == 'Model must have exog attribute set.'
ucm = UnobservedComponents(rand_data.iloc[:101, 0],
level='llevel',
exog=rand_data.iloc[:101, 1:])
ucm.data = None
with pytest.raises(ValueError) as excinfo:
ci = CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=ucm)
assert str(excinfo.value) == 'Model must have data attribute set.'
def test_specifications():
# Clear warnings
structural.__warningregistry__ = {}
endog = [1, 2]
# Test that when nothing specified, a warning is issued and the model that
# is fit is one with irregular=True and nothing else.
warning = SpecificationWarning
match = 'irregular component added'
with pytest.warns(warning, match=match):
mod = UnobservedComponents(endog)
assert_equal(mod.trend_specification, 'irregular')
# Test an invalid string trend specification
with pytest.raises(ValueError):
UnobservedComponents(endog, 'invalid spec')
# Test that if a trend component is specified without a level component,
# a warning is issued and a deterministic level component is added
warning = SpecificationWarning
match = 'Trend component specified without'
with pytest.warns(warning, match=match):
mod = UnobservedComponents(endog, trend=True, irregular=True)
assert_equal(mod.trend_specification, 'deterministic trend')
# Test that if a string specification is provided, a warning is issued if
# the boolean attributes are also specified
trend_attributes = [
'irregular', 'trend', 'stochastic_level', 'stochastic_trend'
]
for attribute in trend_attributes:
kwargs = {attribute: True}
warning = SpecificationWarning
match = 'may be overridden when the trend'
with pytest.warns(warning, match=match):
UnobservedComponents(endog, 'deterministic trend', **kwargs)
# Test that a seasonal with period less than two is invalid
with pytest.raises(ValueError):
UnobservedComponents(endog, seasonal=1)
def test_causal_cto_w_custom_model_and_seasons(rand_data, pre_int_period,
post_int_period):
pre_data = rand_data.loc[pre_int_period[0]: pre_int_period[1], :]
model = UnobservedComponents(endog=pre_data.iloc[:, 0], level='llevel',
exog=pre_data.iloc[:, 1:],
freq_seasonal=[{'period': 4}, {'period': 3}])
ci = CausalImpact(rand_data, pre_int_period, post_int_period, model=model)
assert ci.model.freq_seasonal_periods == [4, 3]
assert ci.model.freq_seasonal_harmonics == [2, 1]
def run(self):
"""Fit the BSTS model to the data.
"""
self.model = UnobservedComponents(
self.data.loc[:self.data_inter - 1, self._obs_col()].values,
exog=self.data.loc[:self.data_inter - 1, self._reg_cols()].values,
level='local linear trend',
seasonal=self.model_args['n_seasons'],
)
self.fit = self.model.fit(
maxiter=self.model_args['max_iter'],
)
def test_recreate_model():
nobs = 100
endog = np.ones(nobs) * 2.0
exog = np.ones(nobs)
levels = [
'irregular', 'ntrend', 'fixed intercept', 'deterministic constant',
'dconstant', 'local level', 'llevel', 'random walk', 'rwalk',
'fixed slope', 'deterministic trend', 'dtrend',
'local linear deterministic trend', 'lldtrend',
'random walk with drift', 'rwdrift', 'local linear trend',
'lltrend', 'smooth trend', 'strend', 'random trend', 'rtrend']
for level in levels:
# Note: have to add in some stochastic component, otherwise we have
# problems with entirely deterministic models
# level + stochastic seasonal
mod = UnobservedComponents(endog, level=level, seasonal=2,
stochastic_seasonal=True, exog=exog)
mod2 = UnobservedComponents(endog, exog=exog, **mod._get_init_kwds())
check_equivalent_models(mod, mod2)
# level + autoregressive
mod = UnobservedComponents(endog, level=level, exog=exog,
autoregressive=1)
mod2 = UnobservedComponents(endog, exog=exog, **mod._get_init_kwds())
check_equivalent_models(mod, mod2)
# level + stochastic cycle
mod = UnobservedComponents(endog, level=level, exog=exog,
cycle=True, stochastic_cycle=True,
damped_cycle=True)
mod2 = UnobservedComponents(endog, exog=exog, **mod._get_init_kwds())
check_equivalent_models(mod, mod2)
def _construct_default_model(self):
"""Constructs default local level unobserved states model with input data.
Returns
-------
model: `UnobservedComponents` built using pre-intervention data as training
data.
"""
data = self.pre_data if self.normed_pre_data is None else self.normed_pre_data
y = data.iloc[:, 0]
X = data.iloc[:, 1:] if data.shape[1] > 1 else None
model = UnobservedComponents(endog=y, level='llevel', exog=X)
return model
def test_misc_exog():
# Tests for missing data
nobs = 20
k_endog = 1
np.random.seed(1208)
endog = np.random.normal(size=(nobs, k_endog))
endog[:4, 0] = np.nan
exog1 = np.random.normal(size=(nobs, 1))
exog2 = np.random.normal(size=(nobs, 2))
index = pd.date_range('1970-01-01', freq='QS', periods=nobs)
endog_pd = pd.DataFrame(endog, index=index)
exog1_pd = pd.Series(exog1.squeeze(), index=index)
exog2_pd = pd.DataFrame(exog2, index=index)
models = [
UnobservedComponents(endog, 'llevel', exog=exog1),
UnobservedComponents(endog, 'llevel', exog=exog2),
UnobservedComponents(endog, 'llevel', exog=exog2),
UnobservedComponents(endog_pd, 'llevel', exog=exog1_pd),
UnobservedComponents(endog_pd, 'llevel', exog=exog2_pd),
UnobservedComponents(endog_pd, 'llevel', exog=exog2_pd),
]
for mod in models:
# Smoke tests
mod.start_params
res = mod.fit(disp=False)
res.summary()
res.predict()
res.predict(dynamic=True)
res.get_prediction()
oos_exog = np.random.normal(size=(1, mod.k_exog))
res.forecast(steps=1, exog=oos_exog)
res.get_forecast(steps=1, exog=oos_exog)
# Smoke tests for invalid exog
oos_exog = np.random.normal(size=(1))
with pytest.raises(ValueError):
res.forecast(steps=1, exog=oos_exog)
oos_exog = np.random.normal(size=(2, mod.k_exog))
with pytest.raises(ValueError):
res.forecast(steps=1, exog=oos_exog)
oos_exog = np.random.normal(size=(1, mod.k_exog + 1))
with pytest.raises(ValueError):
res.forecast(steps=1, exog=oos_exog)
# Test invalid model specifications
with pytest.raises(ValueError):
UnobservedComponents(endog, 'llevel', exog=np.zeros((10, 4)))
def test_apply_results():
endog = np.arange(100)
exog = np.ones_like(endog)
params = [1., 1., 0.1, 1.]
mod1 = UnobservedComponents(endog[:50], 'llevel', exog=exog[:50])
res1 = mod1.smooth(params)
mod2 = UnobservedComponents(endog[50:], 'llevel', exog=exog[50:])
res2 = mod2.smooth(params)
res3 = res2.apply(endog[:50], exog=exog[:50])
assert_equal(res1.specification, res3.specification)
for attr in [
'nobs', 'llf', 'llf_obs', 'loglikelihood_burn',
'cov_params_default'
]:
assert_equal(getattr(res3, attr), getattr(res1, attr))
for attr in [
'filtered_state', 'filtered_state_cov', 'predicted_state',
'predicted_state_cov', 'forecasts', 'forecasts_error',
'forecasts_error_cov', 'standardized_forecasts_error',
'forecasts_error_diffuse_cov', 'predicted_diffuse_state_cov',
'scaled_smoothed_estimator', 'scaled_smoothed_estimator_cov',
'smoothing_error', 'smoothed_state', 'smoothed_state_cov',
'smoothed_state_autocov', 'smoothed_measurement_disturbance',
'smoothed_state_disturbance',
'smoothed_measurement_disturbance_cov',
'smoothed_state_disturbance_cov'
]:
assert_equal(getattr(res3, attr), getattr(res1, attr))
assert_allclose(res3.forecast(10, exog=np.ones(10)),
res1.forecast(10, exog=np.ones(10)))
def test_mle_reg(use_exact_diffuse):
endog = np.arange(100) * 1.0
exog = endog * 2
# Make the fit not-quite-perfect
endog[::2] += 0.01
endog[1::2] -= 0.01
with warnings.catch_warnings(record=True):
mod1 = UnobservedComponents(endog,
irregular=True,
exog=exog,
mle_regression=False,
use_exact_diffuse=use_exact_diffuse)
res1 = mod1.fit(disp=-1)
mod2 = UnobservedComponents(endog,
irregular=True,
exog=exog,
mle_regression=True,
use_exact_diffuse=use_exact_diffuse)
res2 = mod2.fit(disp=-1)
assert_allclose(res1.regression_coefficients.filtered[0, -1],
0.5,
atol=1e-5)
assert_allclose(res2.params[1], 0.5, atol=1e-5)
# When the regression component is part of the state vector with exact
# diffuse initialization, we have two diffuse observations
if use_exact_diffuse:
print(res1.predicted_diffuse_state_cov)
assert_equal(res1.nobs_diffuse, 2)
assert_equal(res2.nobs_diffuse, 0)
else:
assert_equal(res1.loglikelihood_burn, 1)
assert_equal(res2.loglikelihood_burn, 0)
def test_causal_cto_w_custom_model(rand_data, pre_int_period, post_int_period):
pre_data = rand_data.loc[pre_int_period[0]: pre_int_period[1], :]
model = UnobservedComponents(endog=pre_data.iloc[:, 0], level='llevel',
exog=pre_data.iloc[:, 1:])
ci = CausalImpact(rand_data, pre_int_period, post_int_period, model=model)
assert ci.model.endog_names == 'y'
assert ci.model.exog_names == ['x1', 'x2']
assert ci.model.k_endog == 1
assert ci.model.level
assert ci.model.trend_specification == 'local level'
assert isinstance(ci.trained_model, UnobservedComponentsResultsWrapper)
assert ci.trained_model.nobs == len(pre_data)
def construct_model(data, model_args={}):
"""Specifies the model and performs inference. Inference means using a
technique that combines Kalman Filters with Maximum Likelihood Estimators
methods to fit the parameters that best explain the observed data.
Args:
data: time series of response variable and optional covariates
model_args: optional list of additional model arguments
Returns:
An Unobserved Components Model, as returned by UnobservedComponents()
"""
from statsmodels.tsa.statespace.structural import UnobservedComponents
y = data.iloc[:, 0]
observations_ill_conditioned(y)
#LocalLevel specification of statespace
ss = {}
ss["endog"] = y.values
ss["level"] = "llevel"
# No regression?
if len(data.columns) == 1:
mod = UnobservedComponents(**ss)
return mod
else:
# Static regression
if not model_args.get("dynamic_regression"):
ss["exog"] = data.iloc[:, 1:].values
mod = UnobservedComponents(**ss)
return mod
# Dynamic regression
else:
raise NotImplementedError()
def get_referenced_model(model, endog, exog):
"""
Buils an `UnobservedComponents` model using as reference the input `model`. This is
mainly used for building models to make simulations of time series.
Args
----
model: `UnobservedComponents`.
Template model that is used as reference to build a new one with new `endog`
and `exog` variables.
endog: pandas.Series.
New endog value to be used in model.
exog: pandas.Series.
New exog value to be used in model.
Returns
-------
ref_model: `UnobservedComponents`.
New model built from input `model` setup.
"""
args = {}
args['level'] = model.level
args['trend'] = model.trend
args['seasonal'] = model.seasonal_periods
args['freq_seasonal'] = [{
'period': period,
'harmonics': h
} for (
period,
h) in zip(model.freq_seasonal_periods, model.freq_seasonal_harmonics)]
args['cycle'] = model.cycle
args['ar'] = model.ar_order
args['exog'] = exog
args['endog'] = endog
args['irregular'] = model.irregular
args['stochastic_level'] = model.stochastic_level
args['stochastic_trend'] = model.stochastic_trend
args['stochastic_seasonal'] = model.stochastic_seasonal
args['stochastic_freq_seasonal'] = model.stochastic_freq_seasonal
args['stochastic_cycle'] = model.stochastic_cycle
args['damped_cycle'] = model.damped_cycle
cycle_bounds = model.cycle_frequency_bound
lower_cycle_bound = 2 * np.pi / cycle_bounds[1]
upper_cycle_bound = 2 * np.pi / cycle_bounds[0] if cycle_bounds[
0] > 0 else np.inf
args['cycle_period_bounds'] = (lower_cycle_bound, upper_cycle_bound)
ref_model = UnobservedComponents(**args)
return ref_model
def _get_default_model(self):
"""Constructs default local level unobserved states model using input data and
`self.model_args`.
Returns
-------
model: `UnobservedComponents` built using pre-intervention data as training
data.
"""
data = self.pre_data if self.normed_pre_data is None else self.normed_pre_data
y = data.iloc[:, 0]
X = data.iloc[:, 1:] if data.shape[1] > 1 else None
freq_seasonal = self.model_args.get('nseasons')
model = UnobservedComponents(endog=y, level='llevel', exog=X,
freq_seasonal=freq_seasonal)
return model
def __init__(self, endog, **kwargs):
self.exog = exog = kwargs.pop("exog", None)
self.kwargs = kwargs
assert endog.ndim == 3
assert exog is None or exog.ndim in (2, 3)
self.n_models = endog.shape[0]
self.models = []
for series_idx in range(self.n_models):
exog = (
self.exog[series_idx, ...]
if self.exog is not None and self.exog.ndim == 3
else self.exog
)
m = UnobservedComponents(endog[series_idx, ...], exog=exog, **kwargs)
self.models += [m]
def run(self, return_df=False):
"""Fit the BSTS model to the data.
"""
self._model = UnobservedComponents(
self.data.loc[:self.data_inter - 1, self._obs_col()].values,
exog=self.data.loc[:self.data_inter - 1, self._reg_cols()].values,
level='local linear trend',
seasonal=self.model_args['n_seasons'],
)
self._fit = self._model.fit(
maxiter=self.model_args['max_iter'],
)
self._get_estimates()
self._get_difference_estimates()
self._get_cumulative_estimates()
if return_df:
return self.result
def simulated_y(self):
"""
In order to process lower and upper boundaries for different metrics we simulate
several responses for `y` using parameters trained during the fitting phase.
Returns
-------
simulations: np.array
Array where each row is a simulation of the response variable whose shape is
(n simulations, n points in post period).
"""
if self._simulated_y is None:
simulations = []
# For more information about the `trend` and how it works, please refer to:
# https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.structural.UnobservedComponents.html
trend = self.model.trend_specification
y = np.zeros(len(self.post_data))
exog_data = self.post_data if self.mu_sig is None else self.normed_post_data
X = exog_data.iloc[:, 1:] if exog_data.shape[1] > 1 else None
model = UnobservedComponents(y, level=trend, exog=X)
# `params` is related to the parameters found when fitting the Kalman filter
# from the observed time series.
params = self.trained_model.params
predicted_state = self.trained_model.predicted_state[..., -1]
predicted_state_cov = self.trained_model.predicted_state_cov[...,
-1]
for _ in range(self.n_sims):
initial_state = np.random.multivariate_normal(
predicted_state, predicted_state_cov)
sim = model.simulate(params,
len(self.post_data),
initial_state=initial_state)
if self.mu_sig:
sim = sim * self.mu_sig[1] + self.mu_sig[0]
simulations.append(sim)
self._simulated_y = np.array(simulations)
return self._simulated_y
else:
return self._simulated_y
def test_simulated_y_custom_model():
np.random.seed(1)
ar = np.r_[1, 0.9]
ma = np.array([1])
arma_process = ArmaProcess(ar, ma)
X = 100 + arma_process.generate_sample(nsample=100)
y = 1.2 * X + np.random.normal(size=(100))
data = pd.DataFrame({'y': y, 'X': X}, columns=['y', 'X'])
intervention_idx = 70
normed_pre_data, _ = standardize(data.iloc[:intervention_idx])
model = UnobservedComponents(
endog=normed_pre_data['y'].iloc[0:intervention_idx],
level='llevel',
exog=normed_pre_data['X'].iloc[0:intervention_idx])
ci = CausalImpact(data, [0, 69], [70, 99], model=model)
assert ci.simulated_y.shape == (1000, 30)
lower, upper = np.percentile(ci.simulated_y.mean(axis=1), [5, 95])
assert lower > 119
assert upper < 121
def run(self, max_iter=1000, return_df=False):
"""Fit the BSTS model to the data.
:param int max_iter: max number of iterations in UnobservedComponents.fit (maximum likelihood estimator)
:param bool return_df: set to `True` if you want this method to return the dataframe of model results
:return: None or pandas.DataFrame of results
"""
self._model = UnobservedComponents(
self.data.loc[:self._inter_index - 1,
self._obs_col()].values,
exog=self.data.loc[:self._inter_index - 1,
self._reg_cols()].values,
level='local linear trend',
seasonal=self.n_seasons,
)
self._fit = self._model.fit(maxiter=max_iter)
self._get_estimates()
self._get_difference_estimates()
self._get_cumulative_estimates()
if return_df:
return self.result
def test_start_params():
# Test that the behavior is correct for multiple exogenous and / or
# autoregressive components
# Parameters
nobs = int(1e4)
beta = np.r_[10, -2]
phi = np.r_[0.5, 0.1]
# Generate data
np.random.seed(1234)
exog = np.c_[np.ones(nobs), np.arange(nobs)*1.0]
eps = np.random.normal(size=nobs)
endog = np.zeros(nobs+2)
for t in range(1, nobs):
endog[t+1] = phi[0] * endog[t] + phi[1] * endog[t-1] + eps[t]
endog = endog[2:]
endog += np.dot(exog, beta)
# Now just test that the starting parameters are approximately what they
# ought to be (could make this arbitrarily precise by increasing nobs,
# but that would slow down the test for no real gain)
mod = UnobservedComponents(endog, exog=exog, autoregressive=2)
assert_allclose(mod.start_params, [1., 0.5, 0.1, 10, -2], atol=1e-1)
def test_matrices_somewhat_complicated_model():
values = dta.copy()
model = UnobservedComponents(values['unemp'],
level='lltrend',
freq_seasonal=[{'period': 4},
{'period': 9, 'harmonics': 3}],
cycle=True,
cycle_period_bounds=[2, 30],
damped_cycle=True,
stochastic_freq_seasonal=[True, False],
stochastic_cycle=True
)
# Selected parameters
params = [1, # irregular_var
3, 4, # lltrend parameters: level_var, trend_var
5, # freq_seasonal parameters: freq_seasonal_var_0
# cycle parameters: cycle_var, cycle_freq, cycle_damp
6, 2*np.pi/30., .9
]
model.update(params)
# Check scalar properties
assert_equal(model.k_states, 2 + 4 + 6 + 2)
assert_equal(model.k_state_cov, 2 + 1 + 0 + 1)
assert_equal(model.loglikelihood_burn, 2 + 4 + 6 + 2)
assert_allclose(model.ssm.k_posdef, 2 + 4 + 0 + 2)
assert_equal(model.k_params, len(params))
# Check the statespace model matrices against hand-constructed answers
# We group the terms by the component
expected_design = np.r_[[1, 0],
[1, 0, 1, 0],
[1, 0, 1, 0, 1, 0],
[1, 0]].reshape(1, 14)
assert_allclose(model.ssm.design[:, :, 0], expected_design)
expected_transition = __direct_sum([
np.array([[1, 1],
[0, 1]]),
np.array([[0, 1, 0, 0],
[-1, 0, 0, 0],
[0, 0, -1, 0],
[0, 0, 0, -1]]),
np.array([[np.cos(2*np.pi*1/9.), np.sin(2*np.pi*1/9.), 0, 0, 0, 0],
[-np.sin(2*np.pi*1/9.), np.cos(2*np.pi*1/9.), 0, 0, 0, 0],
[0, 0, np.cos(2*np.pi*2/9.), np.sin(2*np.pi*2/9.), 0, 0],
[0, 0, -np.sin(2*np.pi*2/9.), np.cos(2*np.pi*2/9.), 0, 0],
[0, 0, 0, 0, np.cos(2*np.pi/3.), np.sin(2*np.pi/3.)],
[0, 0, 0, 0, -np.sin(2*np.pi/3.), np.cos(2*np.pi/3.)]]),
np.array([[.9*np.cos(2*np.pi/30.), .9*np.sin(2*np.pi/30.)],
[-.9*np.sin(2*np.pi/30.), .9*np.cos(2*np.pi/30.)]])
])
assert_allclose(
model.ssm.transition[:, :, 0], expected_transition, atol=1e-7)
# Since the second seasonal term is not stochastic,
# the dimensionality of the state disturbance is 14 - 6 = 8
expected_selection = np.zeros((14, 14 - 6))
expected_selection[0:2, 0:2] = np.eye(2)
expected_selection[2:6, 2:6] = np.eye(4)
expected_selection[-2:, -2:] = np.eye(2)
assert_allclose(model.ssm.selection[:, :, 0], expected_selection)
expected_state_cov = __direct_sum([
np.diag(params[1:3]),
np.eye(4)*params[3],
np.eye(2)*params[4]
])
assert_allclose(model.ssm.state_cov[:, :, 0], expected_state_cov)
def run_ucm(name):
true = getattr(results_structural, name)
for model in true['models']:
kwargs = model.copy()
kwargs.update(true['kwargs'])
# Make a copy of the data
values = dta.copy()
freq = kwargs.pop('freq', None)
if freq is not None:
values.index = pd.date_range(start='1959-01-01', periods=len(dta),
freq=freq)
# Test pandas exog
if 'exog' in kwargs:
# Default value here is pd.Series object
exog = np.log(values['realgdp'])
# Also allow a check with a 1-dim numpy array
if kwargs['exog'] == 'numpy':
exog = exog.values.squeeze()
kwargs['exog'] = exog
# Create the model
mod = UnobservedComponents(values['unemp'], **kwargs)
# Smoke test for starting parameters, untransform, transform
# Also test that transform and untransform are inverses
mod.start_params
roundtrip = mod.transform_params(
mod.untransform_params(mod.start_params))
assert_allclose(mod.start_params, roundtrip)
# Fit the model at the true parameters
res_true = mod.filter(true['params'])
# Check that the cycle bounds were computed correctly
freqstr = freq[0] if freq is not None else values.index.freqstr[0]
if 'cycle_period_bounds' in kwargs:
cycle_period_bounds = kwargs['cycle_period_bounds']
elif freqstr == 'A':
cycle_period_bounds = (1.5, 12)
elif freqstr == 'Q':
cycle_period_bounds = (1.5*4, 12*4)
elif freqstr == 'M':
cycle_period_bounds = (1.5*12, 12*12)
else:
# If we have no information on data frequency, require the
# cycle frequency to be between 0 and pi
cycle_period_bounds = (2, np.inf)
# Test that the cycle frequency bound is correct
assert_equal(mod.cycle_frequency_bound,
(2*np.pi / cycle_period_bounds[1],
2*np.pi / cycle_period_bounds[0]))
# Test that the likelihood is correct
rtol = true.get('rtol', 1e-7)
atol = true.get('atol', 0)
assert_allclose(res_true.llf, true['llf'], rtol=rtol, atol=atol)
# Optional smoke test for plot_components
try:
import matplotlib.pyplot as plt
try:
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
except ImportError:
pass
fig = plt.figure()
res_true.plot_components(fig=fig)
except ImportError:
pass
# Now fit the model via MLE
with warnings.catch_warnings(record=True):
res = mod.fit(disp=-1)
# If we found a higher likelihood, no problem; otherwise check
# that we're very close to that found by R
if res.llf <= true['llf']:
assert_allclose(res.llf, true['llf'], rtol=1e-4)
# Smoke test for summary
res.summary()
def fit(self, X, y=None):
# Perform top percentile ceiling
self.X = X
mode = self.mode
if '*f' in self.mode:
self.X = np.minimum(X, np.percentile(X, 75))
mode = self.mode.partition('*f')[0]
# Perform transformation if specified by *transformation
if '*ln' in self.mode:
self.X = np.log(np.array(X) + 1)
mode = self.mode.partition('*ln')[0]
elif '*bc' in self.mode:
transformer = pm.preprocessing.BoxCoxEndogTransformer()
self.X = transformer.fit_transform(y=X)
self.transformer = transformer
mode = self.mode.partition('*bc')[0]
try:
if mode == 'll':
# Local Level
model = LocalLevel(self.X)
self.res_ = model.fit(disp=False)
self.k_exog = None
elif mode == 'lla':
endog = X[2:]
exog = np.column_stack((X[1:-1], X[:-2]))
self.k_exog = exog.shape[1]
model = UnobservedComponents(endog=endog,
exog=exog,
level='local level')
self.res_ = model.fit(disp=False)
elif mode == 'lls':
self.k_exog = None
model = SARIMAX(endog=self.X,
order=(2, 0, 0),
trend='c',
measurement_error=True)
self.res_ = model.fit(disp=False)
elif mode == 'llt':
# Local Linear Trend
model = UnobservedComponents(endog=self.X,
level='local linear trend')
self.res_ = model.fit(disp=False)
elif mode == 'llc':
# Local Level Cycle
model = UnobservedComponents(endog=self.X,
level='local level',
cycle=True,
stochastic_cycle=True)
self.res_ = model.fit(disp=False)
elif mode == 'arima':
self.res_ = pm.auto_arima(self.X,
start_p=1,
start_q=1,
start_P=1,
start_Q=1,
max_p=5,
max_q=5,
max_P=5,
max_Q=5,
seasonal=True,
stepwise=True,
suppress_warnings=True,
D=10,
max_D=10,
error_action='ignore')
elif mode == 'rw1':
# For RW model
self.res_ = None
self.converged = False
except np.linalg.LinAlgError:
# Some kalman filter error ==> Use random walk
print(f'Convergence failed for {mode}')
self.converged = False
return self
try:
self.converged = self.res_.mle_retvals['converged']
except AttributeError:
if mode == 'arima':
self.converged = True # auto ARIMA from pmdarima should always converge
return self
def test_custom_model_fit(rand_data, pre_int_period, post_int_period,
monkeypatch):
fit_mock = mock.Mock()
monkeypatch.setattr(
'causalimpact.main.CausalImpact._process_posterior_inferences',
mock.Mock())
pre_data = rand_data.loc[pre_int_period[0]:pre_int_period[1], :]
model = UnobservedComponents(endog=pre_data.iloc[:, 0],
level='llevel',
exog=pre_data.iloc[:, 1:])
model.fit = fit_mock
CausalImpact(rand_data, pre_int_period, post_int_period, model=model)
fit_mock.assert_called_with(bounds=[(None, None), (0.01 / 1.2, 0.01 * 1.2),
(None, None), (None, None)],
disp=False,
nseasons=[],
standardize=True)
CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=model,
disp=True)
fit_mock.assert_called_with(bounds=[(None, None), (0.01 / 1.2, 0.01 * 1.2),
(None, None), (None, None)],
disp=True,
nseasons=[],
standardize=True)
CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=model,
disp=True,
prior_level_sd=0.01)
fit_mock.assert_called_with(bounds=[(None, None), (0.01 / 1.2, 0.01 * 1.2),
(None, None), (None, None)],
disp=True,
prior_level_sd=0.01,
nseasons=[],
standardize=True)
CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=model,
disp=True,
prior_level_sd=None)
fit_mock.assert_called_with(bounds=[(None, None), (None, None),
(None, None), (None, None)],
disp=True,
prior_level_sd=None,
nseasons=[],
standardize=True)
model = UnobservedComponents(endog=pre_data.iloc[:, 0],
level='llevel',
exog=pre_data.iloc[:, 1:],
freq_seasonal=[{
'period': 3
}])
model.fit = fit_mock
CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=model,
disp=True,
prior_level_sd=0.001)
fit_mock.assert_called_with(bounds=[
(None, None), (0.001 / 1.2, 0.001 * 1.2), (None, None), (None, None),
(None, None)
],
disp=True,
prior_level_sd=0.001,
nseasons=[],
standardize=True)
model = UnobservedComponents(endog=pre_data.iloc[:, 0],
level=True,
exog=pre_data.iloc[:, 1],
trend=True,
seasonal=3,
stochastic_level=True)
model.fit = fit_mock
CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=model,
disp=True,
prior_level_sd=0.001)
fit_mock.assert_called_with(bounds=[(0.001 / 1.2, 0.001 * 1.2),
(None, None), (None, None)],
disp=True,
prior_level_sd=0.001,
nseasons=[],
standardize=True)
new_pre_data = rand_data.loc[pre_int_period[0]:pre_int_period[1],
['y', 'x1']]
model = UnobservedComponents(endog=new_pre_data.iloc[:, 0],
level='llevel',
exog=new_pre_data.iloc[:, 1:])
model.fit = fit_mock
CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=model,
disp=False)
fit_mock.assert_called_with(bounds=[(None, None), (0.01 / 1.2, 0.01 * 1.2),
(None, None)],
disp=False,
nseasons=[],
standardize=True)
model = UnobservedComponents(endog=new_pre_data.iloc[:, 0],
level='dtrend',
exog=new_pre_data.iloc[:, 1:])
model.fit = fit_mock
CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=model,
disp=False)
fit_mock.assert_called_with(bounds=[(None, None), (None, None)],
disp=False,
nseasons=[],
standardize=True)
model = UnobservedComponents(endog=new_pre_data.iloc[:, 0],
level='lltrend',
exog=new_pre_data.iloc[:, 1:])
model.fit = fit_mock
CausalImpact(rand_data,
pre_int_period,
post_int_period,
model=model,
disp=False)
fit_mock.assert_called_with(bounds=[(None, None), (0.01 / 1.2, 0.01 * 1.2),
(None, None), (None, None)],
disp=False,
nseasons=[],
standardize=True)