def test_smoothed_state_obs_weights_univariate_singular(singular, periods,
reset_randomstate):
# Tests for the univariate case when the forecast error covariance matrix
# is singular (so the multivariate approach cannot be used, and the use of
# pinv in computing the weights becomes actually operative)
endog = np.zeros((10, 2))
endog[6, 0] = np.nan
endog[7, :] = np.nan
endog[8, 1] = np.nan
mod = TVSS(endog)
mod.ssm.initialize_known([1.2, 0.8], np.eye(2) * 0)
if singular == 'both':
mod['obs_cov', ..., :periods] = 0
else:
mod['obs_cov', 0, 1, :periods] = 0
mod['obs_cov', 1, 0, :periods] = 0
mod['obs_cov', singular, singular, :periods] = 0
mod['state_cov', :, :, :periods] = 0
mod.ssm.filter_univariate = True
res = mod.smooth([])
# Make sure we actually have singular covariance matrices in the periods
# specified
for i in range(periods):
eigvals = np.linalg.eigvalsh(res.forecasts_error_cov[..., i])
assert_equal(np.min(eigvals), 0)
# Compute the desiried weights
n = mod.nobs
m = mod.k_states
p = mod.k_endog
desired = np.zeros((n, n, m, p)) * np.nan
# Here we manually compute the weights by adjusting one observation at a
# time
for j in range(n):
for i in range(p):
if np.isnan(endog[j, i]):
desired[:, j, :, i] = np.nan
else:
y = endog.copy()
y[j, i] = 1.0
tmp_mod = mod.clone(y)
tmp_mod.ssm.initialize_known([1.2, 0.8], np.eye(2) * 0)
tmp_mod.ssm.filter_univariate = True
tmp_res = tmp_mod.smooth([])
desired[:, j, :, i] = (tmp_res.smoothed_state.T
- res.smoothed_state.T)
actual, _, _ = tools.compute_smoothed_state_weights(res)
assert_allclose(actual, desired, atol=1e-12)
def test_compute_t_compute_j(compute_j, compute_t, reset_randomstate):
# Tests for the collapsed case
endog = np.zeros((10, 6))
endog[2, :] = np.nan
endog[6, 0] = np.nan
endog[7, :] = np.nan
endog[8, 1] = np.nan
mod = TVSS(endog)
mod['obs_intercept'] = np.zeros((6, 1))
mod.ssm.initialize_known([1.2, 0.8], np.eye(2))
mod.ssm.filter_collapsed = True
res = mod.smooth([])
# Compute the desiried weights
n = mod.nobs
m = mod.k_states
p = mod.k_endog
desired = np.zeros((n, n, m, p)) * np.nan
# Here we manually compute the weights by adjusting one observation at a
# time
for j in range(n):
for i in range(p):
if np.isnan(endog[j, i]):
desired[:, j, :, i] = np.nan
else:
y = endog.copy()
y[j, i] = 1.0
tmp_mod = mod.clone(y)
tmp_mod['obs_intercept'] = np.zeros((6, 1))
tmp_mod.ssm.initialize_known([1.2, 0.8], np.eye(2))
mod.ssm.filter_collapsed = True
tmp_res = tmp_mod.smooth([])
desired[:, j, :, i] = (tmp_res.smoothed_state.T
- res.smoothed_state.T)
actual, _, _ = tools.compute_smoothed_state_weights(
res, compute_t=compute_t, compute_j=compute_j)
compute_t = np.atleast_1d(compute_t)
compute_j = np.atleast_1d(compute_j)
for t in np.arange(10):
if t not in compute_t:
desired[t, :] = np.nan
for j in np.arange(10):
if j not in compute_j:
desired[:, j] = np.nan
assert_allclose(actual, desired, atol=1e-12)
def test_smoothed_state_obs_weights_collapsed(reset_randomstate):
# Tests for the collapsed case
endog = np.zeros((20, 6))
endog[2, :] = np.nan
endog[6, 0] = np.nan
endog[7, :] = np.nan
endog[8, 1] = np.nan
mod = TVSS(endog)
mod['obs_intercept'] = np.zeros((6, 1))
mod.ssm.initialize_known([1.2, 0.8], np.eye(2))
mod.ssm.filter_collapsed = True
res = mod.smooth([])
# Compute the desiried weights
n = mod.nobs
m = mod.k_states
p = mod.k_endog
desired = np.zeros((n, n, m, p)) * np.nan
# Here we manually compute the weights by adjusting one observation at a
# time
for j in range(n):
for i in range(p):
if np.isnan(endog[j, i]):
desired[:, j, :, i] = np.nan
else:
y = endog.copy()
y[j, i] = 1.0
tmp_mod = mod.clone(y)
tmp_mod['obs_intercept'] = np.zeros((6, 1))
tmp_mod.ssm.initialize_known([1.2, 0.8], np.eye(2))
mod.ssm.filter_collapsed = True
tmp_res = tmp_mod.smooth([])
desired[:, j, :, i] = (tmp_res.smoothed_state.T
- res.smoothed_state.T)
desired_state_intercept_weights = np.zeros((n, n, m, m)) * np.nan
# Here we manually compute the weights by adjusting one state intercept
# at a time
for j in range(n):
for ell in range(m):
tmp_mod = mod.clone(endog)
tmp_mod['obs_intercept'] = np.zeros((6, 1))
tmp_mod.ssm.initialize_known([1.2, 0.8], np.eye(2))
mod.ssm.filter_collapsed = True
if tmp_mod['state_intercept'].ndim == 1:
si = tmp_mod['state_intercept']
tmp_mod['state_intercept'] = np.zeros((mod.k_states, mod.nobs))
tmp_mod['state_intercept', :, :] = si[:, None]
tmp_mod['state_intercept', ell, j] += 1.0
tmp_res = tmp_mod.ssm.smooth()
desired_state_intercept_weights[:, j, :, ell] = (
tmp_res.smoothed_state.T - res.smoothed_state.T)
actual, actual_state_intercept_weights, _ = (
tools.compute_smoothed_state_weights(res))
assert_allclose(actual, desired, atol=1e-12)
assert_allclose(actual_state_intercept_weights,
desired_state_intercept_weights, atol=1e-12)
def test_smoothed_state_obs_weights_TVSS(univariate, diffuse,
reset_randomstate):
endog = np.zeros((10, 3))
# One simple way to introduce more diffuse periods is to have fully missing
# observations at the beginning
if diffuse == 4:
endog[:3] = np.nan
endog[6, 0] = np.nan
endog[7, :] = np.nan
endog[8, 1] = np.nan
mod = TVSS(endog)
prior_mean = np.array([1.2, 0.8])
prior_cov = np.eye(2)
if not diffuse:
mod.ssm.initialize_known(prior_mean, prior_cov)
if univariate:
mod.ssm.filter_univariate = True
res = mod.smooth([])
# Compute the desiried weights
n = mod.nobs
m = mod.k_states
p = mod.k_endog
desired = np.zeros((n, n, m, p)) * np.nan
# Here we manually compute the weights by adjusting one observation at a
# time
for j in range(n):
for i in range(p):
if np.isnan(endog[j, i]):
desired[:, j, :, i] = np.nan
else:
y = endog.copy()
y[j, i] = 1.0
tmp_mod = mod.clone(y)
if not diffuse:
tmp_mod.ssm.initialize_known(prior_mean, prior_cov)
if univariate:
tmp_mod.ssm.filter_univariate = True
tmp_res = tmp_mod.smooth([])
desired[:, j, :, i] = (tmp_res.smoothed_state.T
- res.smoothed_state.T)
desired_state_intercept_weights = np.zeros((n, n, m, m)) * np.nan
# Here we manually compute the weights by adjusting one state intercept
# at a time
for j in range(n):
for ell in range(m):
tmp_mod = mod.clone(endog)
if not diffuse:
tmp_mod.ssm.initialize_known(prior_mean, prior_cov)
if univariate:
tmp_mod.ssm.filter_univariate = True
if tmp_mod['state_intercept'].ndim == 1:
si = tmp_mod['state_intercept']
tmp_mod['state_intercept'] = np.zeros((mod.k_states, mod.nobs))
tmp_mod['state_intercept', :, :] = si[:, None]
tmp_mod['state_intercept', ell, j] += 1.0
tmp_res = tmp_mod.ssm.smooth()
desired_state_intercept_weights[:, j, :, ell] = (
tmp_res.smoothed_state.T - res.smoothed_state.T)
desired_prior_weights = np.zeros((n, m, m)) * np.nan
if not diffuse:
for i in range(m):
a = prior_mean.copy()
a[i] += 1
tmp_mod = mod.clone(endog)
tmp_mod.ssm.initialize_known(a, prior_cov)
tmp_res = tmp_mod.smooth([])
desired_prior_weights[:, :, i] = (tmp_res.smoothed_state.T
- res.smoothed_state.T)
if not diffuse:
mod.ssm.initialize_known(prior_mean, prior_cov)
actual, actual_state_intercept_weights, actual_prior_weights = (
tools.compute_smoothed_state_weights(res))
d = res.nobs_diffuse
assert_equal(d, diffuse)
if diffuse:
assert_allclose(actual[:d], np.nan, atol=1e-12)
assert_allclose(actual[:, :d], np.nan, atol=1e-12)
assert_allclose(actual_state_intercept_weights[:d], np.nan)
assert_allclose(actual_state_intercept_weights[:, :d], np.nan)
assert_allclose(actual_prior_weights, np.nan)
else:
# Test that the weights are the same
assert_allclose(actual_prior_weights, desired_prior_weights,
atol=1e-12)
# In the non-diffuse case, we can actually use the weights along with
# the prior and observations to compute the smoothed state directly,
# and then compare that to what was returned by the usual Kalman
# smoothing routines
# Note that TVSS sets the state intercept to zeros, so this does not
# test that, although those weights are tested separately, see above
# and below.
contribution_prior = np.nansum(
actual_prior_weights * prior_mean[None, None, :], axis=2)
contribution_endog = np.nansum(
actual * (endog - mod['obs_intercept'].T)[None, :, None, :],
axis=(1, 3))
computed_smoothed_state = contribution_prior + contribution_endog
assert_allclose(computed_smoothed_state, res.smoothed_state.T)
assert_allclose(actual[d:, d:], desired[d:, d:], atol=1e-12)
assert_allclose(actual_state_intercept_weights[d:, d:],
desired_state_intercept_weights[d:, d:], atol=1e-12)