def test_summary(): dates = pd.date_range(start='1980-01-01', end='1984-01-01', freq='AS') endog = pd.Series([1,2,3,4,5], index=dates) mod = MLEModel(endog, **kwargs) res = mod.filter([]) # Get the summary txt = str(res.summary()) # Test res.summary when the model has dates assert_equal(re.search('Sample:\s+01-01-1980', txt) is not None, True) assert_equal(re.search('\s+- 01-01-1984', txt) is not None, True) # Test res.summary when `model_name` was not provided assert_equal(re.search('Model:\s+MLEModel', txt) is not None, True) # Smoke test that summary still works when diagnostic tests fail with warnings.catch_warnings(): warnings.simplefilter("ignore") res.filter_results._standardized_forecasts_error[:] = np.nan res.summary() res.filter_results._standardized_forecasts_error = 1 res.summary() res.filter_results._standardized_forecasts_error = 'a' res.summary()
def setup_class(cls, which='mixed', *args, **kwargs): # Data dta = datasets.macrodata.load_pandas().data dta.index = pd.date_range(start='1959-01-01', end='2009-7-01', freq='QS') obs = np.log(dta[['realgdp','realcons','realinv']]).diff().iloc[1:] * 400 if which == 'all': obs.iloc[:50, :] = np.nan obs.iloc[119:130, :] = np.nan elif which == 'partial': obs.iloc[0:50, 0] = np.nan obs.iloc[119:130, 0] = np.nan elif which == 'mixed': obs.iloc[0:50, 0] = np.nan obs.iloc[19:70, 1] = np.nan obs.iloc[39:90, 2] = np.nan obs.iloc[119:130, 0] = np.nan obs.iloc[119:130, 2] = np.nan mod = cls.create_model(obs, **kwargs) cls.model = mod.ssm n_disturbance_variates = ( (cls.model.k_endog + cls.model.k_posdef) * cls.model.nobs ) np.random.seed(1234) dv = np.random.normal(size=n_disturbance_variates) isv = np.random.normal(size=cls.model.k_states) # Collapsed filtering, smoothing, and simulation smoothing cls.model.filter_collapsed = True cls.results_b = cls.model.smooth() cls.sim_b = cls.model.simulation_smoother() cls.sim_b.simulate(disturbance_variates=dv, initial_state_variates=isv) # Conventional filtering, smoothing, and simulation smoothing cls.model.filter_collapsed = False cls.results_a = cls.model.smooth() cls.sim_a = cls.model.simulation_smoother() cls.sim_a.simulate(disturbance_variates=dv, initial_state_variates=isv) # Create the model with augmented state space kwargs.pop('filter_collapsed', None) mod = MLEModel(obs, k_states=4, k_posdef=2, **kwargs) mod['design', :3, :2] = np.array([[-32.47143586, 17.33779024], [-7.40264169, 1.69279859], [-209.04702853, 125.2879374]]) mod['obs_cov'] = np.diag( np.array([0.0622668, 1.95666886, 58.37473642])) mod['transition', :2, :2] = np.array([[0.29935707, 0.33289005], [-0.7639868, 1.2844237]]) mod['transition', 2:, :2] = np.eye(2) mod['selection', :2, :2] = np.eye(2) mod['state_cov'] = np.array([[1.2, -0.25], [-0.25, 1.1]]) mod.initialize_approximate_diffuse(1e6) cls.augmented_model = mod.ssm cls.augmented_results = mod.ssm.smooth()
def setup_class(cls, which='none', **kwargs): # Results path = current_path + os.sep + 'results/results_smoothing_generalobscov_R.csv' cls.desired = pd.read_csv(path) # Data dta = datasets.macrodata.load_pandas().data dta.index = pd.date_range(start='1959-01-01', end='2009-7-01', freq='QS') obs = dta[['realgdp','realcons','realinv']].diff().iloc[1:] if which == 'all': obs.iloc[:50, :] = np.nan obs.iloc[119:130, :] = np.nan elif which == 'partial': obs.iloc[0:50, 0] = np.nan obs.iloc[119:130, 0] = np.nan elif which == 'mixed': obs.iloc[0:50, 0] = np.nan obs.iloc[19:70, 1] = np.nan obs.iloc[39:90, 2] = np.nan obs.iloc[119:130, 0] = np.nan obs.iloc[119:130, 2] = np.nan # Create the model mod = MLEModel(obs, k_states=3, k_posdef=3, **kwargs) mod['design'] = np.eye(3) mod['obs_cov'] = np.array([[ 609.0746647855, 0. , 0. ], [ 0. , 1.8774916622, 0. ], [ 0. , 0. , 124.6768281675]]) mod['transition'] = np.array([[-0.8110473405, 1.8005304445, 1.0215975772], [-1.9846632699, 2.4091302213, 1.9264449765], [ 0.9181658823, -0.2442384581, -0.6393462272]]) mod['selection'] = np.eye(3) mod['state_cov'] = np.array([[ 1552.9758843938, 612.7185121905, 877.6157204992], [ 612.7185121905, 467.8739411204, 70.608037339 ], [ 877.6157204992, 70.608037339 , 900.5440385836]]) mod.initialize_approximate_diffuse(1e6) cls.model = mod.ssm # Conventional filtering, smoothing, and simulation smoothing cls.model.filter_conventional = True cls.conventional_results = cls.model.smooth() n_disturbance_variates = ( (cls.model.k_endog + cls.model.k_posdef) * cls.model.nobs ) cls.conventional_sim = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states) ) # Univariate filtering, smoothing, and simulation smoothing cls.model.filter_univariate = True cls.univariate_results = cls.model.smooth() cls.univariate_sim = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states) )
def setup_class(cls, which, dtype=float, alternate_timing=False, **kwargs): # Results path = os.path.join(current_path, 'results', 'results_smoothing_generalobscov_R.csv') cls.desired = pd.read_csv(path) # Data dta = datasets.macrodata.load_pandas().data dta.index = pd.date_range(start='1959-01-01', end='2009-7-01', freq='QS') obs = dta[['realgdp', 'realcons', 'realinv']].diff().iloc[1:] if which == 'all': obs.iloc[:50, :] = np.nan obs.iloc[119:130, :] = np.nan elif which == 'partial': obs.iloc[0:50, 0] = np.nan obs.iloc[119:130, 0] = np.nan elif which == 'mixed': obs.iloc[0:50, 0] = np.nan obs.iloc[19:70, 1] = np.nan obs.iloc[39:90, 2] = np.nan obs.iloc[119:130, 0] = np.nan obs.iloc[119:130, 2] = np.nan # Create the model mod = MLEModel(obs, k_states=3, k_posdef=3, **kwargs) mod['design'] = np.eye(3) X = (np.arange(9) + 1).reshape((3, 3)) / 10. mod['obs_cov'] = np.dot(X, X.T) mod['transition'] = np.eye(3) mod['selection'] = np.eye(3) mod['state_cov'] = np.eye(3) mod.initialize_approximate_diffuse(1e6) cls.model = mod.ssm # Conventional filtering, smoothing, and simulation smoothing cls.model.filter_conventional = True cls.conventional_results = cls.model.smooth() n_disturbance_variates = ( (cls.model.k_endog + cls.model.k_posdef) * cls.model.nobs ) cls.conventional_sim = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states) ) # Univariate filtering, smoothing, and simulation smoothing cls.model.filter_univariate = True cls.univariate_results = cls.model.smooth() cls.univariate_sim = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states) )
def test_diagnostics_nile_durbinkoopman(): # Test the diagnostic tests using the Nile dataset. Results are from # Durbin and Koopman (2012); parameter values reported on page 37; test # statistics on page 40 niledata = nile.data.load_pandas().data niledata.index = pd.date_range('1871-01-01', '1970-01-01', freq='AS') mod = MLEModel(niledata['volume'], k_states=1, initialization='approximate_diffuse', initial_variance=1e15, loglikelihood_burn=1) mod.ssm['design', 0, 0] = 1 mod.ssm['obs_cov', 0, 0] = 15099. mod.ssm['transition', 0, 0] = 1 mod.ssm['selection', 0, 0] = 1 mod.ssm['state_cov', 0, 0] = 1469.1 res = mod.filter([]) # Test Ljung-Box # Note: only 3 digits provided in the reference paper actual = res.test_serial_correlation(method='ljungbox', lags=9)[0, 0, -1] assert_allclose(actual, [8.84], atol=1e-2) # Test Jarque-Bera # Note: The book reports 0.09 for Kurtosis, because it is reporting the # statistic less the mean of the Kurtosis distribution (which is 3). norm = res.test_normality(method='jarquebera')[0] actual = [norm[0], norm[2], norm[3]] assert_allclose(actual, [0.05, -0.03, 3.09], atol=1e-2) # Test Heteroskedasticity # Note: only 2 digits provided in the book actual = res.test_heteroskedasticity(method='breakvar')[0, 0] assert_allclose(actual, [0.61], atol=1e-2)
def test_diagnostics_nile_eviews(): # Test the diagnostic tests using the Nile dataset. Results are from # "Fitting State Space Models with EViews" (Van den Bossche 2011, # Journal of Statistical Software). # For parameter values, see Figure 2 # For Ljung-Box and Jarque-Bera statistics and p-values, see Figure 5 # The Heteroskedasticity statistic is not provided in this paper. niledata = nile.data.load_pandas().data niledata.index = pd.date_range('1871-01-01', '1970-01-01', freq='AS') mod = MLEModel(niledata['volume'], k_states=1, initialization='approximate_diffuse', initial_variance=1e15, loglikelihood_burn=1) mod.ssm['design', 0, 0] = 1 mod.ssm['obs_cov', 0, 0] = np.exp(9.600350) mod.ssm['transition', 0, 0] = 1 mod.ssm['selection', 0, 0] = 1 mod.ssm['state_cov', 0, 0] = np.exp(7.348705) res = mod.filter([]) # Test Ljung-Box # Note: only 3 digits provided in the reference paper actual = res.test_serial_correlation(method='ljungbox', lags=10)[0, :, -1] assert_allclose(actual, [13.117, 0.217], atol=1e-3) # Test Jarque-Bera actual = res.test_normality(method='jarquebera')[0, :2] assert_allclose(actual, [0.041686, 0.979373], atol=1e-5)
def create_model(cls, obs, **kwargs): # Create the model with typical state space mod = MLEModel(obs, k_states=2, k_posdef=2, **kwargs) mod['design'] = np.array([[-32.47143586, 17.33779024], [-7.40264169, 1.69279859], [-209.04702853, 125.2879374]]) mod['obs_cov'] = np.diag(np.array([0.0622668, 1.95666886, 58.37473642])) mod['transition'] = np.array([[0.29935707, 0.33289005], [-0.7639868, 1.2844237]]) mod['selection'] = np.eye(2) mod['state_cov'] = np.array([[1.2, -0.25], [-0.25, 1.1]]) mod.initialize_approximate_diffuse(1e6) return mod
def test_params(): mod = MLEModel([1,2], **kwargs) # By default start_params raises NotImplementedError assert_raises(NotImplementedError, lambda: mod.start_params) # But param names are by default an empty array assert_equal(mod.param_names, []) # We can set them in the object if we want mod._start_params = [1] mod._param_names = ['a'] assert_equal(mod.start_params, [1]) assert_equal(mod.param_names, ['a'])
def test_params(): mod = MLEModel([1, 2], **kwargs) # By default start_params raises NotImplementedError assert_raises(NotImplementedError, lambda: mod.start_params) # But param names are by default an empty array assert_equal(mod.param_names, []) # We can set them in the object if we want mod._start_params = [1] mod._param_names = ['a'] assert_equal(mod.start_params, [1]) assert_equal(mod.param_names, ['a'])
def test_forecast(): # Numpy mod = MLEModel([1,2], **kwargs) res = mod.filter([]) forecast = res.forecast(steps=10) assert_allclose(forecast, np.ones((10,)) * 2) assert_allclose(res.get_forecast(steps=10).predicted_mean, forecast) # Pandas index = pd.date_range('1960-01-01', periods=2, freq='MS') mod = MLEModel(pd.Series([1,2], index=index), **kwargs) res = mod.filter([]) assert_allclose(res.forecast(steps=10), np.ones((10,)) * 2) assert_allclose(res.forecast(steps='1960-12-01'), np.ones((10,)) * 2) assert_allclose(res.get_forecast(steps=10).predicted_mean, np.ones((10,)) * 2)
def test_summary(): dates = pd.date_range(start='1980-01-01', end='1984-01-01', freq='AS') endog = pd.TimeSeries([1,2,3,4,5], index=dates) mod = MLEModel(endog, **kwargs) res = mod.filter([]) # Get the summary txt = str(res.summary()) # Test res.summary when the model has dates assert_equal(re.search('Sample:\s+01-01-1980', txt) is not None, True) assert_equal(re.search('\s+- 01-01-1984', txt) is not None, True) # Test res.summary when `model_name` was not provided assert_equal(re.search('Model:\s+MLEModel', txt) is not None, True)
def test_summary(): dates = pd.date_range(start='1980-01-01', end='1984-01-01', freq='AS') endog = pd.Series([1,2,3,4,5], index=dates) mod = MLEModel(endog, **kwargs) res = mod.filter([]) # Get the summary txt = str(res.summary()) # Test res.summary when the model has dates assert_equal(re.search('Sample:\s+01-01-1980', txt) is not None, True) assert_equal(re.search('\s+- 01-01-1984', txt) is not None, True) # Test res.summary when `model_name` was not provided assert_equal(re.search('Model:\s+MLEModel', txt) is not None, True)
def create_model(cls, obs, **kwargs): # Create the model with typical state space mod = MLEModel(obs, k_states=2, k_posdef=2, **kwargs) mod['design'] = np.array([[-32.47143586, 17.33779024], [-7.40264169, 1.69279859], [-209.04702853, 125.2879374]]) mod['obs_cov'] = np.diag( np.array([0.0622668, 1.95666886, 58.37473642])) mod['transition'] = np.array([[0.29935707, 0.33289005], [-0.7639868, 1.2844237]]) mod['selection'] = np.eye(2) mod['state_cov'] = np.array([[1.2, -0.25], [-0.25, 1.1]]) mod.initialize_approximate_diffuse(1e6) return mod
def test_init_matrices_time_invariant(): # Test setting state space system matrices in __init__, with time-invariant # matrices k_endog = 2 k_states = 3 k_posdef = 1 endog = np.zeros((10, 2)) obs_intercept = np.arange(k_endog) * 1.0 design = np.reshape( np.arange(k_endog * k_states) * 1.0, (k_endog, k_states)) obs_cov = np.reshape(np.arange(k_endog**2) * 1.0, (k_endog, k_endog)) state_intercept = np.arange(k_states) * 1.0 transition = np.reshape(np.arange(k_states**2) * 1.0, (k_states, k_states)) selection = np.reshape( np.arange(k_states * k_posdef) * 1.0, (k_states, k_posdef)) state_cov = np.reshape(np.arange(k_posdef**2) * 1.0, (k_posdef, k_posdef)) mod = MLEModel(endog, k_states=k_states, k_posdef=k_posdef, obs_intercept=obs_intercept, design=design, obs_cov=obs_cov, state_intercept=state_intercept, transition=transition, selection=selection, state_cov=state_cov) assert_allclose(mod['obs_intercept'], obs_intercept) assert_allclose(mod['design'], design) assert_allclose(mod['obs_cov'], obs_cov) assert_allclose(mod['state_intercept'], state_intercept) assert_allclose(mod['transition'], transition) assert_allclose(mod['selection'], selection) assert_allclose(mod['state_cov'], state_cov)
def setup_class(cls, dtype=float, alternate_timing=False, **kwargs): cls.results = results_kalman_filter.uc_bi # GDP and Unemployment, Quarterly, 1948.1 - 1995.3 data = pd.DataFrame( cls.results['data'], index=pd.date_range('1947-01-01', '1995-07-01', freq='QS'), columns=['GDP', 'UNEMP'] )[4:] data['GDP'] = np.log(data['GDP']) data['UNEMP'] = (data['UNEMP']/100) data['X'] = np.exp(data['GDP']) * data['UNEMP'] k_states = 2 cls.mlemodel = MLEModel(data, k_states=k_states, **kwargs) cls.model = cls.mlemodel.ssm if alternate_timing: cls.model.timing_init_filtered = True # Statespace representation cls.model['selection'] = np.eye(cls.model.k_states) # Update matrices with test parameters cls.model['design'] = np.array([[0.5, 0.2], [0, 0.8], [1, -0.5]]) cls.model['transition'] = np.array([[0.4, 0.5], [1, 0]]) cls.model['obs_cov'] = np.diag([0.2, 1.1, 0.5]) cls.model['state_cov'] = np.diag([2., 1]) # Initialization cls.model.initialize_approximate_diffuse()
def check_stationary_initialization_2dim(dtype=float): endog = np.zeros(10, dtype=dtype) # 2-dimensional example mod = MLEModel(endog, k_states=2, k_posdef=2) mod.ssm.initialize_stationary() intercept = np.array([2.3, -10.2], dtype=dtype) phi = np.array([[0.8, 0.1], [-0.2, 0.7]], dtype=dtype) sigma2 = np.array([[1.4, -0.2], [-0.2, 4.5]], dtype=dtype) mod['state_intercept'] = intercept mod['transition'] = phi mod['selection'] = np.eye(2).astype(dtype) mod['state_cov'] = sigma2 mod.ssm._initialize_filter() mod.ssm._initialize_state() _statespace = mod.ssm._statespace initial_state = np.array(_statespace.initial_state) initial_state_cov = np.array(_statespace.initial_state_cov) desired = np.linalg.solve(np.eye(2).astype(dtype) - phi, intercept) assert_allclose(initial_state, desired) desired = solve_discrete_lyapunov(phi, sigma2) # precision reductions only required for single precision float / complex assert_allclose(initial_state_cov, desired, atol=1e-5)
def check_stationary_initialization_1dim(dtype=float): endog = np.zeros(10, dtype=dtype) # 1-dimensional example mod = MLEModel(endog, k_states=1, k_posdef=1) mod.ssm.initialize_stationary() intercept = np.array([2.3], dtype=dtype) phi = np.diag([0.9]).astype(dtype) sigma2 = np.diag([1.3]).astype(dtype) mod['state_intercept'] = intercept mod['transition'] = phi mod['selection'] = np.eye(1).astype(dtype) mod['state_cov'] = sigma2 mod.ssm._initialize_filter() mod.ssm._initialize_state() _statespace = mod.ssm._statespace initial_state = np.array(_statespace.initial_state) initial_state_cov = np.array(_statespace.initial_state_cov) # precision reductions only required for float complex case # mean = intercept + phi * mean # intercept = (1 - phi) * mean # mean = intercept / (1 - phi) assert_allclose(initial_state, intercept / (1 - phi[0, 0])) desired = np.linalg.inv(np.eye(1) - phi).dot(intercept) assert_allclose(initial_state, desired) # var = phi**2 var + sigma2 # var = sigma2 / (1 - phi**2) assert_allclose(initial_state_cov, sigma2 / (1 - phi**2)) assert_allclose(initial_state_cov, solve_discrete_lyapunov(phi, sigma2))
def test_filter(): endog = np.array([1., 2.]) mod = MLEModel(endog, **kwargs) # Test return of ssm object res = mod.filter([], return_ssm=True) assert_equal(isinstance(res, kalman_filter.FilterResults), True) # Test return of full results object res = mod.filter([]) assert_equal(isinstance(res, MLEResultsWrapper), True) assert_equal(res.cov_type, 'opg') # Test return of full results object, specific covariance type res = mod.filter([], cov_type='oim') assert_equal(isinstance(res, MLEResultsWrapper), True) assert_equal(res.cov_type, 'oim')
def test_predict(): dates = pd.date_range(start='1980-01-01', end='1981-01-01', freq='AS') endog = pd.TimeSeries([1,2], index=dates) mod = MLEModel(endog, **kwargs) res = mod.filter([]) # Test that predict with start=None, end=None does prediction with full # dataset assert_equal(res.predict().shape, (mod.k_endog, mod.nobs)) # Test a string value to the dynamic option assert_allclose(res.predict(dynamic='1981-01-01'), res.predict()) # Test an invalid date string value to the dynamic option assert_raises(ValueError, res.predict, dynamic='1982-01-01') # Test predict with full results assert_equal(isinstance(res.predict(full_results=True), kalman_filter.FilterResults), True)
def test_predict(): dates = pd.date_range(start='1980-01-01', end='1981-01-01', freq='AS') endog = pd.TimeSeries([1, 2], index=dates) mod = MLEModel(endog, **kwargs) res = mod.filter([]) # Test that predict with start=None, end=None does prediction with full # dataset assert_equal(res.predict().shape, (mod.k_endog, mod.nobs)) # Test a string value to the dynamic option assert_allclose(res.predict(dynamic='1981-01-01'), res.predict()) # Test an invalid date string value to the dynamic option assert_raises(ValueError, res.predict, dynamic='1982-01-01') # Test predict with full results assert_equal( isinstance(res.predict(full_results=True), kalman_filter.FilterResults), True)
def test_predict(): dates = pd.date_range(start='1980-01-01', end='1981-01-01', freq='AS') endog = pd.TimeSeries([1,2], index=dates) mod = MLEModel(endog, **kwargs) res = mod.filter([]) # Test that predict with start=None, end=None does prediction with full # dataset predict = res.predict() assert_equal(predict.shape, (mod.nobs,)) assert_allclose(res.get_prediction().predicted_mean, predict) # Test a string value to the dynamic option assert_allclose(res.predict(dynamic='1981-01-01'), res.predict()) # Test an invalid date string value to the dynamic option assert_raises(ValueError, res.predict, dynamic='1982-01-01') # Test for passing a string to predict when dates are not set mod = MLEModel([1,2], **kwargs) res = mod.filter([]) assert_raises(ValueError, res.predict, dynamic='string')
def test_summary(): dates = pd.date_range(start='1980-01-01', end='1984-01-01', freq='AS') endog = pd.Series([1,2,3,4,5], index=dates) mod = MLEModel(endog, **kwargs) res = mod.filter([]) # Get the summary txt = str(res.summary()) # Test res.summary when the model has dates assert_equal(re.search('Sample:\s+01-01-1980', txt) is not None, True) assert_equal(re.search('\s+- 01-01-1984', txt) is not None, True) # Test res.summary when `model_name` was not provided assert_equal(re.search('Model:\s+MLEModel', txt) is not None, True) # Smoke test that summary still works when diagnostic tests fail res.filter_results._standardized_forecasts_error[:] = np.nan res.summary() res.filter_results._standardized_forecasts_error = 1 res.summary() res.filter_results._standardized_forecasts_error = 'a' res.summary()
def check_endog(endog, nobs=2, k_endog=1, **kwargs): # create the model mod = MLEModel(endog, **kwargs) # the data directly available in the model is the Statsmodels version of # the data; it should be 2-dim, C-contiguous, long-shaped: # (nobs, k_endog) == (2, 1) assert_equal(mod.endog.ndim, 2) assert_equal(mod.endog.flags['C_CONTIGUOUS'], True) assert_equal(mod.endog.shape, (nobs, k_endog)) # the data in the `ssm` object is the state space version of the data; it # should be 2-dim, F-contiguous, wide-shaped (k_endog, nobs) == (1, 2) # and it should share data with mod.endog assert_equal(mod.ssm.endog.ndim, 2) assert_equal(mod.ssm.endog.flags['F_CONTIGUOUS'], True) assert_equal(mod.ssm.endog.shape, (k_endog, nobs)) assert_equal(mod.ssm.endog.base is mod.endog, True) return mod
def test_predict(): dates = pd.date_range(start='1980-01-01', end='1981-01-01', freq='AS') endog = pd.Series([1, 2], index=dates) mod = MLEModel(endog, **kwargs) res = mod.filter([]) # Test that predict with start=None, end=None does prediction with full # dataset predict = res.predict() assert_equal(predict.shape, (mod.nobs, )) assert_allclose(res.get_prediction().predicted_mean, predict) # Test a string value to the dynamic option assert_allclose(res.predict(dynamic='1981-01-01'), res.predict()) # Test an invalid date string value to the dynamic option # assert_raises(ValueError, res.predict, dynamic='1982-01-01') # Test for passing a string to predict when dates are not set mod = MLEModel([1, 2], **kwargs) res = mod.filter([]) assert_raises(KeyError, res.predict, dynamic='string')
def test_transform(): # The transforms in MLEModel are noops mod = MLEModel([1, 2], **kwargs) # Test direct transform, untransform assert_allclose(mod.transform_params([2, 3]), [2, 3]) assert_allclose(mod.untransform_params([2, 3]), [2, 3]) # Smoke test for transformation in `filter`, `update`, `loglike`, # `loglikeobs` mod.filter([], transformed=False) mod.update([], transformed=False) mod.loglike([], transformed=False) mod.loglikeobs([], transformed=False) # Note that mod is an SARIMAX instance, and the two parameters are # variances mod, _ = get_dummy_mod(fit=False) # Test direct transform, untransform assert_allclose(mod.transform_params([2, 3]), [4, 9]) assert_allclose(mod.untransform_params([4, 9]), [2, 3]) # Test transformation in `filter` res = mod.filter([2, 3], transformed=True) assert_allclose(res.params, [2, 3]) res = mod.filter([2, 3], transformed=False) assert_allclose(res.params, [4, 9])
def test_from_formula(): assert_raises(NotImplementedError, lambda: MLEModel.from_formula(1, 2, 3))
def test_from_formula(): assert_raises(NotImplementedError, lambda: MLEModel.from_formula(1,2,3))
def test_transform(): # The transforms in MLEModel are noops mod = MLEModel([1,2], **kwargs) # Test direct transform, untransform assert_allclose(mod.transform_params([2, 3]), [2, 3]) assert_allclose(mod.untransform_params([2, 3]), [2, 3]) # Smoke test for transformation in `filter`, `update`, `loglike`, # `loglikeobs` mod.filter([], transformed=False) mod.update([], transformed=False) mod.loglike([], transformed=False) mod.loglikeobs([], transformed=False) # Note that mod is an SARIMAX instance, and the two parameters are # variances mod, _ = get_dummy_mod(fit=False) # Test direct transform, untransform assert_allclose(mod.transform_params([2, 3]), [4, 9]) assert_allclose(mod.untransform_params([4, 9]), [2, 3]) # Test transformation in `filter` res = mod.filter([2, 3], transformed=True) assert_allclose(res.params, [2, 3]) res = mod.filter([2, 3], transformed=False) assert_allclose(res.params, [4, 9])
def test_basic_endog(): # Test various types of basic python endog inputs (e.g. lists, scalars...) # Check cannot call with non-array_like # fails due to checks in Statsmodels base classes assert_raises(ValueError, MLEModel, endog=1, k_states=1) assert_raises(ValueError, MLEModel, endog='a', k_states=1) assert_raises(ValueError, MLEModel, endog=True, k_states=1) # Check behavior with different types mod = MLEModel([1], **kwargs) res = mod.filter([]) assert_equal(res.filter_results.endog, [[1]]) mod = MLEModel([1.], **kwargs) res = mod.filter([]) assert_equal(res.filter_results.endog, [[1]]) mod = MLEModel([True], **kwargs) res = mod.filter([]) assert_equal(res.filter_results.endog, [[1]]) mod = MLEModel(['a'], **kwargs) # raises error due to inability coerce string to numeric assert_raises(ValueError, mod.filter, []) # Check that a different iterable tpyes give the expected result endog = [1., 2.] mod = check_endog(endog, **kwargs) mod.filter([]) endog = [[1.], [2.]] mod = check_endog(endog, **kwargs) mod.filter([]) endog = (1., 2.) mod = check_endog(endog, **kwargs) mod.filter([])
def test_basic_endog(): # Test various types of basic python endog inputs (e.g. lists, scalars...) # Check cannot call with non-array-like # fails due to checks in Statsmodels base classes assert_raises(ValueError, MLEModel, endog=1, k_states=1) assert_raises(ValueError, MLEModel, endog='a', k_states=1) assert_raises(ValueError, MLEModel, endog=True, k_states=1) # Check behavior with different types mod = MLEModel([1], **kwargs) res = mod.filter([]) assert_equal(res.filter_results.endog, [[1]]) mod = MLEModel([1.], **kwargs) res = mod.filter([]) assert_equal(res.filter_results.endog, [[1]]) mod = MLEModel([True], **kwargs) res = mod.filter([]) assert_equal(res.filter_results.endog, [[1]]) mod = MLEModel(['a'], **kwargs) # raises error due to inability coerce string to numeric assert_raises(ValueError, mod.filter, []) # Check that a different iterable tpyes give the expected result endog = [1.,2.] mod = check_endog(endog, **kwargs) mod.filter([]) endog = [[1.],[2.]] mod = check_endog(endog, **kwargs) mod.filter([]) endog = (1.,2.) mod = check_endog(endog, **kwargs) mod.filter([])
def setup_class(cls, which='none', **kwargs): # Results path = os.path.join(current_path, 'results', 'results_smoothing_generalobscov_R.csv') cls.desired = pd.read_csv(path) # Data dta = datasets.macrodata.load_pandas().data dta.index = pd.date_range(start='1959-01-01', end='2009-7-01', freq='QS') obs = dta[['realgdp', 'realcons', 'realinv']].diff().iloc[1:] if which == 'all': obs.iloc[:50, :] = np.nan obs.iloc[119:130, :] = np.nan elif which == 'partial': obs.iloc[0:50, 0] = np.nan obs.iloc[119:130, 0] = np.nan elif which == 'mixed': obs.iloc[0:50, 0] = np.nan obs.iloc[19:70, 1] = np.nan obs.iloc[39:90, 2] = np.nan obs.iloc[119:130, 0] = np.nan obs.iloc[119:130, 2] = np.nan # Create the model mod = MLEModel(obs, k_states=3, k_posdef=3, **kwargs) mod['design'] = np.eye(3) mod['obs_cov'] = np.array([ [609.0746647855, 0., 0.], [0., 1.8774916622, 0.], [0., 0., 124.6768281675]]) mod['transition'] = np.array([ [-0.8110473405, 1.8005304445, 1.0215975772], [-1.9846632699, 2.4091302213, 1.9264449765], [0.9181658823, -0.2442384581, -0.6393462272]]) mod['selection'] = np.eye(3) mod['state_cov'] = np.array([ [1552.9758843938, 612.7185121905, 877.6157204992], [612.7185121905, 467.8739411204, 70.608037339], [877.6157204992, 70.608037339, 900.5440385836]]) mod.initialize_approximate_diffuse(1e6) cls.model = mod.ssm # Conventional filtering, smoothing, and simulation smoothing cls.model.filter_conventional = True cls.conventional_results = cls.model.smooth() n_disturbance_variates = ( (cls.model.k_endog + cls.model.k_posdef) * cls.model.nobs ) cls.conventional_sim = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states) ) # Univariate filtering, smoothing, and simulation smoothing cls.model.filter_univariate = True cls.univariate_results = cls.model.smooth() cls.univariate_sim = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states) )
def setup_class(cls, dtype=float, alternate_timing=False, **kwargs): cls.true = results_kalman_filter.uc_bi cls.true_states = pd.DataFrame(cls.true['states']) # GDP and Unemployment, Quarterly, 1948.1 - 1995.3 data = pd.DataFrame(cls.true['data'], index=pd.date_range('1947-01-01', '1995-07-01', freq='QS'), columns=['GDP', 'UNEMP'])[4:] data['GDP'] = np.log(data['GDP']) data['UNEMP'] = (data['UNEMP'] / 100) k_states = 6 cls.mlemodel = MLEModel(data, k_states=k_states, **kwargs) cls.model = cls.mlemodel.ssm # Statespace representation cls.model.design[:, :, 0] = [[1, 1, 0, 0, 0, 0], [0, 0, 0, 0, 0, 1]] cls.model.transition[([0, 0, 1, 1, 2, 3, 4, 5], [0, 4, 1, 2, 1, 2, 4, 5], [0, 0, 0, 0, 0, 0, 0, 0])] = [1, 1, 0, 0, 1, 1, 1, 1] cls.model.selection = np.eye(cls.model.k_states) # Update matrices with given parameters (sigma_v, sigma_e, sigma_w, sigma_vl, sigma_ec, phi_1, phi_2, alpha_1, alpha_2, alpha_3) = np.array(cls.true['parameters'], ) cls.model.design[([1, 1, 1], [1, 2, 3], [0, 0, 0])] = [alpha_1, alpha_2, alpha_3] cls.model.transition[([1, 1], [1, 2], [0, 0])] = [phi_1, phi_2] cls.model.obs_cov[1, 1, 0] = sigma_ec**2 cls.model.state_cov[np.diag_indices(k_states) + (np.zeros(k_states, dtype=int), )] = [ sigma_v**2, sigma_e**2, 0, 0, sigma_w**2, sigma_vl**2 ] # Initialization initial_state = np.zeros((k_states, )) initial_state_cov = np.eye(k_states) * 100 # Initialization: cls.modification if not alternate_timing: initial_state_cov = np.dot( np.dot(cls.model.transition[:, :, 0], initial_state_cov), cls.model.transition[:, :, 0].T) else: cls.model.timing_init_filtered = True cls.model.initialize_known(initial_state, initial_state_cov) # Conventional filtering, smoothing, and simulation smoothing cls.model.filter_conventional = True cls.conventional_results = cls.model.smooth() n_disturbance_variates = ((cls.model.k_endog + cls.model.k_posdef) * cls.model.nobs) cls.conventional_sim = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states)) # Univariate filtering, smoothing, and simulation smoothing cls.model.filter_univariate = True cls.univariate_results = cls.model.smooth() cls.univariate_sim = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states))
def test_forecast(): mod = MLEModel([1,2], **kwargs) res = mod.filter([]) assert_allclose(res.forecast(steps=10), [[2]*10])
def test_numpy_endog(): # Test various types of numpy endog inputs # Check behavior of the link maintained between passed `endog` and # `mod.endog` arrays endog = np.array([1., 2.]) mod = MLEModel(endog, **kwargs) assert_equal(mod.endog.base is not mod.data.orig_endog, True) assert_equal(mod.endog.base is not endog, True) assert_equal(mod.data.orig_endog.base is not endog, True) endog[0] = 2 # there is no link to mod.endog assert_equal(mod.endog, np.r_[1, 2].reshape(2,1)) # there remains a link to mod.data.orig_endog assert_equal(mod.data.orig_endog, endog) # Check behavior with different memory layouts / shapes # Example (failure): 0-dim array endog = np.array(1.) # raises error due to len(endog) failing in Statsmodels base classes assert_raises(TypeError, check_endog, endog, **kwargs) # Example : 1-dim array, both C- and F-contiguous, length 2 endog = np.array([1.,2.]) assert_equal(endog.ndim, 1) assert_equal(endog.flags['C_CONTIGUOUS'], True) assert_equal(endog.flags['F_CONTIGUOUS'], True) assert_equal(endog.shape, (2,)) mod = check_endog(endog, **kwargs) mod.filter([]) # Example : 2-dim array, C-contiguous, long-shaped: (nobs, k_endog) endog = np.array([1., 2.]).reshape(2, 1) assert_equal(endog.ndim, 2) assert_equal(endog.flags['C_CONTIGUOUS'], True) assert_equal(endog.flags['F_CONTIGUOUS'], False) assert_equal(endog.shape, (2, 1)) mod = check_endog(endog, **kwargs) mod.filter([]) # Example : 2-dim array, C-contiguous, wide-shaped: (k_endog, nobs) endog = np.array([1., 2.]).reshape(1, 2) assert_equal(endog.ndim, 2) assert_equal(endog.flags['C_CONTIGUOUS'], True) assert_equal(endog.flags['F_CONTIGUOUS'], False) assert_equal(endog.shape, (1, 2)) # raises error because arrays are always interpreted as # (nobs, k_endog), which means that k_endog=2 is incompatibile with shape # of design matrix (1, 1) assert_raises(ValueError, check_endog, endog, **kwargs) # Example : 2-dim array, F-contiguous, long-shaped (nobs, k_endog) endog = np.array([1., 2.]).reshape(1, 2).transpose() assert_equal(endog.ndim, 2) assert_equal(endog.flags['C_CONTIGUOUS'], False) assert_equal(endog.flags['F_CONTIGUOUS'], True) assert_equal(endog.shape, (2, 1)) mod = check_endog(endog, **kwargs) mod.filter([]) # Example : 2-dim array, F-contiguous, wide-shaped (k_endog, nobs) endog = np.array([1., 2.]).reshape(2, 1).transpose() assert_equal(endog.ndim, 2) assert_equal(endog.flags['C_CONTIGUOUS'], False) assert_equal(endog.flags['F_CONTIGUOUS'], True) assert_equal(endog.shape, (1, 2)) # raises error because arrays are always interpreted as # (nobs, k_endog), which means that k_endog=2 is incompatibile with shape # of design matrix (1, 1) assert_raises(ValueError, check_endog, endog, **kwargs) # Example (failure): 3-dim array endog = np.array([1., 2.]).reshape(2, 1, 1) # raises error due to direct ndim check in Statsmodels base classes assert_raises(ValueError, check_endog, endog, **kwargs) # Example : np.array with 2 columns # Update kwargs for k_endog=2 kwargs2 = { 'k_states': 1, 'design': [[1], [0.]], 'obs_cov': [[1, 0], [0, 1]], 'transition': [[1]], 'selection': [[1]], 'state_cov': [[1]], 'initialization': 'approximate_diffuse' } endog = np.array([[1., 2.], [3., 4.]]) mod = check_endog(endog, k_endog=2, **kwargs2) mod.filter([])
def test_forecast(): mod = MLEModel([1, 2], **kwargs) res = mod.filter([]) assert_allclose(res.forecast(steps=10), [[2] * 10])
def test_numpy_endog(): # Test various types of numpy endog inputs # Check behavior of the link maintained between passed `endog` and # `mod.endog` arrays endog = np.array([1., 2.]) mod = MLEModel(endog, **kwargs) assert_equal(mod.endog.base is not mod.data.orig_endog, True) assert_equal(mod.endog.base is not endog, True) assert_equal(mod.data.orig_endog.base is not endog, True) endog[0] = 2 # there is no link to mod.endog assert_equal(mod.endog, np.r_[1, 2].reshape(2, 1)) # there remains a link to mod.data.orig_endog assert_equal(mod.data.orig_endog, endog) # Check behavior with different memory layouts / shapes # Example (failure): 0-dim array endog = np.array(1.) # raises error due to len(endog) failing in Statsmodels base classes assert_raises(TypeError, check_endog, endog, **kwargs) # Example : 1-dim array, both C- and F-contiguous, length 2 endog = np.array([1., 2.]) assert_equal(endog.ndim, 1) assert_equal(endog.flags['C_CONTIGUOUS'], True) assert_equal(endog.flags['F_CONTIGUOUS'], True) assert_equal(endog.shape, (2, )) mod = check_endog(endog, **kwargs) mod.filter([]) # Example : 2-dim array, C-contiguous, long-shaped: (nobs, k_endog) endog = np.array([1., 2.]).reshape(2, 1) assert_equal(endog.ndim, 2) assert_equal(endog.flags['C_CONTIGUOUS'], True) # On newer numpy (>= 0.10), this array is (rightly) both C and F contiguous # assert_equal(endog.flags['F_CONTIGUOUS'], False) assert_equal(endog.shape, (2, 1)) mod = check_endog(endog, **kwargs) mod.filter([]) # Example : 2-dim array, C-contiguous, wide-shaped: (k_endog, nobs) endog = np.array([1., 2.]).reshape(1, 2) assert_equal(endog.ndim, 2) assert_equal(endog.flags['C_CONTIGUOUS'], True) # On newer numpy (>= 0.10), this array is (rightly) both C and F contiguous # assert_equal(endog.flags['F_CONTIGUOUS'], False) assert_equal(endog.shape, (1, 2)) # raises error because arrays are always interpreted as # (nobs, k_endog), which means that k_endog=2 is incompatibile with shape # of design matrix (1, 1) assert_raises(ValueError, check_endog, endog, **kwargs) # Example : 2-dim array, F-contiguous, long-shaped (nobs, k_endog) endog = np.array([1., 2.]).reshape(1, 2).transpose() assert_equal(endog.ndim, 2) # On newer numpy (>= 0.10), this array is (rightly) both C and F contiguous # assert_equal(endog.flags['C_CONTIGUOUS'], False) assert_equal(endog.flags['F_CONTIGUOUS'], True) assert_equal(endog.shape, (2, 1)) mod = check_endog(endog, **kwargs) mod.filter([]) # Example : 2-dim array, F-contiguous, wide-shaped (k_endog, nobs) endog = np.array([1., 2.]).reshape(2, 1).transpose() assert_equal(endog.ndim, 2) # On newer numpy (>= 0.10), this array is (rightly) both C and F contiguous # assert_equal(endog.flags['C_CONTIGUOUS'], False) assert_equal(endog.flags['F_CONTIGUOUS'], True) assert_equal(endog.shape, (1, 2)) # raises error because arrays are always interpreted as # (nobs, k_endog), which means that k_endog=2 is incompatibile with shape # of design matrix (1, 1) assert_raises(ValueError, check_endog, endog, **kwargs) # Example (failure): 3-dim array endog = np.array([1., 2.]).reshape(2, 1, 1) # raises error due to direct ndim check in Statsmodels base classes assert_raises(ValueError, check_endog, endog, **kwargs) # Example : np.array with 2 columns # Update kwargs for k_endog=2 kwargs2 = { 'k_states': 1, 'design': [[1], [0.]], 'obs_cov': [[1, 0], [0, 1]], 'transition': [[1]], 'selection': [[1]], 'state_cov': [[1]], 'initialization': 'approximate_diffuse' } endog = np.array([[1., 2.], [3., 4.]]) mod = check_endog(endog, k_endog=2, **kwargs2) mod.filter([])
def setup_class(cls, which='mixed', *args, **kwargs): # Data dta = datasets.macrodata.load_pandas().data dta.index = pd.date_range(start='1959-01-01', end='2009-7-01', freq='QS') obs = np.log(dta[['realgdp','realcons','realinv']]).diff().iloc[1:] * 400 if which == 'all': obs.iloc[:50, :] = np.nan obs.iloc[119:130, :] = np.nan elif which == 'partial': obs.iloc[0:50, 0] = np.nan obs.iloc[119:130, 0] = np.nan elif which == 'mixed': obs.iloc[0:50, 0] = np.nan obs.iloc[19:70, 1] = np.nan obs.iloc[39:90, 2] = np.nan obs.iloc[119:130, 0] = np.nan obs.iloc[119:130, 2] = np.nan # Create the model with typical state space mod = MLEModel(obs, k_states=2, k_posdef=2, **kwargs) mod['design'] = np.array([[-32.47143586, 17.33779024], [-7.40264169, 1.69279859], [-209.04702853, 125.2879374]]) mod['obs_cov'] = np.diag( np.array([0.0622668, 1.95666886, 58.37473642])) mod['transition'] = np.array([[0.29935707, 0.33289005], [-0.7639868, 1.2844237]]) mod['selection'] = np.eye(2) mod['state_cov'] = np.array([[1.2, -0.25], [-0.25, 1.1]]) mod.initialize_approximate_diffuse(1e6) cls.model = mod.ssm n_disturbance_variates = ( (cls.model.k_endog + cls.model.k_posdef) * cls.model.nobs ) # Collapsed filtering, smoothing, and simulation smoothing cls.model.filter_collapsed = True cls.results_b = cls.model.smooth() cls.sim_b = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states) ) # Conventional filtering, smoothing, and simulation smoothing cls.model.filter_collapsed = False cls.results_a = cls.model.smooth() cls.sim_a = cls.model.simulation_smoother( disturbance_variates=np.zeros(n_disturbance_variates), initial_state_variates=np.zeros(cls.model.k_states) ) # Create the model with augmented state space kwargs.pop('filter_collapsed', None) mod = MLEModel(obs, k_states=4, k_posdef=2, **kwargs) mod['design', :3, :2] = np.array([[-32.47143586, 17.33779024], [-7.40264169, 1.69279859], [-209.04702853, 125.2879374]]) mod['obs_cov'] = np.diag( np.array([0.0622668, 1.95666886, 58.37473642])) mod['transition', :2, :2] = np.array([[0.29935707, 0.33289005], [-0.7639868, 1.2844237]]) mod['transition', 2:, :2] = np.eye(2) mod['selection', :2, :2] = np.eye(2) mod['state_cov'] = np.array([[1.2, -0.25], [-0.25, 1.1]]) mod.initialize_approximate_diffuse(1e6) cls.augmented_model = mod.ssm cls.augmented_results = mod.ssm.smooth()