def test_phreg(self): np.random.seed(8742) n = 300 x1 = np.random.normal(size=n) x2 = np.random.normal(size=n) event_time = np.random.exponential(size=n) * np.exp(x1) obs_time = np.random.exponential(size=n) time = np.where(event_time < obs_time, event_time, obs_time) status = np.where(time == event_time, 1, 0) df = pd.DataFrame({"time": time, "status": status, "x1": x1, "x2": x2}) df.loc[10:40, 'time'] = np.nan df.loc[10:40, 'status'] = np.nan df.loc[30:50, 'x1'] = np.nan df.loc[40:60, 'x2'] = np.nan from statsmodels.duration.hazard_regression import PHReg idata = mice.MICEData(df) idata.set_imputer("time", "0 + x1 + x2", model_class=PHReg, init_kwds={"status": mice.PatsyFormula("status")}, predict_kwds={"pred_type": "hr"}) x = idata.next_sample() assert (isinstance(x, pd.DataFrame))
def test_MICE1(self): df = gendat() imp_data = mice.MICEData(df) mi = mice.MICE("y ~ x1 + x2 + x1:x2", sm.OLS, imp_data) from statsmodels.regression.linear_model import RegressionResultsWrapper for j in range(3): x = mi.next_sample() assert (issubclass(x.__class__, RegressionResultsWrapper))
def test_MICE(self): df = gendat() imp_data = mice.MICEData(df) mi = mice.MICE("y ~ x1 + x2 + x1:x2", sm.OLS, imp_data) result = mi.fit(1, 3) assert (issubclass(result.__class__, mice.MICEResults)) # Smoke test for results smr = result.summary()
def test_plot_imputed_hist(self): df = gendat() imp_data = mice.MICEData(df) imp_data.update_all() plt.clf() for plot_points in False, True: fig = imp_data.plot_imputed_hist('x4') fig.get_axes()[0].set_title('plot_imputed_hist') close_or_save(pdf, fig)
def test_fit_obs(self): df = gendat() imp_data = mice.MICEData(df) imp_data.update_all() plt.clf() for plot_points in False, True: fig = imp_data.plot_fit_obs('x4', plot_points=plot_points) fig.get_axes()[0].set_title('plot_fit_scatterplot') close_or_save(pdf, fig)
def test_MICE2(self): from statsmodels.genmod.generalized_linear_model import GLMResultsWrapper df = gendat() imp_data = mice.MICEData(df) mi = mice.MICE("x3 ~ x1 + x2", sm.GLM, imp_data, init_kwds={"family": sm.families.Binomial()}) for j in range(3): x = mi.next_sample() assert(isinstance(x, GLMResultsWrapper)) assert(isinstance(x.family, sm.families.Binomial))
def test_plot_missing_pattern(self): df = gendat() imp_data = mice.MICEData(df) for row_order in "pattern", "raw": for hide_complete_rows in False, True: for color_row_patterns in False, True: plt.clf() fig = imp_data.plot_missing_pattern(row_order=row_order, hide_complete_rows=hide_complete_rows, color_row_patterns=color_row_patterns) close_or_save(pdf, fig)
def test_next_sample(self): df = gendat() imp_data = mice.MICEData(df) all_x = [] for j in range(2): x = imp_data.next_sample() assert (isinstance(x, pd.DataFrame)) assert_equal(df.shape, x.shape) all_x.append(x) # The returned dataframes are all the same object assert (all_x[0] is all_x[1])
def test_set_imputer(self): """ Test with specified perturbation method. """ from statsmodels.regression.linear_model import RegressionResultsWrapper from statsmodels.genmod.generalized_linear_model import GLMResultsWrapper df = gendat() orig = df.copy() mx = pd.notnull(df) nrow, ncol = df.shape imp_data = mice.MICEData(df) imp_data.set_imputer('x1', 'x3 + x4 + x3*x4') imp_data.set_imputer('x2', 'x4 + I(x5**2)') imp_data.set_imputer('x3', model_class=sm.GLM, init_kwds={"family": sm.families.Binomial()}) imp_data.update_all() assert_equal(imp_data.data.shape[0], nrow) assert_equal(imp_data.data.shape[1], ncol) assert_allclose(orig[mx], imp_data.data[mx]) for j in range(1, 6): if j == 3: assert_equal(isinstance(imp_data.models['x3'], sm.GLM), True) assert_equal( isinstance(imp_data.models['x3'].family, sm.families.Binomial), True) assert_equal( isinstance(imp_data.results['x3'], GLMResultsWrapper), True) else: assert_equal(isinstance(imp_data.models['x%d' % j], sm.OLS), True) assert_equal( isinstance(imp_data.results['x%d' % j], RegressionResultsWrapper), True) fml = 'x1 ~ x3 + x4 + x3*x4' assert_equal(imp_data.conditional_formula['x1'], fml) fml = 'x4 ~ x1 + x2 + x3 + x5 + y' assert_equal(imp_data.conditional_formula['x4'], fml) assert_equal(imp_data._cycle_order, ['x5', 'x3', 'x4', 'y', 'x2', 'x1'])
def test_combine(self): np.random.seed(3897) x1 = np.random.normal(size=300) x2 = np.random.normal(size=300) y = x1 + x2 + np.random.normal(size=300) x1[0:100] = np.nan x2[250:] = np.nan df = pd.DataFrame({"x1": x1, "x2": x2, "y": y}) idata = mice.MICEData(df) mi = mice.MICE("y ~ x1 + x2", sm.OLS, idata, n_skip=20) result = mi.fit(10, 20) fmi = np.asarray([0.1920533, 0.1587287, 0.33174032]) assert_allclose(result.frac_miss_info, fmi, atol=1e-5) params = np.asarray([-0.05397474, 0.97273307, 1.01652293]) assert_allclose(result.params, params, atol=1e-5) tvalues = np.asarray([-0.84781698, 15.10491582, 13.59998039]) assert_allclose(result.tvalues, tvalues, atol=1e-5)
def test_pertmeth(self): """ Test with specified perturbation method. """ df = gendat() orig = df.copy() mx = pd.notnull(df) nrow, ncol = df.shape for pert_meth in "gaussian", "boot": imp_data = mice.MICEData(df, perturbation_method=pert_meth) for k in range(2): imp_data.update_all() assert_equal(imp_data.data.shape[0], nrow) assert_equal(imp_data.data.shape[1], ncol) assert_allclose(orig[mx], imp_data.data[mx]) assert_equal(imp_data._cycle_order, ['x5', 'x3', 'x4', 'y', 'x2', 'x1'])
def test_default(self): """ Test with all defaults. """ df = gendat() orig = df.copy() mx = pd.notnull(df) imp_data = mice.MICEData(df) nrow, ncol = df.shape assert_allclose(imp_data.ix_miss['x1'], np.arange(60)) assert_allclose(imp_data.ix_obs['x1'], np.arange(60, 200)) assert_allclose(imp_data.ix_miss['x2'], np.arange(40)) assert_allclose(imp_data.ix_miss['x3'], np.arange(10, 30, 2)) assert_allclose( imp_data.ix_obs['x3'], np.concatenate((np.arange(10), np.arange(11, 30, 2), np.arange(30, 200)))) for k in range(3): imp_data.update_all() assert_equal(imp_data.data.shape[0], nrow) assert_equal(imp_data.data.shape[1], ncol) assert_allclose(orig[mx], imp_data.data[mx]) fml = 'x1 ~ x2 + x3 + x4 + x5 + y' assert_equal(imp_data.conditional_formula['x1'], fml) assert_equal(imp_data._cycle_order, ['x5', 'x3', 'x4', 'y', 'x2', 'x1']) # Should make a copy assert (not (df is imp_data.data)) (endog_obs, exog_obs, exog_miss, predict_obs_kwds, predict_miss_kwds) = imp_data.get_split_data('x3') assert_equal(len(endog_obs), 190) assert_equal(exog_obs.shape, [190, 6]) assert_equal(exog_miss.shape, [10, 6])