def test_dtype_object():
# GH#880
X = np.random.random((40, 2))
df = pd.DataFrame(X)
df[2] = np.random.randint(2, size=40).astype('object')
df['constant'] = 1
y = pd.Series(np.random.randint(2, size=40))
with pytest.raises(ValueError):
sm_data.handle_data(y, df)
def test_extra_kwargs_2d(self):
sigma = np.random.random((25, 25))
sigma = sigma + sigma.T - np.diag(np.diag(sigma))
data = sm_data.handle_data(self.y, self.X, 'drop', sigma=sigma)
idx = ~np.isnan(np.c_[self.y, self.X]).any(axis=1)
sigma = sigma[idx][:, idx]
np.testing.assert_array_equal(data.sigma, sigma)
def setup_class(cls):
cls.endog = endog = pd.DataFrame(np.random.random((10, 4)),
columns=['y_1', 'y_2', 'y_3', 'y_4'])
exog = pd.DataFrame(np.random.random((10, 2)), columns=['x_1', 'x_2'])
exog.insert(0, 'const', 1)
cls.exog = exog
cls.data = sm_data.handle_data(cls.endog, cls.exog)
nrows = 10
nvars = 3
neqs = 4
cls.col_input = np.random.random(nvars)
cls.col_result = pd.Series(cls.col_input, index=exog.columns)
cls.row_input = np.random.random(nrows)
cls.row_result = pd.Series(cls.row_input, index=exog.index)
cls.cov_input = np.random.random((nvars, nvars))
cls.cov_result = pd.DataFrame(cls.cov_input,
index=exog.columns,
columns=exog.columns)
cls.cov_eq_input = np.random.random((neqs, neqs))
cls.cov_eq_result = pd.DataFrame(cls.cov_eq_input,
index=endog.columns,
columns=endog.columns)
cls.col_eq_input = np.random.random((nvars, neqs))
cls.col_eq_result = pd.DataFrame(cls.col_eq_input,
index=exog.columns,
columns=endog.columns)
cls.xnames = ['const', 'x_1', 'x_2']
cls.ynames = ['y_1', 'y_2', 'y_3', 'y_4']
cls.row_labels = cls.exog.index
def setup_class(cls):
super(TestArrays1dExog, cls).setup_class()
cls.endog = np.random.random(10)
exog = np.random.random(10)
cls.data = sm_data.handle_data(cls.endog, exog)
cls.exog = exog[:, None]
cls.xnames = ['x1']
cls.ynames = 'y'
def test_labels(self):
2, 10, 14 # WTF
labels = pd.Index([
0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21,
22, 23, 24
])
data = sm_data.handle_data(self.y, self.X, 'drop')
assert data.row_labels.equals(labels)
def setup_class(cls):
super(TestStructarrays, cls).setup_class()
cls.endog = np.random.random(9).view([('y_1', 'f8')]).view(np.recarray)
exog = np.random.random(9 * 3).view([('const', 'f8'), ('x_1', 'f8'),
('x_2', 'f8')]).view(np.recarray)
exog['const'] = 1
cls.exog = exog
cls.data = sm_data.handle_data(cls.endog, cls.exog)
cls.xnames = ['const', 'x_1', 'x_2']
cls.ynames = 'y_1'
def test_drop(self):
y = self.y
X = self.X
combined = np.c_[y, X]
idx = ~np.isnan(combined).any(axis=1)
y = y[idx]
X = X[idx]
data = sm_data.handle_data(self.y, self.X, 'drop')
np.testing.assert_array_equal(data.endog, y)
np.testing.assert_array_equal(data.exog, X)
def test_drop(self):
y = self.y
X = self.X
combined = np.c_[y, X]
idx = ~np.isnan(combined).any(axis=1)
y = y.loc[idx]
X = X.loc[idx]
data = sm_data.handle_data(self.y, self.X, 'drop')
np.testing.assert_array_equal(data.endog, y.values)
tm.assert_series_equal(data.orig_endog, self.y.loc[idx])
np.testing.assert_array_equal(data.exog, X.values)
tm.assert_frame_equal(data.orig_exog, self.X.loc[idx])
def setup_class(cls):
cls.endog = np.random.random(10)
cls.exog = np.c_[np.ones(10), np.random.random((10, 2))]
cls.data = sm_data.handle_data(cls.endog, cls.exog)
nrows = 10
nvars = 3
cls.col_result = cls.col_input = np.random.random(nvars)
cls.row_result = cls.row_input = np.random.random(nrows)
cls.cov_result = cls.cov_input = np.random.random((nvars, nvars))
cls.xnames = ['const', 'x1', 'x2']
cls.ynames = 'y'
cls.row_labels = None
def setup_class(cls):
cls.endog = np.random.random((10, 4))
cls.exog = np.c_[np.ones(10), np.random.random((10, 2))]
cls.data = sm_data.handle_data(cls.endog, cls.exog)
nrows = 10
nvars = 3
neqs = 4
cls.col_result = cls.col_input = np.random.random(nvars)
cls.row_result = cls.row_input = np.random.random(nrows)
cls.cov_result = cls.cov_input = np.random.random((nvars, nvars))
cls.cov_eq_result = cls.cov_eq_input = np.random.random((neqs, neqs))
cls.col_eq_result = cls.col_eq_input = np.array((neqs, nvars))
cls.xnames = ['const', 'x1', 'x2']
cls.ynames = ['y1', 'y2', 'y3', 'y4']
cls.row_labels = None
def setup_class(cls):
cls.endog = pd.Series(np.random.random(10), name='y_1')
exog = pd.Series(np.random.random(10), name='x_1')
cls.exog = exog
cls.data = sm_data.handle_data(cls.endog, cls.exog)
nrows = 10
nvars = 1
cls.col_input = np.random.random(nvars)
cls.col_result = pd.Series(cls.col_input, index=[exog.name])
cls.row_input = np.random.random(nrows)
cls.row_result = pd.Series(cls.row_input, index=exog.index)
cls.cov_input = np.random.random((nvars, nvars))
cls.cov_result = pd.DataFrame(cls.cov_input,
index=[exog.name],
columns=[exog.name])
cls.xnames = ['x_1']
cls.ynames = 'y_1'
cls.row_labels = cls.exog.index
def setup_class(cls):
cls.endog = pd.DataFrame(np.random.random(10), columns=['y_1'])
exog = pd.DataFrame(np.random.random((10, 2)), columns=['x1', 'x2'])
exog.insert(0, 'const', 1)
cls.exog = exog.values.tolist()
cls.data = sm_data.handle_data(cls.endog, cls.exog)
nrows = 10
nvars = 3
cls.col_input = np.random.random(nvars)
cls.col_result = pd.Series(cls.col_input, index=exog.columns)
cls.row_input = np.random.random(nrows)
cls.row_result = pd.Series(cls.row_input, index=exog.index)
cls.cov_input = np.random.random((nvars, nvars))
cls.cov_result = pd.DataFrame(cls.cov_input,
index=exog.columns,
columns=exog.columns)
cls.xnames = ['const', 'x1', 'x2']
cls.ynames = 'y_1'
cls.row_labels = cls.endog.index
def test_endog_only_raise(self):
with pytest.raises(Exception):
sm_data.handle_data(self.y, None, 'raise')
def test_none(self):
data = sm_data.handle_data(self.y, self.X, 'none', hasconst=False)
np.testing.assert_array_equal(data.endog, self.y.values)
np.testing.assert_array_equal(data.exog, self.X.values)
def test_endog_only_drop(self):
y = self.y
y = y[~np.isnan(y)]
data = sm_data.handle_data(self.y, None, 'drop')
np.testing.assert_array_equal(data.endog, y)
def test_raise_no_missing(self):
# smoke test for GH#1700
sm_data.handle_data(np.random.random(20), np.random.random((20, 2)),
'raise')
def test_endog_only_drop(self):
y = self.y
y = y.dropna()
data = sm_data.handle_data(self.y, None, 'drop')
np.testing.assert_array_equal(data.endog, y.values)
def test_mv_endog(self):
y = self.X
y = y.loc[~np.isnan(y.values).any(axis=1)]
data = sm_data.handle_data(self.X, None, 'drop')
np.testing.assert_array_equal(data.endog, y.values)
def test_pandas_noconstant(self):
exog = self.data.exog.copy()
data = sm_data.handle_data(self.data.endog, exog)
np.testing.assert_equal(data.k_constant, 0)
np.testing.assert_equal(data.const_idx, None)
def test_pandas_constant(self):
exog = self.data.exog.copy()
exog['const'] = 1
data = sm_data.handle_data(self.data.endog, exog)
np.testing.assert_equal(data.k_constant, 1)
np.testing.assert_equal(data.const_idx, 6)
def setup_class(cls):
super(TestArrays2dEndog, cls).setup_class()
cls.endog = np.random.random((10, 1))
cls.exog = np.c_[np.ones(10), np.random.random((10, 2))]
cls.data = sm_data.handle_data(cls.endog, cls.exog)
def test_array_noconstant(self):
exog = self.data.exog.copy()
data = sm_data.handle_data(self.data.endog.values, exog.values)
np.testing.assert_equal(data.k_constant, 0)
np.testing.assert_equal(data.const_idx, None)
def test_extra_kwargs_1d(self):
weights = np.random.random(25)
data = sm_data.handle_data(self.y, self.X, 'drop', weights=weights)
idx = ~np.isnan(np.c_[self.y, self.X]).any(axis=1)
weights = weights[idx]
np.testing.assert_array_equal(data.weights, weights)
def test_formula_missing_extra_arrays():
np.random.seed(1)
# because patsy can't turn off missing data-handling as of 0.3.0, we need
# separate tests to make sure that missing values are handled correctly
# when going through formulas
# there is a handle_formula_data step
# then there is the regular handle_data step
# see GH#2083
# the untested cases are endog/exog have missing. extra has missing.
# endog/exog are fine. extra has missing.
# endog/exog do or do not have missing and extra has wrong dimension
y = np.random.randn(10)
y_missing = y.copy()
y_missing[[2, 5]] = np.nan
X = np.random.randn(10)
X_missing = X.copy()
X_missing[[1, 3]] = np.nan
weights = np.random.uniform(size=10)
weights_missing = weights.copy()
weights_missing[[6]] = np.nan
weights_wrong_size = np.random.randn(12)
data = {
'y': y,
'X': X,
'y_missing': y_missing,
'X_missing': X_missing,
'weights': weights,
'weights_missing': weights_missing
}
data = pd.DataFrame.from_dict(data)
data['constant'] = 1
formula = 'y_missing ~ X_missing'
((endog, exog), missing_idx,
design_info) = handle_formula_data(data,
None,
formula,
depth=2,
missing='drop')
kwargs = {
'missing_idx': missing_idx,
'missing': 'drop',
'weights': data['weights_missing']
}
model_data = sm_data.handle_data(endog, exog, **kwargs)
data_nona = data.dropna()
np.testing.assert_equal(data_nona['y'].values, model_data.endog)
np.testing.assert_equal(data_nona[['constant', 'X']].values,
model_data.exog)
np.testing.assert_equal(data_nona['weights'].values, model_data.weights)
tmp = handle_formula_data(data, None, formula, depth=2, missing='drop')
(endog, exog), missing_idx, design_info = tmp
weights_2d = np.random.randn(10, 10)
weights_2d[[8, 7], [7, 8]] = np.nan # symmetric missing values
kwargs.update({'weights': weights_2d, 'missing_idx': missing_idx})
model_data2 = sm_data.handle_data(endog, exog, **kwargs)
good_idx = [0, 4, 6, 9]
np.testing.assert_equal(data.loc[good_idx, 'y'], model_data2.endog)
np.testing.assert_equal(data.loc[good_idx, ['constant', 'X']],
model_data2.exog)
np.testing.assert_equal(weights_2d[good_idx][:, good_idx],
model_data2.weights)
tmp = handle_formula_data(data, None, formula, depth=2, missing='drop')
(endog, exog), missing_idx, design_info = tmp
kwargs.update({'weights': weights_wrong_size, 'missing_idx': missing_idx})
with pytest.raises(ValueError):
sm_data.handle_data(endog, exog, **kwargs)
def test_raise_no_missing(self):
# smoke test for GH#1700
sm_data.handle_data(pd.Series(np.random.random(20)),
pd.DataFrame(np.random.random((20, 2))), 'raise')
def test_raise(self):
with pytest.raises(Exception):
sm_data.handle_data(self.y, self.X, 'raise')
def setup_class(cls):
super(TestLists, cls).setup_class()
cls.endog = np.random.random(10).tolist()
cls.exog = np.c_[np.ones(10), np.random.random((10, 2))].tolist()
cls.data = sm_data.handle_data(cls.endog, cls.exog)
