def test_cache():
gen = generate_data(2,
True,
2,
factor_format="pandas",
ncont=0,
cont_interactions=1)
first = len(_VARIABLE_CACHE)
mod = AbsorbingLS(gen.y,
gen.x,
absorb=gen.absorb.iloc[:, :1],
interactions=gen.interactions)
mod.fit()
second = len(_VARIABLE_CACHE)
mod = AbsorbingLS(gen.y,
gen.x,
absorb=gen.absorb,
interactions=gen.interactions)
mod.fit()
third = len(_VARIABLE_CACHE)
assert third - second == 1
assert second - first == 1
mod = AbsorbingLS(gen.y,
gen.x,
absorb=gen.absorb,
interactions=gen.interactions)
mod.fit()
fourth = len(_VARIABLE_CACHE)
assert fourth - third == 0
def test_drop_missing():
gen = generate_data(2,
True,
2,
factor_format="pandas",
ncont=0,
cont_interactions=1)
gen.y[::53] = np.nan
gen.x[::79] = np.nan
with pytest.warns(MissingValueWarning):
AbsorbingLS(gen.y,
gen.x,
absorb=gen.absorb,
interactions=gen.interactions)
gen = generate_data(2,
True,
2,
factor_format="pandas",
ncont=0,
cont_interactions=1)
for col in gen.absorb:
gen.absorb[col] = gen.absorb[col].astype("int64").astype("object")
col_iloc = gen.absorb.columns.get_loc(col)
gen.absorb.iloc[::91, col_iloc] = np.nan
gen.absorb[col] = pd.Categorical(gen.absorb[col].to_numpy())
with pytest.warns(MissingValueWarning):
AbsorbingLS(gen.y,
gen.x,
absorb=gen.absorb,
interactions=gen.interactions)
def test_center_cov_arg():
gen = generate_data(
2, True, 2, factor_format="pandas", ncont=0, cont_interactions=1
)
mod = AbsorbingLS(gen.y, gen.x, absorb=gen.absorb, interactions=gen.interactions)
res = mod.fit(center=True)
assert "center" not in res.cov_config
def test_options(random_gen):
absorb = random_gen.randint(0, 10, size=1000)
x = random_gen.standard_normal((1000, 3))
y = random_gen.standard_normal((1000))
dfd = {f"x{i}": pd.Series(x[:, i]) for i in range(3)}
dfd.update({"c": pd.Series(absorb, dtype="category"), "y": pd.Series(y)})
df = pd.DataFrame(dfd)
y = df.y
x = df.iloc[:, :3]
mod = AbsorbingLS(y, x, absorb=df[["c"]], drop_absorbed=True)
mod.fit(absorb_options={"drop_singletons": False})
mod.fit(absorb_options={"atol": 1e-7, "btol": 1e-7}, method="lsmr")
mod = AbsorbingLS(y, x[["x0", "x1"]], absorb=df[["x2", "c"]], drop_absorbed=True)
with pytest.raises(RuntimeError, match="HDFE has been"):
mod.fit(absorb_options={"atol": 1e-7, "btol": 1e-7}, method="hdfe")
def test_instrments():
gen = generate_data(
2, True, 2, factor_format="pandas", ncont=0, cont_interactions=1
)
mod = AbsorbingLS(
gen.y, gen.x, absorb=gen.absorb.iloc[:, :1], interactions=gen.interactions
)
assert mod.instruments.shape[1] == 0
def test_smoke(data):
mod = AbsorbingLS(
data.y,
data.x,
absorb=data.absorb,
interactions=data.interactions,
weights=data.weights,
)
res = mod.fit()
assert isinstance(res.summary, Summary)
assert isinstance(str(res.summary), str)
def test_fully_absorb(random_gen):
absorb = random_gen.randint(0, 10, size=1000)
x = random_gen.standard_normal((1000, 3))
y = random_gen.standard_normal((1000))
dfd = {f"x{i}": pd.Series(x[:, i]) for i in range(3)}
dfd.update({"c": pd.Series(absorb, dtype="category"), "y": pd.Series(y)})
df = pd.DataFrame(dfd)
y = df.y
x = pd.get_dummies(df.c, drop_first=False)
mod = AbsorbingLS(y, x, absorb=df[["c"]], drop_absorbed=True)
with pytest.raises(ValueError, match="All columns in exog"):
mod.fit()
def test_drop_absorb(random_gen):
absorb = random_gen.randint(0, 10, size=1000)
x = random_gen.standard_normal((1000, 3))
y = random_gen.standard_normal((1000))
dfd = {f"x{i}": pd.Series(x[:, i]) for i in range(3)}
dfd.update({"c": pd.Series(absorb, dtype="category"), "y": pd.Series(y)})
df = pd.DataFrame(dfd)
y = df.y
x = df.iloc[:, :3]
x = pd.concat([x, pd.get_dummies(df.c).iloc[:, :2]], axis=1)
mod = AbsorbingLS(y, x, absorb=df[["c"]], drop_absorbed=True)
with pytest.warns(AbsorbingEffectWarning):
res = mod.fit()
assert len(res.params) == 3
assert all(f"x{i}" in res.params for i in range(3))
assert isinstance(str(res.summary), str)
mod = AbsorbingLS(y, x, absorb=df[["c"]])
with pytest.raises(AbsorbingEffectError):
mod.fit()
mod = AbsorbingLS(y, x.iloc[:, -2:], absorb=df[["c"]])
with pytest.raises(AbsorbingEffectError):
mod.fit()
def test_lsmr_options(random_gen):
absorb = random_gen.randint(0, 10, size=1000)
x = random_gen.standard_normal((1000, 3))
y = random_gen.standard_normal((1000))
dfd = {f"x{i}": pd.Series(x[:, i]) for i in range(3)}
dfd.update({"c": pd.Series(absorb, dtype="category"), "y": pd.Series(y)})
df = pd.DataFrame(dfd)
y = df.y
x = df.iloc[:, :3]
mod = AbsorbingLS(y, x, absorb=df[["c"]], drop_absorbed=True)
with pytest.warns(FutureWarning, match="lsmr_options"):
mod.fit(lsmr_options={})
with pytest.raises(ValueError, match="absorb_options cannot"):
mod.fit(lsmr_options={}, absorb_options={})
def test_against_ols(ols_data):
mod = AbsorbingLS(
ols_data.y,
ols_data.x,
absorb=ols_data.absorb,
interactions=ols_data.interactions,
weights=ols_data.weights,
)
res = mod.fit()
absorb = []
has_dummy = False
if ols_data.absorb is not None:
absorb.append(ols_data.absorb.cont.to_numpy())
if ols_data.absorb.cat.shape[1] > 0:
dummies = dummy_matrix(ols_data.absorb.cat, precondition=False)[0]
assert isinstance(dummies, sp.csc_matrix)
absorb.append(dummies.A)
has_dummy = ols_data.absorb.cat.shape[1] > 0
if ols_data.interactions is not None:
for interact in ols_data.interactions:
absorb.append(interact.sparse.A)
_x = ols_data.x
if absorb:
absorb = np.column_stack(absorb)
if np.any(np.ptp(_x, 0) == 0) and has_dummy:
if ols_data.weights is None:
absorb = annihilate(absorb, np.ones((absorb.shape[0], 1)))
else:
root_w = np.sqrt(mod.weights.ndarray)
wabsorb = annihilate(root_w * absorb, root_w)
absorb = (1.0 / root_w) * wabsorb
rank = np.linalg.matrix_rank(absorb)
if rank < absorb.shape[1]:
a, b = np.linalg.eig(absorb.T @ absorb)
order = np.argsort(a)[::-1]
a, b = a[order], b[:, order]
z = absorb @ b
absorb = z[:, :rank]
_x = np.column_stack([_x, absorb])
ols_mod = _OLS(ols_data.y, _x, weights=ols_data.weights)
ols_res = ols_mod.fit()
assert_results_equal(ols_res, res)
def test_absorbing_exceptions(random_gen):
with pytest.raises(TypeError):
absorbed = random_gen.standard_normal((NOBS, 2))
assert isinstance(absorbed, np.ndarray)
AbsorbingLS(
random_gen.standard_normal(NOBS),
random_gen.standard_normal((NOBS, 2)),
absorb=absorbed,
)
with pytest.raises(ValueError):
AbsorbingLS(
random_gen.standard_normal(NOBS), random_gen.standard_normal((NOBS - 1, 2))
)
with pytest.raises(ValueError):
AbsorbingLS(
random_gen.standard_normal(NOBS),
random_gen.standard_normal((NOBS, 2)),
absorb=pd.DataFrame(random_gen.standard_normal((NOBS - 1, 1))),
)
with pytest.raises(ValueError):
AbsorbingLS(
random_gen.standard_normal(NOBS),
random_gen.standard_normal((NOBS, 2)),
interactions=random_cat(10, NOBS - 1, frame=True, rs=random_gen),
)
mod = AbsorbingLS(
random_gen.standard_normal(NOBS),
random_gen.standard_normal((NOBS, 2)),
interactions=random_cat(10, NOBS, frame=True, rs=random_gen),
)
with pytest.raises(RuntimeError):
assert isinstance(mod.absorbed_dependent, pd.DataFrame)
with pytest.raises(RuntimeError):
assert isinstance(mod.absorbed_exog, pd.DataFrame)
with pytest.raises(TypeError):
interactions = random_gen.randint(0, 10, size=(NOBS, 2))
assert isinstance(interactions, np.ndarray)
AbsorbingLS(
random_gen.standard_normal(NOBS),
random_gen.standard_normal((NOBS, 2)),
interactions=interactions,
)
def test_absorbing_exceptions(rs):
with pytest.raises(TypeError):
AbsorbingLS(
rs.standard_normal(NOBS),
rs.standard_normal((NOBS, 2)),
absorb=rs.standard_normal((NOBS, 2)),
)
with pytest.raises(ValueError):
AbsorbingLS(rs.standard_normal(NOBS), rs.standard_normal(
(NOBS - 1, 2)))
with pytest.raises(ValueError):
AbsorbingLS(
rs.standard_normal(NOBS),
rs.standard_normal((NOBS, 2)),
absorb=pd.DataFrame(rs.standard_normal((NOBS - 1, 1))),
)
with pytest.raises(ValueError):
AbsorbingLS(
rs.standard_normal(NOBS),
rs.standard_normal((NOBS, 2)),
interactions=random_cat(10, NOBS - 1, frame=True, rs=rs),
)
mod = AbsorbingLS(
rs.standard_normal(NOBS),
rs.standard_normal((NOBS, 2)),
interactions=random_cat(10, NOBS, frame=True, rs=rs),
)
with pytest.raises(RuntimeError):
mod.absorbed_dependent
with pytest.raises(RuntimeError):
mod.absorbed_exog
with pytest.raises(TypeError):
AbsorbingLS(
rs.standard_normal(NOBS),
rs.standard_normal((NOBS, 2)),
interactions=rs.randint(0, 10, size=(NOBS, 2)),
)