def _first_time_fit(
self,
use_cache: bool,
absorb_options: Optional[Dict[str, Union[bool, str, ArrayLike, None,
Dict[str, Any]]]],
method: str,
) -> None:
weights = (cast(Float64Array, self.weights.ndarray)
if self._is_weighted else None)
use_hdfe = weights is None and method in ("auto", "hdfe")
use_hdfe = use_hdfe and not self._absorb_inter.cont.shape[1]
use_hdfe = use_hdfe and not self._interaction_list
if not use_hdfe and method == "hdfe":
raise RuntimeError(
"HDFE has been set as the method but the model cannot be estimated "
"using HDFE. HDFE requires that the model is unweighted and that the "
"absorbed regressors include only fixed effects (dummy variables)."
)
areg = AbsorbingRegressor(
cat=self._absorb_inter.cat,
cont=self._absorb_inter.cont,
interactions=self._interaction_list,
weights=weights,
)
areg_constant = areg.has_constant
self._regressors = areg.regressors
self._num_params += areg.approx_rank
# Do not double count intercept-like terms
self._has_constant = self._has_constant_exog or areg_constant
self._num_params -= min(self._has_constant_exog, areg_constant)
self._regressors_hash = areg.hash
self._constant_absorbed = self._has_constant_exog and areg_constant
dep = self._dependent.ndarray
exog = cast(Float64Array, self._exog.ndarray)
root_w = sqrt(self._weight_data.ndarray)
dep = root_w * dep
exog = root_w * exog
denom = root_w.T @ root_w
mu_dep = (root_w.T @ dep) / denom
mu_exog = (root_w.T @ exog) / denom
absorb_options = {} if absorb_options is None else absorb_options
assert isinstance(self._regressors, sp.csc_matrix)
if self._regressors.shape[1] > 0:
if use_hdfe:
from pyhdfe import create
absorb_options["drop_singletons"] = False
algo = create(self._absorb_inter.cat, **absorb_options)
dep_exog = column_stack((dep, exog))
resids = algo.residualize(dep_exog)
dep_resid = resids[:, :1]
exog_resid = resids[:, 1:]
else:
self._regressors = preconditioner(self._regressors)[0]
dep_exog = column_stack((dep, exog))
resid = lsmr_annihilate(
self._regressors,
dep_exog,
use_cache,
self._regressors_hash,
**absorb_options,
)
dep_resid = resid[:, :1]
exog_resid = resid[:, 1:]
else:
dep_resid = dep
exog_resid = exog
if self._constant_absorbed:
dep_resid += root_w * mu_dep
exog_resid += root_w * mu_exog
if not self._drop_absorbed:
check_absorbed(exog_resid, self.exog.cols, exog)
else:
ncol = exog_resid.shape[1]
retain = not_absorbed(exog_resid)
if not retain:
raise ValueError(
"All columns in exog have been fully absorbed by the "
"included effects. This model cannot be estimated.")
elif len(retain) < ncol:
drop = set(range(ncol)).difference(retain)
dropped = ", ".join([str(self.exog.cols[i]) for i in drop])
warnings.warn(
absorbing_warn_msg.format(absorbed_variables=dropped),
AbsorbingEffectWarning,
)
exog_resid = exog_resid[:, retain]
self._columns = [self._columns[i] for i in retain]
self._absorbed_dependent = DataFrame(
dep_resid,
index=self._dependent.pandas.index,
columns=self._dependent.pandas.columns,
)
self._absorbed_exog = DataFrame(exog_resid,
index=self._exog.pandas.index,
columns=self._columns)
def fit(
self,
*,
cov_type: str = "robust",
debiased: bool = False,
lsmr_options: Optional[Dict[str, Union[float, bool]]] = None,
use_cache: bool = True,
**cov_config: Any,
) -> AbsorbingLSResults:
"""
Estimate model parameters
Parameters
----------
cov_type : str, optional
Name of covariance estimator to use. Supported covariance
estimators are:
* 'unadjusted', 'homoskedastic' - Classic homoskedastic inference
* 'robust', 'heteroskedastic' - Heteroskedasticity robust inference
* 'kernel' - Heteroskedasticity and autocorrelation robust
inference
* 'cluster' - One-way cluster dependent inference.
Heteroskedasticity robust
debiased : bool, optional
Flag indicating whether to debiased the covariance estimator using
a degree of freedom adjustment.
**cov_config
Additional parameters to pass to covariance estimator. The list
of optional parameters differ according to ``cov_type``. See
the documentation of the alternative covariance estimators for
the complete list of available commands.
lsmr_options : dict
Dictionary of options to pass to scipy.sparse.linalg.lsmr
use_cache : bool
Flag indicating whether the variables, once purged from the
absorbed variables and interactions, should be stored in the cache,
and retrieved if available. Cache can dramatically speed up
re-fitting large models when the set of absorbed variables and
interactions are identical.
Returns
-------
AbsorbingLSResults
Results container
Notes
-----
Additional covariance parameters depend on specific covariance used.
The see the docstring of specific covariance estimator for a list of
supported options. Defaults are used if no covariance configuration
is provided.
If use_cache is True, then variables are hashed based on their
contents using either a 64 bit value (if xxhash is installed) or
a 256 bit value. This allows variables to be reused in different
models if the set of absorbing variables and interactions is held
constant.
See also
--------
linearmodels.iv.covariance.HomoskedasticCovariance
linearmodels.iv.covariance.HeteroskedasticCovariance
linearmodels.iv.covariance.KernelCovariance
linearmodels.iv.covariance.ClusteredCovariance
"""
if self._absorbed_dependent is None:
self._first_time_fit(use_cache, lsmr_options)
self._x = exog_resid = to_numpy(self.absorbed_exog)
dep_resid = to_numpy(self.absorbed_dependent)
if self._exog.shape[1] == 0:
params = empty((0, 1))
else:
if exog_resid.shape[1]:
check_absorbed(exog_resid, self.exog.cols)
params = lstsq(exog_resid, dep_resid, rcond=None)[0]
self._num_params += exog_resid.shape[1]
cov_estimator = COVARIANCE_ESTIMATORS[cov_type]
cov_config["debiased"] = debiased
cov_config["kappa"] = 0.0
cov_config_copy = {k: v for k, v in cov_config.items()}
if "center" in cov_config_copy:
del cov_config_copy["center"]
cov_estimator_inst = cov_estimator(exog_resid, dep_resid, exog_resid,
params, **cov_config_copy)
results = {"kappa": 0.0, "liml_kappa": 0.0}
pe = self._post_estimation(params, cov_estimator_inst, cov_type)
results.update(pe)
results["df_model"] = self._num_params
return AbsorbingLSResults(results, self)
def test_all_absorbed_exception():
x_orig = np.random.standard_normal((200, 3))
x = x_orig * 1e-32
with pytest.raises(AbsorbingEffectError,
match="All exog variables have been"):
check_absorbed(x, ["a", "b", "c"], x_orig)
def _first_time_fit(
self, use_cache: bool,
lsmr_options: Optional[Dict[str, Union[float, bool]]]) -> None:
weights = self.weights.ndarray if self._is_weighted else None
areg = AbsorbingRegressor(
cat=self._absorb_inter.cat,
cont=self._absorb_inter.cont,
interactions=self._interaction_list,
weights=weights,
)
areg_constant = areg.has_constant
self._regressors = preconditioner(areg.regressors)[0]
self._num_params += areg.approx_rank
# Do not double count intercept-like terms
self._has_constant = self._has_constant_exog or areg_constant
self._num_params -= min(self._has_constant_exog, areg_constant)
self._regressors_hash = areg.hash
self._constant_absorbed = self._has_constant_exog and areg_constant
dep = self._dependent.ndarray
exog = self._exog.ndarray
root_w = sqrt(self._weight_data.ndarray)
dep = root_w * dep
exog = root_w * exog
denom = root_w.T @ root_w
mu_dep = (root_w.T @ dep) / denom
mu_exog = (root_w.T @ exog) / denom
lsmr_options = {} if lsmr_options is None else lsmr_options
assert isinstance(self._regressors, sp.csc_matrix)
if self._regressors.shape[1] > 0:
dep_resid = lsmr_annihilate(self._regressors, dep, use_cache,
self._regressors_hash, **lsmr_options)
exog_resid = lsmr_annihilate(self._regressors, exog, use_cache,
self._regressors_hash, **lsmr_options)
else:
dep_resid = dep
exog_resid = exog
if self._constant_absorbed:
dep_resid += root_w * mu_dep
exog_resid += root_w * mu_exog
if not self._drop_absorbed:
check_absorbed(exog_resid, self.exog.cols, exog)
else:
ncol = exog_resid.shape[1]
retain = not_absorbed(exog_resid)
if not retain:
raise ValueError(
"All columns in exog have been fully absorbed by the "
"included effects. This model cannot be estimated.")
elif len(retain) < ncol:
drop = set(range(ncol)).difference(retain)
dropped = ", ".join([str(self.exog.cols[i]) for i in drop])
warnings.warn(
absorbing_warn_msg.format(absorbed_variables=dropped),
AbsorbingEffectWarning,
)
exog_resid = exog_resid[:, retain]
self._columns = [self._columns[i] for i in retain]
self._absorbed_dependent = DataFrame(
dep_resid,
index=self._dependent.pandas.index,
columns=self._dependent.pandas.columns,
)
self._absorbed_exog = DataFrame(exog_resid,
index=self._exog.pandas.index,
columns=self._columns)