def _fit_mle(self):
"""Fits MLE of the common part of the model.
This used to be called in the class instantiation, but there is no need to fit the GLM when
there are no automatic priors. So this method is only called when needed.
"""
missing = "drop" if self.model.dropna else "none"
try:
self.mle = GLM(
endog=self.model.response.data,
exog=self.dm,
family=self.model.family.smfamily(self.model.family.smlink),
missing=missing,
).fit()
except PerfectSeparationError as error:
msg = "Perfect separation detected, automatic priors are not available. "
msg += "Please indicate priors manually."
raise PerfectSeparationError(msg) from error
except:
print("Unexpected error:", sys.exc_info()[0])
raise
def _fit_pirls(self, alpha, start_params=None, maxiter=100, tol=1e-8,
scale=None, cov_type='nonrobust', cov_kwds=None, use_t=None,
weights=None):
"""fit model with penalized reweighted least squares
"""
# TODO: this currently modifies several attributes
# self.scale, self.scaletype, self.mu, self.weights
# self.data_weights,
# and possibly self._offset_exposure
# several of those might not be necessary, e.g. mu and weights
# alpha = alpha * len(y) * self.scale / 100
# TODO: we need to rescale alpha
endog = self.endog
wlsexog = self.exog # smoother.basis
spl_s = self.penal.penalty_matrix(alpha=alpha)
nobs, n_columns = wlsexog.shape
# TODO what are these values?
if weights is None:
self.data_weights = np.array([1.] * nobs)
else:
self.data_weights = weights
if not hasattr(self, '_offset_exposure'):
self._offset_exposure = 0
self.scaletype = scale
# TODO: check default scale types
# self.scaletype = 'dev'
# during iteration
self.scale = 1
if start_params is None:
mu = self.family.starting_mu(endog)
lin_pred = self.family.predict(mu)
else:
lin_pred = np.dot(wlsexog, start_params) + self._offset_exposure
mu = self.family.fitted(lin_pred)
dev = self.family.deviance(endog, mu)
history = dict(params=[None, start_params], deviance=[np.inf, dev])
converged = False
criterion = history['deviance']
# This special case is used to get the likelihood for a specific
# params vector.
if maxiter == 0:
mu = self.family.fitted(lin_pred)
self.scale = self.estimate_scale(mu)
wls_results = lm.RegressionResults(self, start_params, None)
iteration = 0
for iteration in range(maxiter):
# TODO: is this equivalent to point 1 of page 136:
# w = 1 / (V(mu) * g'(mu)) ?
self.weights = self.data_weights * self.family.weights(mu)
# TODO: is this equivalent to point 1 of page 136:
# z = g(mu)(y - mu) + X beta ?
wlsendog = (lin_pred + self.family.link.deriv(mu) * (endog - mu)
- self._offset_exposure)
# this defines the augmented matrix point 2a on page 136
wls_results = penalized_wls(wlsendog, wlsexog, spl_s, self.weights)
lin_pred = np.dot(wlsexog, wls_results.params).ravel()
lin_pred += self._offset_exposure
mu = self.family.fitted(lin_pred)
# We don't need to update scale in GLM/LEF models
# We might need it in dispersion models.
# self.scale = self.estimate_scale(mu)
history = self._update_history(wls_results, mu, history)
if endog.squeeze().ndim == 1 and np.allclose(mu - endog, 0):
msg = "Perfect separation detected, results not available"
raise PerfectSeparationError(msg)
# TODO need atol, rtol
# args of _check_convergence: (criterion, iteration, atol, rtol)
converged = _check_convergence(criterion, iteration, tol, 0)
if converged:
break
self.mu = mu
self.scale = self.estimate_scale(mu)
glm_results = GLMGamResults(self, wls_results.params,
wls_results.normalized_cov_params,
self.scale,
cov_type=cov_type, cov_kwds=cov_kwds,
use_t=use_t)
glm_results.method = "PIRLS"
history['iteration'] = iteration + 1
glm_results.fit_history = history
glm_results.converged = converged
return GLMGamResultsWrapper(glm_results)
def _fit_irls_sparse(self,
start_params=None,
maxiter=50,
tol=1e-3,
scale=None,
cov_type='nonrobust',
cov_kwds=None,
use_t=None,
**kwargs):
"""Fit a GLM using IRLS.
Fit a generalized linear mode (GLM) for a given family using
iteratively reweighted least squares (IRLS).
"""
if not scipy.sparse.issparse(self.exog):
raise ValueError("Matrix not sparse")
endog = self.endog
wlsexog = self.exog
if start_params is None:
mu = self.family.starting_mu(self.endog)
lin_pred = self.family.predict(mu)
else:
# This is a hack for a faster warm start
start_params[start_params > 1e2] = 1e2
start_params[start_params < -1e2] = -1e2
lin_pred = wlsexog.dot(start_params) + self._offset_exposure
mu = self.family.fitted(lin_pred)
dev = self.family.deviance(self.endog[self.endog[:, 0] > 0, :],
mu[self.endog[:, 0] > 0, :])
if np.isnan(dev):
if not (start_params is None):
# This is a hack for a faster warm start
start_params[start_params > 1e1] = 1e1
start_params[start_params < -1e1] = -1e1
lin_pred = wlsexog.dot(start_params) + self._offset_exposure
mu = self.family.fitted(lin_pred)
dev = self.family.deviance(self.endog, mu)
if np.isnan(dev):
dump_path = os.path.join(
self.tmp_dir,
'tmpdump.' + time.asctime().replace(' ', '_') + '.dat')
pickle.dump([lin_pred, mu, endog, exog, start_params],
open(dump_path, 'w'))
raise ValueError(
"The first guess on the deviance function returned a "
" nan. This could be a boundary problem and should "
" be reported. Please try to restart omniCLIP. "
"Parameter dump at: " + dump_path)
else:
raise ValueError(
"The first guess on the deviance function returned a "
" nan. This could be a boundary problem and should "
" be reported. Please try to restart omniCLIP. ")
# first guess on the deviance is assumed to be scaled by 1.
# params are none to start, so they line up with the deviance
history = dict(params=[None, start_params], deviance=[np.inf, dev])
converged = False
criterion = history['deviance']
# This special case is used to get the likelihood for a specific
# params vector.
if maxiter == 0:
pass
for iteration in range(maxiter):
self.weights = self.data_weights * self.family.weights(mu)
wlsendog = (lin_pred + self.family.link.deriv(mu) *
(self.endog - mu) - self._offset_exposure)
W = scipy.sparse.diags(self.weights[:, 0], 0)
# Compute x for current interation
temp_mat = wlsexog.transpose().dot(W)
lu = sla.splu(csc_matrix(temp_mat.dot(wlsexog)))
wls_results = lu.solve(temp_mat.dot(wlsendog))
wls_results[wls_results > 1e2] = 1e2
wls_results[wls_results < -1e2] = -1e2
lin_pred = self.exog.dot(wls_results) + self._offset_exposure
mu = self.family.fitted(lin_pred)
history['mu'] = mu
history['params'].append(wls_results)
temp_endog = self.endog[:]
temp_endog[temp_endog < 0] = 0
history['deviance'].append(self.family.deviance(self.endog, mu))
if endog.squeeze().ndim == 1 and np.allclose(mu - endog, 0):
msg = "Perfect separation detected, results not available"
raise PerfectSeparationError(msg)
converged = _check_convergence(criterion, iteration, tol)
if converged:
break
self.mu = mu
history['iteration'] = iteration + 1
return [wls_results, history]
def fit(self, start_params=None, maxiter=100, method='IRLS', tol=1e-8,
scale=None):
"""
Fits a generalized linear model for a given family.
parameters
----------
maxiter : int, optional
Default is 100.
method : string
Default is 'IRLS' for iteratively reweighted least squares. This
is currently the only method available for GLM fit.
scale : string or float, optional
`scale` can be 'X2', 'dev', or a float
The default value is None, which uses `X2` for Gamma, Gaussian,
and Inverse Gaussian.
`X2` is Pearson's chi-square divided by `df_resid`.
The default is 1 for the Binomial and Poisson families.
`dev` is the deviance divided by df_resid
tol : float
Convergence tolerance. Default is 1e-8.
start_params : array-like, optional
Initial guess of the solution for the loglikelihood maximization.
The default is family-specific and is given by the
``family.starting_mu(endog)``. If start_params is given then the
initial mean will be calculated as ``np.dot(exog, start_params)``.
"""
endog = self.endog
if endog.ndim > 1 and endog.shape[1] == 2:
data_weights = endog.sum(1) # weights are total trials
else:
data_weights = np.ones((endog.shape[0]))
self.data_weights = data_weights
if np.shape(self.data_weights) == () and self.data_weights > 1:
self.data_weights = self.data_weights * np.ones((endog.shape[0]))
self.scaletype = scale
if isinstance(self.family, families.Binomial):
# this checks what kind of data is given for Binomial.
# family will need a reference to endog if this is to be removed from
# preprocessing
self.endog = self.family.initialize(self.endog)
if hasattr(self, 'offset'):
offset = self.offset
elif hasattr(self, 'exposure'):
offset = self.exposure
else:
offset = 0
#TODO: would there ever be both and exposure and an offset?
wlsexog = self.exog
if start_params is None:
mu = self.family.starting_mu(self.endog)
else:
mu = self.family.fitted(np.dot(wlsexog, start_params))
eta = self.family.predict(mu)
dev = self.family.deviance(self.endog, mu)
if np.isnan(dev):
raise ValueError("The first guess on the deviance function "
"returned a nan. This could be a boundary "
" problem and should be reported.")
# first guess on the deviance is assumed to be scaled by 1.
# params are none to start, so they line up with the deviance
history = dict(params=[None, start_params], deviance=[np.inf, dev])
iteration = 0
converged = 0
criterion = history['deviance']
while not converged:
self.weights = data_weights*self.family.weights(mu)
wlsendog = (eta + self.family.link.deriv(mu) * (self.endog-mu)
- offset)
wls_results = lm.WLS(wlsendog, wlsexog, self.weights).fit()
eta = np.dot(self.exog, wls_results.params) + offset
mu = self.family.fitted(eta)
history = self._update_history(wls_results, mu, history)
self.scale = self.estimate_scale(mu)
iteration += 1
if endog.squeeze().ndim == 1 and np.allclose(mu - endog, 0):
msg = "Perfect separation detected, results not available"
raise PerfectSeparationError(msg)
converged = _check_convergence(criterion, iteration, tol, maxiter)
self.mu = mu
glm_results = GLMResults(self, wls_results.params,
wls_results.normalized_cov_params,
self.scale)
history['iteration'] = iteration
glm_results.fit_history = history
return GLMResultsWrapper(glm_results)
def fit(self,
start_params=None,
maxiter=100,
method='IRLS',
tol=1e-8,
scale=None,
cov_type='nonrobust',
cov_kwds=None,
use_t=None,
**kwargs):
"""
Fits a generalized linear model for a given family.
parameters
----------
maxiter : int, optional
Default is 100.
method : string
Default is 'IRLS' for iteratively reweighted least squares. This
is currently the only method available for GLM fit.
scale : string or float, optional
`scale` can be 'X2', 'dev', or a float
The default value is None, which uses `X2` for Gamma, Gaussian,
and Inverse Gaussian.
`X2` is Pearson's chi-square divided by `df_resid`.
The default is 1 for the Binomial and Poisson families.
`dev` is the deviance divided by df_resid
tol : float
Convergence tolerance. Default is 1e-8.
start_params : array-like, optional
Initial guess of the solution for the loglikelihood maximization.
The default is family-specific and is given by the
``family.starting_mu(endog)``. If start_params is given then the
initial mean will be calculated as ``np.dot(exog, start_params)``.
Notes
-----
This method does not take any extra undocumented ``kwargs``.
"""
endog = self.endog
if endog.ndim > 1 and endog.shape[1] == 2:
data_weights = endog.sum(1) # weights are total trials
else:
data_weights = np.ones((endog.shape[0]))
self.data_weights = data_weights
if np.shape(self.data_weights) == () and self.data_weights > 1:
self.data_weights = self.data_weights * np.ones((endog.shape[0]))
self.scaletype = scale
if isinstance(self.family, families.Binomial):
# this checks what kind of data is given for Binomial.
# family will need a reference to endog if this is to be removed from
# preprocessing
self.endog = self.family.initialize(self.endog)
# Construct a combined offset/exposure term. Note that
# exposure has already been logged if present.
offset_exposure = 0.
if hasattr(self, 'offset'):
offset_exposure = self.offset
if hasattr(self, 'exposure'):
offset_exposure = offset_exposure + self.exposure
self._offset_exposure = offset_exposure
wlsexog = self.exog
if start_params is None:
mu = self.family.starting_mu(self.endog)
lin_pred = self.family.predict(mu)
else:
lin_pred = np.dot(wlsexog, start_params) + offset_exposure
mu = self.family.fitted(lin_pred)
dev = self.family.deviance(self.endog, mu)
if np.isnan(dev):
raise ValueError("The first guess on the deviance function "
"returned a nan. This could be a boundary "
" problem and should be reported.")
# first guess on the deviance is assumed to be scaled by 1.
# params are none to start, so they line up with the deviance
history = dict(params=[None, start_params], deviance=[np.inf, dev])
converged = False
criterion = history['deviance']
# This special case is used to get the likelihood for a specific
# params vector.
if maxiter == 0:
mu = self.family.fitted(lin_pred)
self.scale = self.estimate_scale(mu)
wls_results = lm.RegressionResults(self, start_params, None)
iteration = 0
for iteration in range(maxiter):
self.weights = data_weights * self.family.weights(mu)
wlsendog = (lin_pred + self.family.link.deriv(mu) *
(self.endog - mu) - offset_exposure)
wls_results = lm.WLS(wlsendog, wlsexog, self.weights).fit()
lin_pred = np.dot(self.exog, wls_results.params) + offset_exposure
mu = self.family.fitted(lin_pred)
history = self._update_history(wls_results, mu, history)
self.scale = self.estimate_scale(mu)
if endog.squeeze().ndim == 1 and np.allclose(mu - endog, 0):
msg = "Perfect separation detected, results not available"
raise PerfectSeparationError(msg)
converged = _check_convergence(criterion, iteration, tol)
if converged:
break
self.mu = mu
glm_results = GLMResults(self,
wls_results.params,
wls_results.normalized_cov_params,
self.scale,
cov_type=cov_type,
cov_kwds=cov_kwds,
use_t=use_t)
history['iteration'] = iteration + 1
glm_results.fit_history = history
return GLMResultsWrapper(glm_results)