def test_hess(self):
# NOTE: I had to overwrite this to lessen the tolerance
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
# NOTE: notice the accuracy below and the epsilon changes
# this doesn't work well for score -> hessian with non-cs step
# it's a little better around the optimum
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=True)
assert_almost_equal(he, hefd, decimal=4)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
1e-9,
centered=False)
assert_almost_equal(he, hefd, decimal=2)
hescs = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hescs, decimal=DEC8)
hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike)
assert_almost_equal(he, hecs, decimal=5)
# NOTE: these just don't work well
# hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3)
# assert_almost_equal(he, hecs, decimal=1)
# hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4)
# assert_almost_equal(he, hecs, decimal=0)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4)
assert_almost_equal(he, hecs, decimal=0)
def test_dtypes():
def f(x):
return 2 * x
desired = np.array([[2, 0], [0, 2]])
assert_allclose(numdiff.approx_fprime(np.array([1, 2]), f), desired)
assert_allclose(numdiff.approx_fprime(np.array([1., 2.]), f), desired)
assert_allclose(numdiff.approx_fprime(np.array([1. + 0j, 2. + 0j]), f),
desired)
def test_grad_fun1_fd(self):
for test_params in self.params:
gtrue = self.gradtrue(test_params)
fun = self.fun()
epsilon = 1e-6
gfd = numdiff.approx_fprime(test_params,
fun,
epsilon=epsilon,
args=self.args)
gfd += numdiff.approx_fprime(test_params,
fun,
epsilon=-epsilon,
args=self.args)
gfd /= 2.
assert_almost_equal(gtrue, gfd, decimal=DEC6)
def test_deriv(link):
# Check link function derivatives using numeric differentiation.
np.random.seed(24235)
for k in range(10): # TODO: WTF does looping over k do here?
p = np.random.uniform(0, 1)
d = link.deriv(p)
da = nd.approx_fprime(np.r_[p], link)
assert_allclose(d, da, rtol=1e-6, atol=1e-6)
def test_score(self):
for test_params in self.params:
sc = self.mod.score(test_params)
scfd = numdiff.approx_fprime(test_params.ravel(), self.mod.loglike)
assert_almost_equal(sc, scfd, decimal=1)
sccs = numdiff.approx_fprime_cs(test_params.ravel(),
self.mod.loglike)
assert_almost_equal(sc, sccs, decimal=11)
def check_score_hessian(results):
# GH#4620
# compare models core and hessian with numerical derivatives
params = results.params
# avoid checking score at MLE, score close to zero
sc = results.model.score(params * 0.98, scale=1)
# cs currently (0.9) does not work for all families
# sc2 = approx_fprime_cs(params * 0.98, results.model.loglike)
llfunc = lambda x: results.model.loglike(x, scale=1)
sc2 = approx_fprime(params * 0.98, llfunc)
assert_allclose(sc, sc2, rtol=0.05)
hess = results.model.hessian(params, scale=1)
hess2 = approx_hess(params, llfunc)
assert_allclose(hess, hess2, rtol=0.05)
scfunc = lambda x: results.model.score(x, scale=1)
hess3 = approx_fprime(params, scfunc)
assert_allclose(hess, hess3, rtol=0.05)
def deriv2(self, p):
"""
Second derivative of the link function g''(p)
implemented through numerical differentiation
"""
from sm2.tools.numdiff import approx_fprime
p = np.atleast_1d(p)
# Note: special function for norm.ppf does not support complex
return np.diag(approx_fprime(p, self.deriv, centered=True))
def test_hess(self):
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
# NOTE: notice the accuracy below
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=True)
assert_allclose(he, hefd, rtol=1e-9)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=False)
assert_almost_equal(he, hefd, decimal=4)
hescs = numdiff.approx_fprime_cs(test_params.ravel(),
self.mod.score)
assert_allclose(he, hescs, rtol=1e-13)
hecs = numdiff.approx_hess_cs(test_params.ravel(),
self.mod.loglike)
assert_allclose(he, hecs, rtol=1e-9)
# NOTE: Look at the lack of precision - default epsilon not always
# best
grad = self.mod.score(test_params)
hecs, gradcs = numdiff.approx_hess1(test_params,
self.mod.loglike,
1e-6,
return_grad=True)
assert_almost_equal(he, hecs, decimal=1)
assert_almost_equal(grad, gradcs, decimal=1)
hecs, gradcs = numdiff.approx_hess2(test_params,
self.mod.loglike,
1e-4,
return_grad=True)
assert_almost_equal(he, hecs, decimal=3)
assert_almost_equal(grad, gradcs, decimal=1)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-5)
assert_almost_equal(he, hecs, decimal=4)
def test_grad_fun1_fdc(self):
for test_params in self.params:
gtrue = self.gradtrue(test_params)
fun = self.fun()
gfd = numdiff.approx_fprime(test_params,
fun,
epsilon=1e-8,
args=self.args,
centered=True)
assert_almost_equal(gtrue, gfd, decimal=DEC5)
def score(self, params):
"""
Return the gradient of the loglikelihood at params.
Parameters
----------
params : array-like
The parameter values at which to evaluate the score function.
Notes
-----
Returns numerical gradient.
"""
loglike = self.loglike
return approx_fprime(params, loglike, epsilon=1e-8)
def test_deriv2(link):
# Check link function second derivatives using numeric differentiation.
np.random.seed(24235)
# TODO: Resolve errors with the numeric derivatives
if link is probit:
raise pytest.skip()
for k in range(10): # TODO: WTF does looping over k do here?
p = np.random.uniform(0, 1)
p = np.clip(p, 0.01, 0.99)
if link is cauchy:
p = np.clip(p, 0.03, 0.97)
d = link.deriv2(p)
da = nd.approx_fprime(np.r_[p], link.deriv)
assert_allclose(d, da, rtol=1e-6, atol=1e-6)
def score_obs(self, params):
"""
Generic Zero Inflated model score (gradient) vector of the
log-likelihood
Parameters
----------
params : array-like
The parameters of the model
Returns
-------
score : ndarray, 1-D
The score vector of the model, i.e. the first derivative of the
loglikelihood function, evaluated at `params`
"""
params_infl = params[:self.k_inflate]
params_main = params[self.k_inflate:]
y = self.endog
w = self.model_infl.predict(params_infl)
w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
score_main = self.model_main.score_obs(params_main)
llf_main = self.model_main.loglikeobs(params_main)
llf = self.loglikeobs(params)
zero_idx = np.nonzero(y == 0)[0]
nonzero_idx = np.nonzero(y)[0]
dldp = np.zeros((self.exog.shape[0], self.k_exog), dtype=np.float64)
dldw = np.zeros_like(self.exog_infl, dtype=np.float64)
dldp[zero_idx, :] = (score_main[zero_idx].T *
(1 - (w[zero_idx]) / np.exp(llf[zero_idx]))).T
dldp[nonzero_idx, :] = score_main[nonzero_idx]
if self.inflation == 'logit':
dldw[zero_idx, :] = (self.exog_infl[zero_idx].T * w[zero_idx] *
(1 - w[zero_idx]) *
(1 - np.exp(llf_main[zero_idx])) /
np.exp(llf[zero_idx])).T
dldw[nonzero_idx, :] = -(self.exog_infl[nonzero_idx].T *
w[nonzero_idx]).T
elif self.inflation == 'probit':
# TODO: Maybe do this _before_ all the junk above?
return approx_fprime(params, self.loglikeobs)
return np.hstack((dldw, dldp))
def test_pdf_equiv():
# Check that two implementations of MNLogit.pdf agree
nobs = 10**4
J = 8
k_exog = 10
np.random.seed(8)
exog = np.random.randn(nobs, k_exog)
endog = np.random.randint(0, J, size=nobs)
# wendog = pd.get_dummies(endog).values
model = MNLogit(endog, exog)
params = np.random.random((model.K * (model.J - 1)))
params = params.reshape(model.K, -1, order='F')
Xb = model.exog.dot(params)
pdf1 = model.pdf(Xb)
pdf2 = mnlogit_pdf(Xb)
assert_allclose(pdf1, pdf2)
# Check that differentiating cdf matches pdf
pdf1 = np.zeros((J - 1, J - 1))
for k in range(J - 1):
func = lambda x: model.cdf(x.reshape(1, -1))[0][k] # noqa:E731
pdf1[k, :] = approx_fprime(Xb[0], func)
pdf2 = model.pdf(Xb[:1])[0]
# Differentiating model.cdf ignores the "base" column of wendog; i.e.
# computes (J-1)x(J-1) partials. model.pdf computes JxJ partials, but
# the non-overlapping ones should be redundant because the partials
# should sum to zero over both columns and rows.
assert_allclose(pdf2.sum(0), 0, atol=1e-14)
assert_allclose(pdf2.sum(1), 0, atol=1e-14)
# We can therefore drop the inconveniently mismatched row and column
pdf3 = pdf2[1:, :-1].T
# TODO: clarify which axis means what
assert_allclose(pdf3, pdf1, atol=5e-8)
def margeff_cov_params(model, params, exog, cov_params, at, derivative,
dummy_ind, count_ind, method, J):
"""
Computes the variance-covariance of marginal effects by the delta method.
Parameters
----------
model : model instance
The model that returned the fitted results. Its pdf method is used
for computing the Jacobian of discrete variables in dummy_ind and
count_ind
params : array-like
estimated model parameters
exog : array-like
exogenous variables at which to calculate the derivative
cov_params : array-like
The variance-covariance of the parameters
at : str
Options are:
- 'overall', The average of the marginal effects at each
observation.
- 'mean', The marginal effects at the mean of each regressor.
- 'median', The marginal effects at the median of each regressor.
- 'zero', The marginal effects at zero for each regressor.
- 'all', The marginal effects at each observation.
Only overall has any effect here.you
derivative : function or array-like
If a function, it returns the marginal effects of the model with
respect to the exogenous variables evaluated at exog. Expected to be
called derivative(params, exog). This will be numerically
differentiated. Otherwise, it can be the Jacobian of the marginal
effects with respect to the parameters.
dummy_ind : array-like
Indices of the columns of exog that contain dummy variables
count_ind : array-like
Indices of the columns of exog that contain count variables
Notes
-----
For continuous regressors, the variance-covariance is given by
Asy. Var[MargEff] = [d margeff / d params] V [d margeff / d params]'
where V is the parameter variance-covariance.
The outer Jacobians are computed via numerical differentiation if
derivative is a function.
"""
if callable(derivative):
from sm2.tools.numdiff import approx_fprime_cs
params = params.ravel('F') # for Multinomial
try:
jacobian_mat = approx_fprime_cs(params,
derivative,
args=(exog, method))
except TypeError: # stats.norm.cdf doesn't take complex values
from sm2.tools.numdiff import approx_fprime
jacobian_mat = approx_fprime(params,
derivative,
args=(exog, method))
if at == 'overall':
jacobian_mat = np.mean(jacobian_mat, axis=1)
else:
jacobian_mat = jacobian_mat.squeeze() # exog was 2d row vector
if dummy_ind is not None:
jacobian_mat = _margeff_cov_params_dummy(model, jacobian_mat,
params, exog, dummy_ind,
method, J)
if count_ind is not None:
jacobian_mat = _margeff_cov_params_count(model, jacobian_mat,
params, exog, count_ind,
method, J)
else:
jacobian_mat = derivative
# NOTE: this won't go through for at == 'all'
return np.dot(np.dot(jacobian_mat, cov_params), jacobian_mat.T)