def test_hess(self):
#NOTE: I had to overwrite this to lessen the tolerance
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
#NOTE: notice the accuracy below and the epsilon changes
# this doesn't work well for score -> hessian with non-cs step
# it's a little better around the optimum
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params, self.mod.score,
centered=True)
assert_almost_equal(he, hefd, decimal=4)
hefd = numdiff.approx_fprime(test_params, self.mod.score, 1e-9,
centered=False)
assert_almost_equal(he, hefd, decimal=2)
hescs = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hescs, decimal=DEC8)
hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike)
assert_almost_equal(he, hecs, decimal=5)
#NOTE: these just don't work well
#hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3)
#assert_almost_equal(he, hecs, decimal=1)
#hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4)
#assert_almost_equal(he, hecs, decimal=0)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4)
assert_almost_equal(he, hecs, decimal=0)
def test_hess(self):
#NOTE: I had to overwrite this to lessen the tolerance
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
#NOTE: notice the accuracy below and the epsilon changes
# this does not work well for score -> hessian with non-cs step
# it's a little better around the optimum
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=True)
assert_almost_equal(he, hefd, decimal=4)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
1e-9,
centered=False)
assert_almost_equal(he, hefd, decimal=2)
hescs = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hescs, decimal=DEC8)
hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike)
assert_almost_equal(he, hecs, decimal=5)
#NOTE: these just do not work well
#hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3)
#assert_almost_equal(he, hecs, decimal=1)
#hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4)
#assert_almost_equal(he, hecs, decimal=0)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4)
assert_almost_equal(he, hecs, decimal=0)
def hessian(self, params, *args, **kwargs):
"""
Hessian matrix of the likelihood function, evaluated at the given
parameters.
Parameters
----------
params : array_like
Array of parameters at which to evaluate the hessian.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
-------
hessian : array
Hessian matrix evaluated at `params`
Notes
-----
This is a numerical approximation.
Both \*args and \*\*kwargs are necessary because the optimizer from
`fit` must call this function and only supports passing arguments via
\*args (for example `scipy.optimize.fmin_l_bfgs`).
"""
nargs = len(args)
if nargs < 1:
kwargs.setdefault('average_loglike', True)
if nargs < 2:
kwargs.setdefault('transformed', False)
if nargs < 3:
kwargs.setdefault('set_params', False)
initial_state = kwargs.pop('initial_state', None)
initial_state_cov = kwargs.pop('initial_state_cov', None)
if initial_state is not None and initial_state_cov is not None:
# If initialization is stationary, we don't want to recalculate the
# initial_state_cov for each new set of parameters here
initialization = self.initialization
_initial_state = self._initial_state
_initial_state_cov = self._initial_state_cov
_initial_variance = self._initial_variance
self.initialize_known(initial_state, initial_state_cov)
hessian = approx_hess_cs(params,
self.loglike,
epsilon=1e-9,
args=args,
kwargs=kwargs)
if initial_state is not None and initial_state_cov is not None:
# Reset the initialization
self.initialization = initialization
self._initial_state = _initial_state
self._initial_state_cov = _initial_state_cov
self._initial_variance = _initial_variance
return hessian
def test_hess_fun1_cs(self):
for test_params in self.params:
#hetrue = 0
hetrue = self.hesstrue(test_params)
if hetrue is not None: #Hessian does not work for 2d return of fun
fun = self.fun()
hecs = numdiff.approx_hess_cs(test_params, fun, args=self.args)
assert_almost_equal(hetrue, hecs, decimal=DEC6)
def test_hess_fun1_cs(self):
for test_params in self.params:
#hetrue = 0
hetrue = self.hesstrue(test_params)
if not hetrue is None: #Hessian doesn't work for 2d return of fun
fun = self.fun()
hecs = numdiff.approx_hess_cs(test_params, fun, args=self.args)
assert_almost_equal(hetrue, hecs, decimal=DEC6)
def hessian(self, params, *args, **kwargs):
"""
Hessian matrix of the likelihood function, evaluated at the given
parameters.
Parameters
----------
params : array_like
Array of parameters at which to evaluate the hessian.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
-------
hessian : array
Hessian matrix evaluated at `params`
Notes
-----
This is a numerical approximation.
Both \*args and \*\*kwargs are necessary because the optimizer from
`fit` must call this function and only supports passing arguments via
\*args (for example `scipy.optimize.fmin_l_bfgs`).
"""
nargs = len(args)
if nargs < 1:
kwargs.setdefault('average_loglike', True)
if nargs < 2:
kwargs.setdefault('transformed', False)
if nargs < 3:
kwargs.setdefault('set_params', False)
initial_state = kwargs.pop('initial_state', None)
initial_state_cov = kwargs.pop('initial_state_cov', None)
if initial_state is not None and initial_state_cov is not None:
# If initialization is stationary, we don't want to recalculate the
# initial_state_cov for each new set of parameters here
initialization = self.initialization
_initial_state = self._initial_state
_initial_state_cov = self._initial_state_cov
_initial_variance = self._initial_variance
self.initialize_known(initial_state, initial_state_cov)
hessian = approx_hess_cs(params, self.loglike, epsilon=1e-9, args=args,
kwargs=kwargs)
if initial_state is not None and initial_state_cov is not None:
# Reset the initialization
self.initialization = initialization
self._initial_state = _initial_state
self._initial_state_cov = _initial_state_cov
self._initial_variance = _initial_variance
return hessian
def calculate_se(x, init_dict, X1, X0, Z1, Z0, Y1, Y0, num_treated,
num_untreated):
"""This function calculates the standard errors for given parameterization via an
approximation of the hessian matrix.
"""
num_ind = Y1.shape[0] + Y0.shape[0]
x0 = x.copy()
warning = None
if init_dict["ESTIMATION"]["maxiter"] == 0:
se = [np.nan] * len(x0)
hess_inv = np.full((len(x0), len(x0)), np.nan)
conf_interval = [[np.nan, np.nan]] * len(x0)
p_values, t_values = len(x0) * [np.nan], len(x0) * [np.nan]
else:
norm_value = norm.ppf(0.975)
# Calculate the hessian matrix, check if it is p
hess = approx_hess_cs(
x0,
log_likelihood,
args=(init_dict, X1, X0, Z1, Z0, Y1, Y0, num_treated,
num_untreated),
)
try:
hess_inv = np.linalg.inv(hess)
se = np.sqrt(np.diag(hess_inv) / num_ind)
aux = norm_value * se
upper = np.add(x0, aux)
lower = np.subtract(x0, aux)
se = se.copy()
hess_inv = hess_inv
conf_interval = [[lower[i], upper[i]] for i in range(len(lower))]
p_values, t_values = calculate_p_values(se, x0, num_ind)
except LinAlgError:
se = [np.nan] * len(x0)
hess_inv = np.full((len(x0), len(x0)), np.nan)
conf_interval = [[np.nan, np.nan]] * len(x0)
t_values = len(se) * [np.nan]
p_values = len(se) * [np.nan]
# Check if standard errors are defined, if not add warning message
if False in np.isfinite(se):
warning = [
"The estimation process was not able to provide standard errors for"
" the estimation results, because the approximation \n "
" of the hessian matrix "
"leads to a singular Matrix.\n"
]
return se, hess_inv, conf_interval, p_values, t_values, warning
def _hessian_cs(self, params, *args, **kwargs):
"""
Hessian matrix computed by second-order complex-step differentiation
on the `loglike` function.
"""
transformed = (args[0] if len(args) > 0 else kwargs.get(
'transformed', False))
f = lambda params, **kwargs: self.loglike(params, **kwargs) / self.nobs
hessian = approx_hess_cs(params,
f,
kwargs={'transformed': transformed})
return hessian
def _hessian_cs(self, params, *args, **kwargs):
"""
Hessian matrix computed by second-order complex-step differentiation
on the `loglike` function.
"""
transformed = (
args[0] if len(args) > 0 else kwargs.get('transformed', False)
)
f = lambda params, **kwargs: self.loglike(params, **kwargs) / self.nobs
hessian = approx_hess_cs(params, f, kwargs={
'transformed': transformed
})
return hessian
def _hessian_cs(self, params, *args, **kwargs):
"""
Hessian matrix computed by second-order complex-step differentiation
on the `loglike` function.
"""
nargs = len(args)
if nargs < 1:
kwargs.setdefault('average_loglike', True)
if nargs < 2:
kwargs.setdefault('transformed', False)
if nargs < 3:
kwargs.setdefault('set_params', False)
self.update(params)
return approx_hess_cs(params, self.loglike, args=args, kwargs=kwargs)
def _hessian_complex_step(self, params, **kwargs):
"""
Hessian matrix computed by second-order complex-step differentiation
on the `loglike` function.
"""
# the default epsilon can be too small
epsilon = _get_epsilon(params, 3., None, len(params))
kwargs['transformed'] = True
kwargs['complex_step'] = True
hessian = approx_hess_cs(
params, self.loglike, epsilon=epsilon, kwargs=kwargs)
# TODO: changed this to nobs_effective, has to be changed when merging
# with statespace mlemodel
return hessian / (self.nobs_effective)
def _hessian_cs(self, params, *args, **kwargs):
"""
Hessian matrix computed by second-order complex-step differentiation
on the `loglike` function.
"""
nargs = len(args)
if nargs < 1:
kwargs.setdefault('average_loglike', True)
if nargs < 2:
kwargs.setdefault('transformed', False)
if nargs < 3:
kwargs.setdefault('set_params', False)
self.update(params)
return approx_hess_cs(params, self.loglike, args=args, kwargs=kwargs)
def test_hess(self):
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
#NOTE: notice the accuracy below
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=True)
assert_allclose(he, hefd, rtol=1e-9)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=False)
assert_almost_equal(he, hefd, decimal=4)
hescs = numdiff.approx_fprime_cs(test_params.ravel(),
self.mod.score)
assert_allclose(he, hescs, rtol=1e-13)
hecs = numdiff.approx_hess_cs(test_params.ravel(),
self.mod.loglike)
assert_allclose(he, hecs, rtol=1e-9)
#NOTE: Look at the lack of precision - default epsilon not always
#best
grad = self.mod.score(test_params)
hecs, gradcs = numdiff.approx_hess1(test_params,
self.mod.loglike,
1e-6,
return_grad=True)
assert_almost_equal(he, hecs, decimal=1)
assert_almost_equal(grad, gradcs, decimal=1)
hecs, gradcs = numdiff.approx_hess2(test_params,
self.mod.loglike,
1e-4,
return_grad=True)
assert_almost_equal(he, hecs, decimal=3)
assert_almost_equal(grad, gradcs, decimal=1)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-5)
assert_almost_equal(he, hecs, decimal=4)
def calculate_se(rslt, init_dict, data_frame):
"""This function calculates the standard errors for given parameterization via an approximation
of the hessian matrix."""
x0 = rslt['AUX']['x_internal']
if init_dict['ESTIMATION']['maxiter'] == 0:
rslt['AUX']['standard_errors'] = [np.nan] * len(x0)
rslt['AUX']['hess_inv'] = '---'
rslt['AUX']['confidence_intervals'] = [[np.nan, np.nan]] * len(x0)
else:
# Calculate the hessian matrix, check if it is p
hess = approx_hess_cs(x0, log_likelihood, args=(init_dict, data_frame))
try:
hess_inv = np.linalg.inv(hess)
se = np.sqrt(np.diag(hess_inv) / data_frame.shape[0])
rslt['AUX']['standard_errors'] = se
rslt['AUX']['hess_inv'] = hess_inv
rslt['AUX']['confidence_intervals'] = []
for counter, param in enumerate(x0):
upper = param + norm.ppf(0.975) * se[counter]
lower = param - norm.ppf(0.975) * se[counter]
rslt['AUX']['confidence_intervals'] += [[lower, upper]]
except LinAlgError:
rslt['AUX']['standard_errors'] = [np.nan] * len(x0)
rslt['AUX']['hess_inv'] = '---'
rslt['AUX']['confidence_intervals'] = [[np.nan, np.nan]] * len(x0)
# Check if standard errors are defined, if not add warning message
if False in np.isfinite(rslt['AUX']['standard_errors']):
rslt['warning'] += [
'The estimation process was not able to provide standard errors for'
' the estimation results, because the approximation \n '
' of the hessian matrix '
'leads to a singular Matrix'
]
return rslt
def test_hess(self):
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
#NOTE: notice the accuracy below
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params, self.mod.score,
centered=True)
assert_allclose(he, hefd, rtol=1e-9)
hefd = numdiff.approx_fprime(test_params, self.mod.score,
centered=False)
assert_almost_equal(he, hefd, decimal=4)
hescs = numdiff.approx_fprime_cs(test_params.ravel(),
self.mod.score)
assert_allclose(he, hescs, rtol=1e-13)
hecs = numdiff.approx_hess_cs(test_params.ravel(),
self.mod.loglike)
assert_allclose(he, hecs, rtol=1e-9)
#NOTE: Look at the lack of precision - default epsilon not always
#best
grad = self.mod.score(test_params)
hecs, gradcs = numdiff.approx_hess1(test_params, self.mod.loglike,
1e-6, return_grad=True)
assert_almost_equal(he, hecs, decimal=1)
assert_almost_equal(grad, gradcs, decimal=1)
hecs, gradcs = numdiff.approx_hess2(test_params, self.mod.loglike,
1e-4, return_grad=True)
assert_almost_equal(he, hecs, decimal=3)
assert_almost_equal(grad, gradcs, decimal=1)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-5)
assert_almost_equal(he, hecs, decimal=4)
x = np.arange(nobs * 3).reshape(nobs, -1)
x = np.random.randn(nobs, 3)
xk = np.array([1, 2, 3])
xk = np.array([1., 1., 1.])
#xk = np.zeros(3)
beta = xk
y = np.dot(x, beta) + 0.1 * np.random.randn(nobs)
xkols = np.dot(np.linalg.pinv(x), y)
print approx_fprime((1, 2, 3), fun, epsilon, x)
gradtrue = x.sum(0)
print x.sum(0)
gradcs = approx_fprime_cs((1, 2, 3), fun, (x, ), h=1.0e-20)
print gradcs, maxabs(gradcs, gradtrue)
print approx_hess_cs((1, 2, 3), fun, (x, ),
h=1.0e-20) #this is correctly zero
print approx_hess_cs((1, 2, 3), fun2,
(y, x), h=1.0e-20) - 2 * np.dot(x.T, x)
print numdiff.approx_hess(xk, fun2, 1e-3, (y, x))[0] - 2 * np.dot(x.T, x)
gt = (-x * 2 * (y - np.dot(x, [1, 2, 3]))[:, None])
g = approx_fprime_cs((1, 2, 3), fun1, (y, x),
h=1.0e-20) #.T #this shouldn't be transposed
gd = numdiff.approx_fprime((1, 2, 3), fun1, epsilon, (y, x))
print maxabs(g, gt)
print maxabs(gd, gt)
import statsmodels.api as sm
data = sm.datasets.spector.load()
x = np.arange(nobs * 3).reshape(nobs, -1)
x = np.random.randn(nobs, 3)
xk = np.array([1, 2, 3])
xk = np.array([1., 1., 1.])
#xk = np.zeros(3)
beta = xk
y = np.dot(x, beta) + 0.1 * np.random.randn(nobs)
xkols = np.dot(np.linalg.pinv(x), y)
print(approx_fprime((1, 2, 3), fun, epsilon, x))
gradtrue = x.sum(0)
print(x.sum(0))
gradcs = approx_fprime_cs((1, 2, 3), fun, (x, ), h=1.0e-20)
print(gradcs, maxabs(gradcs, gradtrue))
print(approx_hess_cs((1, 2, 3), fun, (x, ),
h=1.0e-20)) #this is correctly zero
print(
approx_hess_cs((1, 2, 3), fun2, (y, x), h=1.0e-20) -
2 * np.dot(x.T, x))
print(numdiff.approx_hess(xk, fun2, 1e-3, (y, x))[0] - 2 * np.dot(x.T, x))
gt = (-x * 2 * (y - np.dot(x, [1, 2, 3]))[:, None])
g = approx_fprime_cs((1, 2, 3), fun1, (y, x),
h=1.0e-20) #.T #this should not be transposed
gd = numdiff.approx_fprime((1, 2, 3), fun1, epsilon, (y, x))
print(maxabs(g, gt))
print(maxabs(gd, gt))
data = sm.datasets.spector.load()
data.exog = sm.add_constant(data.exog, prepend=False)
x = np.arange(nobs*3).reshape(nobs,-1)
x = np.random.randn(nobs,3)
xk = np.array([1,2,3])
xk = np.array([1.,1.,1.])
#xk = np.zeros(3)
beta = xk
y = np.dot(x, beta) + 0.1*np.random.randn(nobs)
xkols = np.dot(np.linalg.pinv(x),y)
print(approx_fprime((1,2,3),fun,epsilon,x))
gradtrue = x.sum(0)
print(x.sum(0))
gradcs = approx_fprime_cs((1,2,3), fun, (x,), h=1.0e-20)
print(gradcs, maxabs(gradcs, gradtrue))
print(approx_hess_cs((1,2,3), fun, (x,), h=1.0e-20)) #this is correctly zero
print(approx_hess_cs((1,2,3), fun2, (y,x), h=1.0e-20)-2*np.dot(x.T, x))
print(numdiff.approx_hess(xk,fun2,1e-3, (y,x))[0] - 2*np.dot(x.T, x))
gt = (-x*2*(y-np.dot(x, [1,2,3]))[:,None])
g = approx_fprime_cs((1,2,3), fun1, (y,x), h=1.0e-20)#.T #this shouldn't be transposed
gd = numdiff.approx_fprime((1,2,3),fun1,epsilon,(y,x))
print(maxabs(g, gt))
print(maxabs(gd, gt))
import statsmodels.api as sm
data = sm.datasets.spector.load(as_pandas=False)
data.exog = sm.add_constant(data.exog, prepend=False)
