def test_hess(self):
#NOTE: I had to overwrite this to lessen the tolerance
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
#NOTE: notice the accuracy below and the epsilon changes
# this doesn't work well for score -> hessian with non-cs step
# it's a little better around the optimum
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params, self.mod.score,
centered=True)
assert_almost_equal(he, hefd, decimal=4)
hefd = numdiff.approx_fprime(test_params, self.mod.score, 1e-9,
centered=False)
assert_almost_equal(he, hefd, decimal=2)
hescs = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hescs, decimal=DEC8)
hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike)
assert_almost_equal(he, hecs, decimal=5)
#NOTE: these just don't work well
#hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3)
#assert_almost_equal(he, hecs, decimal=1)
#hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4)
#assert_almost_equal(he, hecs, decimal=0)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4)
assert_almost_equal(he, hecs, decimal=0)
def test_hess(self):
#NOTE: I had to overwrite this to lessen the tolerance
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
#NOTE: notice the accuracy below and the epsilon changes
# this does not work well for score -> hessian with non-cs step
# it's a little better around the optimum
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=True)
assert_almost_equal(he, hefd, decimal=4)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
1e-9,
centered=False)
assert_almost_equal(he, hefd, decimal=2)
hescs = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hescs, decimal=DEC8)
hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike)
assert_almost_equal(he, hecs, decimal=5)
#NOTE: these just do not work well
#hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3)
#assert_almost_equal(he, hecs, decimal=1)
#hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4)
#assert_almost_equal(he, hecs, decimal=0)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4)
assert_almost_equal(he, hecs, decimal=0)
def linear_approximation(self, steady_state=None):
''' Given a nonlinear rational expectations model in the form:
psi_1[x(t+1),x(t)] = psi_2[x(t+1),x(t)]
this method returns the linear approximation of the model with matrices a and b such that:
a * y(t+1) = b * y(t)
where y(t) = x(t) - x is the log deviation of the vector x from its steady state value.
Args:
steady_state: (Pandas Series or numpy array or list)
Returns:
None
Attributes:
log_linear: (bool) Whether the model is log-linear. Sets to False.
a: (Numpy ndarray)
b: (Numpy ndarray)
'''
# Set log_linear attribute
self.log_linear = False
# Warn if steady state attribute ss has not been assigned
if steady_state is None:
try:
steady_state = self.ss
except:
raise ValueError(
'You must specify a steady state for the model before attempting to linearize.'
)
# Compute approximation
def equilibrium(vars_fwd, vars_cur):
vars_fwd = pd.Series(vars_fwd, index=self.names['variables'])
vars_cur = pd.Series(vars_cur, index=self.names['variables'])
equilibrium_left = self.equilibrium_fun(vars_fwd, vars_cur,
self.parameters)
equilibrium_right = np.ones(len(self.names['variables']))
return equilibrium_left - equilibrium_right
equilibrium_fwd = lambda fwd: equilibrium(fwd, steady_state)
equilibrium_cur = lambda cur: equilibrium(steady_state, cur)
# Assign attributes
self.a = approx_fprime_cs(steady_state.ravel(), equilibrium_fwd)
self.b = -approx_fprime_cs(steady_state.ravel(), equilibrium_cur)
def grad(self, params=None, **kwds):
"""First derivative, jacobian of func evaluated at params.
Parameters
----------
params : None or ndarray
Values at which gradient is evaluated. If params is None, then
the attached params are used.
TODO: should we drop this
kwds : keyword arguments
This keyword arguments are used without changes in the calulation
of numerical derivatives. These are only used if a `deriv` function
was not provided.
Returns
-------
grad : ndarray
gradient or jacobian of the function
"""
if params is None:
params = self.params
if self._grad is not None:
return self._grad(params)
else:
# copied from discrete_margins
try:
from statsmodels.tools.numdiff import approx_fprime_cs
jac = approx_fprime_cs(params, self.fun, **kwds)
except TypeError: # norm.cdf doesn't take complex values
from statsmodels.tools.numdiff import approx_fprime
jac = approx_fprime(params, self.fun, **kwds)
return jac
def score(self, params, *args, **kwargs):
"""
Compute the score function at params.
Parameters
----------
params : array_like
Array of parameters at which to evaluate the score.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
----------
score : array
Score, evaluated at `params`.
Notes
-----
This is a numerical approximation, calculated using first-order complex
step differentiation on the `loglike` method.
Both \*args and \*\*kwargs are necessary because the optimizer from
`fit` must call this function and only supports passing arguments via
\*args (for example `scipy.optimize.fmin_l_bfgs`).
"""
nargs = len(args)
if nargs < 1:
kwargs.setdefault('average_loglike', True)
if nargs < 2:
kwargs.setdefault('transformed', False)
if nargs < 3:
kwargs.setdefault('set_params', False)
return approx_fprime_cs(params, self.loglike, args=args, kwargs=kwargs)
def score(self, params, *args, **kwargs):
"""
Compute the score function at params.
Parameters
----------
params : array_like
Array of parameters at which to evaluate the score.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
----------
score : array
Score, evaluated at `params`.
Notes
-----
This is a numerical approximation, calculated using first-order complex
step differentiation on the `loglike` method.
Both \*args and \*\*kwargs are necessary because the optimizer from
`fit` must call this function and only supports passing arguments via
\*args (for example `scipy.optimize.fmin_l_bfgs`).
"""
nargs = len(args)
if nargs < 1:
kwargs.setdefault('average_loglike', True)
if nargs < 2:
kwargs.setdefault('transformed', False)
if nargs < 3:
kwargs.setdefault('set_params', False)
return approx_fprime_cs(params, self.loglike, args=args, kwargs=kwargs)
def deriv(self, mu):
"""
Derivative of the variance function v'(mu)
"""
from statsmodels.tools.numdiff import approx_fprime_cs
# TODO: diag workaround proplem with numdiff for 1d
return np.diag(approx_fprime_cs(mu, self))
def score(self, params, *args, **kwargs):
"""
Compute the score function at params.
Parameters
----------
params : array_like
Array of parameters at which to evaluate the score.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
----------
score : array
Score, evaluated at `params`.
Notes
-----
This is a numerical approximation, calculated using first-order complex
step differentiation on the `loglike` method.
Both \*args and \*\*kwargs are necessary because the optimizer from
`fit` must call this function and only supports passing arguments via
\*args (for example `scipy.optimize.fmin_l_bfgs`).
"""
transformed = (
args[0] if len(args) > 0 else kwargs.get('transformed', False)
)
score = approx_fprime_cs(params, self.loglike, kwargs={
'transformed': transformed
})
return score
def deriv(self, mu):
"""
Derivative of the variance function v'(mu)
"""
from statsmodels.tools.numdiff import approx_fprime_cs
# TODO: diag workaround proplem with numdiff for 1d
return np.diag(approx_fprime_cs(mu, self))
def score(self, params, *args, **kwargs):
"""
Compute the score function at params.
Parameters
----------
params : array_like
Array of parameters at which to evaluate the score.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
----------
score : array
Score, evaluated at `params`.
Notes
-----
This is a numerical approximation, calculated using first-order complex
step differentiation on the `loglike` method.
Both \*args and \*\*kwargs are necessary because the optimizer from
`fit` must call this function and only supports passing arguments via
\*args (for example `scipy.optimize.fmin_l_bfgs`).
"""
transformed = (args[0] if len(args) > 0 else kwargs.get(
'transformed', False))
score = approx_fprime_cs(params,
self.loglike,
kwargs={'transformed': transformed})
return score
def deriv2(self, p):
"""Second derivative of the link function g''(p)
implemented through numerical differentiation
"""
from statsmodels.tools.numdiff import approx_fprime_cs
# TODO: workaround proplem with numdiff for 1d
return np.diag(approx_fprime_cs(p, self.deriv))
def test_grad_fun1_cs(self):
for test_params in self.params:
#gtrue = self.x.sum(0)
gtrue = self.gradtrue(test_params)
fun = self.fun()
gcs = numdiff.approx_fprime_cs(test_params, fun, args=self.args)
assert_almost_equal(gtrue, gcs, decimal=DEC13)
def test_grad_fun1_cs(self):
for test_params in self.params:
#gtrue = self.x.sum(0)
gtrue = self.gradtrue(test_params)
fun = self.fun()
gcs = numdiff.approx_fprime_cs(test_params, fun, args=self.args)
assert_almost_equal(gtrue, gcs, decimal=DEC13)
def _score_complex_step(self, params, **kwargs):
# the default epsilon can be too small
# inversion_method = INVERT_UNIVARIATE | SOLVE_LU
epsilon = _get_epsilon(params, 2., None, len(params))
kwargs['transformed'] = True
kwargs['complex_step'] = True
return approx_fprime_cs(params, self.loglike, epsilon=epsilon,
kwargs=kwargs)
def deriv2(self, p):
"""Second derivative of the link function g''(p)
implemented through numerical differentiation
"""
from statsmodels.tools.numdiff import approx_fprime_cs
# TODO: workaround proplem with numdiff for 1d
return np.diag(approx_fprime_cs(p, self.deriv))
def score(self, params, *args, **kwargs):
"""
Compute the score function at params.
Parameters
----------
params : array_like
Array of parameters at which to evaluate the score.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
----------
score : array
Score, evaluated at `params`.
Notes
-----
This is a numerical approximation.
Both \*args and \*\*kwargs are necessary because the optimizer from
`fit` must call this function and only supports passing arguments via
\*args (for example `scipy.optimize.fmin_l_bfgs`).
"""
nargs = len(args)
if nargs < 1:
kwargs.setdefault('average_loglike', True)
if nargs < 2:
kwargs.setdefault('transformed', False)
if nargs < 3:
kwargs.setdefault('set_params', False)
initial_state = kwargs.pop('initial_state', None)
initial_state_cov = kwargs.pop('initial_state_cov', None)
if initial_state is not None and initial_state_cov is not None:
# If initialization is stationary, we don't want to recalculate the
# initial_state_cov for each new set of parameters here
initialization = self.initialization
_initial_state = self._initial_state
_initial_state_cov = self._initial_state_cov
_initial_variance = self._initial_variance
self.initialize_known(initial_state, initial_state_cov)
score = approx_fprime_cs(params,
self.loglike,
epsilon=1e-9,
args=args,
kwargs=kwargs)
if initial_state is not None and initial_state_cov is not None:
# Reset the initialization
self.initialization = initialization
self._initial_state = _initial_state
self._initial_state_cov = _initial_state_cov
self._initial_variance = _initial_variance
return score
def test_score(self):
for test_params in self.params:
sc = self.mod.score(test_params)
scfd = numdiff.approx_fprime(test_params.ravel(), self.mod.loglike)
assert_almost_equal(sc, scfd, decimal=1)
sccs = numdiff.approx_fprime_cs(test_params.ravel(),
self.mod.loglike)
assert_almost_equal(sc, sccs, decimal=11)
def deriv(self, t, *args):
"""First derivative of the dependence function
implemented through numerical differentiation
"""
# TODO: workaround proplem with numdiff for 1d
t = np.atleast_1d(t)
# return np.diag(approx_fprime(t, self.evaluate, args=args))
return np.diag(approx_fprime_cs(t, self.evaluate, args=args))
def test_score(self):
for test_params in self.params:
sc = self.mod.score(test_params)
scfd = numdiff.approx_fprime(test_params.ravel(),
self.mod.loglike)
assert_almost_equal(sc, scfd, decimal=1)
sccs = numdiff.approx_fprime_cs(test_params.ravel(),
self.mod.loglike)
assert_almost_equal(sc, sccs, decimal=11)
def score(self, params, *args, **kwargs):
"""
Compute the score function at params.
Parameters
----------
params : array_like
Array of parameters at which to evaluate the score.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
----------
score : array
Score, evaluated at `params`.
Notes
-----
This is a numerical approximation.
Both \*args and \*\*kwargs are necessary because the optimizer from
`fit` must call this function and only supports passing arguments via
\*args (for example `scipy.optimize.fmin_l_bfgs`).
"""
nargs = len(args)
if nargs < 1:
kwargs.setdefault('average_loglike', True)
if nargs < 2:
kwargs.setdefault('transformed', False)
if nargs < 3:
kwargs.setdefault('set_params', False)
initial_state = kwargs.pop('initial_state', None)
initial_state_cov = kwargs.pop('initial_state_cov', None)
if initial_state is not None and initial_state_cov is not None:
# If initialization is stationary, we don't want to recalculate the
# initial_state_cov for each new set of parameters here
initialization = self.initialization
_initial_state = self._initial_state
_initial_state_cov = self._initial_state_cov
_initial_variance = self._initial_variance
self.initialize_known(initial_state, initial_state_cov)
score = approx_fprime_cs(params, self.loglike, epsilon=1e-9, args=args,
kwargs=kwargs)
if initial_state is not None and initial_state_cov is not None:
# Reset the initialization
self.initialization = initialization
self._initial_state = _initial_state
self._initial_state_cov = _initial_state_cov
self._initial_variance = _initial_variance
return score
def test_hess(self):
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
#NOTE: notice the accuracy below
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=True)
assert_allclose(he, hefd, rtol=1e-9)
hefd = numdiff.approx_fprime(test_params,
self.mod.score,
centered=False)
assert_almost_equal(he, hefd, decimal=4)
hescs = numdiff.approx_fprime_cs(test_params.ravel(),
self.mod.score)
assert_allclose(he, hescs, rtol=1e-13)
hecs = numdiff.approx_hess_cs(test_params.ravel(),
self.mod.loglike)
assert_allclose(he, hecs, rtol=1e-9)
#NOTE: Look at the lack of precision - default epsilon not always
#best
grad = self.mod.score(test_params)
hecs, gradcs = numdiff.approx_hess1(test_params,
self.mod.loglike,
1e-6,
return_grad=True)
assert_almost_equal(he, hecs, decimal=1)
assert_almost_equal(grad, gradcs, decimal=1)
hecs, gradcs = numdiff.approx_hess2(test_params,
self.mod.loglike,
1e-4,
return_grad=True)
assert_almost_equal(he, hecs, decimal=3)
assert_almost_equal(grad, gradcs, decimal=1)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-5)
assert_almost_equal(he, hecs, decimal=4)
def jac_predict(self, params):
'''jacobian of prediction function using complex step derivative
This assumes that the predict function does not use complex variable
but is designed to do so.
'''
from statsmodels.tools.numdiff import approx_fprime_cs
jaccs_err = approx_fprime_cs(params, self._predict)
return jaccs_err
def jac_predict(self, params):
'''jacobian of prediction function using complex step derivative
This assumes that the predict function does not use complex variable
but is designed to do so.
'''
from statsmodels.tools.numdiff import approx_fprime_cs
jaccs_err = approx_fprime_cs(params, self._predict)
return jaccs_err
def __call__(self, x, *args, **kwds):
x = np.atleast_1d(x)
if self.method.startswith('complex'):
grad = approx_fprime_cs(x, self.fun, self.step, args, kwds)
else:
grad = approx_fprime(x,
self.fun,
self.step,
args,
kwds,
centered=self.method.startswith('central'))
return grad
def test14():
"""This test checks wether our gradient functions work properly."""
constr = {"AGENTS": 10000, "DETERMINISTIC": False}
for _ in range(10):
generate_random_dict(constr)
init_dict = read("test.grmpy.yml")
print(init_dict["AUX"])
df = simulate("test.grmpy.yml")
D, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict)
num_treated = X1.shape[1]
num_untreated = X1.shape[1] + X0.shape[1]
x0 = start_values(init_dict, D, X1, X0, Z1, Z0, Y1, Y0, "init")
x0_back = backward_transformation(x0)
llh_gradient_approx = approx_fprime_cs(
x0_back,
log_likelihood,
args=(X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated, None,
False),
)
llh_gradient = gradient_hessian(x0_back, X1, X0, Z1, Z0, Y1, Y0)
min_inter_approx = approx_fprime_cs(
x0,
minimizing_interface,
args=(X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated, None,
False),
)
_, min_inter_gradient = log_likelihood(x0_back, X1, X0, Z1, Z0, Y1, Y0,
num_treated, num_untreated,
None, True)
np.testing.assert_array_almost_equal(min_inter_approx,
min_inter_gradient,
decimal=5)
np.testing.assert_array_almost_equal(llh_gradient_approx,
llh_gradient,
decimal=5)
cleanup()
def score_numdiff(self, params, pen_weight=None, method='fd', **kwds):
"""score based on finite difference derivative
"""
if pen_weight is None:
pen_weight = self.pen_weight
loglike = lambda p: self.loglike(p, pen_weight=pen_weight, **kwds)
if method == 'cs':
return approx_fprime_cs(params, loglike)
elif method == 'fd':
return approx_fprime(params, loglike, centered=True)
else:
raise ValueError('method not recognized, should be "fd" or "cs"')
def arma_scoreobs(endog,
ar_params=None,
ma_params=None,
sigma2=1,
prefix=None):
"""
Compute the score per observation (gradient of the loglikelihood function)
Parameters
----------
endog : ndarray
The observed time-series process.
ar_params : ndarray, optional
Autoregressive coefficients, not including the zero lag.
ma_params : ndarray, optional
Moving average coefficients, not including the zero lag, where the sign
convention assumes the coefficients are part of the lag polynomial on
the right-hand-side of the ARMA definition (i.e. they have the same
sign from the usual econometrics convention in which the coefficients
are on the right-hand-side of the ARMA definition).
sigma2 : ndarray, optional
The ARMA innovation variance. Default is 1.
prefix : str, optional
The BLAS prefix associated with the datatype. Default is to find the
best datatype based on given input. This argument is typically only
used internally.
Returns
----------
scoreobs : array
Score per observation, evaluated at the given parameters.
Notes
-----
This is a numerical approximation, calculated using first-order complex
step differentiation on the `arma_loglike` method.
"""
ar_params = [] if ar_params is None else ar_params
ma_params = [] if ma_params is None else ma_params
p = len(ar_params)
q = len(ma_params)
def func(params):
return arma_loglikeobs(endog, params[:p], params[p:p + q],
params[p + q:])
params0 = np.r_[ar_params, ma_params, sigma2]
epsilon = _get_epsilon(params0, 2., None, len(params0))
return approx_fprime_cs(params0, func, epsilon)
def score_numdiff(self, params, pen_weight=None, method='fd', **kwds):
"""score based on finite difference derivative
"""
if pen_weight is None:
pen_weight = self.pen_weight
loglike = lambda p: self.loglike(p, pen_weight=pen_weight, **kwds)
if method == 'cs':
return approx_fprime_cs(params, loglike)
elif method == 'fd':
return approx_fprime(params, loglike, centered=True)
else:
raise ValueError('method not recognized, should be "fd" or "cs"')
def test_hess(self):
for test_params in self.params:
he = self.mod.hessian(test_params)
hefd = numdiff.approx_fprime_cs(test_params, self.mod.score)
assert_almost_equal(he, hefd, decimal=DEC8)
#NOTE: notice the accuracy below
assert_almost_equal(he, hefd, decimal=7)
hefd = numdiff.approx_fprime(test_params, self.mod.score,
centered=True)
assert_allclose(he, hefd, rtol=1e-9)
hefd = numdiff.approx_fprime(test_params, self.mod.score,
centered=False)
assert_almost_equal(he, hefd, decimal=4)
hescs = numdiff.approx_fprime_cs(test_params.ravel(),
self.mod.score)
assert_allclose(he, hescs, rtol=1e-13)
hecs = numdiff.approx_hess_cs(test_params.ravel(),
self.mod.loglike)
assert_allclose(he, hecs, rtol=1e-9)
#NOTE: Look at the lack of precision - default epsilon not always
#best
grad = self.mod.score(test_params)
hecs, gradcs = numdiff.approx_hess1(test_params, self.mod.loglike,
1e-6, return_grad=True)
assert_almost_equal(he, hecs, decimal=1)
assert_almost_equal(grad, gradcs, decimal=1)
hecs, gradcs = numdiff.approx_hess2(test_params, self.mod.loglike,
1e-4, return_grad=True)
assert_almost_equal(he, hecs, decimal=3)
assert_almost_equal(grad, gradcs, decimal=1)
hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-5)
assert_almost_equal(he, hecs, decimal=4)
def arma_scoreobs(endog, ar_params=None, ma_params=None, sigma2=1,
prefix=None):
"""
Compute the score per observation (gradient of the loglikelihood function)
Parameters
----------
endog : ndarray
The observed time-series process.
ar_params : ndarray, optional
Autoregressive coefficients, not including the zero lag.
ma_params : ndarray, optional
Moving average coefficients, not including the zero lag, where the sign
convention assumes the coefficients are part of the lag polynomial on
the right-hand-side of the ARMA definition (i.e. they have the same
sign from the usual econometrics convention in which the coefficients
are on the right-hand-side of the ARMA definition).
sigma2 : ndarray, optional
The ARMA innovation variance. Default is 1.
prefix : str, optional
The BLAS prefix associated with the datatype. Default is to find the
best datatype based on given input. This argument is typically only
used internally.
Returns
----------
scoreobs : array
Score per observation, evaluated at the given parameters.
Notes
-----
This is a numerical approximation, calculated using first-order complex
step differentiation on the `arma_loglike` method.
"""
ar_params = [] if ar_params is None else ar_params
ma_params = [] if ma_params is None else ma_params
p = len(ar_params)
q = len(ma_params)
def func(params):
return arma_loglikeobs(endog, params[:p], params[p:p + q],
params[p + q:])
params0 = np.r_[ar_params, ma_params, sigma2]
epsilon = _get_epsilon(params0, 2., None, len(params0))
return approx_fprime_cs(params0, func, epsilon)
def test_ev_copula(case):
# check ev copulas, cdf and transform against R `evd` package
ev_tr, v1, v2, args, res1 = case
res = copula_bv_ev([v1, v2], ev_tr, args=args)
assert_allclose(res, res1, rtol=1e-13)
# check derivatives of dependence function
if ev_tr in (trev.transform_bilogistic, trev.transform_tev):
return
d1_res = approx_fprime_cs(np.array([v1, v2]), ev_tr.evaluate, args=args)
d1_res = np.diag(d1_res)
d1 = ev_tr.deriv(np.array([v1, v2]), *args)
assert_allclose(d1, d1_res, rtol=1e-8)
d1_res = approx_hess(np.array([0.5]), ev_tr.evaluate, args=args)
d1_res = np.diag(d1_res)
d1 = ev_tr.deriv2(0.5, *args)
assert_allclose(d1, d1_res, rtol=1e-7)
def test_vectorized():
def f(x):
return 2 * x
desired = np.array([2, 2])
# vectorized parameter, column vector
p = np.array([[1, 2]]).T
assert_allclose(_approx_fprime_scalar(p, f), desired[:, None], rtol=1e-8)
assert_allclose(_approx_fprime_scalar(p.squeeze(), f), desired, rtol=1e-8)
assert_allclose(_approx_fprime_cs_scalar(p, f),
desired[:, None],
rtol=1e-8)
assert_allclose(_approx_fprime_cs_scalar(p.squeeze(), f),
desired,
rtol=1e-8)
# check 2-d row, see #7680
# not allowed/implemented for approx_fprime, raises broadcast ValueError
# assert_allclose(approx_fprime(p.T, f), desired, rtol=1e-8)
# similar as used in MarkovSwitching unit test
assert_allclose(approx_fprime_cs(p.T, f).squeeze(), desired, rtol=1e-8)
def score_obs(self, params, **kwargs):
"""
Compute the score per observation, evaluated at params
Parameters
----------
params : array_like
Array of parameters at which to evaluate the score.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
----------
score : array (nobs, k_vars)
Score per observation, evaluated at `params`.
Notes
-----
This is a numerical approximation, calculated using first-order complex
step differentiation on the `loglikeobs` method.
"""
self.update(params)
return approx_fprime_cs(params, self.loglikeobs, kwargs=kwargs)
def transform_jacobian(self, unconstrained):
"""
Jacobian matrix for the parameter transformation function
Parameters
----------
unconstrained : array_like
Array of unconstrained parameters used by the optimizer.
Returns
-------
jacobian : array
Jacobian matrix of the transformation, evaluated at `unconstrained`
Notes
-----
This is a numerical approximation.
See Also
--------
transform_params
"""
return approx_fprime_cs(unconstrained, self.transform_params)
def transform_jacobian(self, unconstrained):
"""
Jacobian matrix for the parameter transformation function
Parameters
----------
unconstrained : array_like
Array of unconstrained parameters used by the optimizer.
Returns
-------
jacobian : array
Jacobian matrix of the transformation, evaluated at `unconstrained`
Notes
-----
This is a numerical approximation.
See Also
--------
transform_params
"""
return approx_fprime_cs(unconstrained, self.transform_params)
def score_obs(self, params, **kwargs):
"""
Compute the score per observation, evaluated at params
Parameters
----------
params : array_like
Array of parameters at which to evaluate the score.
*args, **kwargs
Additional arguments to the `loglike` method.
Returns
----------
score : array (nobs, k_vars)
Score per observation, evaluated at `params`.
Notes
-----
This is a numerical approximation, calculated using first-order complex
step differentiation on the `loglikeobs` method.
"""
self.update(params)
return approx_fprime_cs(params, self.loglikeobs, kwargs=kwargs)
def score(self, params):
return approx_fprime_cs(params, self.loglike)
def test_partials_logistic():
# Here we compare to analytic derivatives and to finite-difference
# approximations
logistic = markov_switching._logistic
partials_logistic = markov_switching._partials_logistic
# For a number, logistic(x) = np.exp(x) / (1 + np.exp(x))
# Then d/dx = logistix(x) - logistic(x)**2
cases = [0, 10., -4]
for x in cases:
assert_allclose(partials_logistic(x), logistic(x) - logistic(x)**2)
assert_allclose(partials_logistic(x), approx_fprime_cs([x], logistic))
# For a vector, logistic(x) returns
# np.exp(x[i]) / (1 + np.sum(np.exp(x[:]))) for each i
# Then d logistic(x[i]) / dx[i] = (logistix(x) - logistic(x)**2)[i]
# And d logistic(x[i]) / dx[j] = -(logistic(x[i]) * logistic[x[j]])
cases = [[1.], [0, 1.], [-2, 3., 1.2, -30.]]
for x in cases:
evaluated = np.atleast_1d(logistic(x))
partials = np.diag(evaluated - evaluated**2)
for i in range(len(x)):
for j in range(i):
partials[i, j] = partials[j, i] = -evaluated[i] * evaluated[j]
assert_allclose(partials_logistic(x), partials)
assert_allclose(partials_logistic(x), approx_fprime_cs(x, logistic))
# For a 2-dim, logistic(x) returns
# np.exp(x[i, t]) / (1 + np.sum(np.exp(x[:, t]))) for each i, each t
# but squeezed
case = [[1.]]
evaluated = logistic(case)
partial = [evaluated - evaluated**2]
assert_allclose(partials_logistic(case), partial)
assert_allclose(partials_logistic(case), approx_fprime_cs(case, logistic))
# # Here, np.array(case) is 2x1, so it is interpreted as i=0, 1 and t=0
case = [[0], [1.]]
evaluated = logistic(case)[:, 0]
partials = np.diag(evaluated - evaluated**2)
partials[0, 1] = partials[1, 0] = -np.multiply(*evaluated)
assert_allclose(partials_logistic(case)[:, :, 0], partials)
assert_allclose(partials_logistic(case),
approx_fprime_cs(np.squeeze(case), logistic)[..., None])
# Here, np.array(case) is 1x2, so it is interpreted as i=0 and t=0, 1
case = [[0, 1.]]
evaluated = logistic(case)
partials = (evaluated - evaluated**2)[None, ...]
assert_allclose(partials_logistic(case), partials)
assert_allclose(partials_logistic(case),
approx_fprime_cs(case, logistic).T)
# For a 3-dim, logistic(x) returns
# np.exp(x[i, j, t]) / (1 + np.sum(np.exp(x[:, j, t])))
# for each i, each j, each t
case = np.arange(2 * 3 * 4).reshape(2, 3, 4)
evaluated = logistic(case)
partials = partials_logistic(case)
for t in range(4):
for j in range(3):
desired = np.diag(evaluated[:, j, t] - evaluated[:, j, t]**2)
desired[0, 1] = desired[1, 0] = -np.multiply(*evaluated[:, j, t])
assert_allclose(partials[..., j, t], desired)
def margeff_cov_params(model, params, exog, cov_params, at, derivative,
dummy_ind, count_ind, method, J):
"""
Computes the variance-covariance of marginal effects by the delta method.
Parameters
----------
model : model instance
The model that returned the fitted results. Its pdf method is used
for computing the Jacobian of discrete variables in dummy_ind and
count_ind
params : array-like
estimated model parameters
exog : array-like
exogenous variables at which to calculate the derivative
cov_params : array-like
The variance-covariance of the parameters
at : str
Options are:
- 'overall', The average of the marginal effects at each
observation.
- 'mean', The marginal effects at the mean of each regressor.
- 'median', The marginal effects at the median of each regressor.
- 'zero', The marginal effects at zero for each regressor.
- 'all', The marginal effects at each observation.
Only overall has any effect here.you
derivative : function or array-like
If a function, it returns the marginal effects of the model with
respect to the exogenous variables evaluated at exog. Expected to be
called derivative(params, exog). This will be numerically
differentiated. Otherwise, it can be the Jacobian of the marginal
effects with respect to the parameters.
dummy_ind : array-like
Indices of the columns of exog that contain dummy variables
count_ind : array-like
Indices of the columns of exog that contain count variables
Notes
-----
For continuous regressors, the variance-covariance is given by
Asy. Var[MargEff] = [d margeff / d params] V [d margeff / d params]'
where V is the parameter variance-covariance.
The outer Jacobians are computed via numerical differentiation if
derivative is a function.
"""
if callable(derivative):
from statsmodels.tools.numdiff import approx_fprime_cs
params = params.ravel('F') # for Multinomial
try:
jacobian_mat = approx_fprime_cs(params,
derivative,
args=(exog, method))
except TypeError: # norm.cdf doesn't take complex values
from statsmodels.tools.numdiff import approx_fprime
jacobian_mat = approx_fprime(params,
derivative,
args=(exog, method))
if at == 'overall':
jacobian_mat = np.mean(jacobian_mat, axis=1)
else:
jacobian_mat = jacobian_mat.squeeze() # exog was 2d row vector
if dummy_ind is not None:
jacobian_mat = _margeff_cov_params_dummy(model, jacobian_mat,
params, exog, dummy_ind,
method, J)
if count_ind is not None:
jacobian_mat = _margeff_cov_params_count(model, jacobian_mat,
params, exog, count_ind,
method, J)
else:
jacobian_mat = derivative
#NOTE: this won't go through for at == 'all'
return np.dot(np.dot(jacobian_mat, cov_params), jacobian_mat.T)
def calculate_se(x, maxiter, X1, X0, Z1, Z0, Y1, Y0):
"""This function computes the standard errors of the parameters by approximating the
Jacobian of the gradient function. Based on that it computes the confidence
Parameters
----------
x: numpy.array
Parameter values
maxiter: float
maximum number of iterations
X1: numpy.array
Outcome related regressors of the treated individuals
X0: numpy.array
Outcome related regressors of the untreated individuals
Z1: numpy.array
Choice related regressors of the treated individuals
Z0: numpy.array
Choice related regressors of the untreated individuals
Y1: numpy.array
Outcomes of the treated individuals
Y0: numpy.array
Outcomes of the untreated individuals
Returns
------
se: numpy.array
Standard errors of the parameters
hess_inv: numpy.array
Inverse hessian matrix evaluated at the parameter vector
conf_interval: numpy.array
Confidence intervals of the parameters
p_values: numpy.array
p-values of the parameters
t_values: numpy.array
t-values of the parameters
warning: str
Warning message if the approximated hessian matrix is not invertible
"""
num_ind = Y1.shape[0] + Y0.shape[0]
x0 = x.copy()
warning = None
if maxiter == 0:
se = [np.nan] * len(x0)
hess_inv = np.full((len(x0), len(x0)), np.nan)
conf_interval = [[np.nan, np.nan]] * len(x0)
p_values, t_values = len(x0) * [np.nan], len(x0) * [np.nan]
else:
norm_value = norm.ppf(0.975)
# Calculate the hessian matrix, check if it is p
hess = approx_fprime_cs(x0,
gradient_hessian,
args=(X1, X0, Z1, Z0, Y1, Y0))
try:
hess_inv = np.linalg.inv(hess)
se = np.sqrt(np.diag(hess_inv) / num_ind)
aux = norm_value * se
hess_inv = hess_inv
conf_interval = np.vstack((np.subtract(x0, aux), np.add(x0,
aux))).T
t_values = np.divide(x0, se)
p_values = 2 * (1 - t.cdf(np.abs(t_values), df=num_ind - len(x0)))
except LinAlgError:
se = np.full(len(x0), np.nan)
hess_inv = np.full((len(x0), len(x0)), np.nan)
conf_interval = np.full((len(x0), 2), np.nan)
t_values = np.full(len(x0), np.nan)
p_values = np.full(len(x0), np.nan)
# Check if standard errors are defined, if not add warning message
if False in np.isfinite(se):
warning = [
"The estimation process was not able to provide standard errors for"
" the estimation results, because the approximation of the hessian "
"matrix leads to a singular Matrix."
]
return (
se,
hess_inv,
conf_interval[:, 0],
conf_interval[:, 1],
p_values,
t_values,
warning,
)
def margeff_cov_params(model, params, exog, cov_params, at, derivative,
dummy_ind, count_ind, method, J):
"""
Computes the variance-covariance of marginal effects by the delta method.
Parameters
----------
model : model instance
The model that returned the fitted results. Its pdf method is used
for computing the Jacobian of discrete variables in dummy_ind and
count_ind
params : array-like
estimated model parameters
exog : array-like
exogenous variables at which to calculate the derivative
cov_params : array-like
The variance-covariance of the parameters
at : str
Options are:
- 'overall', The average of the marginal effects at each
observation.
- 'mean', The marginal effects at the mean of each regressor.
- 'median', The marginal effects at the median of each regressor.
- 'zero', The marginal effects at zero for each regressor.
- 'all', The marginal effects at each observation.
Only overall has any effect here.you
derivative : function or array-like
If a function, it returns the marginal effects of the model with
respect to the exogenous variables evaluated at exog. Expected to be
called derivative(params, exog). This will be numerically
differentiated. Otherwise, it can be the Jacobian of the marginal
effects with respect to the parameters.
dummy_ind : array-like
Indices of the columns of exog that contain dummy variables
count_ind : array-like
Indices of the columns of exog that contain count variables
Notes
-----
For continuous regressors, the variance-covariance is given by
Asy. Var[MargEff] = [d margeff / d params] V [d margeff / d params]'
where V is the parameter variance-covariance.
The outer Jacobians are computed via numerical differentiation if
derivative is a function.
"""
if callable(derivative):
from statsmodels.tools.numdiff import approx_fprime_cs
params = params.ravel('F') # for Multinomial
try:
jacobian_mat = approx_fprime_cs(params, derivative,
args=(exog,method))
except TypeError, err: #norm.cdf doesn't take complex values
from statsmodels.tools.numdiff import approx_fprime1
jacobian_mat = approx_fprime1(params, derivative,
args=(exog,method))
if at == 'overall':
jacobian_mat = np.mean(jacobian_mat, axis=1)
else:
jacobian_mat = jacobian_mat.squeeze() # exog was 2d row vector
if dummy_ind is not None:
jacobian_mat = _margeff_cov_params_dummy(model, jacobian_mat,
params, exog, dummy_ind, method, J)
if count_ind is not None:
jacobian_mat = _margeff_cov_params_count(model, jacobian_mat,
params, exog, count_ind, method, J)
def test_partials_logistic():
# Here we compare to analytic derivatives and to finite-difference
# approximations
logistic = markov_switching._logistic
partials_logistic = markov_switching._partials_logistic
# For a number, logistic(x) = np.exp(x) / (1 + np.exp(x))
# Then d/dx = logistix(x) - logistic(x)**2
cases = [0, 10., -4]
for x in cases:
assert_allclose(partials_logistic(x), logistic(x) - logistic(x)**2)
assert_allclose(partials_logistic(x), approx_fprime_cs([x], logistic))
# For a vector, logistic(x) returns
# np.exp(x[i]) / (1 + np.sum(np.exp(x[:]))) for each i
# Then d logistic(x[i]) / dx[i] = (logistix(x) - logistic(x)**2)[i]
# And d logistic(x[i]) / dx[j] = -(logistic(x[i]) * logistic[x[j]])
cases = [[1.], [0, 1.], [-2, 3., 1.2, -30.]]
for x in cases:
evaluated = np.atleast_1d(logistic(x))
partials = np.diag(evaluated - evaluated**2)
for i in range(len(x)):
for j in range(i):
partials[i, j] = partials[j, i] = -evaluated[i] * evaluated[j]
assert_allclose(partials_logistic(x), partials)
assert_allclose(partials_logistic(x), approx_fprime_cs(x, logistic))
# For a 2-dim, logistic(x) returns
# np.exp(x[i, t]) / (1 + np.sum(np.exp(x[:, t]))) for each i, each t
# but squeezed
case = [[1.]]
evaluated = logistic(case)
partial = [evaluated - evaluated**2]
assert_allclose(partials_logistic(case), partial)
assert_allclose(partials_logistic(case), approx_fprime_cs(case, logistic))
# # Here, np.array(case) is 2x1, so it is interpreted as i=0, 1 and t=0
case = [[0], [1.]]
evaluated = logistic(case)[:, 0]
partials = np.diag(evaluated - evaluated**2)
partials[0, 1] = partials[1, 0] = -np.multiply(*evaluated)
assert_allclose(partials_logistic(case)[:, :, 0], partials)
assert_allclose(partials_logistic(case),
approx_fprime_cs(np.squeeze(case), logistic)[..., None])
# Here, np.array(case) is 1x2, so it is interpreted as i=0 and t=0, 1
case = [[0, 1.]]
evaluated = logistic(case)
partials = (evaluated - evaluated**2)[None, ...]
assert_allclose(partials_logistic(case), partials)
assert_allclose(partials_logistic(case),
approx_fprime_cs(case, logistic).T)
# For a 3-dim, logistic(x) returns
# np.exp(x[i, j, t]) / (1 + np.sum(np.exp(x[:, j, t])))
# for each i, each j, each t
case = np.arange(2*3*4).reshape(2, 3, 4)
evaluated = logistic(case)
partials = partials_logistic(case)
for t in range(4):
for j in range(3):
desired = np.diag(evaluated[:, j, t] - evaluated[:, j, t]**2)
desired[0, 1] = desired[1, 0] = -np.multiply(*evaluated[:, j, t])
assert_allclose(partials[..., j, t], desired)
epsilon = 1e-6
nobs = 200
x = np.arange(nobs*3).reshape(nobs,-1)
x = np.random.randn(nobs,3)
xk = np.array([1,2,3])
xk = np.array([1.,1.,1.])
#xk = np.zeros(3)
beta = xk
y = np.dot(x, beta) + 0.1*np.random.randn(nobs)
xkols = np.dot(np.linalg.pinv(x),y)
print(approx_fprime((1,2,3),fun,epsilon,x))
gradtrue = x.sum(0)
print(x.sum(0))
gradcs = approx_fprime_cs((1,2,3), fun, (x,), h=1.0e-20)
print(gradcs, maxabs(gradcs, gradtrue))
print(approx_hess_cs((1,2,3), fun, (x,), h=1.0e-20)) #this is correctly zero
print(approx_hess_cs((1,2,3), fun2, (y,x), h=1.0e-20)-2*np.dot(x.T, x))
print(numdiff.approx_hess(xk,fun2,1e-3, (y,x))[0] - 2*np.dot(x.T, x))
gt = (-x*2*(y-np.dot(x, [1,2,3]))[:,None])
g = approx_fprime_cs((1,2,3), fun1, (y,x), h=1.0e-20)#.T #this shouldn't be transposed
gd = numdiff.approx_fprime((1,2,3),fun1,epsilon,(y,x))
print(maxabs(g, gt))
print(maxabs(gd, gt))
import statsmodels.api as sm
epsilon = 1e-6
nobs = 200
x = np.arange(nobs * 3).reshape(nobs, -1)
x = np.random.randn(nobs, 3)
xk = np.array([1, 2, 3])
xk = np.array([1., 1., 1.])
#xk = np.zeros(3)
beta = xk
y = np.dot(x, beta) + 0.1 * np.random.randn(nobs)
xkols = np.dot(np.linalg.pinv(x), y)
print(approx_fprime((1, 2, 3), fun, epsilon, x))
gradtrue = x.sum(0)
print(x.sum(0))
gradcs = approx_fprime_cs((1, 2, 3), fun, (x, ), h=1.0e-20)
print(gradcs, maxabs(gradcs, gradtrue))
print(approx_hess_cs((1, 2, 3), fun, (x, ),
h=1.0e-20)) #this is correctly zero
print(
approx_hess_cs((1, 2, 3), fun2, (y, x), h=1.0e-20) -
2 * np.dot(x.T, x))
print(numdiff.approx_hess(xk, fun2, 1e-3, (y, x))[0] - 2 * np.dot(x.T, x))
gt = (-x * 2 * (y - np.dot(x, [1, 2, 3]))[:, None])
g = approx_fprime_cs((1, 2, 3), fun1, (y, x),
h=1.0e-20) #.T #this should not be transposed
gd = numdiff.approx_fprime((1, 2, 3), fun1, epsilon, (y, x))
print(maxabs(g, gt))
print(maxabs(gd, gt))
