def test_hess(self): #NOTE: I had to overwrite this to lessen the tolerance for test_params in self.params: he = self.mod.hessian(test_params) hefd = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hefd, decimal=DEC8) #NOTE: notice the accuracy below and the epsilon changes # this doesn't work well for score -> hessian with non-cs step # it's a little better around the optimum assert_almost_equal(he, hefd, decimal=7) hefd = numdiff.approx_fprime(test_params, self.mod.score, centered=True) assert_almost_equal(he, hefd, decimal=4) hefd = numdiff.approx_fprime(test_params, self.mod.score, 1e-9, centered=False) assert_almost_equal(he, hefd, decimal=2) hescs = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hescs, decimal=DEC8) hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike) assert_almost_equal(he, hecs, decimal=5) #NOTE: these just don't work well #hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3) #assert_almost_equal(he, hecs, decimal=1) #hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4) #assert_almost_equal(he, hecs, decimal=0) hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4) assert_almost_equal(he, hecs, decimal=0)
def test_hess(self): #NOTE: I had to overwrite this to lessen the tolerance for test_params in self.params: he = self.mod.hessian(test_params) hefd = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hefd, decimal=DEC8) #NOTE: notice the accuracy below and the epsilon changes # this does not work well for score -> hessian with non-cs step # it's a little better around the optimum assert_almost_equal(he, hefd, decimal=7) hefd = numdiff.approx_fprime(test_params, self.mod.score, centered=True) assert_almost_equal(he, hefd, decimal=4) hefd = numdiff.approx_fprime(test_params, self.mod.score, 1e-9, centered=False) assert_almost_equal(he, hefd, decimal=2) hescs = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hescs, decimal=DEC8) hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike) assert_almost_equal(he, hecs, decimal=5) #NOTE: these just do not work well #hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3) #assert_almost_equal(he, hecs, decimal=1) #hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4) #assert_almost_equal(he, hecs, decimal=0) hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4) assert_almost_equal(he, hecs, decimal=0)
def linear_approximation(self, steady_state=None): ''' Given a nonlinear rational expectations model in the form: psi_1[x(t+1),x(t)] = psi_2[x(t+1),x(t)] this method returns the linear approximation of the model with matrices a and b such that: a * y(t+1) = b * y(t) where y(t) = x(t) - x is the log deviation of the vector x from its steady state value. Args: steady_state: (Pandas Series or numpy array or list) Returns: None Attributes: log_linear: (bool) Whether the model is log-linear. Sets to False. a: (Numpy ndarray) b: (Numpy ndarray) ''' # Set log_linear attribute self.log_linear = False # Warn if steady state attribute ss has not been assigned if steady_state is None: try: steady_state = self.ss except: raise ValueError( 'You must specify a steady state for the model before attempting to linearize.' ) # Compute approximation def equilibrium(vars_fwd, vars_cur): vars_fwd = pd.Series(vars_fwd, index=self.names['variables']) vars_cur = pd.Series(vars_cur, index=self.names['variables']) equilibrium_left = self.equilibrium_fun(vars_fwd, vars_cur, self.parameters) equilibrium_right = np.ones(len(self.names['variables'])) return equilibrium_left - equilibrium_right equilibrium_fwd = lambda fwd: equilibrium(fwd, steady_state) equilibrium_cur = lambda cur: equilibrium(steady_state, cur) # Assign attributes self.a = approx_fprime_cs(steady_state.ravel(), equilibrium_fwd) self.b = -approx_fprime_cs(steady_state.ravel(), equilibrium_cur)
def grad(self, params=None, **kwds): """First derivative, jacobian of func evaluated at params. Parameters ---------- params : None or ndarray Values at which gradient is evaluated. If params is None, then the attached params are used. TODO: should we drop this kwds : keyword arguments This keyword arguments are used without changes in the calulation of numerical derivatives. These are only used if a `deriv` function was not provided. Returns ------- grad : ndarray gradient or jacobian of the function """ if params is None: params = self.params if self._grad is not None: return self._grad(params) else: # copied from discrete_margins try: from statsmodels.tools.numdiff import approx_fprime_cs jac = approx_fprime_cs(params, self.fun, **kwds) except TypeError: # norm.cdf doesn't take complex values from statsmodels.tools.numdiff import approx_fprime jac = approx_fprime(params, self.fun, **kwds) return jac
def score(self, params, *args, **kwargs): """ Compute the score function at params. Parameters ---------- params : array_like Array of parameters at which to evaluate the score. *args, **kwargs Additional arguments to the `loglike` method. Returns ---------- score : array Score, evaluated at `params`. Notes ----- This is a numerical approximation, calculated using first-order complex step differentiation on the `loglike` method. Both \*args and \*\*kwargs are necessary because the optimizer from `fit` must call this function and only supports passing arguments via \*args (for example `scipy.optimize.fmin_l_bfgs`). """ nargs = len(args) if nargs < 1: kwargs.setdefault('average_loglike', True) if nargs < 2: kwargs.setdefault('transformed', False) if nargs < 3: kwargs.setdefault('set_params', False) return approx_fprime_cs(params, self.loglike, args=args, kwargs=kwargs)
def deriv(self, mu): """ Derivative of the variance function v'(mu) """ from statsmodels.tools.numdiff import approx_fprime_cs # TODO: diag workaround proplem with numdiff for 1d return np.diag(approx_fprime_cs(mu, self))
def score(self, params, *args, **kwargs): """ Compute the score function at params. Parameters ---------- params : array_like Array of parameters at which to evaluate the score. *args, **kwargs Additional arguments to the `loglike` method. Returns ---------- score : array Score, evaluated at `params`. Notes ----- This is a numerical approximation, calculated using first-order complex step differentiation on the `loglike` method. Both \*args and \*\*kwargs are necessary because the optimizer from `fit` must call this function and only supports passing arguments via \*args (for example `scipy.optimize.fmin_l_bfgs`). """ transformed = ( args[0] if len(args) > 0 else kwargs.get('transformed', False) ) score = approx_fprime_cs(params, self.loglike, kwargs={ 'transformed': transformed }) return score
def score(self, params, *args, **kwargs): """ Compute the score function at params. Parameters ---------- params : array_like Array of parameters at which to evaluate the score. *args, **kwargs Additional arguments to the `loglike` method. Returns ---------- score : array Score, evaluated at `params`. Notes ----- This is a numerical approximation, calculated using first-order complex step differentiation on the `loglike` method. Both \*args and \*\*kwargs are necessary because the optimizer from `fit` must call this function and only supports passing arguments via \*args (for example `scipy.optimize.fmin_l_bfgs`). """ transformed = (args[0] if len(args) > 0 else kwargs.get( 'transformed', False)) score = approx_fprime_cs(params, self.loglike, kwargs={'transformed': transformed}) return score
def deriv2(self, p): """Second derivative of the link function g''(p) implemented through numerical differentiation """ from statsmodels.tools.numdiff import approx_fprime_cs # TODO: workaround proplem with numdiff for 1d return np.diag(approx_fprime_cs(p, self.deriv))
def test_grad_fun1_cs(self): for test_params in self.params: #gtrue = self.x.sum(0) gtrue = self.gradtrue(test_params) fun = self.fun() gcs = numdiff.approx_fprime_cs(test_params, fun, args=self.args) assert_almost_equal(gtrue, gcs, decimal=DEC13)
def _score_complex_step(self, params, **kwargs): # the default epsilon can be too small # inversion_method = INVERT_UNIVARIATE | SOLVE_LU epsilon = _get_epsilon(params, 2., None, len(params)) kwargs['transformed'] = True kwargs['complex_step'] = True return approx_fprime_cs(params, self.loglike, epsilon=epsilon, kwargs=kwargs)
def score(self, params, *args, **kwargs): """ Compute the score function at params. Parameters ---------- params : array_like Array of parameters at which to evaluate the score. *args, **kwargs Additional arguments to the `loglike` method. Returns ---------- score : array Score, evaluated at `params`. Notes ----- This is a numerical approximation. Both \*args and \*\*kwargs are necessary because the optimizer from `fit` must call this function and only supports passing arguments via \*args (for example `scipy.optimize.fmin_l_bfgs`). """ nargs = len(args) if nargs < 1: kwargs.setdefault('average_loglike', True) if nargs < 2: kwargs.setdefault('transformed', False) if nargs < 3: kwargs.setdefault('set_params', False) initial_state = kwargs.pop('initial_state', None) initial_state_cov = kwargs.pop('initial_state_cov', None) if initial_state is not None and initial_state_cov is not None: # If initialization is stationary, we don't want to recalculate the # initial_state_cov for each new set of parameters here initialization = self.initialization _initial_state = self._initial_state _initial_state_cov = self._initial_state_cov _initial_variance = self._initial_variance self.initialize_known(initial_state, initial_state_cov) score = approx_fprime_cs(params, self.loglike, epsilon=1e-9, args=args, kwargs=kwargs) if initial_state is not None and initial_state_cov is not None: # Reset the initialization self.initialization = initialization self._initial_state = _initial_state self._initial_state_cov = _initial_state_cov self._initial_variance = _initial_variance return score
def test_score(self): for test_params in self.params: sc = self.mod.score(test_params) scfd = numdiff.approx_fprime(test_params.ravel(), self.mod.loglike) assert_almost_equal(sc, scfd, decimal=1) sccs = numdiff.approx_fprime_cs(test_params.ravel(), self.mod.loglike) assert_almost_equal(sc, sccs, decimal=11)
def deriv(self, t, *args): """First derivative of the dependence function implemented through numerical differentiation """ # TODO: workaround proplem with numdiff for 1d t = np.atleast_1d(t) # return np.diag(approx_fprime(t, self.evaluate, args=args)) return np.diag(approx_fprime_cs(t, self.evaluate, args=args))
def test_hess(self): for test_params in self.params: he = self.mod.hessian(test_params) hefd = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hefd, decimal=DEC8) #NOTE: notice the accuracy below assert_almost_equal(he, hefd, decimal=7) hefd = numdiff.approx_fprime(test_params, self.mod.score, centered=True) assert_allclose(he, hefd, rtol=1e-9) hefd = numdiff.approx_fprime(test_params, self.mod.score, centered=False) assert_almost_equal(he, hefd, decimal=4) hescs = numdiff.approx_fprime_cs(test_params.ravel(), self.mod.score) assert_allclose(he, hescs, rtol=1e-13) hecs = numdiff.approx_hess_cs(test_params.ravel(), self.mod.loglike) assert_allclose(he, hecs, rtol=1e-9) #NOTE: Look at the lack of precision - default epsilon not always #best grad = self.mod.score(test_params) hecs, gradcs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-6, return_grad=True) assert_almost_equal(he, hecs, decimal=1) assert_almost_equal(grad, gradcs, decimal=1) hecs, gradcs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4, return_grad=True) assert_almost_equal(he, hecs, decimal=3) assert_almost_equal(grad, gradcs, decimal=1) hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-5) assert_almost_equal(he, hecs, decimal=4)
def jac_predict(self, params): '''jacobian of prediction function using complex step derivative This assumes that the predict function does not use complex variable but is designed to do so. ''' from statsmodels.tools.numdiff import approx_fprime_cs jaccs_err = approx_fprime_cs(params, self._predict) return jaccs_err
def __call__(self, x, *args, **kwds): x = np.atleast_1d(x) if self.method.startswith('complex'): grad = approx_fprime_cs(x, self.fun, self.step, args, kwds) else: grad = approx_fprime(x, self.fun, self.step, args, kwds, centered=self.method.startswith('central')) return grad
def test14(): """This test checks wether our gradient functions work properly.""" constr = {"AGENTS": 10000, "DETERMINISTIC": False} for _ in range(10): generate_random_dict(constr) init_dict = read("test.grmpy.yml") print(init_dict["AUX"]) df = simulate("test.grmpy.yml") D, X1, X0, Z1, Z0, Y1, Y0 = process_data(df, init_dict) num_treated = X1.shape[1] num_untreated = X1.shape[1] + X0.shape[1] x0 = start_values(init_dict, D, X1, X0, Z1, Z0, Y1, Y0, "init") x0_back = backward_transformation(x0) llh_gradient_approx = approx_fprime_cs( x0_back, log_likelihood, args=(X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated, None, False), ) llh_gradient = gradient_hessian(x0_back, X1, X0, Z1, Z0, Y1, Y0) min_inter_approx = approx_fprime_cs( x0, minimizing_interface, args=(X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated, None, False), ) _, min_inter_gradient = log_likelihood(x0_back, X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated, None, True) np.testing.assert_array_almost_equal(min_inter_approx, min_inter_gradient, decimal=5) np.testing.assert_array_almost_equal(llh_gradient_approx, llh_gradient, decimal=5) cleanup()
def score_numdiff(self, params, pen_weight=None, method='fd', **kwds): """score based on finite difference derivative """ if pen_weight is None: pen_weight = self.pen_weight loglike = lambda p: self.loglike(p, pen_weight=pen_weight, **kwds) if method == 'cs': return approx_fprime_cs(params, loglike) elif method == 'fd': return approx_fprime(params, loglike, centered=True) else: raise ValueError('method not recognized, should be "fd" or "cs"')
def arma_scoreobs(endog, ar_params=None, ma_params=None, sigma2=1, prefix=None): """ Compute the score per observation (gradient of the loglikelihood function) Parameters ---------- endog : ndarray The observed time-series process. ar_params : ndarray, optional Autoregressive coefficients, not including the zero lag. ma_params : ndarray, optional Moving average coefficients, not including the zero lag, where the sign convention assumes the coefficients are part of the lag polynomial on the right-hand-side of the ARMA definition (i.e. they have the same sign from the usual econometrics convention in which the coefficients are on the right-hand-side of the ARMA definition). sigma2 : ndarray, optional The ARMA innovation variance. Default is 1. prefix : str, optional The BLAS prefix associated with the datatype. Default is to find the best datatype based on given input. This argument is typically only used internally. Returns ---------- scoreobs : array Score per observation, evaluated at the given parameters. Notes ----- This is a numerical approximation, calculated using first-order complex step differentiation on the `arma_loglike` method. """ ar_params = [] if ar_params is None else ar_params ma_params = [] if ma_params is None else ma_params p = len(ar_params) q = len(ma_params) def func(params): return arma_loglikeobs(endog, params[:p], params[p:p + q], params[p + q:]) params0 = np.r_[ar_params, ma_params, sigma2] epsilon = _get_epsilon(params0, 2., None, len(params0)) return approx_fprime_cs(params0, func, epsilon)
def test_ev_copula(case): # check ev copulas, cdf and transform against R `evd` package ev_tr, v1, v2, args, res1 = case res = copula_bv_ev([v1, v2], ev_tr, args=args) assert_allclose(res, res1, rtol=1e-13) # check derivatives of dependence function if ev_tr in (trev.transform_bilogistic, trev.transform_tev): return d1_res = approx_fprime_cs(np.array([v1, v2]), ev_tr.evaluate, args=args) d1_res = np.diag(d1_res) d1 = ev_tr.deriv(np.array([v1, v2]), *args) assert_allclose(d1, d1_res, rtol=1e-8) d1_res = approx_hess(np.array([0.5]), ev_tr.evaluate, args=args) d1_res = np.diag(d1_res) d1 = ev_tr.deriv2(0.5, *args) assert_allclose(d1, d1_res, rtol=1e-7)
def test_vectorized(): def f(x): return 2 * x desired = np.array([2, 2]) # vectorized parameter, column vector p = np.array([[1, 2]]).T assert_allclose(_approx_fprime_scalar(p, f), desired[:, None], rtol=1e-8) assert_allclose(_approx_fprime_scalar(p.squeeze(), f), desired, rtol=1e-8) assert_allclose(_approx_fprime_cs_scalar(p, f), desired[:, None], rtol=1e-8) assert_allclose(_approx_fprime_cs_scalar(p.squeeze(), f), desired, rtol=1e-8) # check 2-d row, see #7680 # not allowed/implemented for approx_fprime, raises broadcast ValueError # assert_allclose(approx_fprime(p.T, f), desired, rtol=1e-8) # similar as used in MarkovSwitching unit test assert_allclose(approx_fprime_cs(p.T, f).squeeze(), desired, rtol=1e-8)
def score_obs(self, params, **kwargs): """ Compute the score per observation, evaluated at params Parameters ---------- params : array_like Array of parameters at which to evaluate the score. *args, **kwargs Additional arguments to the `loglike` method. Returns ---------- score : array (nobs, k_vars) Score per observation, evaluated at `params`. Notes ----- This is a numerical approximation, calculated using first-order complex step differentiation on the `loglikeobs` method. """ self.update(params) return approx_fprime_cs(params, self.loglikeobs, kwargs=kwargs)
def transform_jacobian(self, unconstrained): """ Jacobian matrix for the parameter transformation function Parameters ---------- unconstrained : array_like Array of unconstrained parameters used by the optimizer. Returns ------- jacobian : array Jacobian matrix of the transformation, evaluated at `unconstrained` Notes ----- This is a numerical approximation. See Also -------- transform_params """ return approx_fprime_cs(unconstrained, self.transform_params)
def score(self, params): return approx_fprime_cs(params, self.loglike)
def test_partials_logistic(): # Here we compare to analytic derivatives and to finite-difference # approximations logistic = markov_switching._logistic partials_logistic = markov_switching._partials_logistic # For a number, logistic(x) = np.exp(x) / (1 + np.exp(x)) # Then d/dx = logistix(x) - logistic(x)**2 cases = [0, 10., -4] for x in cases: assert_allclose(partials_logistic(x), logistic(x) - logistic(x)**2) assert_allclose(partials_logistic(x), approx_fprime_cs([x], logistic)) # For a vector, logistic(x) returns # np.exp(x[i]) / (1 + np.sum(np.exp(x[:]))) for each i # Then d logistic(x[i]) / dx[i] = (logistix(x) - logistic(x)**2)[i] # And d logistic(x[i]) / dx[j] = -(logistic(x[i]) * logistic[x[j]]) cases = [[1.], [0, 1.], [-2, 3., 1.2, -30.]] for x in cases: evaluated = np.atleast_1d(logistic(x)) partials = np.diag(evaluated - evaluated**2) for i in range(len(x)): for j in range(i): partials[i, j] = partials[j, i] = -evaluated[i] * evaluated[j] assert_allclose(partials_logistic(x), partials) assert_allclose(partials_logistic(x), approx_fprime_cs(x, logistic)) # For a 2-dim, logistic(x) returns # np.exp(x[i, t]) / (1 + np.sum(np.exp(x[:, t]))) for each i, each t # but squeezed case = [[1.]] evaluated = logistic(case) partial = [evaluated - evaluated**2] assert_allclose(partials_logistic(case), partial) assert_allclose(partials_logistic(case), approx_fprime_cs(case, logistic)) # # Here, np.array(case) is 2x1, so it is interpreted as i=0, 1 and t=0 case = [[0], [1.]] evaluated = logistic(case)[:, 0] partials = np.diag(evaluated - evaluated**2) partials[0, 1] = partials[1, 0] = -np.multiply(*evaluated) assert_allclose(partials_logistic(case)[:, :, 0], partials) assert_allclose(partials_logistic(case), approx_fprime_cs(np.squeeze(case), logistic)[..., None]) # Here, np.array(case) is 1x2, so it is interpreted as i=0 and t=0, 1 case = [[0, 1.]] evaluated = logistic(case) partials = (evaluated - evaluated**2)[None, ...] assert_allclose(partials_logistic(case), partials) assert_allclose(partials_logistic(case), approx_fprime_cs(case, logistic).T) # For a 3-dim, logistic(x) returns # np.exp(x[i, j, t]) / (1 + np.sum(np.exp(x[:, j, t]))) # for each i, each j, each t case = np.arange(2 * 3 * 4).reshape(2, 3, 4) evaluated = logistic(case) partials = partials_logistic(case) for t in range(4): for j in range(3): desired = np.diag(evaluated[:, j, t] - evaluated[:, j, t]**2) desired[0, 1] = desired[1, 0] = -np.multiply(*evaluated[:, j, t]) assert_allclose(partials[..., j, t], desired)
def margeff_cov_params(model, params, exog, cov_params, at, derivative, dummy_ind, count_ind, method, J): """ Computes the variance-covariance of marginal effects by the delta method. Parameters ---------- model : model instance The model that returned the fitted results. Its pdf method is used for computing the Jacobian of discrete variables in dummy_ind and count_ind params : array-like estimated model parameters exog : array-like exogenous variables at which to calculate the derivative cov_params : array-like The variance-covariance of the parameters at : str Options are: - 'overall', The average of the marginal effects at each observation. - 'mean', The marginal effects at the mean of each regressor. - 'median', The marginal effects at the median of each regressor. - 'zero', The marginal effects at zero for each regressor. - 'all', The marginal effects at each observation. Only overall has any effect here.you derivative : function or array-like If a function, it returns the marginal effects of the model with respect to the exogenous variables evaluated at exog. Expected to be called derivative(params, exog). This will be numerically differentiated. Otherwise, it can be the Jacobian of the marginal effects with respect to the parameters. dummy_ind : array-like Indices of the columns of exog that contain dummy variables count_ind : array-like Indices of the columns of exog that contain count variables Notes ----- For continuous regressors, the variance-covariance is given by Asy. Var[MargEff] = [d margeff / d params] V [d margeff / d params]' where V is the parameter variance-covariance. The outer Jacobians are computed via numerical differentiation if derivative is a function. """ if callable(derivative): from statsmodels.tools.numdiff import approx_fprime_cs params = params.ravel('F') # for Multinomial try: jacobian_mat = approx_fprime_cs(params, derivative, args=(exog, method)) except TypeError: # norm.cdf doesn't take complex values from statsmodels.tools.numdiff import approx_fprime jacobian_mat = approx_fprime(params, derivative, args=(exog, method)) if at == 'overall': jacobian_mat = np.mean(jacobian_mat, axis=1) else: jacobian_mat = jacobian_mat.squeeze() # exog was 2d row vector if dummy_ind is not None: jacobian_mat = _margeff_cov_params_dummy(model, jacobian_mat, params, exog, dummy_ind, method, J) if count_ind is not None: jacobian_mat = _margeff_cov_params_count(model, jacobian_mat, params, exog, count_ind, method, J) else: jacobian_mat = derivative #NOTE: this won't go through for at == 'all' return np.dot(np.dot(jacobian_mat, cov_params), jacobian_mat.T)
def calculate_se(x, maxiter, X1, X0, Z1, Z0, Y1, Y0): """This function computes the standard errors of the parameters by approximating the Jacobian of the gradient function. Based on that it computes the confidence Parameters ---------- x: numpy.array Parameter values maxiter: float maximum number of iterations X1: numpy.array Outcome related regressors of the treated individuals X0: numpy.array Outcome related regressors of the untreated individuals Z1: numpy.array Choice related regressors of the treated individuals Z0: numpy.array Choice related regressors of the untreated individuals Y1: numpy.array Outcomes of the treated individuals Y0: numpy.array Outcomes of the untreated individuals Returns ------ se: numpy.array Standard errors of the parameters hess_inv: numpy.array Inverse hessian matrix evaluated at the parameter vector conf_interval: numpy.array Confidence intervals of the parameters p_values: numpy.array p-values of the parameters t_values: numpy.array t-values of the parameters warning: str Warning message if the approximated hessian matrix is not invertible """ num_ind = Y1.shape[0] + Y0.shape[0] x0 = x.copy() warning = None if maxiter == 0: se = [np.nan] * len(x0) hess_inv = np.full((len(x0), len(x0)), np.nan) conf_interval = [[np.nan, np.nan]] * len(x0) p_values, t_values = len(x0) * [np.nan], len(x0) * [np.nan] else: norm_value = norm.ppf(0.975) # Calculate the hessian matrix, check if it is p hess = approx_fprime_cs(x0, gradient_hessian, args=(X1, X0, Z1, Z0, Y1, Y0)) try: hess_inv = np.linalg.inv(hess) se = np.sqrt(np.diag(hess_inv) / num_ind) aux = norm_value * se hess_inv = hess_inv conf_interval = np.vstack((np.subtract(x0, aux), np.add(x0, aux))).T t_values = np.divide(x0, se) p_values = 2 * (1 - t.cdf(np.abs(t_values), df=num_ind - len(x0))) except LinAlgError: se = np.full(len(x0), np.nan) hess_inv = np.full((len(x0), len(x0)), np.nan) conf_interval = np.full((len(x0), 2), np.nan) t_values = np.full(len(x0), np.nan) p_values = np.full(len(x0), np.nan) # Check if standard errors are defined, if not add warning message if False in np.isfinite(se): warning = [ "The estimation process was not able to provide standard errors for" " the estimation results, because the approximation of the hessian " "matrix leads to a singular Matrix." ] return ( se, hess_inv, conf_interval[:, 0], conf_interval[:, 1], p_values, t_values, warning, )
def margeff_cov_params(model, params, exog, cov_params, at, derivative, dummy_ind, count_ind, method, J): """ Computes the variance-covariance of marginal effects by the delta method. Parameters ---------- model : model instance The model that returned the fitted results. Its pdf method is used for computing the Jacobian of discrete variables in dummy_ind and count_ind params : array-like estimated model parameters exog : array-like exogenous variables at which to calculate the derivative cov_params : array-like The variance-covariance of the parameters at : str Options are: - 'overall', The average of the marginal effects at each observation. - 'mean', The marginal effects at the mean of each regressor. - 'median', The marginal effects at the median of each regressor. - 'zero', The marginal effects at zero for each regressor. - 'all', The marginal effects at each observation. Only overall has any effect here.you derivative : function or array-like If a function, it returns the marginal effects of the model with respect to the exogenous variables evaluated at exog. Expected to be called derivative(params, exog). This will be numerically differentiated. Otherwise, it can be the Jacobian of the marginal effects with respect to the parameters. dummy_ind : array-like Indices of the columns of exog that contain dummy variables count_ind : array-like Indices of the columns of exog that contain count variables Notes ----- For continuous regressors, the variance-covariance is given by Asy. Var[MargEff] = [d margeff / d params] V [d margeff / d params]' where V is the parameter variance-covariance. The outer Jacobians are computed via numerical differentiation if derivative is a function. """ if callable(derivative): from statsmodels.tools.numdiff import approx_fprime_cs params = params.ravel('F') # for Multinomial try: jacobian_mat = approx_fprime_cs(params, derivative, args=(exog,method)) except TypeError, err: #norm.cdf doesn't take complex values from statsmodels.tools.numdiff import approx_fprime1 jacobian_mat = approx_fprime1(params, derivative, args=(exog,method)) if at == 'overall': jacobian_mat = np.mean(jacobian_mat, axis=1) else: jacobian_mat = jacobian_mat.squeeze() # exog was 2d row vector if dummy_ind is not None: jacobian_mat = _margeff_cov_params_dummy(model, jacobian_mat, params, exog, dummy_ind, method, J) if count_ind is not None: jacobian_mat = _margeff_cov_params_count(model, jacobian_mat, params, exog, count_ind, method, J)
def test_partials_logistic(): # Here we compare to analytic derivatives and to finite-difference # approximations logistic = markov_switching._logistic partials_logistic = markov_switching._partials_logistic # For a number, logistic(x) = np.exp(x) / (1 + np.exp(x)) # Then d/dx = logistix(x) - logistic(x)**2 cases = [0, 10., -4] for x in cases: assert_allclose(partials_logistic(x), logistic(x) - logistic(x)**2) assert_allclose(partials_logistic(x), approx_fprime_cs([x], logistic)) # For a vector, logistic(x) returns # np.exp(x[i]) / (1 + np.sum(np.exp(x[:]))) for each i # Then d logistic(x[i]) / dx[i] = (logistix(x) - logistic(x)**2)[i] # And d logistic(x[i]) / dx[j] = -(logistic(x[i]) * logistic[x[j]]) cases = [[1.], [0, 1.], [-2, 3., 1.2, -30.]] for x in cases: evaluated = np.atleast_1d(logistic(x)) partials = np.diag(evaluated - evaluated**2) for i in range(len(x)): for j in range(i): partials[i, j] = partials[j, i] = -evaluated[i] * evaluated[j] assert_allclose(partials_logistic(x), partials) assert_allclose(partials_logistic(x), approx_fprime_cs(x, logistic)) # For a 2-dim, logistic(x) returns # np.exp(x[i, t]) / (1 + np.sum(np.exp(x[:, t]))) for each i, each t # but squeezed case = [[1.]] evaluated = logistic(case) partial = [evaluated - evaluated**2] assert_allclose(partials_logistic(case), partial) assert_allclose(partials_logistic(case), approx_fprime_cs(case, logistic)) # # Here, np.array(case) is 2x1, so it is interpreted as i=0, 1 and t=0 case = [[0], [1.]] evaluated = logistic(case)[:, 0] partials = np.diag(evaluated - evaluated**2) partials[0, 1] = partials[1, 0] = -np.multiply(*evaluated) assert_allclose(partials_logistic(case)[:, :, 0], partials) assert_allclose(partials_logistic(case), approx_fprime_cs(np.squeeze(case), logistic)[..., None]) # Here, np.array(case) is 1x2, so it is interpreted as i=0 and t=0, 1 case = [[0, 1.]] evaluated = logistic(case) partials = (evaluated - evaluated**2)[None, ...] assert_allclose(partials_logistic(case), partials) assert_allclose(partials_logistic(case), approx_fprime_cs(case, logistic).T) # For a 3-dim, logistic(x) returns # np.exp(x[i, j, t]) / (1 + np.sum(np.exp(x[:, j, t]))) # for each i, each j, each t case = np.arange(2*3*4).reshape(2, 3, 4) evaluated = logistic(case) partials = partials_logistic(case) for t in range(4): for j in range(3): desired = np.diag(evaluated[:, j, t] - evaluated[:, j, t]**2) desired[0, 1] = desired[1, 0] = -np.multiply(*evaluated[:, j, t]) assert_allclose(partials[..., j, t], desired)
epsilon = 1e-6 nobs = 200 x = np.arange(nobs*3).reshape(nobs,-1) x = np.random.randn(nobs,3) xk = np.array([1,2,3]) xk = np.array([1.,1.,1.]) #xk = np.zeros(3) beta = xk y = np.dot(x, beta) + 0.1*np.random.randn(nobs) xkols = np.dot(np.linalg.pinv(x),y) print(approx_fprime((1,2,3),fun,epsilon,x)) gradtrue = x.sum(0) print(x.sum(0)) gradcs = approx_fprime_cs((1,2,3), fun, (x,), h=1.0e-20) print(gradcs, maxabs(gradcs, gradtrue)) print(approx_hess_cs((1,2,3), fun, (x,), h=1.0e-20)) #this is correctly zero print(approx_hess_cs((1,2,3), fun2, (y,x), h=1.0e-20)-2*np.dot(x.T, x)) print(numdiff.approx_hess(xk,fun2,1e-3, (y,x))[0] - 2*np.dot(x.T, x)) gt = (-x*2*(y-np.dot(x, [1,2,3]))[:,None]) g = approx_fprime_cs((1,2,3), fun1, (y,x), h=1.0e-20)#.T #this shouldn't be transposed gd = numdiff.approx_fprime((1,2,3),fun1,epsilon,(y,x)) print(maxabs(g, gt)) print(maxabs(gd, gt)) import statsmodels.api as sm
epsilon = 1e-6 nobs = 200 x = np.arange(nobs * 3).reshape(nobs, -1) x = np.random.randn(nobs, 3) xk = np.array([1, 2, 3]) xk = np.array([1., 1., 1.]) #xk = np.zeros(3) beta = xk y = np.dot(x, beta) + 0.1 * np.random.randn(nobs) xkols = np.dot(np.linalg.pinv(x), y) print(approx_fprime((1, 2, 3), fun, epsilon, x)) gradtrue = x.sum(0) print(x.sum(0)) gradcs = approx_fprime_cs((1, 2, 3), fun, (x, ), h=1.0e-20) print(gradcs, maxabs(gradcs, gradtrue)) print(approx_hess_cs((1, 2, 3), fun, (x, ), h=1.0e-20)) #this is correctly zero print( approx_hess_cs((1, 2, 3), fun2, (y, x), h=1.0e-20) - 2 * np.dot(x.T, x)) print(numdiff.approx_hess(xk, fun2, 1e-3, (y, x))[0] - 2 * np.dot(x.T, x)) gt = (-x * 2 * (y - np.dot(x, [1, 2, 3]))[:, None]) g = approx_fprime_cs((1, 2, 3), fun1, (y, x), h=1.0e-20) #.T #this should not be transposed gd = numdiff.approx_fprime((1, 2, 3), fun1, epsilon, (y, x)) print(maxabs(g, gt)) print(maxabs(gd, gt))