def test_hess(self): #NOTE: I had to overwrite this to lessen the tolerance for test_params in self.params: he = self.mod.hessian(test_params) hefd = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hefd, decimal=DEC8) #NOTE: notice the accuracy below and the epsilon changes # this doesn't work well for score -> hessian with non-cs step # it's a little better around the optimum assert_almost_equal(he, hefd, decimal=7) hefd = numdiff.approx_fprime(test_params, self.mod.score, centered=True) assert_almost_equal(he, hefd, decimal=4) hefd = numdiff.approx_fprime(test_params, self.mod.score, 1e-9, centered=False) assert_almost_equal(he, hefd, decimal=2) hescs = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hescs, decimal=DEC8) hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike) assert_almost_equal(he, hecs, decimal=5) #NOTE: these just don't work well #hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3) #assert_almost_equal(he, hecs, decimal=1) #hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4) #assert_almost_equal(he, hecs, decimal=0) hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4) assert_almost_equal(he, hecs, decimal=0)
def test_hess(self): #NOTE: I had to overwrite this to lessen the tolerance for test_params in self.params: he = self.mod.hessian(test_params) hefd = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hefd, decimal=DEC8) #NOTE: notice the accuracy below and the epsilon changes # this does not work well for score -> hessian with non-cs step # it's a little better around the optimum assert_almost_equal(he, hefd, decimal=7) hefd = numdiff.approx_fprime(test_params, self.mod.score, centered=True) assert_almost_equal(he, hefd, decimal=4) hefd = numdiff.approx_fprime(test_params, self.mod.score, 1e-9, centered=False) assert_almost_equal(he, hefd, decimal=2) hescs = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hescs, decimal=DEC8) hecs = numdiff.approx_hess_cs(test_params, self.mod.loglike) assert_almost_equal(he, hecs, decimal=5) #NOTE: these just do not work well #hecs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-3) #assert_almost_equal(he, hecs, decimal=1) #hecs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4) #assert_almost_equal(he, hecs, decimal=0) hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-4) assert_almost_equal(he, hecs, decimal=0)
def hessian(self, params, *args, **kwargs): """ Hessian matrix of the likelihood function, evaluated at the given parameters. Parameters ---------- params : array_like Array of parameters at which to evaluate the hessian. *args, **kwargs Additional arguments to the `loglike` method. Returns ------- hessian : array Hessian matrix evaluated at `params` Notes ----- This is a numerical approximation. Both \*args and \*\*kwargs are necessary because the optimizer from `fit` must call this function and only supports passing arguments via \*args (for example `scipy.optimize.fmin_l_bfgs`). """ nargs = len(args) if nargs < 1: kwargs.setdefault('average_loglike', True) if nargs < 2: kwargs.setdefault('transformed', False) if nargs < 3: kwargs.setdefault('set_params', False) initial_state = kwargs.pop('initial_state', None) initial_state_cov = kwargs.pop('initial_state_cov', None) if initial_state is not None and initial_state_cov is not None: # If initialization is stationary, we don't want to recalculate the # initial_state_cov for each new set of parameters here initialization = self.initialization _initial_state = self._initial_state _initial_state_cov = self._initial_state_cov _initial_variance = self._initial_variance self.initialize_known(initial_state, initial_state_cov) hessian = approx_hess_cs(params, self.loglike, epsilon=1e-9, args=args, kwargs=kwargs) if initial_state is not None and initial_state_cov is not None: # Reset the initialization self.initialization = initialization self._initial_state = _initial_state self._initial_state_cov = _initial_state_cov self._initial_variance = _initial_variance return hessian
def test_hess_fun1_cs(self): for test_params in self.params: #hetrue = 0 hetrue = self.hesstrue(test_params) if hetrue is not None: #Hessian does not work for 2d return of fun fun = self.fun() hecs = numdiff.approx_hess_cs(test_params, fun, args=self.args) assert_almost_equal(hetrue, hecs, decimal=DEC6)
def test_hess_fun1_cs(self): for test_params in self.params: #hetrue = 0 hetrue = self.hesstrue(test_params) if not hetrue is None: #Hessian doesn't work for 2d return of fun fun = self.fun() hecs = numdiff.approx_hess_cs(test_params, fun, args=self.args) assert_almost_equal(hetrue, hecs, decimal=DEC6)
def calculate_se(x, init_dict, X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated): """This function calculates the standard errors for given parameterization via an approximation of the hessian matrix. """ num_ind = Y1.shape[0] + Y0.shape[0] x0 = x.copy() warning = None if init_dict["ESTIMATION"]["maxiter"] == 0: se = [np.nan] * len(x0) hess_inv = np.full((len(x0), len(x0)), np.nan) conf_interval = [[np.nan, np.nan]] * len(x0) p_values, t_values = len(x0) * [np.nan], len(x0) * [np.nan] else: norm_value = norm.ppf(0.975) # Calculate the hessian matrix, check if it is p hess = approx_hess_cs( x0, log_likelihood, args=(init_dict, X1, X0, Z1, Z0, Y1, Y0, num_treated, num_untreated), ) try: hess_inv = np.linalg.inv(hess) se = np.sqrt(np.diag(hess_inv) / num_ind) aux = norm_value * se upper = np.add(x0, aux) lower = np.subtract(x0, aux) se = se.copy() hess_inv = hess_inv conf_interval = [[lower[i], upper[i]] for i in range(len(lower))] p_values, t_values = calculate_p_values(se, x0, num_ind) except LinAlgError: se = [np.nan] * len(x0) hess_inv = np.full((len(x0), len(x0)), np.nan) conf_interval = [[np.nan, np.nan]] * len(x0) t_values = len(se) * [np.nan] p_values = len(se) * [np.nan] # Check if standard errors are defined, if not add warning message if False in np.isfinite(se): warning = [ "The estimation process was not able to provide standard errors for" " the estimation results, because the approximation \n " " of the hessian matrix " "leads to a singular Matrix.\n" ] return se, hess_inv, conf_interval, p_values, t_values, warning
def _hessian_cs(self, params, *args, **kwargs): """ Hessian matrix computed by second-order complex-step differentiation on the `loglike` function. """ transformed = (args[0] if len(args) > 0 else kwargs.get( 'transformed', False)) f = lambda params, **kwargs: self.loglike(params, **kwargs) / self.nobs hessian = approx_hess_cs(params, f, kwargs={'transformed': transformed}) return hessian
def _hessian_cs(self, params, *args, **kwargs): """ Hessian matrix computed by second-order complex-step differentiation on the `loglike` function. """ transformed = ( args[0] if len(args) > 0 else kwargs.get('transformed', False) ) f = lambda params, **kwargs: self.loglike(params, **kwargs) / self.nobs hessian = approx_hess_cs(params, f, kwargs={ 'transformed': transformed }) return hessian
def _hessian_cs(self, params, *args, **kwargs): """ Hessian matrix computed by second-order complex-step differentiation on the `loglike` function. """ nargs = len(args) if nargs < 1: kwargs.setdefault('average_loglike', True) if nargs < 2: kwargs.setdefault('transformed', False) if nargs < 3: kwargs.setdefault('set_params', False) self.update(params) return approx_hess_cs(params, self.loglike, args=args, kwargs=kwargs)
def _hessian_complex_step(self, params, **kwargs): """ Hessian matrix computed by second-order complex-step differentiation on the `loglike` function. """ # the default epsilon can be too small epsilon = _get_epsilon(params, 3., None, len(params)) kwargs['transformed'] = True kwargs['complex_step'] = True hessian = approx_hess_cs( params, self.loglike, epsilon=epsilon, kwargs=kwargs) # TODO: changed this to nobs_effective, has to be changed when merging # with statespace mlemodel return hessian / (self.nobs_effective)
def test_hess(self): for test_params in self.params: he = self.mod.hessian(test_params) hefd = numdiff.approx_fprime_cs(test_params, self.mod.score) assert_almost_equal(he, hefd, decimal=DEC8) #NOTE: notice the accuracy below assert_almost_equal(he, hefd, decimal=7) hefd = numdiff.approx_fprime(test_params, self.mod.score, centered=True) assert_allclose(he, hefd, rtol=1e-9) hefd = numdiff.approx_fprime(test_params, self.mod.score, centered=False) assert_almost_equal(he, hefd, decimal=4) hescs = numdiff.approx_fprime_cs(test_params.ravel(), self.mod.score) assert_allclose(he, hescs, rtol=1e-13) hecs = numdiff.approx_hess_cs(test_params.ravel(), self.mod.loglike) assert_allclose(he, hecs, rtol=1e-9) #NOTE: Look at the lack of precision - default epsilon not always #best grad = self.mod.score(test_params) hecs, gradcs = numdiff.approx_hess1(test_params, self.mod.loglike, 1e-6, return_grad=True) assert_almost_equal(he, hecs, decimal=1) assert_almost_equal(grad, gradcs, decimal=1) hecs, gradcs = numdiff.approx_hess2(test_params, self.mod.loglike, 1e-4, return_grad=True) assert_almost_equal(he, hecs, decimal=3) assert_almost_equal(grad, gradcs, decimal=1) hecs = numdiff.approx_hess3(test_params, self.mod.loglike, 1e-5) assert_almost_equal(he, hecs, decimal=4)
def calculate_se(rslt, init_dict, data_frame): """This function calculates the standard errors for given parameterization via an approximation of the hessian matrix.""" x0 = rslt['AUX']['x_internal'] if init_dict['ESTIMATION']['maxiter'] == 0: rslt['AUX']['standard_errors'] = [np.nan] * len(x0) rslt['AUX']['hess_inv'] = '---' rslt['AUX']['confidence_intervals'] = [[np.nan, np.nan]] * len(x0) else: # Calculate the hessian matrix, check if it is p hess = approx_hess_cs(x0, log_likelihood, args=(init_dict, data_frame)) try: hess_inv = np.linalg.inv(hess) se = np.sqrt(np.diag(hess_inv) / data_frame.shape[0]) rslt['AUX']['standard_errors'] = se rslt['AUX']['hess_inv'] = hess_inv rslt['AUX']['confidence_intervals'] = [] for counter, param in enumerate(x0): upper = param + norm.ppf(0.975) * se[counter] lower = param - norm.ppf(0.975) * se[counter] rslt['AUX']['confidence_intervals'] += [[lower, upper]] except LinAlgError: rslt['AUX']['standard_errors'] = [np.nan] * len(x0) rslt['AUX']['hess_inv'] = '---' rslt['AUX']['confidence_intervals'] = [[np.nan, np.nan]] * len(x0) # Check if standard errors are defined, if not add warning message if False in np.isfinite(rslt['AUX']['standard_errors']): rslt['warning'] += [ 'The estimation process was not able to provide standard errors for' ' the estimation results, because the approximation \n ' ' of the hessian matrix ' 'leads to a singular Matrix' ] return rslt
x = np.arange(nobs * 3).reshape(nobs, -1) x = np.random.randn(nobs, 3) xk = np.array([1, 2, 3]) xk = np.array([1., 1., 1.]) #xk = np.zeros(3) beta = xk y = np.dot(x, beta) + 0.1 * np.random.randn(nobs) xkols = np.dot(np.linalg.pinv(x), y) print approx_fprime((1, 2, 3), fun, epsilon, x) gradtrue = x.sum(0) print x.sum(0) gradcs = approx_fprime_cs((1, 2, 3), fun, (x, ), h=1.0e-20) print gradcs, maxabs(gradcs, gradtrue) print approx_hess_cs((1, 2, 3), fun, (x, ), h=1.0e-20) #this is correctly zero print approx_hess_cs((1, 2, 3), fun2, (y, x), h=1.0e-20) - 2 * np.dot(x.T, x) print numdiff.approx_hess(xk, fun2, 1e-3, (y, x))[0] - 2 * np.dot(x.T, x) gt = (-x * 2 * (y - np.dot(x, [1, 2, 3]))[:, None]) g = approx_fprime_cs((1, 2, 3), fun1, (y, x), h=1.0e-20) #.T #this shouldn't be transposed gd = numdiff.approx_fprime((1, 2, 3), fun1, epsilon, (y, x)) print maxabs(g, gt) print maxabs(gd, gt) import statsmodels.api as sm data = sm.datasets.spector.load()
x = np.arange(nobs * 3).reshape(nobs, -1) x = np.random.randn(nobs, 3) xk = np.array([1, 2, 3]) xk = np.array([1., 1., 1.]) #xk = np.zeros(3) beta = xk y = np.dot(x, beta) + 0.1 * np.random.randn(nobs) xkols = np.dot(np.linalg.pinv(x), y) print(approx_fprime((1, 2, 3), fun, epsilon, x)) gradtrue = x.sum(0) print(x.sum(0)) gradcs = approx_fprime_cs((1, 2, 3), fun, (x, ), h=1.0e-20) print(gradcs, maxabs(gradcs, gradtrue)) print(approx_hess_cs((1, 2, 3), fun, (x, ), h=1.0e-20)) #this is correctly zero print( approx_hess_cs((1, 2, 3), fun2, (y, x), h=1.0e-20) - 2 * np.dot(x.T, x)) print(numdiff.approx_hess(xk, fun2, 1e-3, (y, x))[0] - 2 * np.dot(x.T, x)) gt = (-x * 2 * (y - np.dot(x, [1, 2, 3]))[:, None]) g = approx_fprime_cs((1, 2, 3), fun1, (y, x), h=1.0e-20) #.T #this should not be transposed gd = numdiff.approx_fprime((1, 2, 3), fun1, epsilon, (y, x)) print(maxabs(g, gt)) print(maxabs(gd, gt)) data = sm.datasets.spector.load() data.exog = sm.add_constant(data.exog, prepend=False)
x = np.arange(nobs*3).reshape(nobs,-1) x = np.random.randn(nobs,3) xk = np.array([1,2,3]) xk = np.array([1.,1.,1.]) #xk = np.zeros(3) beta = xk y = np.dot(x, beta) + 0.1*np.random.randn(nobs) xkols = np.dot(np.linalg.pinv(x),y) print(approx_fprime((1,2,3),fun,epsilon,x)) gradtrue = x.sum(0) print(x.sum(0)) gradcs = approx_fprime_cs((1,2,3), fun, (x,), h=1.0e-20) print(gradcs, maxabs(gradcs, gradtrue)) print(approx_hess_cs((1,2,3), fun, (x,), h=1.0e-20)) #this is correctly zero print(approx_hess_cs((1,2,3), fun2, (y,x), h=1.0e-20)-2*np.dot(x.T, x)) print(numdiff.approx_hess(xk,fun2,1e-3, (y,x))[0] - 2*np.dot(x.T, x)) gt = (-x*2*(y-np.dot(x, [1,2,3]))[:,None]) g = approx_fprime_cs((1,2,3), fun1, (y,x), h=1.0e-20)#.T #this shouldn't be transposed gd = numdiff.approx_fprime((1,2,3),fun1,epsilon,(y,x)) print(maxabs(g, gt)) print(maxabs(gd, gt)) import statsmodels.api as sm data = sm.datasets.spector.load(as_pandas=False) data.exog = sm.add_constant(data.exog, prepend=False)