def comp_sep(A_ev, d, invN, A_dB_ev, comp_of_dB, *minimize_args, **minimize_kwargs): """ Perform component separation Build the (inverse) spectral likelihood and minimize it to estimate the parameters of the mixing matrix. Separate the components using the best-fit mixing matrix. Parameters ---------- A_ev : function The evaluator of the mixing matrix. It takes a float or an array as argument and returns the mixing matrix, a ndarray with shape *(..., n_freq, n_comp)* d: ndarray The data vector. Shape *(..., n_freq)*. invN: ndarray or None The inverse noise matrix. Shape *(..., n_freq, n_freq)*. A_dB_ev : function The evaluator of the derivative of the mixing matrix. It returns a list, each entry is the derivative with respect to a different parameter. comp_of_dB: list of IndexExpression It allows to provide as output of *A_dB_ev* only the non-zero columns *A*. ``A_dB_ev(x)[i]`` is assumed to be the derivative of ``A[comp_of_dB[i]]``. minimize_args: list Positional arguments to be passed to `scipy.optimize.minimize`. At this moment it just contains *x0*, the initial guess for the spectral parameters minimize_kwargs: dict Keyword arguments to be passed to `scipy.optimize.minimize`. A good choice for most cases is ``minimize_kwargs = {'tol': 1, options: {'disp': True}}``. *tol* depends on both the solver and your signal to noise: it should ensure that the difference between the best fit -logL and and the minimum is well less then 1, without exagereting (a difference of 1e-4 is useless). *disp* also triggers a verbose callback that monitors the convergence. Returns ------- result : scipy.optimze.OptimizeResult (dict) Result of the spectral likelihood maximisation It is the output of `scipy.optimize.minimize`, plus some extra. It includes - **x**: *(ndarray)* - the best-fit spectra indices - **Sigma**: *(ndarray)* - the semi-analytic covariance of the best-fit spectra indices patch. - **s**: *(ndarray)* - Separated components, Shape *(..., n_comp)* - **invAtNA** : *(ndarray)* - Covariance of the separated components. Shape *(..., n_comp, n_comp)* Note ---- The *...* in the arguments denote any extra set of dimension. They have to be compatible among different arguments in the `numpy` broadcasting sense. """ # If mixing matrix is fixed, separate and return if isinstance(A_ev, np.ndarray): res = sp.optimize.OptimizeResult() res.s, (u_e_v, L) = Wd(A_ev, d, invN, True) res.invAtNA = _invAtNA_svd(u_e_v) if L is not None: d = _mtv(L, d) res.chi = d - _As_svd(u_e_v, res.s) return res else: # Mixing matrix has free paramters: check that x0 was provided assert minimize_args assert len(minimize_args[0]) # Check input if A_dB_ev is not None: A_dB_ev, comp_of_dB = _A_dB_ev_and_comp_of_dB_as_compatible_list( A_dB_ev, comp_of_dB, minimize_args[0]) if 'options' in minimize_kwargs and 'disp' in minimize_kwargs['options']: disp = minimize_kwargs['options']['disp'] else: disp = False # Prepare functions for minimize fun, jac, last_values = _build_bound_inv_logL_and_logL_dB( A_ev, d, invN, A_dB_ev, comp_of_dB) minimize_kwargs['jac'] = jac # Gather minmize arguments if disp and 'callback' not in minimize_kwargs: minimize_kwargs['callback'] = verbose_callback() # Likelihood maximization res = sp.optimize.minimize(fun, *minimize_args, **minimize_kwargs) # Gather results u_e_v_last, A_dB_last, x_last, pw_d = last_values if not np.all(x_last[0] == res.x): fun(res.x) # Make sure that last_values refer to the minimum res.s = _Wd_svd(u_e_v_last[0], pw_d[0]) res.invAtNA = _invAtNA_svd(u_e_v_last[0]) res.chi = pw_d[0] - _As_svd(u_e_v_last[0], res.s) if A_dB_ev is None: fisher = numdifftools.Hessian(fun)(res.x) # TODO: something cheaper else: fisher = _fisher_logL_dB_dB_svd(u_e_v_last[0], res.s, A_dB_last[0], comp_of_dB) As_dB = (_mv(A_dB_i, res.s[comp_of_dB_i]) for A_dB_i, comp_of_dB_i in zip(A_dB_last[0], comp_of_dB)) res.chi_dB = [] for comp_of_dB_i, As_dB_i in zip(comp_of_dB, As_dB): freq_of_dB = comp_of_dB_i[:-1] + (slice(None), ) res.chi_dB.append( np.sum(res.chi[freq_of_dB] * As_dB_i, -1) / np.linalg.norm(As_dB_i, axis=-1)) try: res.Sigma = np.linalg.inv(fisher) except np.linalg.LinAlgError: res.Sigma = fisher * np.nan res.Sigma_inv = fisher return res
dX = np.stack([dX1, dX2, dX3], axis=2) sigmas = np.array([sigma_mup, sigma_r, sigma_beta]) Sigma = est.all_covariances(dX, sigmas) # burn-in for jit # calculate log=likelihood from this return est.log_likelihood(Y, Sigma) test = return_loglike(estimates) #%% import numdifftools as nd dfun = nd.Hessian(return_loglike, step = 1e-05) #Hessian_mat = dfun([eps_x, Fvac]) Hessian_mat = dfun([rho_mup, rho_r, rho_beta, sigma_mup, sigma_r, sigma_beta, eps_x, Fvac_factor]) print(Hessian_mat) #%% Fisher obs = Y.flatten().size I = - Hessian_mat var = np.linalg.inv(I ) stds = np.sqrt( abs(np.diag(var)) ) var_covar = abs(np.linalg.inv(I ) ) # std. errors print(stds)
def area_hess(self, params): hess_matrix = nd.Hessian(self.area)(params) return hess_matrix
def _ci_delta(self): # Calculate the variance-covariance matrix using the # hessian from numdifftools # This is used to obtain confidence intervals for the estimators and # the return values for several return values. # # More info about the delta method can be found on: # - Coles, Stuart: "An Introduction to Statistical Modeling of # Extreme Values", Springer (2001) # - https://en.wikipedia.org/wiki/Delta_method # data c = -self.c # We negate the shape to avoid inconsistency problems!? loc = self.loc scale = self.scale hess = _ndt.Hessian(self._nnlf) T = _np.arange(0.1, 5000, 0.1) sT = -_np.log(1. - self.frec / T) sT2 = self.distr.isf(self.frec / T) # VarCovar matrix and confidence values for estimators and return values # Confidence interval for return values (up values and down values) ci_Tu = _np.zeros(sT.shape) ci_Td = _np.zeros(sT.shape) if c: # If c then we are calculating GEV confidence intervals varcovar = _np.linalg.inv(hess([c, loc, scale])) self.params_ci = OrderedDict() se = _np.sqrt(_np.diag(varcovar)) self._se = se self.params_ci['shape'] = (self.c - _st.norm.ppf(1 - self.ci / 2) * se[0], self.c + _st.norm.ppf(1 - self.ci / 2) * se[0]) self.params_ci['location'] = ( self.loc - _st.norm.ppf(1 - self.ci / 2) * se[1], self.loc + _st.norm.ppf(1 - self.ci / 2) * se[1]) self.params_ci['scale'] = (self.scale - _st.norm.ppf(1 - self.ci / 2) * se[2], self.scale + _st.norm.ppf(1 - self.ci / 2) * se[2]) for i, val in enumerate(sT2): gradZ = [ scale * (c**-2) * (1 - sT[i]**(-c)) - scale * (c**-1) * (sT[i]**-c) * _np.log(sT[i]), 1, -(1 - sT[i]**(-c)) / c ] se = _np.dot(_np.dot(gradZ, varcovar), _np.array(gradZ).T) ci_Tu[i] = val + _st.norm.ppf(1 - self.ci / 2) * _np.sqrt(se) ci_Td[i] = val - _st.norm.ppf(1 - self.ci / 2) * _np.sqrt(se) else: # else then we are calculating Gumbel confidence intervals varcovar = _np.linalg.inv(hess([loc, scale])) self.params_ci = OrderedDict() se = _np.sqrt(_np.diag(varcovar)) self._se = se self.params_ci['shape'] = (0, 0) self.params_ci['location'] = ( self.loc - _st.norm.ppf(1 - self.ci / 2) * se[0], self.loc + _st.norm.ppf(1 - self.ci / 2) * se[0]) self.params_ci['scale'] = (self.scale - _st.norm.ppf(1 - self.ci / 2) * se[1], self.scale + _st.norm.ppf(1 - self.ci / 2) * se[1]) for i, val in enumerate(sT2): gradZ = [1, -_np.log(sT[i])] se = _np.dot(_np.dot(gradZ, varcovar), _np.array(gradZ).T) ci_Tu[i] = val + _st.norm.ppf(1 - self.ci / 2) * _np.sqrt(se) ci_Td[i] = val - _st.norm.ppf(1 - self.ci / 2) * _np.sqrt(se) self._ci_Tu = ci_Tu self._ci_Td = ci_Td
def _optimize_fit(self, obj_type=None, **kwargs): if obj_type == self.neg_loglik: method = 'MLE' else: method = 'PML' # Starting values phi = self.latent_variables.get_z_starting_values() phi = kwargs.get('start', phi).copy() # If user supplied # Optimize using L-BFGS-B p = optimize.minimize(obj_type, phi, method='L-BFGS-B') theta, Y, scores, states, states_var, X_names = self._categorize_model_output( p.x) # Check that matrix is non-singular; act accordingly try: ihessian = np.linalg.inv(nd.Hessian(obj_type)(p.x)) ses = np.power(np.abs(np.diag(ihessian)), 0.5) self.latent_variables.set_z_values(p.x, method, ses, None) # Change this in future try: latent_variables_store = self.latent_variables.copy() except: latent_variables_store = self.latent_variables return MLEResults(data_name=self.data_name, X_names=X_names, model_name=self.model_name, model_type=self.model_type, latent_variables=latent_variables_store, results=p, data=Y, index=self.index, multivariate_model=self.multivariate_model, objective_object=obj_type, method=method, ihessian=ihessian, signal=theta, scores=scores, z_hide=self._z_hide, max_lag=self.max_lag, states=states, states_var=states_var) except: self.latent_variables.set_z_values(p.x, method, None, None) # Change this in future try: latent_variables_store = self.latent_variables.copy() except: latent_variables_store = self.latent_variables return MLEResults(data_name=self.data_name, X_names=X_names, model_name=self.model_name, model_type=self.model_type, latent_variables=latent_variables_store, results=p, data=Y, index=self.index, multivariate_model=self.multivariate_model, objective_object=obj_type, method=method, ihessian=None, signal=theta, scores=scores, z_hide=self._z_hide, max_lag=self.max_lag, states=states, states_var=states_var)
def area_hess(self, params): """! @brief Compute hessian of gaussian area function """ hess_matrix = nd.Hessian(self.area)(params) return hess_matrix
# H = nd.Hessian(logcosh)([float(w[0]), float(w[1])]) # print(np.linalg.eig(H)[0]) #H = nd.Hessian(logcosh)([float(w[0]), float(w[1])]) # save direction vectors of hyperplanes w_list_0 = [] margin_list_0 = [] interval_0 = interval_generator(0, math.pi / 2, math.pi / 100) # interval_0 = [0, math.pi/10, math.pi/9, math.pi/8, math.pi/7, math.pi/6, math.pi/5, math.pi/4, math.pi/3, 2*math.pi/5,3*math.pi/7, math.pi/2] largest_eig_sigmoid = [] for angle in interval_0: w = find_hyperplane_vector(angle=angle) w_list_0.append(w) loss_val_0 = logistic_loss_0(w) loss_val_1 = logistic_loss_1(w) H_max_0 = nd.Hessian(logistic_loss_0)([float(w[0]), float(w[1])]) H_max_1 = nd.Hessian(logistic_loss_1)([float(w[0]), float(w[1])]) largest_eig_sigmoid.append( np.amax(np.linalg.eig(H_max_0)[0]) + np.amax(np.linalg.eig(H_max_1)[0])) for w in w_list_0: margin = min([ abs(np.dot(np.transpose(w), (2, 0))), abs(np.dot(np.transpose(w), (0, 2))) ]) margin_list_0.append(margin) plt.figure(1) plt.subplot(311) plt.plot(interval_0, largest_eig_sigmoid, '^-r') plt.title("angle vs max eig val")
spec_plot.set(xlabel=r'log($\lambda / \lambda_{max}$)', ylabel='bin count') minX, maxX = plt.xlim() minY, maxY = plt.ylim() yval = (maxY + minY) / 2. * 0.6 spec_plot.annotate( 'Sloppiness', xy=(minX, yval), xytext=((maxX + minX) / 2. * 0.8, yval + 0.1), # draws an arrow from one set of coordinates to the other arrowprops=dict(facecolor='black', width=3), # sets style of arrow and colour annotation_clip=False) # This enables the arrow to be outside of the plot spec_plot.annotate( '', xy=((maxX + minX) / 2. * 0.8 + 1.1, yval), xytext=(maxX, yval), # draws an arrow from one set of coordinates to the other arrowprops=dict(facecolor='black', width=3), # sets style of arrow and colour annotation_clip=False) # This enables the arrow to be outside of the plot plt.show() def test_function(x): return 2 * np.sin(x[0]) + 3 * np.cos(x[1]) H = nd.Hessian(test_function)([1.3, 0.3]) print(H) print(os.path.join(os.getcwd(), 'results', 'Sloppiness'))
bounds_error=False, fill_value=1.0) ang_penalty = (penalty_box(rotx) + penalty_box(roty) + penalty_box(rotz)) ypenalty = 0 if no_penalty: ang_penalty = 1.0 return np.array(diff) #* ang_penalty ### Optimize the previously defined function(s) no_rot_res = opti.minimize(cost_function, init, method='SLSQP') no_rot_soln = no_rot_res.x Hfun = ndt.Hessian(cost_function, full_output=True) hessian_ndt, info = Hfun(no_rot_soln) no_rot_pcov = np.linalg.inv(hessian_ndt) for i in [0, 1, 2, 3]: init_rot_2[i] = no_rot_soln[i] rot_res = opti.minimize(cost_function_rot_2, init_rot_2, method='SLSQP') rot_soln = rot_res.x Hfun2 = ndt.Hessian(cost_function_rot_2, full_output=True) hessian_ndt2, info2 = Hfun2(rot_soln) rot_pcov = np.linalg.inv(hessian_ndt2) #test_angles = np.linspace(-45.0, 45.0, 200) #test_cost = []
def hapsb_femaleROHcontam_preload(iid, roh_list, mpileup_path, h5_path1000g, meta_path_ref, folder_out=None, init_c=0.025, trim=0.005, minLen=0.05, conPop=["CEU"], roh_jump=300, e_rate_ref=1e-3, processes=1, n_ref=2504, exclude_pops=["AFR"], prefix=None, logfile=False, cleanup=False): """ Estimating autosomal contamination rate from a list of ROH blocks. Need at least one ROH for inference. Parameters ---------- iid: str IID of the sample. We assume that the mpileup file has the format $iid.chr[1-22].mpileup. roh_list: str Path to a file containing a list of ROH blocks. This file should have the same format as the output of hapROH. mpileup_path: Directory of mpileup files. One file for each autosome. h5_path1000g: str Path to the reference panel. meta_path_ref: str Path to the metadata of reference panel. folder_out: str Directory in which you want the output to reside. If not given, all output files will be in the parent directory of mpileup_path. init_c: float Initial value for the BFGS search. trim: float Trim both ends of inferred ROH blocks (in Morgan). minLen: float Minimum length of ROH blocks to use in estimating contamination (in Morgan). conPop: list of str Contaminant Ancestry. Must correspond to names in the super_pop or pop column in the 1000G metadata file. roh_jump: float Copying jump rate. e_rate_ref: float Haplotype copying error rate. processes: int Number of processes to use. n_ref: int Number of samples in the reference panel. exclude_pops: list of str A list of populations to exclude from the reference panel. prefix: str Prefix of the output and log file. The output will follow $iid.$prefix.hapCON_ROH.txt. And the log file will follow $iid.$prefix.hapCON_ROH.log. logfile: bool Whether to produce a log file. cleanup: bool Whether to delete intermediary HDF5 files generated during this function run. Returns --------- conMLE: float MLE estimate for contamination. se: float Standard error of the estimated contamination rate. """ # should be the same as hapsb_femaleROHcontam, but a faster implementation if not folder_out: folder_out = os.path.dirname(os.path.abspath(mpileup_path)) prepare_path_general(folder_out, iid, prefix, "hapCON_ROH", logfile) chunks = [] with open(roh_list) as f: f.readline() line = f.readline() while line: _, _, StartM, EndM, _, lengthM, _, ch, _, _ = line.strip().split( ',') StartM, EndM, lengthM = float(StartM), float(EndM), float(lengthM) if lengthM >= minLen: chunks.append((ch, StartM + trim, EndM - trim)) print(f'chr{ch}\t{round(StartM, 6)}\t{round(EndM, 6)}') line = f.readline() if len(chunks) > 0: sumROH = 0 for _, start, end in chunks: sumROH += end - start print( f'a total of {len(chunks)} ROH blocks passing filtering threshold found, total length after trimming: {100*sumROH:.3f}cM.' ) hdf5_path = os.path.join(folder_out, "hdf5") if not os.path.exists(hdf5_path): os.makedirs(hdf5_path) print(f'saving hdf5 files in {hdf5_path}') t1 = time.time() prms = [ [os.path.join(mpileup_path, f'{iid}.chr{ch}.mpileup'), h5_path1000g + str(ch) + ".hdf5", iid, -np.inf, np.inf, hdf5_path, False] \ for ch in range(1, 23)] results = multi_run(mpileup2hdf5, prms, processes) e_rate = np.mean(np.array([err for err, _, _ in results])) print(f'finished reading mpileup files, takes {time.time()-t1:.3f}s') print(f'estimated genotyping error: {e_rate:.3f}') # preload hmm models t1 = time.time() hmms = [] for ch, start, end in chunks: path_targets = hdf5_path + "/" + f"{iid}.chr{ch}.hdf5" hmm = preload(iid, ch, start, end, path_targets, h5_path1000g, meta_path_ref, folder_out, conPop=conPop, roh_jump=roh_jump, e_rate=e_rate, e_rate_ref=e_rate_ref, n_ref=n_ref, exclude_pops=exclude_pops) hmms.append(hmm) print( f'{len(chunks)} hmm models loaded, takes {round(time.time()-t1, 3)}s' ) # the actual optimization part kargs = (hmms, processes) res = minimize(hapsb_multiChunk_preload, init_c, args=kargs, method='L-BFGS-B', bounds=[(0, 0.5)]) if not res.success: print( 'L-BFGS-B does not converge. Printing its result log for diagnostic purpose.' ) print(res) print('please treat the final estimate with caution.') Hfun = ndt.Hessian(hapsb_multiChunk_preload, step=1e-4, full_output=True) try: x = res.x[0] h, info = Hfun(x, *kargs) h = h[0][0] if h < 0: print( 'WARNING: Cannot estimate standard error because the likelihood curve is concave up...' ) se = np.nan else: if x > 0: se = math.sqrt(1 / (h)) else: # hessian does not work well at the boundary, use a different approach print( f'use quadracitc interpolation to obtain likelihood confidence interval...' ) step = 1e-6 grad = (hapsb_multiChunk_preload(step, *kargs) - hapsb_multiChunk_preload(0, *kargs)) / step assert (grad > 0) findroot = lambda x, x0, grad, hess: hess * ( x - x0)**2 / 2.0 + (x - x0) * grad - 1.92 findroot_prime = lambda x, x0, grad, hess: (x - x0 ) * hess + grad res = newton(findroot, x, fprime=findroot_prime, args=(x, grad, h)) se = res / 1.96 if prefix: fileName = f'{iid}.{prefix}.hapCON_ROH.txt' else: fileName = f'{iid}.hapCON_ROH.txt' with open(f'{folder_out}/{fileName}', 'w') as f: f.write(f'Method1: Fixing genotyping error rate at {e_rate}\n') f.write(f'\tROH blocks obtained from: {roh_list}\n') f.write(f'\tNumber of ROH blocks found: {len(chunks)}\n') f.write( f'\tTotal length of ROH after trimming: {round(100*sumROH,3)}cM\n' ) f.write( f'\tMLE for contamination using BFGS: {round(x, 6)} ({round(x-1.96*se, 6)} - {round(x+1.96*se, 6)})\n' ) return x, se except AssertionError: print( f'cannot estimate the Hessian of the loglikelihood around {res.x}' ) se = np.nan finally: if cleanup: shutil.rmtree(hdf5_path) print(f'deleted intermediary hdf5 files at {hdf5_path}') return x, se else: print(f'not enough ROH blocks found to estimate contamination...') sys.exit()
def calc_hessian(self, x): return nd.Hessian(self.calc_fun)(x)
def _evaluate(self, x, out, *args, **kwargs): out["F"] = self.func(x) import numdifftools as nd out["dF"] = nd.Gradient(self.func)(x)[None, None, :] out["ddF"] = nd.Hessian(self.func)(x)[None, None, :]
[el for el in all_parameters if el not in parameters_to_test]) parameters_to_test = ['kSTATbinding', 'kSTATunbinding', 'kpa'] # Add jitter to best fit parameters to avoid numerical instability of finding Hessian at functional 0 (?) best_parameters = np.log([ Detailed_Model.parameters[key] * np.random.uniform(0.97, 1.03) for key in parameters_to_test ]) # Now define the X**2 function which we want to compute Hessian of. It is defined using higher order function # since this allows the Hessian to defined dynamically. new_function = function_builder(Detailed_Model, parameters_to_test, list(np.linspace(0, 60, num=61))) # Use numdifftools to compute the Hessian H = nd.Hessian(new_function)(best_parameters) H = np.nan_to_num(H) pickle.dump( H, open(os.path.join(os.getcwd(), 'results', 'Sloppiness', 'Hessian.pkl'), "wb")) print(H) # Compute eigenspectrum and normalize by largest eigenvalue # - small eigenvalues correspond to sloppy parameters while large eigenvalues give stiff parameters # - strictly speaking the eigenvalues correspond to the sloppiness along principal axes (eigenvectors) of H # - the width of the error ellipse along the corresponding axis is 1/sqrt(eval) evals = np.linalg.eigvals(H) evals = np.nan_to_num(evals) normalized_evals = np.divide(evals, max(evals)) log_normalized_evals = log10(np.absolute(normalized_evals))
def masked_curve_fit(f, x, y, p0=None, sigma=None, fixed=None, method='lm', **kw): """ Wrapper around *scipy.optimize.curve_fit* which allows fixed parameters. *fixed* is a list of integer parameter numbers that should be fixed during the fit. The parameter values for the fixed parameters are given in *p0*. The returned *popt* contains the fixed values of these parameters, and *pcov* contains rows/columns of zero for these parameters. Otherwise the interface is the same as *curve_fit*. """ from scipy.optimize import curve_fit # Hide fixed parameters if fixed is not None: p = p0 + 0. fitted = (p == p) fitted[fixed] = False init = p[fitted] def cost(x, *args, **kw): p[fitted] = args return f(x, *p, **kw) else: cost = f init = p0 # Perform fit if method == 'lm': popt, pcov = curve_fit(cost, x, y, p0=init, sigma=sigma, **kw) else: if sigma is None: sigma = 1 def chisq(p): resid = (cost(x, *p, **kw) - y) / sigma v = np.sum(resid**2) / (len(x) - len(p)) return v from scipy.optimize import minimize res = minimize(chisq, init, method=method, options={'maxiter': 1000}) popt = res.x #from scipy.optimize import fmin #popt = fmin(chisq, init, maxiter=10) try: import numdifftools as nd H = nd.Hessian(chisq)(popt) L = np.linalg.cholesky(H) Linv = np.linalg.inv(L) pcov = np.dot(Linv.T.conj(), Linv) except Exception as exc: print(exc) pcov = np.zeros((len(p0), len(p0))) # Restore fixed parameters if fixed is not None: p[fitted] = popt pcov = zero_insert(pcov, fixed) else: p = popt return p, pcov
def eval_QoIHessian(self, mu, xi): def func(xi): return self.eval_QoI(mu, xi) H = nd.Hessian(func)(xi) return H
print he[0] - 2*np.dot(x.T, x) for eps in [1e-3,1e-4,1e-5,1e-6]: print 'eps =', eps print approx_hess(xk,fun2,eps,args)[0] - 2*np.dot(x.T, x) hcs2 = approx_hess_cs2(xk,fun2,args=args) print 'hcs2' print hcs2 - 2*np.dot(x.T, x) hfd3 = approx_hess3(xk,fun2,args=args) print 'hfd3' print hfd3 - 2*np.dot(x.T, x) import numdifftools as nd hnd = nd.Hessian(lambda a: fun2(a, y, x)) hessnd = hnd(xk) print 'numdiff' print hessnd - 2*np.dot(x.T, x) #assert_almost_equal(hessnd, he[0]) gnd = nd.Gradient(lambda a: fun2(a, y, x)) gradnd = gnd(xk) ''' >>> hnd = nd.Hessian(lambda a: fun2(a, x)) >>> hnd(xk) array([[ 216.87702746, -3.41892545, 1.87887281], [ -3.41892545, 180.76379116, -13.74326021], [ 1.87887281, -13.74326021, 198.5155617 ]]) >>> he (array([[ 216.87702746, -3.41892545, 1.87887281], [ -3.41892545, 180.76379116, -13.74326021],
def _optimize_fit(self, obj_type=None, **kwargs): """ This function fits models using Maximum Likelihood or Penalized Maximum Likelihood """ preopt_search = kwargs.get('preopt_search', True) # If user supplied if obj_type == self.neg_loglik: method = 'MLE' else: method = 'PML' # Starting values - check to see if model has preoptimize method, if not, simply use default starting values if preopt_search is True: try: phi = self._preoptimize_model( self.latent_variables.get_z_starting_values(), method) preoptimized = True except: phi = self.latent_variables.get_z_starting_values() preoptimized = False else: preoptimized = False phi = self.latent_variables.get_z_starting_values() phi = kwargs.get('start', phi).copy() # If user supplied # Optimize using L-BFGS-B p = optimize.minimize(obj_type, phi, method='L-BFGS-B', options={'gtol': 1e-8}) if preoptimized is True: p2 = optimize.minimize( obj_type, self.latent_variables.get_z_starting_values(), method='L-BFGS-B', options={'gtol': 1e-8}) if self.neg_loglik(p2.x) < self.neg_loglik(p.x): p = p2 theta, Y, scores, states, states_var, X_names = self._categorize_model_output( p.x) # Check that matrix is non-singular; act accordingly try: ihessian = np.linalg.inv(nd.Hessian(obj_type)(p.x)) ses = np.power(np.abs(np.diag(ihessian)), 0.5) self.latent_variables.set_z_values(p.x, method, ses, None) except: ihessian = None ses = None self.latent_variables.set_z_values(p.x, method, None, None) self.latent_variables.estimation_method = method # Change this in future try: latent_variables_store = self.latent_variables.copy() except: latent_variables_store = self.latent_variables return MLEResults(data_name=self.data_name, X_names=X_names, model_name=self.model_name, model_type=self.model_type, latent_variables=latent_variables_store, results=p, data=Y, index=self.index, multivariate_model=self.multivariate_model, objective_object=obj_type, method=method, ihessian=ihessian, signal=theta, scores=scores, z_hide=self._z_hide, max_lag=self.max_lag, states=states, states_var=states_var)