def run(self): k = 0 pk = ndt.Gradient(self.func)(self.x0) cur_x = self.x0[:] prev_x = cur_x[:] while True: tmp = cur_x[:] cur_x = [ cur_x[i] - self.lambda_by_golden_section(self.x0) * pk[i] for i in range(len(cur_x)) ] self.segments.append([prev_x, cur_x]) prev_x = cur_x[:] gr = ndt.Gradient(self.func)(cur_x) if self.len_vector(gr) < self.eps: break if k + 1 == self.n: self.x0 = cur_x[:] k = 0 pk = ndt.Gradient(self.func)(self.x0) continue b = ((self.len_vector(ndt.Gradient(self.func)(cur_x))**2) / (self.len_vector(ndt.Gradient(self.func)(tmp))**2)) pk = [-gr[i] - b * pk[i] for i in range(len(pk))] self.iterations += 1 self.answer = cur_x self.answer_point = self.func(cur_x) return self.answer
def test_project(self): dihed = self.dihed1 for _ in range(10): coords = 20 * np.random.random((4, 3)) - 10 grad = nd.Gradient(lambda x: dihed.evaluate(x.reshape((-1, 3)))) ref = grad(coords.ravel()) res = dihed.project(coords) self.assertTrue(np.allclose(ref, res)) dihed = self.dihed3 for _ in range(10): coords = 20 * np.random.random((5, 3)) - 10 grad = nd.Gradient(lambda x: dihed.evaluate(x.reshape((-1, 3)))) ref = grad(coords.ravel()) res = dihed.project(coords) self.assertTrue(np.allclose(ref, res)) dihed = self.dihed4 for _ in range(10): coords = 20 * np.random.random((6, 3)) - 10 grad = nd.Gradient(lambda x: dihed.evaluate(x.reshape((-1, 3)))) ref = grad(coords.ravel()) res = dihed.project(coords) self.assertTrue(np.allclose(ref, res))
def vanilla(foo, x0, gamma=1., err=0.00000001, max_iter=500): """ find minimum by vanilla gradient descent x0 = [x,y] start position vector """ x = x0 converged = False steps = max_iter for i in range(max_iter): grad = nd.Gradient(foo)(x) x = x - gamma * grad grad = nd.Gradient(foo)(x) # if the gradient is small enough in every direction, break if (np.abs(grad[0]) < err) & (np.abs(grad[1]) < err): converged = True steps = i break if converged: print( "vanilla gradient descent converged after {} steps".format(steps)) print("the minimum is at {}".format(x)) else: print( "vanilla gradient descent did not converge after {} steps".format( steps)) return x
def gradient(choice, x): if choice == 0: gx = nd.Gradient(toy_problem_0)(x, x) elif choice == 1: gx = nd.Gradient(toy_problem_1)(x, x) elif choice == 2: gx = nd.Gradient(toy_problem_2)(x, x) elif choice == 3: gx = nd.Gradient(toy_problem_3)(x, x) return gx[0][0]
def test_grad_0D_ind_rand(self): masses = np.array([1.] * 4) ric = RIC() ric.add_lin_bend([1, 2, 3], 4) ric.setup(masses) def func(x): ric.construct_b_matrix(None, np.reshape(x, (-1, 3))) res = np.copy(ric.get_val_lin_bends()) return res jac = nd.Jacobian(func, step_nom=[0.01] * 12) # This is buggy! def func0(x): ric.construct_b_matrix(None, np.reshape(x, (-1, 3))) res = np.copy(ric.get_val_lin_bends()) return res[0] def func1(x): ric.construct_b_matrix(None, np.reshape(x, (-1, 3))) res = np.copy(ric.get_val_lin_bends()) return res[1] grad0 = nd.Gradient(func0, step_nom=[0.01] * 12) grad1 = nd.Gradient(func1, step_nom=[0.01] * 12) for coords in 2 * np.random.random((10, 4, 3)) - 1: coords = np.array(coords, dtype=np.float64) res = ric.construct_b_matrix(None, coords) vals = np.degrees(ric.get_val_lin_bends()) #if np.any(np.abs(vals) > 45): continue # HACK: disable the axes updates _inds = np.copy(ric._ric.ric_lin_bend_inds) ric._ric.ric_lin_bend_inds[:] = 0 #ref = jac(coords.flatten()) ref0 = grad0(coords.flatten()) ref1 = grad1(coords.flatten()) # HACK: enbale the axes updates ric._ric.ric_lin_bend_inds = _inds #print coords #print vals #print ref #print ref0 #print ref1 #print res #self.assertLess(np.max(np.abs(ref-res)),1.e-8) self.assertLess(np.max(np.abs(ref0 - res[0])), 1.e-8) self.assertLess(np.max(np.abs(ref1 - res[1])), 1.e-8)
def test_grad_0D_yz(self): masses = np.array([1.] * 3) ric = RIC() ric.add_lin_bend([1, 2, 3], 'yz') ric.setup(masses) def func(x): assert x.dtype == np.float64 ric.construct_b_matrix(None, np.reshape(x, (-1, 3))) res = np.copy(ric.get_val_lin_bends()) assert res.dtype == np.float64 return res jac = nd.Jacobian(func, step_nom=[0.01] * 9) # This is buggy! def func0(x): ric.construct_b_matrix(None, np.reshape(x, (-1, 3))) res = np.copy(ric.get_val_lin_bends()) return res[0] def func1(x): ric.construct_b_matrix(None, np.reshape(x, (-1, 3))) res = np.copy(ric.get_val_lin_bends()) return res[1] grad0 = nd.Gradient(func0, step_nom=[0.01] * 9) grad1 = nd.Gradient(func1, step_nom=[0.01] * 9) for coords in [ [[-1, 0, 0], [0, 0, 0], [1, 0, 0]], # Ref [[-1, -1, -1], [0, 0, 0], [1, 1, 1]], # Diagonal [[0, 0, 0], [1, 1, 1], [2, 2, 2]], # Offset [[-3, 0, 0], [0, 0, 0], [5, 0, 0]], # Longer [[-1, 0, 0], [0, 0, 0], [1, -1, 0]], # -y [[-1, 0, 0], [0, 0, 0], [1, 1, 0]], # +y [[-1, 0, 0], [0, 0, 0], [1, 0, -1]], # -z [[-1, 0, 0], [0, 0, 0], [1, 0, 1]], # +z ]: coords = np.array(coords, dtype=np.float64) #ref = jac(coords.flatten()) ref0 = grad0(coords.flatten()) ref1 = grad1(coords.flatten()) res = ric.construct_b_matrix(None, coords) #print coords #print ref #print ref0 #print ref1 #print res #self.assertLess(np.max(np.abs(ref-res)),1.e-8) self.assertLess(np.max(np.abs(ref0 - res[0])), 1.e-8) self.assertLess(np.max(np.abs(ref1 - res[1])), 1.e-8)
def evaluate_derivatives_numdifftools( self, x, functions=('gradient', 'jacobian', 'hessian'), method='central'): import numdifftools as nd self.logger.reset() def objfun_scalar(x): obj = self.objective_function( x, *self.objective_function_arguments) obj = postprocess_objfun( obj, self.metric, return_type='scalar', method='ndifftools') self.logger.increment() return obj def objfun_vector(x): obj = self.objective_function( x, *self.objective_function_arguments) obj = postprocess_objfun( obj, self.metric, return_type='vector', method='ndifftools') self.logger.increment() return obj # Todo re-order outputs based on functions input g = [] J = [] H = [] if 'gradient' in functions: g = nd.Gradient(objfun_scalar, method=method)(x) if 'jacobian' in functions: J = nd.Jacobian(objfun_vector, method=method)(x) if 'hessian' in functions: H = nd.Hessian(objfun_scalar, method=method)(x) return g, J, H
def test_grad_0D(self): masses = np.array([1.]*4) ric = RIC() ric.add_out_bend([1,2,3,4]) ric.setup(masses) def func(x): ric.construct_b_matrix(None,np.reshape(x,(-1,3))) res = np.copy(ric.get_val_out_bends()) return res grad = nd.Gradient(func,step_nom=[0.01]*12) for coords in [ [[ 0 ,0, 0],[0,1, 0],[-1,-1,0],[1,-1,0]], [[ 0 ,0, 0],[0,1, 0],[-2,-2,0],[1,-1,0]], [[ 0 ,0, 0],[0,1, 1],[-1,-1,0],[1,-1,0]], [[ 0 ,0, 0],[0,1, 0],[-1,-1,0],[1,-1,1]], [[ 0 ,0, 0],[0,1, 1],[-2,-2,1],[2,-2,1]], [[.1,.1,.9],[0,1, 0],[-1,-1,0],[1,-1,0]], ]: coords = np.array(coords,dtype=np.float64) ref = grad(coords.flatten()) res = ric.construct_b_matrix(None,coords)[0,:] self.assertLess(np.max(np.abs(ref-res)),1.e-10)
def test_grad_beta(self): for n in [10**2]: for v in [1, 2, 10]: if v > n: v = n for c in [1, 2, 10]: if c > n: c = n for i in xrange(0, 10**2): X = np.random.randn((n * c)).reshape((n, c)) V = np.random.randn((n * v)).reshape((n, v)) y = np.random.randn((n)) hetlm_mod = hetlm.model(y, X, V) alpha = np.zeros((c)) def likelihood(beta): return hetlm_mod.likelihood(beta, alpha, negative=True) # Compute gradient numerically num_grad = nd.Gradient(likelihood)(np.zeros((v))) testing.assert_almost_equal(num_grad, hetlm_mod.grad_beta( np.zeros((v)), alpha).reshape((v)), decimal=5)
def test_gradient(self): print('Test Gradient') x = self.test_x analytic_solution = para_est.Rosenbrock() h = self.fd_step # Analytic g_analytic = analytic_solution.g(x) # numdifftools g_ndifftools = nd.Gradient(analytic_solution.f_scalar)(x) g_scipy = ndscipy.Gradient(analytic_solution.f_scalar, h)(x) # para_est self.ps.set_objective_function(analytic_solution.f_vector, metric='e') g_ps = self.ps.evaluate_derivatives(x, h, evaluate='gradient') # Log print(' g analytic : {0}'.format(g_analytic)) print(' g_ndifftools: {0}'.format(g_ndifftools)) print(' g scipy : {0}'.format(g_scipy)) print(' g ps : {0}'.format(g_ps)) print(' ps function evaluation count: {0}'.format( self.ps.logger.get_f_eval_count())) self.assertEqual( True, np.all(np.isclose(g_ps, g_analytic, rtol=1.e-14, atol=1.e-2)))
def minimize_step(objF,x0): #hard_quasi_newton Lmyhess = nd.Hessian(objF) #hessian lambda function myhess = Lmyhess(x0) #hessian call at x0 myhessinv = np.linalg.inv(myhess) #inverse #confer #test_identidade = myhess @ myhessinv #test_cond = np.linalg.cond(myhess) #li, U = np.linalg.eig(myhess) #test_pd = li>0 #assert true ture...true Lmygrad=nd.Gradient(objF) #grad lambda function mygrad=Lmygrad(x0) #grad call at x0 #confer #np.isclose(mygrad,mygrad*0.) #print('myhessinv',myhessinv) #print('mygrad',mygrad) step = -myhessinv @ mygrad #broadcasting rules make it like mxn @ nx1 and returns 1d array #print('step',step) #input('step') return step, mygrad, myhessinv
def GaussNewton_STEP(model,par0,XEXP,YEXP,VAREXP,YC,NEXP,NVENT,NVSAI,NPAR): #NOT TESTED for k in range(NEXP): YC[k,:]=model( XEXP[k,:], par0) DYO=YC-YEXP DFP = np.zeros([NEXP,NVSAI, NPAR]) #versao versatil para evitar vetores muito grandes, pode ser 100,1 ou 1,100 ou 10,10 e o resultado dá igual, só muda a gestão de mem for k in range(NEXP): Lmodel = lambda par: model( XEXP[k,:], par, ) Lmygrad=nd.Gradient( Lmodel ) #grad lambda function mygrad=Lmygrad(par0) #grad call at x0 print(mygrad) #input('mygrad') DFP[k,:,:] = mygrad #*1 U = np.zeros([NPAR]) for k in range(NEXP): #POSSO USAR UM FLATTEN EM NEXP*NVSAI U += DFP[k,:,:].T @ ( np.diag(1/VAREXP[k,:]) @ DYO[k,:]) # = 1/2 * grad T=np.zeros([NPAR,NPAR]) #ok for k in range(NEXP): T += DFP[k,:,:].T @ ( np.diag(1/VAREXP[k,:]) @ DFP[k,:,:]) #diag(1/v), se fizer 1/diag(v) vai ter termos infinitos cova = np.linalg.inv(T) # = 2 * hessinv delP = - cova @ U.T #=2hesinv*1/2grad=hesinv*grad return delP
def minimize(objF,x0): n=len(x0) res=1. tol=1e-4 i=0 imax=100 f0=objF(x0) while res>tol and i<imax: xn=x0*1 #copy fn=f0*1 #copy step,mygrad,myhessinv = minimize_step(objF,xn) alpha = minimize_linesearch(objF,xn,step) x0 = xn + step*alpha f0=objF(x0) res = np.linalg.norm(x0-xn)/n #res x #res = np.abs(f0-fn) #res f i+=1 #grad and hess after linesearch Lmyhess = nd.Hessian(objF) #hessian lambda function myhess = Lmyhess(x0) #hessian call at x0 myhessinv = np.linalg.inv(myhess) #inverse Lmygrad=nd.Gradient(objF) #grad lambda function mygrad=Lmygrad(x0) #grad call at x0 #print('i',i) return x0, f0, mygrad, myhessinv,i
def testgradient(self): fun = lambda x: np.sum(x**2) dfun = nd.Gradient(fun) d = dfun([1, 2, 3]) dtrue = [2., 4., 6.] for (di, dit) in zip(d, dtrue): self.assertAlmostEqual(di, dit)
def test_inferNormalConditional_diag(s): dx1 = 2 dx2 = 3 d = dx1 + dx2 H = lie.so2.alg(np.random.rand()) b = mvn.rvs(np.zeros(dx1), 5 * np.eye(dx1)) x2 = mvn.rvs(np.zeros(dx2), 5 * np.eye(dx2)) u = mvn.rvs(np.zeros(d), 5 * np.eye(d)) S = np.diag(np.diag(iw.rvs(2 * d, 10 * np.eye(d)))) Si = np.linalg.inv(S) # analytically infer x1 ~ N(mu, Sigma) mu, Sigma = SED.inferNormalConditional(x2, H, b, u, S) # determine mu as MAP estimate def nllNormalConditional(v): val = np.concatenate((H.dot(v) + b, x2)) t1 = val - u return 0.5 * t1.dot(Si).dot(t1) g = nd.Gradient(nllNormalConditional) map_est = so.minimize(nllNormalConditional, np.zeros(dx1), method='BFGS', jac=g) norm = np.linalg.norm(map_est.x - mu) # print(f'norm: {norm:.12f}') assert norm < 1e-2, f'SED.inferNormalConditional bad, norm {norm:.6f}'
def dmu(self, pars, scale=1e-3, fix_SN_nuisance=False, **kwargs): """ Construct the 2d-array d/dtheta_j mu_i := mu_{i,j} where theta_j is the jth parameter and mu_i is the ith entry corresponding to the ith redshift """ Nsnia = self.x.size dmu_ = np.zeros([self.N, self.Npars]) #self.N=3*740=2220 pars_names_nuisance = ['A', 'X0', 'VX', 'B', 'C0', 'VC', 'M0', 'VM'] pars_names_cosmo = [ s for s in self.pars_names if s not in pars_names_nuisance ] # get the cosmological parameters only and keep their order pars_cosmo_only = [val for par,val in zip(self.pars_names,pars) \ if par in pars_names_cosmo] pars_cosmo_only = np.asarray(pars_cosmo_only) # Get derivatives wrt cosmological parameters # step_size = scale*pars_cosmo_only #only works if param is nonzero! step_size = 1e-3 # for testing only for j, z in enumerate(self.x): # dist mod as a function of pars (fixed z) fd = lambda x: self.get_model(x).dist_mod(z) # Uncomment one only (returns 1d array): # Method 1: Finite forward difference (scipy method; inaccurate; avoid) # grad_distmod = optimize.approx_fprime(pars_cosmo_only, fd, step_size) # Method 2: Finite central difference (accurate but requires special library) grad_distmod = nd.Gradient(fd, step=step_size)(pars_cosmo_only) for df, par in zip(grad_distmod, pars_names_cosmo): dmu_[3 * j, self.ind[par]] = df # derivatives wrt nuisance parameters if fix_SN_nuisance: for par in pars_names_nuisance: dmu_[:, self.ind[par]] = np.array(Nsnia * [0., 0., 0.]) else: # default p = self.get_dict(pars) d = { 'A': [-p['X0'], 0., 0.], 'X0': [-p['A'], 1., 0.], 'B': [+p['C0'], 0., 0.], 'C0': [+p['B'], 0., 1.], 'M0': [1., 0., 0.], 'VM': [0., 0., 0.], 'VX': [0., 0., 0.], 'VC': [0., 0., 0.] } for par, val in d.iteritems(): dmu_[:, self.ind[par]] = np.array(Nsnia * val) return dmu_ # (3*Nsnia,Npars)
def _get_variance(self): if self.profile_par: pvar = self.fit_dist.par_cov[self.i_fixed, :][:, self.i_fixed] else: i_notfixed = self.i_notfixed phatv = self._par if self.profile_x: gradfun = numdifftools.Gradient(self._myinvfun) else: gradfun = numdifftools.Gradient(self._myprbfun) drl = gradfun(phatv[self.i_notfixed]) pcov = self.fit_dist.par_cov[i_notfixed, :][:, i_notfixed] pvar = sum(numpy.dot(drl, pcov) * drl) return pvar
def fit_positions( ot, bh, dt ): for num_iter in range( 1000 ): X = np.array( ot.get_positions().flat ) fun = lambda cx: obj( cx, ot, bh, dt ) g = nd.Gradient( fun, 1e-6 ) h = nd.Hessian( fun, 1e-6 ) M = np.array( h( X ) ) V = np.array( g( X ) ) d = np.linalg.solve( M, V ) norm = np.linalg.norm( d, ord=np.inf ) if norm > 1e-3: d *= 1e-3 / norm print( " sub_iter:", num_iter, "norm:", norm ) ot.set_positions( ot.get_positions() - d.reshape( ( -1, 2 ) ) ) ot.adjust_weights() # error bm = np.array( bh[ -2 ].flat ) b0 = np.array( bh[ -1 ].flat ) bt = 2 * b0 - bm bc = np.array( ot.get_centroids().flat ) dlt = bc - bt ot.set_positions( op ) return 0.5 * np.sum( dlt ** 2 )
def simulation_demo_numeric(log_mu, d, subs_counts, v_emp): eval_f_partial = functools.partial(eval_f_numeric, subs_counts, v_emp) # initialize the guess with the actual simulation parameter values y_guess = numpy.array([log_mu, d], dtype=float) result = scipy.optimize.fmin( eval_f_partial, y_guess, maxiter=10000, maxfun=10000, full_output=True, ) print('fmin result:') print(result) print() y_opt = result[0] log_mu_opt, d_opt = y_opt[0], y_opt[1] mu_opt = numpy.exp(log_mu_opt) cov = scipy.linalg.inv(numdifftools.Hessian(eval_f_partial)(y_opt)) print('maximum likelihood parameter estimates:') print('log mu:', log_mu_opt, end=' ') print('with standard deviation', numpy.sqrt(cov[0, 0])) print('d:', d_opt, end=' ') print('with standard deviation', numpy.sqrt(cov[1, 1])) print() print('gradient:') print(numdifftools.Gradient(eval_f_partial)(y_opt)) print()
def test_pruning_gradient_branch(taxa_encoded, tree): taxa_patterns, pattern_frequencies = group_sequences(taxa_encoded) pattern_frequencies = np.array(pattern_frequencies) dummy_seq = np.repeat(-1, len(list(taxa_patterns.values())[0])) child_branch_lengths, child_sequences, child_leaf_mask, child_indices = get_tables_np(tree, taxa_patterns, dummy_seq) kappa = 1.0 pi = np.ones(4)/4 kappa_ = tt.scalar() pi_ = tt.vector() child_branch_lengths_ = tt.matrix() child_sequences_ = tt.tensor3(dtype='int64') child_leaf_mask_ = tt.matrix(dtype='bool') child_indices_ = tt.matrix(dtype='int64') pattern_frequencies_ = tt.vector(dtype='int64') child_transition_probs_ = hky_transition_probs_mat(kappa_, pi_, child_branch_lengths_) ll_ = phylogenetic_log_likelihood(child_indices_, child_transition_probs_, child_sequences_, child_leaf_mask_, pattern_frequencies_, pi_) f = theano.function([kappa_, pi_, child_branch_lengths_, child_indices_, child_sequences_, child_leaf_mask_, pattern_frequencies_], ll_) grad_ = tt.grad(ll_, child_branch_lengths_) grad_f = theano.function([kappa_, pi_, child_branch_lengths_, child_indices_, child_sequences_, child_leaf_mask_, pattern_frequencies_], grad_) grad_theano = grad_f(kappa, pi, child_branch_lengths, child_indices, child_sequences, child_leaf_mask, pattern_frequencies) branch_lengths_shape = child_branch_lengths.shape partial_f = lambda branch_lengths: f(kappa, pi, branch_lengths.reshape(branch_lengths_shape), child_indices, child_sequences, child_leaf_mask, pattern_frequencies) grad_numeric = nd.Gradient(partial_f)(child_branch_lengths.flatten()) assert_allclose(grad_theano, grad_numeric.reshape(branch_lengths_shape), rtol = GRAD_RTOL)
def GD_min1(f,xlims, ylims): xa = [] i = 0 ax,bx = xlims[0],xlims[1] ay,by = ylims[0],ylims[1] x_now = init(ax,bx,ay,by) gamma = prec converged = False xa.append(x_now) while converged == False or i < N: converged = term_test(x_now,f) if converged == True: break else: df = nd.Gradient(f) x_next = x_now - gamma*df(x_now) a = (x_next - x_now) b = a.T c = (df(x_next)- df(x_now)) gamma = b*c/mag(c)**2 x_now = x_next xa.append(x_now) i += 1 xa = np.array(xa) f_min = f(x_now) #print('best guess min: ',f_min) #print('min xy vals: ', xa) return xa, i
def get_gradient(string, kernel): return_vec = np.zeros(string.shape) kr_wrp = fkr_wrp(kernel) dfun = numdifftools.Gradient(kr_wrp.calc_free_energy) for ind, v in enumerate(string): return_vec[ind] = dfun(v)[0] return return_vec
def numerical_gradient(func: Callable, params: Iterable["zfit.Parameter"]) -> tf.Tensor: """Calculate numerically the gradients of func() with respect to `params`. Args: func (Callable): Function without arguments that depends on `params` params (ZfitParameter): Parameters that `func` implicitly depends on and with respect to which the derivatives will be taken. Returns: `tf.Tensor`: gradients """ params = convert_to_container(params) def wrapped_func(param_values): for param, value in zip(params, param_values): param.assign(value) return func().numpy() param_vals = tf.stack(params) original_vals = [param.read_value() for param in params] grad_func = numdifftools.Gradient(wrapped_func) gradients = tf.py_function(grad_func, inp=[param_vals], Tout=tf.float64) if gradients.shape == (): gradients = tf.reshape(gradients, shape=(1, )) gradients.set_shape((len(param_vals), )) for param, val in zip(params, original_vals): param.set_value(val) return gradients
def test_log_derivative(): for name, machine in machines.items(): print("Machine test: %s" % name) npar = machine.n_par # random visibile state hi = machine.hilbert assert(hi.size > 0) rg = nk.utils.RandomEngine(seed=1234) v = np.zeros(hi.size) for i in range(100): hi.random_vals(v, rg) randpars = 0.1 * (np.random.randn(npar) + 1.0j * np.random.randn(npar)) machine.parameters = randpars der_log = machine.der_log(v) if("Jastrow" in name): assert(np.max(np.imag(der_log)) == approx(0.)) grad = (nd.Gradient(log_val_f, step=1.0e-8)) num_der_log = grad(randpars, machine, v) assert(np.max(np.real(der_log - num_der_log)) == approx(0., rel=1e-4, abs=1e-4)) # The imaginary part is a bit more tricky, there might be an arbitrary phase shift assert( np.max(np.exp(np.imag(der_log - num_der_log) * 1.0j) - 1.0) == approx(0., rel=4e-4, abs=4e-4))
def optimize(self): ''' perform multivariate newton method for function with vector input and scalar output ''' self.notfound = False x_t = self.start_point #Get an approximation to hessian of function H = nd.Hessian(self.func, order=4, step=1e-6) #Get an approximation of Gradient of function g = nd.Gradient(self.func, order=4, step=1e-6) for i in range(self.num_iter): gr = g(x_t) x_tplus1 = x_t - self.step_size * np.dot(np.linalg.inv(H(x_t)), gr) #check for convergence if max(abs(x_tplus1 - x_t)) < self.tol: break x_t = x_tplus1 print("step\t" + str(np.sqrt(gr.dot(gr)))) print(x_t) self.x = x_tplus1 self.max_min = self.func(x_t) return self
def test_LFM_gradient(artificial_data, models): reg_truth = Regressor(ss=models[0]) reg_truth._use_penalty = False reg_truth._use_jacobian = True dt, u, u1, y, *_ = reg_truth._prepare_data(artificial_data, ['To', 'Qh'], 'Ti') reg_lfm = Regressor(ss=models[1]) reg_lfm._use_penalty = False reg_lfm._use_jacobian = True eta_truth = deepcopy(reg_truth.ss.parameters.eta_free) eta_lfm = deepcopy(reg_lfm.ss.parameters.eta_free) grad_truth = reg_truth._eval_dlog_posterior(eta_truth, dt, u, u1, y)[1] grad_lfm = reg_lfm._eval_dlog_posterior(eta_lfm, dt, u, u1, y)[1] fct = nd.Gradient(reg_truth._eval_log_posterior) grad_truth_approx = fct(eta_truth, dt, u, u1, y) assert np.all(eta_truth == eta_lfm) assert ned(grad_truth, grad_lfm) < 1e-7 assert ned(grad_truth, grad_truth_approx) < 1e-7 assert np.all(np.sign(grad_truth) == np.sign(grad_truth_approx)) assert np.all(np.sign(grad_truth) == np.sign(grad_lfm)) assert grad_truth == pytest.approx(grad_truth_approx, rel=1e-6) assert grad_truth == pytest.approx(grad_lfm, rel=1e-6)
def __default_phase_condition(self, current_vec, last_vec): """ Implements a phase condition for a continuation procedure. This implementation returns the integrated product of the current signal, and the previous signal. This phase condition is implemented as standard in AUTO, and generally accepted as being the best choice. It will work out-the-box for any system. Nevertheless, it may be possible to produce a tailor-made, more computationally efficient phase condition, depending on the specific problem of interest. current_vec : continuation vector Continuation vector at the current predictor/corrector step last_vec : continuation vector Continuation vector at the previous predictor/corrector step Returns some float which is zero only when the phase shift between the current and reference signal is minimized. """ current_model = self.discretisor.undiscretise( self.get_discretisation(current_vec), 1) reference_model = self.discretisor.undiscretise( self.get_discretisation(last_vec), 1) reference_gradient = ndt.Gradient(reference_model) return scipy.integrate.quad( lambda t: np.inner(current_model(t), reference_gradient(t)), 0, 1)[0]
def gradient( func, params, method="central", extrapolation=True, func_args=None, func_kwargs=None ): """ Calculate the gradient of *func*. Args: func (function): A function that maps params_sr into a float. params (DataFrame): see :ref:`parmas_df` method (str): The method for the computation of the derivative. Default is central as it gives the highest accuracy. extrapolation (bool): This variable allows to specify the use of the richardson extrapolation. func_args (list): additional positional arguments for func. func_kwargs (dict): additional positional arguments for func. Returns: Series: The index is the index of params_sr. """ if method not in ["central", "forward", "backward"]: raise ValueError("Method has to be in ['central', 'forward', 'backward']") func_args = [] if func_args is None else func_args func_kwargs = {} if func_kwargs is None else func_kwargs internal_func = _create_internal_func(func, params, func_args, func_kwargs) params_value = params["value"].to_numpy() if extrapolation: grad_np = nd.Gradient(internal_func, method=method)(params_value) else: grad_np = _no_extrapolation_gradient(internal_func, params_value, method) return pd.Series(data=grad_np, index=params.index, name="gradient")
def test_grad(): #try with a model class Network(nn.Module): def __init__(self): super(Network, self).__init__() self.linear2 = nn.Linear(2, 1) self.linear2.weight.data.fill_(0.0) self.linear2.weight[0, 0] = 1. self.linear2.weight[0, 1] = 1. self.linear2.weight[1, 1] = 1. self.linear2.weight[1, 2] = 1. def forward(self, x): pax_predict = self.linear2(x) #print(self.linear2.weight.data) return pax_predict f_t = Network() def fun(x): t_x = torch.from_numpy(x).float() f_x = f_t(t_x).detach().numpy() #print(f_x.shape) return f_x ##1d square #torch time_start = time.clock() model = Network() x = torch.ones((5, 2)) print([af.jacobian(model, x[i, :]) for i in range(x.shape[0])]) print([af.hessian(model, x[i, :]) for i in range(x.shape[0])]) time_e = time.clock() - time_start print(time_e) #numerical time_start = time.clock() model = Network() x = np.ones((5, 2)) df = nd.Gradient(fun) H = nd.Hessian(fun) print(list(map(df, x.tolist()))) print(list(map(H, x.tolist()))) time_e = time.clock() - time_start print(time_e) #from mpc time_start = time.clock() model = Network() x = np.ones((5, 2)) print(grad(model, x)) x = torch.ones((5, 2)) print([af.hessian(model, x[i, :]) for i in range(x.shape[0])]) time_e = time.clock() - time_start print(time_e)
def term_test(xk,f): G = nd.Gradient(f) if np.linalg.norm(G(xk)) <= prec: print('term_test: True', np.linalg.norm(G(xk))) return True else: print('term_test: False') return False