def run(self):
k = 0
pk = ndt.Gradient(self.func)(self.x0)
cur_x = self.x0[:]
prev_x = cur_x[:]
while True:
tmp = cur_x[:]
cur_x = [
cur_x[i] - self.lambda_by_golden_section(self.x0) * pk[i]
for i in range(len(cur_x))
]
self.segments.append([prev_x, cur_x])
prev_x = cur_x[:]
gr = ndt.Gradient(self.func)(cur_x)
if self.len_vector(gr) < self.eps:
break
if k + 1 == self.n:
self.x0 = cur_x[:]
k = 0
pk = ndt.Gradient(self.func)(self.x0)
continue
b = ((self.len_vector(ndt.Gradient(self.func)(cur_x))**2) /
(self.len_vector(ndt.Gradient(self.func)(tmp))**2))
pk = [-gr[i] - b * pk[i] for i in range(len(pk))]
self.iterations += 1
self.answer = cur_x
self.answer_point = self.func(cur_x)
return self.answer
def test_project(self):
dihed = self.dihed1
for _ in range(10):
coords = 20 * np.random.random((4, 3)) - 10
grad = nd.Gradient(lambda x: dihed.evaluate(x.reshape((-1, 3))))
ref = grad(coords.ravel())
res = dihed.project(coords)
self.assertTrue(np.allclose(ref, res))
dihed = self.dihed3
for _ in range(10):
coords = 20 * np.random.random((5, 3)) - 10
grad = nd.Gradient(lambda x: dihed.evaluate(x.reshape((-1, 3))))
ref = grad(coords.ravel())
res = dihed.project(coords)
self.assertTrue(np.allclose(ref, res))
dihed = self.dihed4
for _ in range(10):
coords = 20 * np.random.random((6, 3)) - 10
grad = nd.Gradient(lambda x: dihed.evaluate(x.reshape((-1, 3))))
ref = grad(coords.ravel())
res = dihed.project(coords)
self.assertTrue(np.allclose(ref, res))
def vanilla(foo, x0, gamma=1., err=0.00000001, max_iter=500):
"""
find minimum by vanilla gradient descent
x0 = [x,y] start position vector
"""
x = x0
converged = False
steps = max_iter
for i in range(max_iter):
grad = nd.Gradient(foo)(x)
x = x - gamma * grad
grad = nd.Gradient(foo)(x)
# if the gradient is small enough in every direction, break
if (np.abs(grad[0]) < err) & (np.abs(grad[1]) < err):
converged = True
steps = i
break
if converged:
print(
"vanilla gradient descent converged after {} steps".format(steps))
print("the minimum is at {}".format(x))
else:
print(
"vanilla gradient descent did not converge after {} steps".format(
steps))
return x
def gradient(choice, x):
if choice == 0:
gx = nd.Gradient(toy_problem_0)(x, x)
elif choice == 1:
gx = nd.Gradient(toy_problem_1)(x, x)
elif choice == 2:
gx = nd.Gradient(toy_problem_2)(x, x)
elif choice == 3:
gx = nd.Gradient(toy_problem_3)(x, x)
return gx[0][0]
def test_grad_0D_ind_rand(self):
masses = np.array([1.] * 4)
ric = RIC()
ric.add_lin_bend([1, 2, 3], 4)
ric.setup(masses)
def func(x):
ric.construct_b_matrix(None, np.reshape(x, (-1, 3)))
res = np.copy(ric.get_val_lin_bends())
return res
jac = nd.Jacobian(func, step_nom=[0.01] * 12) # This is buggy!
def func0(x):
ric.construct_b_matrix(None, np.reshape(x, (-1, 3)))
res = np.copy(ric.get_val_lin_bends())
return res[0]
def func1(x):
ric.construct_b_matrix(None, np.reshape(x, (-1, 3)))
res = np.copy(ric.get_val_lin_bends())
return res[1]
grad0 = nd.Gradient(func0, step_nom=[0.01] * 12)
grad1 = nd.Gradient(func1, step_nom=[0.01] * 12)
for coords in 2 * np.random.random((10, 4, 3)) - 1:
coords = np.array(coords, dtype=np.float64)
res = ric.construct_b_matrix(None, coords)
vals = np.degrees(ric.get_val_lin_bends())
#if np.any(np.abs(vals) > 45): continue
# HACK: disable the axes updates
_inds = np.copy(ric._ric.ric_lin_bend_inds)
ric._ric.ric_lin_bend_inds[:] = 0
#ref = jac(coords.flatten())
ref0 = grad0(coords.flatten())
ref1 = grad1(coords.flatten())
# HACK: enbale the axes updates
ric._ric.ric_lin_bend_inds = _inds
#print coords
#print vals
#print ref
#print ref0
#print ref1
#print res
#self.assertLess(np.max(np.abs(ref-res)),1.e-8)
self.assertLess(np.max(np.abs(ref0 - res[0])), 1.e-8)
self.assertLess(np.max(np.abs(ref1 - res[1])), 1.e-8)
def test_grad_0D_yz(self):
masses = np.array([1.] * 3)
ric = RIC()
ric.add_lin_bend([1, 2, 3], 'yz')
ric.setup(masses)
def func(x):
assert x.dtype == np.float64
ric.construct_b_matrix(None, np.reshape(x, (-1, 3)))
res = np.copy(ric.get_val_lin_bends())
assert res.dtype == np.float64
return res
jac = nd.Jacobian(func, step_nom=[0.01] * 9) # This is buggy!
def func0(x):
ric.construct_b_matrix(None, np.reshape(x, (-1, 3)))
res = np.copy(ric.get_val_lin_bends())
return res[0]
def func1(x):
ric.construct_b_matrix(None, np.reshape(x, (-1, 3)))
res = np.copy(ric.get_val_lin_bends())
return res[1]
grad0 = nd.Gradient(func0, step_nom=[0.01] * 9)
grad1 = nd.Gradient(func1, step_nom=[0.01] * 9)
for coords in [
[[-1, 0, 0], [0, 0, 0], [1, 0, 0]], # Ref
[[-1, -1, -1], [0, 0, 0], [1, 1, 1]], # Diagonal
[[0, 0, 0], [1, 1, 1], [2, 2, 2]], # Offset
[[-3, 0, 0], [0, 0, 0], [5, 0, 0]], # Longer
[[-1, 0, 0], [0, 0, 0], [1, -1, 0]], # -y
[[-1, 0, 0], [0, 0, 0], [1, 1, 0]], # +y
[[-1, 0, 0], [0, 0, 0], [1, 0, -1]], # -z
[[-1, 0, 0], [0, 0, 0], [1, 0, 1]], # +z
]:
coords = np.array(coords, dtype=np.float64)
#ref = jac(coords.flatten())
ref0 = grad0(coords.flatten())
ref1 = grad1(coords.flatten())
res = ric.construct_b_matrix(None, coords)
#print coords
#print ref
#print ref0
#print ref1
#print res
#self.assertLess(np.max(np.abs(ref-res)),1.e-8)
self.assertLess(np.max(np.abs(ref0 - res[0])), 1.e-8)
self.assertLess(np.max(np.abs(ref1 - res[1])), 1.e-8)
def evaluate_derivatives_numdifftools(
self, x, functions=('gradient', 'jacobian', 'hessian'),
method='central'):
import numdifftools as nd
self.logger.reset()
def objfun_scalar(x):
obj = self.objective_function(
x, *self.objective_function_arguments)
obj = postprocess_objfun(
obj, self.metric, return_type='scalar', method='ndifftools')
self.logger.increment()
return obj
def objfun_vector(x):
obj = self.objective_function(
x, *self.objective_function_arguments)
obj = postprocess_objfun(
obj, self.metric, return_type='vector', method='ndifftools')
self.logger.increment()
return obj
# Todo re-order outputs based on functions input
g = []
J = []
H = []
if 'gradient' in functions:
g = nd.Gradient(objfun_scalar, method=method)(x)
if 'jacobian' in functions:
J = nd.Jacobian(objfun_vector, method=method)(x)
if 'hessian' in functions:
H = nd.Hessian(objfun_scalar, method=method)(x)
return g, J, H
def test_grad_0D(self):
masses = np.array([1.]*4)
ric = RIC()
ric.add_out_bend([1,2,3,4])
ric.setup(masses)
def func(x):
ric.construct_b_matrix(None,np.reshape(x,(-1,3)))
res = np.copy(ric.get_val_out_bends())
return res
grad = nd.Gradient(func,step_nom=[0.01]*12)
for coords in [
[[ 0 ,0, 0],[0,1, 0],[-1,-1,0],[1,-1,0]],
[[ 0 ,0, 0],[0,1, 0],[-2,-2,0],[1,-1,0]],
[[ 0 ,0, 0],[0,1, 1],[-1,-1,0],[1,-1,0]],
[[ 0 ,0, 0],[0,1, 0],[-1,-1,0],[1,-1,1]],
[[ 0 ,0, 0],[0,1, 1],[-2,-2,1],[2,-2,1]],
[[.1,.1,.9],[0,1, 0],[-1,-1,0],[1,-1,0]],
]:
coords = np.array(coords,dtype=np.float64)
ref = grad(coords.flatten())
res = ric.construct_b_matrix(None,coords)[0,:]
self.assertLess(np.max(np.abs(ref-res)),1.e-10)
def test_grad_beta(self):
for n in [10**2]:
for v in [1, 2, 10]:
if v > n:
v = n
for c in [1, 2, 10]:
if c > n:
c = n
for i in xrange(0, 10**2):
X = np.random.randn((n * c)).reshape((n, c))
V = np.random.randn((n * v)).reshape((n, v))
y = np.random.randn((n))
hetlm_mod = hetlm.model(y, X, V)
alpha = np.zeros((c))
def likelihood(beta):
return hetlm_mod.likelihood(beta,
alpha,
negative=True)
# Compute gradient numerically
num_grad = nd.Gradient(likelihood)(np.zeros((v)))
testing.assert_almost_equal(num_grad,
hetlm_mod.grad_beta(
np.zeros((v)),
alpha).reshape((v)),
decimal=5)
def test_gradient(self):
print('Test Gradient')
x = self.test_x
analytic_solution = para_est.Rosenbrock()
h = self.fd_step
# Analytic
g_analytic = analytic_solution.g(x)
# numdifftools
g_ndifftools = nd.Gradient(analytic_solution.f_scalar)(x)
g_scipy = ndscipy.Gradient(analytic_solution.f_scalar, h)(x)
# para_est
self.ps.set_objective_function(analytic_solution.f_vector, metric='e')
g_ps = self.ps.evaluate_derivatives(x, h, evaluate='gradient')
# Log
print(' g analytic : {0}'.format(g_analytic))
print(' g_ndifftools: {0}'.format(g_ndifftools))
print(' g scipy : {0}'.format(g_scipy))
print(' g ps : {0}'.format(g_ps))
print(' ps function evaluation count: {0}'.format(
self.ps.logger.get_f_eval_count()))
self.assertEqual(
True, np.all(np.isclose(g_ps, g_analytic, rtol=1.e-14,
atol=1.e-2)))
def minimize_step(objF,x0): #hard_quasi_newton
Lmyhess = nd.Hessian(objF) #hessian lambda function
myhess = Lmyhess(x0) #hessian call at x0
myhessinv = np.linalg.inv(myhess) #inverse
#confer
#test_identidade = myhess @ myhessinv
#test_cond = np.linalg.cond(myhess)
#li, U = np.linalg.eig(myhess)
#test_pd = li>0 #assert true ture...true
Lmygrad=nd.Gradient(objF) #grad lambda function
mygrad=Lmygrad(x0) #grad call at x0
#confer
#np.isclose(mygrad,mygrad*0.)
#print('myhessinv',myhessinv)
#print('mygrad',mygrad)
step = -myhessinv @ mygrad #broadcasting rules make it like mxn @ nx1 and returns 1d array
#print('step',step)
#input('step')
return step, mygrad, myhessinv
def GaussNewton_STEP(model,par0,XEXP,YEXP,VAREXP,YC,NEXP,NVENT,NVSAI,NPAR): #NOT TESTED
for k in range(NEXP):
YC[k,:]=model( XEXP[k,:], par0)
DYO=YC-YEXP
DFP = np.zeros([NEXP,NVSAI, NPAR]) #versao versatil para evitar vetores muito grandes, pode ser 100,1 ou 1,100 ou 10,10 e o resultado dá igual, só muda a gestão de mem
for k in range(NEXP):
Lmodel = lambda par: model( XEXP[k,:], par, )
Lmygrad=nd.Gradient( Lmodel ) #grad lambda function
mygrad=Lmygrad(par0) #grad call at x0
print(mygrad)
#input('mygrad')
DFP[k,:,:] = mygrad #*1
U = np.zeros([NPAR])
for k in range(NEXP): #POSSO USAR UM FLATTEN EM NEXP*NVSAI
U += DFP[k,:,:].T @ ( np.diag(1/VAREXP[k,:]) @ DYO[k,:])
# = 1/2 * grad
T=np.zeros([NPAR,NPAR]) #ok
for k in range(NEXP):
T += DFP[k,:,:].T @ ( np.diag(1/VAREXP[k,:]) @ DFP[k,:,:]) #diag(1/v), se fizer 1/diag(v) vai ter termos infinitos
cova = np.linalg.inv(T) # = 2 * hessinv
delP = - cova @ U.T #=2hesinv*1/2grad=hesinv*grad
return delP
def minimize(objF,x0):
n=len(x0)
res=1.
tol=1e-4
i=0
imax=100
f0=objF(x0)
while res>tol and i<imax:
xn=x0*1 #copy
fn=f0*1 #copy
step,mygrad,myhessinv = minimize_step(objF,xn)
alpha = minimize_linesearch(objF,xn,step)
x0 = xn + step*alpha
f0=objF(x0)
res = np.linalg.norm(x0-xn)/n #res x
#res = np.abs(f0-fn) #res f
i+=1
#grad and hess after linesearch
Lmyhess = nd.Hessian(objF) #hessian lambda function
myhess = Lmyhess(x0) #hessian call at x0
myhessinv = np.linalg.inv(myhess) #inverse
Lmygrad=nd.Gradient(objF) #grad lambda function
mygrad=Lmygrad(x0) #grad call at x0
#print('i',i)
return x0, f0, mygrad, myhessinv,i
def testgradient(self):
fun = lambda x: np.sum(x**2)
dfun = nd.Gradient(fun)
d = dfun([1, 2, 3])
dtrue = [2., 4., 6.]
for (di, dit) in zip(d, dtrue):
self.assertAlmostEqual(di, dit)
def test_inferNormalConditional_diag(s):
dx1 = 2
dx2 = 3
d = dx1 + dx2
H = lie.so2.alg(np.random.rand())
b = mvn.rvs(np.zeros(dx1), 5 * np.eye(dx1))
x2 = mvn.rvs(np.zeros(dx2), 5 * np.eye(dx2))
u = mvn.rvs(np.zeros(d), 5 * np.eye(d))
S = np.diag(np.diag(iw.rvs(2 * d, 10 * np.eye(d))))
Si = np.linalg.inv(S)
# analytically infer x1 ~ N(mu, Sigma)
mu, Sigma = SED.inferNormalConditional(x2, H, b, u, S)
# determine mu as MAP estimate
def nllNormalConditional(v):
val = np.concatenate((H.dot(v) + b, x2))
t1 = val - u
return 0.5 * t1.dot(Si).dot(t1)
g = nd.Gradient(nllNormalConditional)
map_est = so.minimize(nllNormalConditional,
np.zeros(dx1),
method='BFGS',
jac=g)
norm = np.linalg.norm(map_est.x - mu)
# print(f'norm: {norm:.12f}')
assert norm < 1e-2, f'SED.inferNormalConditional bad, norm {norm:.6f}'
def dmu(self, pars, scale=1e-3, fix_SN_nuisance=False, **kwargs):
"""
Construct the 2d-array
d/dtheta_j mu_i := mu_{i,j}
where theta_j is the jth parameter and mu_i is the ith
entry corresponding to the ith redshift
"""
Nsnia = self.x.size
dmu_ = np.zeros([self.N, self.Npars]) #self.N=3*740=2220
pars_names_nuisance = ['A', 'X0', 'VX', 'B', 'C0', 'VC', 'M0', 'VM']
pars_names_cosmo = [
s for s in self.pars_names if s not in pars_names_nuisance
]
# get the cosmological parameters only and keep their order
pars_cosmo_only = [val for par,val in zip(self.pars_names,pars) \
if par in pars_names_cosmo]
pars_cosmo_only = np.asarray(pars_cosmo_only)
# Get derivatives wrt cosmological parameters
# step_size = scale*pars_cosmo_only #only works if param is nonzero!
step_size = 1e-3 # for testing only
for j, z in enumerate(self.x):
# dist mod as a function of pars (fixed z)
fd = lambda x: self.get_model(x).dist_mod(z)
# Uncomment one only (returns 1d array):
# Method 1: Finite forward difference (scipy method; inaccurate; avoid)
# grad_distmod = optimize.approx_fprime(pars_cosmo_only, fd, step_size)
# Method 2: Finite central difference (accurate but requires special library)
grad_distmod = nd.Gradient(fd, step=step_size)(pars_cosmo_only)
for df, par in zip(grad_distmod, pars_names_cosmo):
dmu_[3 * j, self.ind[par]] = df
# derivatives wrt nuisance parameters
if fix_SN_nuisance:
for par in pars_names_nuisance:
dmu_[:, self.ind[par]] = np.array(Nsnia * [0., 0., 0.])
else: # default
p = self.get_dict(pars)
d = {
'A': [-p['X0'], 0., 0.],
'X0': [-p['A'], 1., 0.],
'B': [+p['C0'], 0., 0.],
'C0': [+p['B'], 0., 1.],
'M0': [1., 0., 0.],
'VM': [0., 0., 0.],
'VX': [0., 0., 0.],
'VC': [0., 0., 0.]
}
for par, val in d.iteritems():
dmu_[:, self.ind[par]] = np.array(Nsnia * val)
return dmu_ # (3*Nsnia,Npars)
def _get_variance(self):
if self.profile_par:
pvar = self.fit_dist.par_cov[self.i_fixed, :][:, self.i_fixed]
else:
i_notfixed = self.i_notfixed
phatv = self._par
if self.profile_x:
gradfun = numdifftools.Gradient(self._myinvfun)
else:
gradfun = numdifftools.Gradient(self._myprbfun)
drl = gradfun(phatv[self.i_notfixed])
pcov = self.fit_dist.par_cov[i_notfixed, :][:, i_notfixed]
pvar = sum(numpy.dot(drl, pcov) * drl)
return pvar
def fit_positions( ot, bh, dt ):
for num_iter in range( 1000 ):
X = np.array( ot.get_positions().flat )
fun = lambda cx: obj( cx, ot, bh, dt )
g = nd.Gradient( fun, 1e-6 )
h = nd.Hessian( fun, 1e-6 )
M = np.array( h( X ) )
V = np.array( g( X ) )
d = np.linalg.solve( M, V )
norm = np.linalg.norm( d, ord=np.inf )
if norm > 1e-3:
d *= 1e-3 / norm
print( " sub_iter:", num_iter, "norm:", norm )
ot.set_positions( ot.get_positions() - d.reshape( ( -1, 2 ) ) )
ot.adjust_weights()
# error
bm = np.array( bh[ -2 ].flat )
b0 = np.array( bh[ -1 ].flat )
bt = 2 * b0 - bm
bc = np.array( ot.get_centroids().flat )
dlt = bc - bt
ot.set_positions( op )
return 0.5 * np.sum( dlt ** 2 )
def simulation_demo_numeric(log_mu, d, subs_counts, v_emp):
eval_f_partial = functools.partial(eval_f_numeric, subs_counts, v_emp)
# initialize the guess with the actual simulation parameter values
y_guess = numpy.array([log_mu, d], dtype=float)
result = scipy.optimize.fmin(
eval_f_partial,
y_guess,
maxiter=10000,
maxfun=10000,
full_output=True,
)
print('fmin result:')
print(result)
print()
y_opt = result[0]
log_mu_opt, d_opt = y_opt[0], y_opt[1]
mu_opt = numpy.exp(log_mu_opt)
cov = scipy.linalg.inv(numdifftools.Hessian(eval_f_partial)(y_opt))
print('maximum likelihood parameter estimates:')
print('log mu:', log_mu_opt, end=' ')
print('with standard deviation', numpy.sqrt(cov[0, 0]))
print('d:', d_opt, end=' ')
print('with standard deviation', numpy.sqrt(cov[1, 1]))
print()
print('gradient:')
print(numdifftools.Gradient(eval_f_partial)(y_opt))
print()
def test_pruning_gradient_branch(taxa_encoded, tree):
taxa_patterns, pattern_frequencies = group_sequences(taxa_encoded)
pattern_frequencies = np.array(pattern_frequencies)
dummy_seq = np.repeat(-1, len(list(taxa_patterns.values())[0]))
child_branch_lengths, child_sequences, child_leaf_mask, child_indices = get_tables_np(tree, taxa_patterns, dummy_seq)
kappa = 1.0
pi = np.ones(4)/4
kappa_ = tt.scalar()
pi_ = tt.vector()
child_branch_lengths_ = tt.matrix()
child_sequences_ = tt.tensor3(dtype='int64')
child_leaf_mask_ = tt.matrix(dtype='bool')
child_indices_ = tt.matrix(dtype='int64')
pattern_frequencies_ = tt.vector(dtype='int64')
child_transition_probs_ = hky_transition_probs_mat(kappa_, pi_, child_branch_lengths_)
ll_ = phylogenetic_log_likelihood(child_indices_, child_transition_probs_, child_sequences_, child_leaf_mask_, pattern_frequencies_, pi_)
f = theano.function([kappa_, pi_, child_branch_lengths_, child_indices_, child_sequences_, child_leaf_mask_, pattern_frequencies_], ll_)
grad_ = tt.grad(ll_, child_branch_lengths_)
grad_f = theano.function([kappa_, pi_, child_branch_lengths_, child_indices_, child_sequences_, child_leaf_mask_, pattern_frequencies_], grad_)
grad_theano = grad_f(kappa, pi, child_branch_lengths, child_indices, child_sequences, child_leaf_mask, pattern_frequencies)
branch_lengths_shape = child_branch_lengths.shape
partial_f = lambda branch_lengths: f(kappa, pi, branch_lengths.reshape(branch_lengths_shape), child_indices, child_sequences, child_leaf_mask, pattern_frequencies)
grad_numeric = nd.Gradient(partial_f)(child_branch_lengths.flatten())
assert_allclose(grad_theano, grad_numeric.reshape(branch_lengths_shape), rtol = GRAD_RTOL)
def GD_min1(f,xlims, ylims):
xa = []
i = 0
ax,bx = xlims[0],xlims[1]
ay,by = ylims[0],ylims[1]
x_now = init(ax,bx,ay,by)
gamma = prec
converged = False
xa.append(x_now)
while converged == False or i < N:
converged = term_test(x_now,f)
if converged == True:
break
else:
df = nd.Gradient(f)
x_next = x_now - gamma*df(x_now)
a = (x_next - x_now)
b = a.T
c = (df(x_next)- df(x_now))
gamma = b*c/mag(c)**2
x_now = x_next
xa.append(x_now)
i += 1
xa = np.array(xa)
f_min = f(x_now)
#print('best guess min: ',f_min)
#print('min xy vals: ', xa)
return xa, i
def get_gradient(string, kernel):
return_vec = np.zeros(string.shape)
kr_wrp = fkr_wrp(kernel)
dfun = numdifftools.Gradient(kr_wrp.calc_free_energy)
for ind, v in enumerate(string):
return_vec[ind] = dfun(v)[0]
return return_vec
def numerical_gradient(func: Callable,
params: Iterable["zfit.Parameter"]) -> tf.Tensor:
"""Calculate numerically the gradients of func() with respect to `params`.
Args:
func (Callable): Function without arguments that depends on `params`
params (ZfitParameter): Parameters that `func` implicitly depends on and with respect to which the
derivatives will be taken.
Returns:
`tf.Tensor`: gradients
"""
params = convert_to_container(params)
def wrapped_func(param_values):
for param, value in zip(params, param_values):
param.assign(value)
return func().numpy()
param_vals = tf.stack(params)
original_vals = [param.read_value() for param in params]
grad_func = numdifftools.Gradient(wrapped_func)
gradients = tf.py_function(grad_func, inp=[param_vals], Tout=tf.float64)
if gradients.shape == ():
gradients = tf.reshape(gradients, shape=(1, ))
gradients.set_shape((len(param_vals), ))
for param, val in zip(params, original_vals):
param.set_value(val)
return gradients
def test_log_derivative():
for name, machine in machines.items():
print("Machine test: %s" % name)
npar = machine.n_par
# random visibile state
hi = machine.hilbert
assert(hi.size > 0)
rg = nk.utils.RandomEngine(seed=1234)
v = np.zeros(hi.size)
for i in range(100):
hi.random_vals(v, rg)
randpars = 0.1 * (np.random.randn(npar) +
1.0j * np.random.randn(npar))
machine.parameters = randpars
der_log = machine.der_log(v)
if("Jastrow" in name):
assert(np.max(np.imag(der_log)) == approx(0.))
grad = (nd.Gradient(log_val_f, step=1.0e-8))
num_der_log = grad(randpars, machine, v)
assert(np.max(np.real(der_log - num_der_log))
== approx(0., rel=1e-4, abs=1e-4))
# The imaginary part is a bit more tricky, there might be an arbitrary phase shift
assert(
np.max(np.exp(np.imag(der_log - num_der_log) * 1.0j) - 1.0) == approx(0., rel=4e-4, abs=4e-4))
def optimize(self):
'''
perform multivariate newton method for function with vector input
and scalar output
'''
self.notfound = False
x_t = self.start_point
#Get an approximation to hessian of function
H = nd.Hessian(self.func, order=4, step=1e-6)
#Get an approximation of Gradient of function
g = nd.Gradient(self.func, order=4, step=1e-6)
for i in range(self.num_iter):
gr = g(x_t)
x_tplus1 = x_t - self.step_size * np.dot(np.linalg.inv(H(x_t)), gr)
#check for convergence
if max(abs(x_tplus1 - x_t)) < self.tol:
break
x_t = x_tplus1
print("step\t" + str(np.sqrt(gr.dot(gr))))
print(x_t)
self.x = x_tplus1
self.max_min = self.func(x_t)
return self
def test_LFM_gradient(artificial_data, models):
reg_truth = Regressor(ss=models[0])
reg_truth._use_penalty = False
reg_truth._use_jacobian = True
dt, u, u1, y, *_ = reg_truth._prepare_data(artificial_data, ['To', 'Qh'],
'Ti')
reg_lfm = Regressor(ss=models[1])
reg_lfm._use_penalty = False
reg_lfm._use_jacobian = True
eta_truth = deepcopy(reg_truth.ss.parameters.eta_free)
eta_lfm = deepcopy(reg_lfm.ss.parameters.eta_free)
grad_truth = reg_truth._eval_dlog_posterior(eta_truth, dt, u, u1, y)[1]
grad_lfm = reg_lfm._eval_dlog_posterior(eta_lfm, dt, u, u1, y)[1]
fct = nd.Gradient(reg_truth._eval_log_posterior)
grad_truth_approx = fct(eta_truth, dt, u, u1, y)
assert np.all(eta_truth == eta_lfm)
assert ned(grad_truth, grad_lfm) < 1e-7
assert ned(grad_truth, grad_truth_approx) < 1e-7
assert np.all(np.sign(grad_truth) == np.sign(grad_truth_approx))
assert np.all(np.sign(grad_truth) == np.sign(grad_lfm))
assert grad_truth == pytest.approx(grad_truth_approx, rel=1e-6)
assert grad_truth == pytest.approx(grad_lfm, rel=1e-6)
def __default_phase_condition(self, current_vec, last_vec):
"""
Implements a phase condition for a continuation procedure. This
implementation returns the integrated product of the current
signal, and the previous signal. This phase condition is
implemented as standard in AUTO, and generally accepted as being
the best choice. It will work out-the-box for any system.
Nevertheless, it may be possible to produce a tailor-made, more
computationally efficient phase condition, depending on the
specific problem of interest.
current_vec : continuation vector
Continuation vector at the current predictor/corrector
step
last_vec : continuation vector
Continuation vector at the previous predictor/corrector
step
Returns some float which is zero only when the phase shift between
the current and reference signal is minimized.
"""
current_model = self.discretisor.undiscretise(
self.get_discretisation(current_vec), 1)
reference_model = self.discretisor.undiscretise(
self.get_discretisation(last_vec), 1)
reference_gradient = ndt.Gradient(reference_model)
return scipy.integrate.quad(
lambda t: np.inner(current_model(t), reference_gradient(t)), 0,
1)[0]
def gradient(
func, params, method="central", extrapolation=True, func_args=None, func_kwargs=None
):
"""
Calculate the gradient of *func*.
Args:
func (function): A function that maps params_sr into a float.
params (DataFrame): see :ref:`parmas_df`
method (str): The method for the computation of the derivative. Default is
central as it gives the highest accuracy.
extrapolation (bool): This variable allows to specify the use of the
richardson extrapolation.
func_args (list): additional positional arguments for func.
func_kwargs (dict): additional positional arguments for func.
Returns:
Series: The index is the index of params_sr.
"""
if method not in ["central", "forward", "backward"]:
raise ValueError("Method has to be in ['central', 'forward', 'backward']")
func_args = [] if func_args is None else func_args
func_kwargs = {} if func_kwargs is None else func_kwargs
internal_func = _create_internal_func(func, params, func_args, func_kwargs)
params_value = params["value"].to_numpy()
if extrapolation:
grad_np = nd.Gradient(internal_func, method=method)(params_value)
else:
grad_np = _no_extrapolation_gradient(internal_func, params_value, method)
return pd.Series(data=grad_np, index=params.index, name="gradient")
def test_grad():
#try with a model
class Network(nn.Module):
def __init__(self):
super(Network, self).__init__()
self.linear2 = nn.Linear(2, 1)
self.linear2.weight.data.fill_(0.0)
self.linear2.weight[0, 0] = 1.
self.linear2.weight[0, 1] = 1.
self.linear2.weight[1, 1] = 1.
self.linear2.weight[1, 2] = 1.
def forward(self, x):
pax_predict = self.linear2(x)
#print(self.linear2.weight.data)
return pax_predict
f_t = Network()
def fun(x):
t_x = torch.from_numpy(x).float()
f_x = f_t(t_x).detach().numpy()
#print(f_x.shape)
return f_x
##1d square
#torch
time_start = time.clock()
model = Network()
x = torch.ones((5, 2))
print([af.jacobian(model, x[i, :]) for i in range(x.shape[0])])
print([af.hessian(model, x[i, :]) for i in range(x.shape[0])])
time_e = time.clock() - time_start
print(time_e)
#numerical
time_start = time.clock()
model = Network()
x = np.ones((5, 2))
df = nd.Gradient(fun)
H = nd.Hessian(fun)
print(list(map(df, x.tolist())))
print(list(map(H, x.tolist())))
time_e = time.clock() - time_start
print(time_e)
#from mpc
time_start = time.clock()
model = Network()
x = np.ones((5, 2))
print(grad(model, x))
x = torch.ones((5, 2))
print([af.hessian(model, x[i, :]) for i in range(x.shape[0])])
time_e = time.clock() - time_start
print(time_e)
def term_test(xk,f):
G = nd.Gradient(f)
if np.linalg.norm(G(xk)) <= prec:
print('term_test: True', np.linalg.norm(G(xk)))
return True
else:
print('term_test: False')
return False
