def numericalCheckVectorGrad(costGradFunction, xx, otherArgs=None):
'''For costGradFunction of the form:
cost, grad = costGradFunction(xx, *otherArgs)
where xx and grad are vectors of the same shape, this function
checks that grad matches a numerically approxmiated grad from
the cost term.
'''
if otherArgs is None:
cost, anaGrad = costGradFunction(xx)
gradFn = Gradient(lambda x: costGradFunction(x)[0])
else:
cost, anaGrad = costGradFunction(xx, *otherArgs)
gradFn = Gradient(lambda x: costGradFunction(x, *otherArgs)[0])
numGrad = gradFn(xx)
#if any(abs(anaGrad - numGrad) > 10 * gradFn.error_estimate) or gradFn.error_estimate.max() > 1e-10:
if abs(anaGrad - numGrad).max() > 1e-8:
print '[ERROR] %s: Possible failure!' % costGradFunction.__name__
print 'cost:', cost
print '\nanalytical grad:', anaGrad
print '\nnumerical grad:', numGrad
print '\nanalytical / numerical:', anaGrad / numGrad
print 'largest error estimate:', gradFn.error_estimate.max()
print 'errors / error_estimate:', abs(anaGrad -
numGrad) / gradFn.error_estimate
else:
print '[OK] %s: Vector analytical gradient matches numerical approximation (max diff: %s).' % (
costGradFunction.__name__, abs(anaGrad - numGrad).max())
def main():
x0 = array([5, 7])
print 'fun(x0) =', fun(x0)
dfun = Gradient(lambda x: fun(x)[0])
print 'dfun(x0) =', dfun(x0)
print
data = random.randn(9, 16)
# Initialize weights WW
W0 = random.randn(16, 9)
tica = TICA(imgShape=(3, 3),
hiddenLayerShape=(4, 4),
neighborhoodSize=1,
lambd=.1,
epsilon=1e-5)
randIdx = random.randint(0, prod(W0.shape), 10)
print 'tica.cost(W0, data) = ', tica.cost(W0, data)
dcost = Gradient(lambda w: tica.cost(w.reshape(16, 9), data)[0])
print 'd(tica.cost) = ... ', dcost(W0.flatten())[randIdx]
pdb.set_trace()
def complex_gradient(func, parameter_vector, step, **gradient_kwargs):
"""
Calculate gradient for complex function by calculating derivative for amplitude and phase seperately.
Using numerical derivative package numdifftools.
Args:
func : function for gradient calculation
parameter_vector (List[float]): values of parameters(real)
step (float) : step of numerical derivative
"""
amplitude, phase = amplitude_and_phase(func(parameter_vector))
def amplitude_func(parameter_vector):
return amplitude_and_phase(func(parameter_vector))[0]
d_amplitude = Gradient(amplitude_func, step=step,
**gradient_kwargs)(parameter_vector).T
def phase_func(parameter_vector):
return amplitude_and_phase(func(parameter_vector))[1]
d_phase = Gradient(phase_func, step=step,
**gradient_kwargs)(parameter_vector).T
return derivative_from_amplitude_and_phase(amplitude, phase, d_amplitude,
d_phase)
def check_dlogp(model, value, domains):
try:
from numdifftools import Gradient
except ImportError:
return
domains = [d[1:-1] for d in domains]
bij = DictToArrayBijection(
ArrayOrdering(model.cont_vars), model.test_point)
if not model.cont_vars:
return
dlogp = bij.mapf(model.dlogpc())
logp = bij.mapf(model.logpc)
ndlogp = Gradient(logp)
for a in its.product(*domains):
pt = Point(dict((
str(var), val) for var, val in zip(model.vars, a)), model=model)
pt = bij.map(pt)
assert_almost_equal(dlogp(pt), ndlogp(pt),
decimal=6, err_msg=str(pt))
def check_dlogp(self, model, value, domain, paramdomains):
try:
from numdifftools import Gradient
except ImportError:
return
if not model.cont_vars:
return
domains = paramdomains.copy()
domains['value'] = domain
bij = DictToArrayBijection(
ArrayOrdering(model.cont_vars), model.test_point)
dlogp = bij.mapf(model.fastdlogp(model.cont_vars))
logp = bij.mapf(model.fastlogp)
def wrapped_logp(x):
try:
return logp(x)
except:
return np.nan
ndlogp = Gradient(wrapped_logp)
for pt in product(domains, n_samples=100):
pt = Point(pt, model=model)
pt = bij.map(pt)
decimals = select_by_precision(float64=6, float32=4)
assert_almost_equal(dlogp(pt), ndlogp(pt), decimal=decimals, err_msg=str(pt))
def check_dlogp(model, value, domain, paramdomains):
try:
from numdifftools import Gradient
except ImportError:
return
domains = paramdomains + [domain]
bij = DictToArrayBijection(
ArrayOrdering(model.cont_vars), model.test_point)
if not model.cont_vars:
return
dlogp = bij.mapf(model.dlogpc())
logp = bij.mapf(model.logpc)
def wrapped_logp(x):
try :
return logp(x)
except:
return np.nan
ndlogp = Gradient(wrapped_logp)
names = list(map(str, model.vars))
for a in product(domains):
pt = Point(zip(names, a), model=model)
pt = bij.map(pt)
assert_almost_equal(dlogp(pt), ndlogp(pt),
decimal=6, err_msg=str(pt))
def testTica():
data = loadFromPklGz('../data/rica_hyv_patches_16.pkl.gz')
data = data[:25, :500] # take just a small slice of data
#pdb.set_trace()
random.seed(0)
tica = TICA(imgShape=(5, 5),
hiddenLayerShape=(4, 5),
shrink=0,
lambd=.05,
epsilon=1e-5)
normData = True
if normData:
# Project each patch to the unit ball
patchNorms = sqrt(sum(data**2, 0) + (1e-8))
data = data / patchNorms
# Initialize weights WW
WW = random.randn(4 * 5, 5 * 5)
WW = (WW.T / sqrt(sum(WW**2, 1))).T
#loadedWW = loadtxt('../octave-randTheta')
#WW = loadedWW
WW = WW.flatten()
cost, sGrad = tica.cost(WW, data)
print 'Current cost is:', cost
print 'Gradient (symbolic) is:', sGrad
nGradFn = Gradient(lambda w: tica.cost(w, data)[0])
nGrad = nGradFn(WW)
print 'Gradient (finite diff) is:', nGrad
print 'diff is', sGrad - nGrad
pdb.set_trace()
def numericalCheckMatrixGrad(costGradFunction, XX, otherArgs):
'''For costGradFunction of the form:
cost, grad = costGradFunction(XX, *otherArgs)
where XX and grad are matrices of the same shape, this function
checks that grad matches a numerically approxmiated grad from
the cost term.
'''
cost, anaGrad = costGradFunction(XX, *otherArgs)
def unflatteningWrapper(func, xflat, xshape, *args):
return func(reshape(xflat, xshape), *args)
gradFn = Gradient(lambda xflat: unflatteningWrapper(
costGradFunction, xflat, XX.shape, *otherArgs)[0])
numGradFlat = gradFn(XX.flatten())
numGrad = reshape(numGradFlat, XX.shape)
#if any(abs(anaGrad - numGrad).flatten() > 10 * gradFn.error_estimate) or gradFn.error_estimate.max() > 1e-10:
if abs(anaGrad - numGrad).max() > 1e-8:
print '[ERROR] %s: Possible failure!' % costGradFunction.__name__
print 'cost:', cost
print '\nanalytical grad:', anaGrad
print '\nnumerical grad:', numGrad
print '\nanalytical / numerical:', anaGrad / numGrad
print 'largest error estimate:', gradFn.error_estimate.max()
print 'errors / error_estimate:', abs(
anaGrad - numGrad).flatten() / gradFn.error_estimate
else:
print '[OK] %s: Matrix analytical gradient matches numerical approximation (max diff: %s).' % (
costGradFunction.__name__, abs(anaGrad - numGrad).max())
def check_gradient(f, x):
print(x, "\n", f(x))
print("# grad2")
grad2 = Gradient(f)(x)
print("# building grad1")
g = grad(f)
print("# computing grad1")
grad1 = g(x)
print("gradient1\n", grad1, "\ngradient2\n", grad2)
np.allclose(grad1, grad2)
# check Hessian vector product
y = np.random.normal(size=x.shape)
gdot = lambda u: np.dot(g(u), y)
hess1, hess2 = grad(gdot)(x), Gradient(gdot)(x)
print("hess1\n", hess1, "\nhess2\n", hess2)
np.allclose(hess1, hess2)
def test_first_derivative_gradient_richardson(
example_function_gradient_fixtures):
f = example_function_gradient_fixtures["func"]
fprime = example_function_gradient_fixtures["func_prime"]
true_grad = fprime(np.ones(3))
numdifftools_grad = Gradient(f, order=2, n=3, method="central")(np.ones(3))
grad = first_derivative(f, np.ones(3), n_steps=3, method="central")
aaae(numdifftools_grad, grad)
aaae(true_grad, grad)
def test_gradients(chunksize):
from numdifftools import Gradient
zfit.run.chunking.active = True
zfit.run.chunking.max_n_points = chunksize
initial1 = 1.0
initial2 = 2
param1 = zfit.Parameter("param1", initial1)
param2 = zfit.Parameter("param2", initial2)
gauss1 = Gauss(param1, 4, obs=obs1)
gauss1.set_norm_range((-5, 5))
gauss2 = Gauss(param2, 5, obs=obs1)
gauss2.set_norm_range((-5, 5))
data1 = zfit.Data.from_tensor(obs=obs1,
tensor=z.constant(1.0, shape=(100, )))
data1.set_data_range((-5, 5))
data2 = zfit.Data.from_tensor(obs=obs1,
tensor=z.constant(1.0, shape=(100, )))
data2.set_data_range((-5, 5))
nll = UnbinnedNLL(model=[gauss1, gauss2], data=[data1, data2])
def loss_func(values):
for val, param in zip(values, nll.get_cache_deps(only_floating=True)):
param.set_value(val)
return nll.value().numpy()
# theoretical, numerical = tf.test.compute_gradient(loss_func, list(params))
gradient1 = nll.gradient(params=param1)
gradient_func = Gradient(loss_func)
# gradient_func = lambda *args, **kwargs: list(gradient_func_numpy(*args, **kwargs))
assert gradient1[0].numpy() == pytest.approx(
gradient_func([param1.numpy()]))
param1.set_value(initial1)
param2.set_value(initial2)
params = [param2, param1]
gradient2 = nll.gradient(params=params)
both_gradients_true = list(
reversed(list(gradient_func([initial1, initial2
])))) # because param2, then param1
assert [g.numpy() for g in gradient2] == pytest.approx(both_gradients_true)
param1.set_value(initial1)
param2.set_value(initial2)
gradient3 = nll.gradient()
assert frozenset(g.numpy() for g in gradient3) == pytest.approx(
frozenset(both_gradients_true))
def est_se(thi: np.ndarray, zz: List[seed.Partition], mod: Model, ctrl: Controls, ome: np.random.Generator,
pool: Parallel) -> Optional[np.ndarray]:
def f_wrap_obj(thi_: np.ndarray) -> np.ndarray:
thi_ = np.max([np.min([thi_, opt_bounds[1]], 0), opt_bounds[0]], 0)
return f_obj(thi_)
f_obj = update_objective(thi, zz, mod, ctrl, ome, pool)
opt_bounds = buffer_bounds(mod.bounds_thi[0], mod.bounds_thi[1], len(thi), ctrl.bound_buffer)
d_thi = np.min([np.abs(thi - opt_bounds[0]), np.abs(thi - opt_bounds[1])])
if d_thi == 0:
return None
info = Hessian(lambda thi_: -np.mean(f_wrap_obj(thi_)))(thi) - np.cov(Gradient(f_wrap_obj)(thi).T)
se = np.linalg.inv(info)
return se if np.all(np.diag(se) > 0) else None
def testCostGrads():
random.seed(0)
for ii in range(10):
xx = random.normal(0, 1, dim) * ii # so first point is origin
cost, sGrad = costFn(xx)
print 'Current cost is:', cost,
#print 'Gradient (symbolic) is:', sGrad
nGradFn = Gradient(lambda xx: costFn(xx)[0])
nGrad = nGradFn(xx)
#print 'Gradient (finite diff) is:', nGrad
#print 'diff is', sGrad - nGrad
print 'norm(diff) is', norm(sGrad - nGrad)
print ' grad err / est err = ', abs(sGrad - nGrad) / nGradFn.error_estimate
def run(self, logp, x_0=None, grad=None, hess=None):
if not callable(logp):
raise ValueError('logp should be callable.')
try:
x_0 = np.atleast_1d(x_0)
assert x_0.ndim == 1
except Exception:
raise ValueError('invalid value for x_0.')
if self._n_sample is None:
n_sample = min(1000, x_0.shape[-1] * 10)
else:
n_sample = self._n_sample
if not callable(hess):
if callable(grad):
def _hess(*args, **kwargs):
foo = Jacobian(grad, **self._hess_options)(*args, **kwargs)
return (foo + foo.T) / 2
hess = _hess
else:
hess = Hessian(logp, **self._hess_options)
if not callable(grad):
grad = Gradient(logp, **self._grad_options)
opt = minimize(fun=lambda x: -logp(x),
x0=x_0,
method=self._optimize_method,
jac=lambda x: -grad(x),
hess=lambda x: -hess(x),
tol=self._optimize_tol,
options=self._optimize_options)
if not opt.success:
warnings.warn(
'the optimization stopped at {}, but maybe it has not '
'converged yet.'.format(opt.x), RuntimeWarning)
x_max = opt.x
f_max = -opt.fun
cov = np.linalg.inv(make_positive(-hess(x_max), self._max_cond))
samples = self._mvn_generator(x_max, cov / self._beta, n_sample)
return LaplaceResult(x_max, f_max, samples, cov, self._beta, opt)
def __init__(self,
logp,
x_0=None,
x_max=None,
f_max=None,
grad=None,
hess=None,
logp_args=()):
if not callable(logp):
raise ValueError('logp should be callable.')
if x_max is None:
self._x_0 = np.atleast_1d(x_0)
if self._x_0.ndim != 1:
raise ValueError('x_0 should be a 1-d array.')
self._x_max = None
self._f_max = None
else:
self._x_0 = None
self._x_max = np.atleast_1d(x_max)
if self._x_max.ndim != 1:
raise ValueError('x_max should be a 1-d array.')
self._f_max = float(f_max) if (f_max is not None) else None
self._logp = logp
self._grad = grad if callable(grad) else Gradient(logp)
if callable(hess):
self._hess = hess
elif callable(grad):
def _hess(*args, **kwargs):
foo = Jacobian(grad)(*args, **kwargs)
return (foo + foo.T) / 2
self._hess = _hess
else:
self._hess = Hessian(logp)
self._logp_args = logp_args
import bayesfast as bf
import numpy as np
from numdifftools import Gradient, Hessdiag
# TODO: finish comparison
p = bf.Pipeline(var_scales=[[-10, 10], [-5, 8], [-4, 6], [-8, 6]],
hard_bounds=[[0, 0], [0, 1], [1, 0], [1, 1]])
x = np.ones(4) * 0.6
p.to_original_grad(x), np.diag(Gradient(p.to_original)(x))
p.to_original_grad2(x), np.diag(Hessdiag(p.to_original)(x))
p.from_original_grad(x), np.diag(Gradient(p.from_original)(x))
p.from_original_grad2(x), np.diag(Hessdiag(p.from_original)(x))
def maximize_likelihood(self, obsTime, shiftStart, shiftEnd, getCI=False):
x0 = np.array((12., 1.))
if not hasattr(obsTime, "__iter__") or len(np.unique(obsTime)) == 1:
optResult = optimize.OptimizeResult()
optResult.x = np.array([obsTime, 1000])
optResult.fun = -np.inf
optResult.success = True
elif not hasattr(obsTime, "__iter__") and len(np.unique(obsTime)) == 0:
optResult = optimize.OptimizeResult()
optResult.x = x0
optResult.fun = np.nan
optResult.success = True
else:
ineqConstr1 = lambda coeff: coeff
ineqConstr2 = lambda coeff: 24 - coeff[0]
fun = partial(self.neg_log_likelihood,
obsTime=obsTime,
shiftStart=shiftStart,
shiftEnd=shiftEnd)
optResult = optimize.differential_evolution(fun, [(0, 24),
(0, 10)],
maxiter=100,
disp=True)
print(optResult)
optResult = optimize.minimize(fun,
x0,
method='SLSQP',
constraints=({
"type": "ineq",
"fun": ineqConstr1
}, {
"type": "ineq",
"fun": ineqConstr2
}),
options={
'disp': False,
'ftol': 1e-08
})
if getCI:
x0 = optResult.x
ffun = lambda x: -fun(x)
jac, hess = Gradient(ffun), Hessian(ffun)
dim = len(x0)
CIs = np.zeros((dim, 2))
fun0 = -optResult.fun
hess0 = hess(x0)
for i, j in product(range(dim), range(2)):
direction = j * 2 - 1
op_result = find_profile_CI_bound(i,
direction,
x0,
ffun,
jac,
hess,
fun0=fun0,
hess0=hess0)
CIs[i, j] = op_result.x[i]
print("Confidence intervals:")
print(CIs)
print("Optimization result:")
print(optResult)
if optResult.fun < 0:
self.neg_log_likelihood(optResult.x, obsTime, shiftStart, shiftEnd)
if not optResult.success:
raise RuntimeError("Optimization was not successful")
self.location, self.kappa = optResult.x
self.AIC = 2 * (optResult.fun + 2)
self.negLL = optResult.fun
print("AIC:", self.AIC)
return optResult
def calculate_gradient(self, x: list):
return Gradient(self.function)(x)
def test_constraint_tg():
a = p.to_original_grad(x)
b = np.diag(Gradient(p.to_original)(x))
assert np.isclose(a, b).all()
def gradient(function, coordinates=None):
if coordinates is None:
return Gradient(function)
else:
return Gradient(function)(coordinates)