def numericalCheckVectorGrad(costGradFunction, xx, otherArgs=None): '''For costGradFunction of the form: cost, grad = costGradFunction(xx, *otherArgs) where xx and grad are vectors of the same shape, this function checks that grad matches a numerically approxmiated grad from the cost term. ''' if otherArgs is None: cost, anaGrad = costGradFunction(xx) gradFn = Gradient(lambda x: costGradFunction(x)[0]) else: cost, anaGrad = costGradFunction(xx, *otherArgs) gradFn = Gradient(lambda x: costGradFunction(x, *otherArgs)[0]) numGrad = gradFn(xx) #if any(abs(anaGrad - numGrad) > 10 * gradFn.error_estimate) or gradFn.error_estimate.max() > 1e-10: if abs(anaGrad - numGrad).max() > 1e-8: print '[ERROR] %s: Possible failure!' % costGradFunction.__name__ print 'cost:', cost print '\nanalytical grad:', anaGrad print '\nnumerical grad:', numGrad print '\nanalytical / numerical:', anaGrad / numGrad print 'largest error estimate:', gradFn.error_estimate.max() print 'errors / error_estimate:', abs(anaGrad - numGrad) / gradFn.error_estimate else: print '[OK] %s: Vector analytical gradient matches numerical approximation (max diff: %s).' % ( costGradFunction.__name__, abs(anaGrad - numGrad).max())
def main(): x0 = array([5, 7]) print 'fun(x0) =', fun(x0) dfun = Gradient(lambda x: fun(x)[0]) print 'dfun(x0) =', dfun(x0) print data = random.randn(9, 16) # Initialize weights WW W0 = random.randn(16, 9) tica = TICA(imgShape=(3, 3), hiddenLayerShape=(4, 4), neighborhoodSize=1, lambd=.1, epsilon=1e-5) randIdx = random.randint(0, prod(W0.shape), 10) print 'tica.cost(W0, data) = ', tica.cost(W0, data) dcost = Gradient(lambda w: tica.cost(w.reshape(16, 9), data)[0]) print 'd(tica.cost) = ... ', dcost(W0.flatten())[randIdx] pdb.set_trace()
def complex_gradient(func, parameter_vector, step, **gradient_kwargs): """ Calculate gradient for complex function by calculating derivative for amplitude and phase seperately. Using numerical derivative package numdifftools. Args: func : function for gradient calculation parameter_vector (List[float]): values of parameters(real) step (float) : step of numerical derivative """ amplitude, phase = amplitude_and_phase(func(parameter_vector)) def amplitude_func(parameter_vector): return amplitude_and_phase(func(parameter_vector))[0] d_amplitude = Gradient(amplitude_func, step=step, **gradient_kwargs)(parameter_vector).T def phase_func(parameter_vector): return amplitude_and_phase(func(parameter_vector))[1] d_phase = Gradient(phase_func, step=step, **gradient_kwargs)(parameter_vector).T return derivative_from_amplitude_and_phase(amplitude, phase, d_amplitude, d_phase)
def check_dlogp(model, value, domains): try: from numdifftools import Gradient except ImportError: return domains = [d[1:-1] for d in domains] bij = DictToArrayBijection( ArrayOrdering(model.cont_vars), model.test_point) if not model.cont_vars: return dlogp = bij.mapf(model.dlogpc()) logp = bij.mapf(model.logpc) ndlogp = Gradient(logp) for a in its.product(*domains): pt = Point(dict(( str(var), val) for var, val in zip(model.vars, a)), model=model) pt = bij.map(pt) assert_almost_equal(dlogp(pt), ndlogp(pt), decimal=6, err_msg=str(pt))
def check_dlogp(self, model, value, domain, paramdomains): try: from numdifftools import Gradient except ImportError: return if not model.cont_vars: return domains = paramdomains.copy() domains['value'] = domain bij = DictToArrayBijection( ArrayOrdering(model.cont_vars), model.test_point) dlogp = bij.mapf(model.fastdlogp(model.cont_vars)) logp = bij.mapf(model.fastlogp) def wrapped_logp(x): try: return logp(x) except: return np.nan ndlogp = Gradient(wrapped_logp) for pt in product(domains, n_samples=100): pt = Point(pt, model=model) pt = bij.map(pt) decimals = select_by_precision(float64=6, float32=4) assert_almost_equal(dlogp(pt), ndlogp(pt), decimal=decimals, err_msg=str(pt))
def check_dlogp(model, value, domain, paramdomains): try: from numdifftools import Gradient except ImportError: return domains = paramdomains + [domain] bij = DictToArrayBijection( ArrayOrdering(model.cont_vars), model.test_point) if not model.cont_vars: return dlogp = bij.mapf(model.dlogpc()) logp = bij.mapf(model.logpc) def wrapped_logp(x): try : return logp(x) except: return np.nan ndlogp = Gradient(wrapped_logp) names = list(map(str, model.vars)) for a in product(domains): pt = Point(zip(names, a), model=model) pt = bij.map(pt) assert_almost_equal(dlogp(pt), ndlogp(pt), decimal=6, err_msg=str(pt))
def testTica(): data = loadFromPklGz('../data/rica_hyv_patches_16.pkl.gz') data = data[:25, :500] # take just a small slice of data #pdb.set_trace() random.seed(0) tica = TICA(imgShape=(5, 5), hiddenLayerShape=(4, 5), shrink=0, lambd=.05, epsilon=1e-5) normData = True if normData: # Project each patch to the unit ball patchNorms = sqrt(sum(data**2, 0) + (1e-8)) data = data / patchNorms # Initialize weights WW WW = random.randn(4 * 5, 5 * 5) WW = (WW.T / sqrt(sum(WW**2, 1))).T #loadedWW = loadtxt('../octave-randTheta') #WW = loadedWW WW = WW.flatten() cost, sGrad = tica.cost(WW, data) print 'Current cost is:', cost print 'Gradient (symbolic) is:', sGrad nGradFn = Gradient(lambda w: tica.cost(w, data)[0]) nGrad = nGradFn(WW) print 'Gradient (finite diff) is:', nGrad print 'diff is', sGrad - nGrad pdb.set_trace()
def numericalCheckMatrixGrad(costGradFunction, XX, otherArgs): '''For costGradFunction of the form: cost, grad = costGradFunction(XX, *otherArgs) where XX and grad are matrices of the same shape, this function checks that grad matches a numerically approxmiated grad from the cost term. ''' cost, anaGrad = costGradFunction(XX, *otherArgs) def unflatteningWrapper(func, xflat, xshape, *args): return func(reshape(xflat, xshape), *args) gradFn = Gradient(lambda xflat: unflatteningWrapper( costGradFunction, xflat, XX.shape, *otherArgs)[0]) numGradFlat = gradFn(XX.flatten()) numGrad = reshape(numGradFlat, XX.shape) #if any(abs(anaGrad - numGrad).flatten() > 10 * gradFn.error_estimate) or gradFn.error_estimate.max() > 1e-10: if abs(anaGrad - numGrad).max() > 1e-8: print '[ERROR] %s: Possible failure!' % costGradFunction.__name__ print 'cost:', cost print '\nanalytical grad:', anaGrad print '\nnumerical grad:', numGrad print '\nanalytical / numerical:', anaGrad / numGrad print 'largest error estimate:', gradFn.error_estimate.max() print 'errors / error_estimate:', abs( anaGrad - numGrad).flatten() / gradFn.error_estimate else: print '[OK] %s: Matrix analytical gradient matches numerical approximation (max diff: %s).' % ( costGradFunction.__name__, abs(anaGrad - numGrad).max())
def check_gradient(f, x): print(x, "\n", f(x)) print("# grad2") grad2 = Gradient(f)(x) print("# building grad1") g = grad(f) print("# computing grad1") grad1 = g(x) print("gradient1\n", grad1, "\ngradient2\n", grad2) np.allclose(grad1, grad2) # check Hessian vector product y = np.random.normal(size=x.shape) gdot = lambda u: np.dot(g(u), y) hess1, hess2 = grad(gdot)(x), Gradient(gdot)(x) print("hess1\n", hess1, "\nhess2\n", hess2) np.allclose(hess1, hess2)
def test_first_derivative_gradient_richardson( example_function_gradient_fixtures): f = example_function_gradient_fixtures["func"] fprime = example_function_gradient_fixtures["func_prime"] true_grad = fprime(np.ones(3)) numdifftools_grad = Gradient(f, order=2, n=3, method="central")(np.ones(3)) grad = first_derivative(f, np.ones(3), n_steps=3, method="central") aaae(numdifftools_grad, grad) aaae(true_grad, grad)
def test_gradients(chunksize): from numdifftools import Gradient zfit.run.chunking.active = True zfit.run.chunking.max_n_points = chunksize initial1 = 1.0 initial2 = 2 param1 = zfit.Parameter("param1", initial1) param2 = zfit.Parameter("param2", initial2) gauss1 = Gauss(param1, 4, obs=obs1) gauss1.set_norm_range((-5, 5)) gauss2 = Gauss(param2, 5, obs=obs1) gauss2.set_norm_range((-5, 5)) data1 = zfit.Data.from_tensor(obs=obs1, tensor=z.constant(1.0, shape=(100, ))) data1.set_data_range((-5, 5)) data2 = zfit.Data.from_tensor(obs=obs1, tensor=z.constant(1.0, shape=(100, ))) data2.set_data_range((-5, 5)) nll = UnbinnedNLL(model=[gauss1, gauss2], data=[data1, data2]) def loss_func(values): for val, param in zip(values, nll.get_cache_deps(only_floating=True)): param.set_value(val) return nll.value().numpy() # theoretical, numerical = tf.test.compute_gradient(loss_func, list(params)) gradient1 = nll.gradient(params=param1) gradient_func = Gradient(loss_func) # gradient_func = lambda *args, **kwargs: list(gradient_func_numpy(*args, **kwargs)) assert gradient1[0].numpy() == pytest.approx( gradient_func([param1.numpy()])) param1.set_value(initial1) param2.set_value(initial2) params = [param2, param1] gradient2 = nll.gradient(params=params) both_gradients_true = list( reversed(list(gradient_func([initial1, initial2 ])))) # because param2, then param1 assert [g.numpy() for g in gradient2] == pytest.approx(both_gradients_true) param1.set_value(initial1) param2.set_value(initial2) gradient3 = nll.gradient() assert frozenset(g.numpy() for g in gradient3) == pytest.approx( frozenset(both_gradients_true))
def est_se(thi: np.ndarray, zz: List[seed.Partition], mod: Model, ctrl: Controls, ome: np.random.Generator, pool: Parallel) -> Optional[np.ndarray]: def f_wrap_obj(thi_: np.ndarray) -> np.ndarray: thi_ = np.max([np.min([thi_, opt_bounds[1]], 0), opt_bounds[0]], 0) return f_obj(thi_) f_obj = update_objective(thi, zz, mod, ctrl, ome, pool) opt_bounds = buffer_bounds(mod.bounds_thi[0], mod.bounds_thi[1], len(thi), ctrl.bound_buffer) d_thi = np.min([np.abs(thi - opt_bounds[0]), np.abs(thi - opt_bounds[1])]) if d_thi == 0: return None info = Hessian(lambda thi_: -np.mean(f_wrap_obj(thi_)))(thi) - np.cov(Gradient(f_wrap_obj)(thi).T) se = np.linalg.inv(info) return se if np.all(np.diag(se) > 0) else None
def testCostGrads(): random.seed(0) for ii in range(10): xx = random.normal(0, 1, dim) * ii # so first point is origin cost, sGrad = costFn(xx) print 'Current cost is:', cost, #print 'Gradient (symbolic) is:', sGrad nGradFn = Gradient(lambda xx: costFn(xx)[0]) nGrad = nGradFn(xx) #print 'Gradient (finite diff) is:', nGrad #print 'diff is', sGrad - nGrad print 'norm(diff) is', norm(sGrad - nGrad) print ' grad err / est err = ', abs(sGrad - nGrad) / nGradFn.error_estimate
def run(self, logp, x_0=None, grad=None, hess=None): if not callable(logp): raise ValueError('logp should be callable.') try: x_0 = np.atleast_1d(x_0) assert x_0.ndim == 1 except Exception: raise ValueError('invalid value for x_0.') if self._n_sample is None: n_sample = min(1000, x_0.shape[-1] * 10) else: n_sample = self._n_sample if not callable(hess): if callable(grad): def _hess(*args, **kwargs): foo = Jacobian(grad, **self._hess_options)(*args, **kwargs) return (foo + foo.T) / 2 hess = _hess else: hess = Hessian(logp, **self._hess_options) if not callable(grad): grad = Gradient(logp, **self._grad_options) opt = minimize(fun=lambda x: -logp(x), x0=x_0, method=self._optimize_method, jac=lambda x: -grad(x), hess=lambda x: -hess(x), tol=self._optimize_tol, options=self._optimize_options) if not opt.success: warnings.warn( 'the optimization stopped at {}, but maybe it has not ' 'converged yet.'.format(opt.x), RuntimeWarning) x_max = opt.x f_max = -opt.fun cov = np.linalg.inv(make_positive(-hess(x_max), self._max_cond)) samples = self._mvn_generator(x_max, cov / self._beta, n_sample) return LaplaceResult(x_max, f_max, samples, cov, self._beta, opt)
def __init__(self, logp, x_0=None, x_max=None, f_max=None, grad=None, hess=None, logp_args=()): if not callable(logp): raise ValueError('logp should be callable.') if x_max is None: self._x_0 = np.atleast_1d(x_0) if self._x_0.ndim != 1: raise ValueError('x_0 should be a 1-d array.') self._x_max = None self._f_max = None else: self._x_0 = None self._x_max = np.atleast_1d(x_max) if self._x_max.ndim != 1: raise ValueError('x_max should be a 1-d array.') self._f_max = float(f_max) if (f_max is not None) else None self._logp = logp self._grad = grad if callable(grad) else Gradient(logp) if callable(hess): self._hess = hess elif callable(grad): def _hess(*args, **kwargs): foo = Jacobian(grad)(*args, **kwargs) return (foo + foo.T) / 2 self._hess = _hess else: self._hess = Hessian(logp) self._logp_args = logp_args
import bayesfast as bf import numpy as np from numdifftools import Gradient, Hessdiag # TODO: finish comparison p = bf.Pipeline(var_scales=[[-10, 10], [-5, 8], [-4, 6], [-8, 6]], hard_bounds=[[0, 0], [0, 1], [1, 0], [1, 1]]) x = np.ones(4) * 0.6 p.to_original_grad(x), np.diag(Gradient(p.to_original)(x)) p.to_original_grad2(x), np.diag(Hessdiag(p.to_original)(x)) p.from_original_grad(x), np.diag(Gradient(p.from_original)(x)) p.from_original_grad2(x), np.diag(Hessdiag(p.from_original)(x))
def maximize_likelihood(self, obsTime, shiftStart, shiftEnd, getCI=False): x0 = np.array((12., 1.)) if not hasattr(obsTime, "__iter__") or len(np.unique(obsTime)) == 1: optResult = optimize.OptimizeResult() optResult.x = np.array([obsTime, 1000]) optResult.fun = -np.inf optResult.success = True elif not hasattr(obsTime, "__iter__") and len(np.unique(obsTime)) == 0: optResult = optimize.OptimizeResult() optResult.x = x0 optResult.fun = np.nan optResult.success = True else: ineqConstr1 = lambda coeff: coeff ineqConstr2 = lambda coeff: 24 - coeff[0] fun = partial(self.neg_log_likelihood, obsTime=obsTime, shiftStart=shiftStart, shiftEnd=shiftEnd) optResult = optimize.differential_evolution(fun, [(0, 24), (0, 10)], maxiter=100, disp=True) print(optResult) optResult = optimize.minimize(fun, x0, method='SLSQP', constraints=({ "type": "ineq", "fun": ineqConstr1 }, { "type": "ineq", "fun": ineqConstr2 }), options={ 'disp': False, 'ftol': 1e-08 }) if getCI: x0 = optResult.x ffun = lambda x: -fun(x) jac, hess = Gradient(ffun), Hessian(ffun) dim = len(x0) CIs = np.zeros((dim, 2)) fun0 = -optResult.fun hess0 = hess(x0) for i, j in product(range(dim), range(2)): direction = j * 2 - 1 op_result = find_profile_CI_bound(i, direction, x0, ffun, jac, hess, fun0=fun0, hess0=hess0) CIs[i, j] = op_result.x[i] print("Confidence intervals:") print(CIs) print("Optimization result:") print(optResult) if optResult.fun < 0: self.neg_log_likelihood(optResult.x, obsTime, shiftStart, shiftEnd) if not optResult.success: raise RuntimeError("Optimization was not successful") self.location, self.kappa = optResult.x self.AIC = 2 * (optResult.fun + 2) self.negLL = optResult.fun print("AIC:", self.AIC) return optResult
def calculate_gradient(self, x: list): return Gradient(self.function)(x)
def test_constraint_tg(): a = p.to_original_grad(x) b = np.diag(Gradient(p.to_original)(x)) assert np.isclose(a, b).all()
def gradient(function, coordinates=None): if coordinates is None: return Gradient(function) else: return Gradient(function)(coordinates)