def updateAg(self):
tm = timer()
pif('updating A and g...')
JT = self.J.T
self.A = JT.dot(self.J)
self.g = JT.dot(-self.r).reshape((-1, 1))
pif('A and g updated in %.2fs' % tm())
def updateJ(self, obj):
tm = timer()
pif('computing Jacobian...')
self.J = obj.J
if self.J is None:
raise Exception("Computing Jacobian failed!")
if sp.issparse(self.J):
tm2 = timer()
self.J = self.J.tocsr()
pif('converted to csr in {}secs'.format(tm2()))
assert (self.J.nnz > 0)
elif ch.VERBOSE:
nonzero = np.count_nonzero(self.J)
pif('Jacobian dense with sparsity %.3f' % (nonzero / self.J.size))
pif('Jacobian (%dx%d) computed in %.2fs' %
(self.J.shape[0], self.J.shape[1], tm()))
if self.J.shape[1] != self.p.size:
raise Exception('Jacobian size mismatch with objective input')
return self.J
def dr_wrt(self, wrt, profiler=None):
'''
Loop over free variables and delete cache for the whole tree after finished each one
'''
if wrt is self.x:
jacs = []
for fvi, freevar in enumerate(self.free_variables):
tm = timer()
if isinstance(freevar, ch.Select):
new_jac = self.obj.dr_wrt(freevar.a, profiler=profiler)
try:
new_jac = new_jac[:, freevar.idxs]
except:
# non-csc sparse matrices may not support column-wise indexing
new_jac = new_jac.tocsc()[:, freevar.idxs]
else:
new_jac = self.obj.dr_wrt(freevar, profiler=profiler)
pif('dx wrt {} in {}sec, sparse: {}'.format(
freevar.short_name, tm(), sp.issparse(new_jac)))
if self._make_dense and sp.issparse(new_jac):
new_jac = new_jac.todense()
if self._make_sparse and not sp.issparse(new_jac):
new_jac = sp.csc_matrix(new_jac)
if new_jac is None:
raise Exception(
'Objective has no derivative wrt free variable {}. '
'You should likely remove it.'.format(fvi))
jacs.append(new_jac)
tm = timer()
utils.dfs_do_func_on_graph(self.obj, clear_cache_single)
pif('dfs_do_func_on_graph in {}sec'.format(tm()))
tm = timer()
J = hstack(jacs)
pif('hstack in {}sec'.format(tm()))
return J
def _superdot(self, lhs, rhs, profiler=None):
try:
if lhs is None:
return None
if rhs is None:
return None
if isinstance(lhs, np.ndarray) and lhs.size == 1:
lhs = lhs.ravel()[0]
if isinstance(rhs, np.ndarray) and rhs.size == 1:
rhs = rhs.ravel()[0]
if isinstance(lhs, numbers.Number) or isinstance(
rhs, numbers.Number):
return lhs * rhs
if isinstance(rhs, LinearOperator):
return LinearOperator((lhs.shape[0], rhs.shape[1]),
lambda x: lhs.dot(rhs.dot(x)))
if isinstance(lhs, LinearOperator):
if sp.issparse(rhs):
return LinearOperator((lhs.shape[0], rhs.shape[1]),
lambda x: lhs.dot(rhs.dot(x)))
else:
# TODO: ?????????????
# return lhs.matmat(rhs)
return lhs.dot(rhs)
# TODO: Figure out how/whether to do this.
tm_maybe_sparse = timer()
lhs, rhs = utils.convert_inputs_to_sparse_if_necessary(lhs, rhs)
if tm_maybe_sparse() > 0.1:
pif('convert_inputs_to_sparse_if_necessary in {}sec'.format(
tm_maybe_sparse()))
if not sp.issparse(lhs) and sp.issparse(rhs):
return rhs.T.dot(lhs.T).T
return lhs.dot(rhs)
except Exception as e:
import sys, traceback
traceback.print_exc(file=sys.stdout)
if DEBUG:
import pdb
pdb.post_mortem()
else:
raise
def updateGN(self):
tm = timer()
if sp.issparse(self.A):
self.A.eliminate_zeros()
pif('sparse solve...sparsity infill is %.3f%% (hessian %dx%d)' %
(100. * self.A.nnz / (self.A.shape[0] * self.A.shape[1]),
self.A.shape[0], self.A.shape[1]))
if self.g.size > 1:
self.d_gn = self.solve(self.A, self.g).ravel()
if np.any(np.isnan(self.d_gn)) or np.any(np.isinf(self.d_gn)):
from scipy.sparse.linalg import lsqr
warnings.warn("sparse solve failed, falling back to lsqr")
self.d_gn = lsqr(self.A, self.g)[0].ravel()
else:
self.d_gn = np.atleast_1d(self.g.ravel()[0] / self.A[0, 0])
pif('sparse solve...done in %.2fs' % tm())
else:
pif('dense solve...')
try:
self.d_gn = np.linalg.solve(self.A, self.g).ravel()
except Exception:
warnings.warn("dense solve failed, falling back to lsqr")
self.d_gn = np.linalg.lstsq(self.A, self.g)[0].ravel()
pif('dense solve...done in %.2fs' % tm())
def dr_wrt(self, wrt, reverse_mode=False, profiler=None):
tm_dr_wrt = timer()
self.called_dr_wrt = True
self._call_on_changed()
drs = []
if wrt in self._cache['drs']:
if DEBUG:
if wrt not in self._cache_info:
self._cache_info[wrt] = 0
self._cache_info[wrt] += 1
self._status = 'cached'
return self._cache['drs'][wrt]
direct_dr = self._compute_dr_wrt_sliced(wrt)
if direct_dr is not None:
drs.append(direct_dr)
if DEBUG:
self._status = 'pending'
propnames = set(_props_for(self.__class__))
for k in set(self.dterms).intersection(
propnames.union(set(self.__dict__.keys()))):
p = getattr(self, k)
if hasattr(p, 'dterms') and p is not wrt:
indirect_dr = None
if reverse_mode:
lhs = self._compute_dr_wrt_sliced(p)
if isinstance(lhs, LinearOperator):
tm_dr_wrt.pause()
dr2 = p.dr_wrt(wrt)
tm_dr_wrt.resume()
indirect_dr = lhs.matmat(dr2) if dr2 != None else None
else:
indirect_dr = p.lmult_wrt(lhs, wrt)
else: # forward mode
tm_dr_wrt.pause()
dr2 = p.dr_wrt(wrt, profiler=profiler)
tm_dr_wrt.resume()
if dr2 is not None:
indirect_dr = self.compute_rop(p, rhs=dr2)
if indirect_dr is not None:
drs.append(indirect_dr)
if len(drs) == 0:
result = None
elif len(drs) == 1:
result = drs[0]
else:
# TODO: ????????
# result = np.sum(x for x in drs)
if not np.any([isinstance(a, LinearOperator) for a in drs]):
result = reduce(lambda x, y: x + y, drs)
else:
result = LinearOperator(
drs[0].shape,
lambda x: reduce(lambda a, b: a.dot(x) + b.dot(x), drs))
# TODO: figure out how/whether to do this.
if result is not None and not sp.issparse(result):
tm_nonzero = timer()
nonzero = np.count_nonzero(result)
if tm_nonzero() > 0.1:
pif('count_nonzero in {}sec'.format(tm_nonzero()))
if nonzero == 0 or hasattr(
result, 'size') and result.size / float(nonzero) >= 10.0:
tm_convert_to_sparse = timer()
result = sp.csc_matrix(result)
import gc
gc.collect()
pif('converting result to sparse in {}sec'.format(
tm_convert_to_sparse()))
if (result is not None) and (not sp.issparse(result)) and (
not isinstance(result, LinearOperator)):
result = np.atleast_2d(result)
# When the number of parents is one, it indicates that
# caching this is probably not useful because not
# more than one parent will likely ask for this same
# thing again in the same iteration of an optimization.
#
# When the number of parents is zero, this is the top
# level object and should be cached; when it's > 1
# cache the combinations of the children.
#
# If we *always* filled in the cache, it would require
# more memory but would occasionally save a little cpu,
# on average.
if len(self._parents.keyrefs()) != 1:
self._cache['drs'][wrt] = result
if DEBUG:
self._status = 'done'
if getattr(self, '_make_dense', False) and sp.issparse(result):
result = result.todense()
if getattr(self, '_make_sparse', False) and not sp.issparse(result):
result = sp.csc_matrix(result)
if tm_dr_wrt() > 0.1:
pif('dx of {} wrt {} in {}sec, sparse: {}'.format(
self.short_name, wrt.short_name, tm_dr_wrt(),
sp.issparse(result)))
return result
def minimize_dogleg(obj,
free_variables,
on_step=None,
maxiter=200,
max_fevals=np.inf,
sparse_solver='spsolve',
disp=True,
e_1=1e-15,
e_2=1e-15,
e_3=0.,
delta_0=None,
treat_as_dense=False):
""""Nonlinear optimization using Powell's dogleg method.
See Lourakis et al, 2005, ICCV '05, "Is Levenberg-Marquardt the
Most Efficient Optimization for Implementing Bundle Adjustment?":
http://www.ics.forth.gr/cvrl/publications/conferences/0201-P0401-lourakis-levenberg.pdf
e_N are stopping conditions:
e_1 is gradient magnatude threshold
e_2 is step size magnatude threshold
e_3 is improvement threshold (as a ratio; 0.1 means it must improve by 10%% at each step)
maxiter and max_fevals are also stopping conditions. Note that they're not quite the same,
as an iteration may evaluate the function more than once.
sparse_solver is the solver to use to calculate the Gauss-Newton step in the common case
that the Jacobian is sparse. It can be 'spsolve' (in which case scipy.sparse.linalg.spsolve
will be used), 'cg' (in which case scipy.sparse.linalg.cg will be used), or any callable
that matches the api of scipy.sparse.linalg.spsolve to solve `A x = b` for x where A is sparse.
cg, uses a Conjugate Gradient method, and will be faster if A is sparse but x is dense.
spsolve will be faster if x is also sparse.
delta_0 defines the initial trust region. Generally speaking, if this is set too low then
the optimization will never really go anywhere (to small a trust region to make any real
progress before running out of iterations) and if it's set too high then the optimization
will diverge immidiately and go wild (such a large trust region that the initial step so
far overshoots that it can't recover). If it's left as None, it will be automatically
estimated on the first iteration; it's always updated at each iteration, so this is treated
only as an initialization.
handle_as_dense explicitly converts all Jacobians of obj to dense matrices
"""
solve = setup_sparse_solver(sparse_solver)
obj, callback = setup_objective(obj,
free_variables,
on_step=on_step,
disp=disp,
make_dense=treat_as_dense)
state = DoglegState(delta=delta_0, solve=solve)
state.p = obj.x.r
#inject profiler if in DEBUG mode
if ch.DEBUG:
from .monitor import DrWrtProfiler
obj.profiler = DrWrtProfiler(obj)
callback()
state.updateJ(obj)
state.r = obj.r
def stop(msg):
if not state.done:
pif(msg)
state.done = True
if np.linalg.norm(state.g, np.inf) < e_1:
stop('stopping because norm(g, np.inf) < %.2e' % e_1)
while not state.done:
state.start_iteration()
while True:
state.update_step()
if state.step_size <= e_2 * np.linalg.norm(state.p):
stop('stopping because of small step size (norm_dl < %.2e)' %
(e_2 * np.linalg.norm(state.p)))
else:
tm = timer()
obj.x = state.p + state.step
trial = state.trial_r(obj.r)
pif('Residuals computed in %.2fs' % tm())
# if the objective function improved, update input parameter estimate.
# Note that the obj.x already has the new parms,
# and we should not set them again to the same (or we'll bust the cache)
if trial.is_improvement:
state.p = state.p + state.step
callback()
if e_3 > 0. and trial.improvement < e_3:
stop('stopping because improvement < %.1e%%' %
(100 * e_3))
else:
state.updateJ(obj)
state.r = trial.r
if np.linalg.norm(state.g, np.inf) < e_1:
stop('stopping because norm(g, np.inf) < %.2e' %
e_1)
else: # Put the old parms back
obj.x = ch.Ch(state.p)
obj.on_changed(
'x') # copies from flat vector to free variables
# update our trust region
state.updateRadius(trial.rho)
if state.delta <= e_2 * np.linalg.norm(state.p):
stop('stopping because trust region is too small')
if state.done or trial.is_improvement or (obj.fevals >=
max_fevals):
break
if state.iteration >= maxiter:
stop(
'stopping because max number of user-specified iterations (%d) has been met'
% maxiter)
elif obj.fevals >= max_fevals:
stop(
'stopping because max number of user-specified func evals (%d) has been met'
% max_fevals)
return obj.free_variables