def __init__(self, iava, dims, dtype="float64"): ncp = get_array_module(iava) # check non-unique pairs (works only with numpy arrays) _checkunique(to_numpy(iava)) # define dimension of data ndims = len(dims) self.dims = dims self.dimsd = [len(iava[1])] + list(dims[2:]) # find indices and weights self.iava_t = ncp.floor(iava[0]).astype(int) self.iava_b = self.iava_t + 1 self.weights_tb = iava[0] - self.iava_t self.iava_l = ncp.floor(iava[1]).astype(int) self.iava_r = self.iava_l + 1 self.weights_lr = iava[1] - self.iava_l # expand dims to weights for nd-arrays if ndims > 2: for _ in range(ndims - 2): self.weights_tb = ncp.expand_dims(self.weights_tb, axis=-1) self.weights_lr = ncp.expand_dims(self.weights_lr, axis=-1) self.shape = (np.prod(np.array(self.dimsd)), np.prod(np.array(self.dims))) self.dtype = np.dtype(dtype) self.explicit = False
def FISTA(Op, data, niter, eps=0.1, alpha=None, eigsiter=None, eigstol=0, tol=1e-10, returninfo=False, show=False, threshkind='soft', perc=None, callback=None): r"""Fast Iterative Shrinkage-Thresholding Algorithm (FISTA). Solve an optimization problem with :math:`L1` regularization function given the operator ``Op`` and data ``y``. The operator can be real or complex, and should ideally be either square :math:`N=M` or underdetermined :math:`N<M`. Parameters ---------- Op : :obj:`pylops.LinearOperator` Operator to invert data : :obj:`numpy.ndarray` Data niter : :obj:`int` Number of iterations eps : :obj:`float`, optional Sparsity damping alpha : :obj:`float`, optional Step size (:math:`\alpha \le 1/\lambda_{max}(\mathbf{Op}^H\mathbf{Op})` guarantees convergence. If ``None``, the maximum eigenvalue is estimated and the optimal step size is chosen. If provided, the condition will not be checked internally). eigsiter : :obj:`int`, optional Number of iterations for eigenvalue estimation if ``alpha=None`` eigstol : :obj:`float`, optional Tolerance for eigenvalue estimation if ``alpha=None`` tol : :obj:`float`, optional Tolerance. Stop iterations if difference between inverted model at subsequent iterations is smaller than ``tol`` returninfo : :obj:`bool`, optional Return info of FISTA solver show : :obj:`bool`, optional Display iterations log threshkind : :obj:`str`, optional Kind of thresholding ('hard', 'soft', 'half', 'soft-percentile', or 'half-percentile' - 'soft' used as default) perc : :obj:`float`, optional Percentile, as percentage of values to be kept by thresholding (to be provided when thresholding is soft-percentile or half-percentile) callback : :obj:`callable`, optional Function with signature (``callback(x)``) to call after each iteration where ``x`` is the current model vector Returns ------- xinv : :obj:`numpy.ndarray` Inverted model niter : :obj:`int` Number of effective iterations cost : :obj:`numpy.ndarray`, optional History of cost function Raises ------ NotImplementedError If ``threshkind`` is different from hard, soft, half, soft-percentile, or half-percentile ValueError If ``perc=None`` when ``threshkind`` is soft-percentile or half-percentile See Also -------- OMP: Orthogonal Matching Pursuit (OMP). ISTA: Iterative Shrinkage-Thresholding Algorithm (ISTA). SPGL1: Spectral Projected-Gradient for L1 norm (SPGL1). SplitBregman: Split Bregman for mixed L2-L1 norms. Notes ----- Solves the following optimization problem for the operator :math:`\mathbf{Op}` and the data :math:`\mathbf{d}`: .. math:: J = ||\mathbf{d} - \mathbf{Op} \mathbf{x}||_2^2 + \epsilon ||\mathbf{x}||_p using the Fast Iterative Shrinkage-Thresholding Algorithm (FISTA) [1]_, where :math:`p=0, 1, 1/2`. This is a modified version of ISTA solver with improved convergence properties and limited additional computational cost. Similarly to the ISTA solver, the choice of the thresholding algorithm to apply at every iteration is based on the choice of :math:`p`. .. [1] Beck, A., and Teboulle, M., “A Fast Iterative Shrinkage-Thresholding Algorithm for Linear Inverse Problems”, SIAM Journal on Imaging Sciences, vol. 2, pp. 183-202. 2009. """ if not threshkind in ['hard', 'soft', 'half', 'hard-percentile', 'soft-percentile', 'half-percentile']: raise NotImplementedError('threshkind should be hard, soft, half,' 'hard-percentile, soft-percentile, ' 'or half-percentile') if threshkind in ['hard-percentile', 'soft-percentile', 'half-percentile'] and perc is None: raise ValueError('Provide a percentile when choosing hard-percentile,' 'soft-percentile, or half-percentile thresholding') # choose thresholding function if threshkind == 'soft': threshf = _softthreshold elif threshkind == 'hard': threshf = _hardthreshold elif threshkind == 'half': threshf = _halfthreshold elif threshkind == 'hard-percentile': threshf = _hardthreshold_percentile elif threshkind == 'soft-percentile': threshf = _softthreshold_percentile else: threshf = _halfthreshold_percentile # identify backend to use ncp = get_array_module(data) if show: tstart = time.time() print('FISTA optimization (%s thresholding)\n' '-----------------------------------------------------------\n' 'The Operator Op has %d rows and %d cols\n' 'eps = %10e\ttol = %10e\tniter = %d' % (threshkind, Op.shape[0], Op.shape[1], eps, tol, niter)) # step size if alpha is None: if not isinstance(Op, LinearOperator): Op = LinearOperator(Op, explicit=False) # compute largest eigenvalues of Op^H * Op Op1 = LinearOperator(Op.H * Op, explicit=False) if get_module_name(ncp) == 'numpy': maxeig = np.abs(Op1.eigs(neigs=1, symmetric=True, niter=eigsiter, **dict(tol=eigstol, which='LM')))[0] else: maxeig = np.abs(power_iteration(Op1, niter=eigsiter, tol=eigstol, dtype=Op1.dtype, backend='cupy')[0]) alpha = 1. / maxeig # define threshold thresh = eps * alpha * 0.5 if show: if perc is None: print('alpha = %10e\tthresh = %10e' % (alpha, thresh)) else: print('alpha = %10e\tperc = %.1f' % (alpha, perc)) print('-----------------------------------------------------------\n') head1 = ' Itn x[0] r2norm r12norm xupdate' print(head1) # initialize model and cost function xinv = ncp.zeros(int(Op.shape[1]), dtype=Op.dtype) zinv = xinv.copy() t = 1 if returninfo: cost = np.zeros(niter + 1) # iterate for iiter in range(niter): xinvold = xinv.copy() # compute residual resz = data - Op.matvec(zinv) # compute gradient grad = alpha * Op.rmatvec(resz) # update inverted model xinv_unthesh = zinv + grad if perc is None: xinv = threshf(xinv_unthesh, thresh) else: xinv = threshf(xinv_unthesh, 100 - perc) # update auxiliary coefficients told = t t = (1. + np.sqrt(1. + 4. * t ** 2)) / 2. zinv = xinv + ((told - 1.) / t) * (xinv - xinvold) # model update xupdate = np.linalg.norm(xinv - xinvold) if returninfo or show: costdata = 0.5 * np.linalg.norm(data - Op.matvec(xinv)) ** 2 costreg = eps * np.linalg.norm(xinv, ord=1) if returninfo: cost[iiter] = costdata + costreg # run callback if callback is not None: callback(xinv) if show: if iiter < 10 or niter - iiter < 10 or iiter % 10 == 0: msg = '%6g %12.5e %10.3e %9.3e %10.3e' % \ (iiter + 1, to_numpy(xinv[:2])[0], costdata, costdata + costreg, xupdate) print(msg) # check tolerance if xupdate < tol: niter = iiter break # get values pre-threshold at locations where xinv is different from zero # xinv = np.where(xinv != 0, xinv_unthesh, xinv) if show: print('\nIterations = %d Total time (s) = %.2f' % (niter, time.time() - tstart)) print('---------------------------------------------------------\n') if returninfo: return xinv, niter, cost[:niter] else: return xinv, niter
def dottest(Op, nr, nc, tol=1e-6, complexflag=0, raiseerror=True, verb=False, backend='numpy'): r"""Dot test. Generate random vectors :math:`\mathbf{u}` and :math:`\mathbf{v}` and perform dot-test to verify the validity of forward and adjoint operators. This test can help to detect errors in the operator implementation. Parameters ---------- Op : :obj:`pylops.LinearOperator` Linear operator to test. nr : :obj:`int` Number of rows of operator (i.e., elements in data) nc : :obj:`int` Number of columns of operator (i.e., elements in model) tol : :obj:`float`, optional Dottest tolerance complexflag : :obj:`bool`, optional generate random vectors with real (0) or complex numbers (1: only model, 2: only data, 3:both) raiseerror : :obj:`bool`, optional Raise error or simply return ``False`` when dottest fails verb : :obj:`bool`, optional Verbosity backend : :obj:`str`, optional Backend used for dot test computations (``numpy`` or ``cupy``). This parameter will be used to choose how to create the random vectors. Raises ------ ValueError If dot-test is not verified within chosen tolerance. Notes ----- A dot-test is mathematical tool used in the development of numerical linear operators. More specifically, a correct implementation of forward and adjoint for a linear operator should verify the following *equality* within a numerical tolerance: .. math:: (\mathbf{Op}*\mathbf{u})^H*\mathbf{v} = \mathbf{u}^H*(\mathbf{Op}^H*\mathbf{v}) """ ncp = get_module(backend) if complexflag in (0, 2): u = ncp.random.randn(nc) else: u = ncp.random.randn(nc) + 1j * ncp.random.randn(nc) if complexflag in (0, 1): v = ncp.random.randn(nr) else: v = ncp.random.randn(nr) + 1j * ncp.random.randn(nr) y = Op.matvec(u) # Op * u x = Op.rmatvec(v) # Op'* v if complexflag == 0: yy = ncp.dot(y, v) # (Op * u)' * v xx = ncp.dot(u, x) # u' * (Op' * v) else: yy = ncp.vdot(y, v) # (Op * u)' * v xx = ncp.vdot(u, x) # u' * (Op' * v) # convert back to numpy (in case cupy arrays where used), make into a numpy # array and extract the first element. This is ugly but allows to handle # complex numbers in subsequent prints also when using cupy arrays. xx, yy = np.array([to_numpy(xx)])[0], np.array([to_numpy(yy)])[0] # evaluate if dot test is passed if complexflag == 0: if np.abs((yy - xx) / ((yy + xx + 1e-15) / 2)) < tol: if verb: print('Dot test passed, v^T(Opu)=%f - u^T(Op^Tv)=%f' % (yy, xx)) return True else: if raiseerror: raise ValueError( 'Dot test failed, v^T(Opu)=%f - u^T(Op^Tv)=%f' % (yy, xx)) if verb: print('Dot test failed, v^T(Opu)=%f - u^T(Op^Tv)=%f' % (yy, xx)) return False else: checkreal = np.abs((np.real(yy) - np.real(xx)) / ((np.real(yy) + np.real(xx) + 1e-15) / 2)) < tol checkimag = np.abs((np.real(yy) - np.real(xx)) / ((np.real(yy) + np.real(xx) + 1e-15) / 2)) < tol if checkreal and checkimag: if verb: print('Dot test passed, v^T(Opu)=%f%+fi - u^T(Op^Tv)=%f%+fi' % (yy.real, yy.imag, xx.real, xx.imag)) return True else: if raiseerror: raise ValueError('Dot test failed, v^H(Opu)=%f%+fi ' '- u^H(Op^Hv)=%f%+fi' % (yy.real, yy.imag, xx.real, xx.imag)) if verb: print('Dot test failed, v^H(Opu)=%f%+fi - u^H(Op^Hv)=%f%+fi' % (yy.real, yy.imag, xx.real, xx.imag)) return False
def ISTA(Op, data, niter, eps=0.1, alpha=None, eigsiter=None, eigstol=0, tol=1e-10, monitorres=False, returninfo=False, show=False, threshkind='soft', perc=None, callback=None): r"""Iterative Shrinkage-Thresholding Algorithm (ISTA). Solve an optimization problem with :math:`Lp, \quad p=0, 1/2, 1` regularization, given the operator ``Op`` and data ``y``. The operator can be real or complex, and should ideally be either square :math:`N=M` or underdetermined :math:`N<M`. Parameters ---------- Op : :obj:`pylops.LinearOperator` Operator to invert data : :obj:`numpy.ndarray` Data niter : :obj:`int` Number of iterations eps : :obj:`float`, optional Sparsity damping alpha : :obj:`float`, optional Step size (:math:`\alpha \le 1/\lambda_{max}(\mathbf{Op}^H\mathbf{Op})` guarantees convergence. If ``None``, the maximum eigenvalue is estimated and the optimal step size is chosen. If provided, the condition will not be checked internally). eigsiter : :obj:`float`, optional Number of iterations for eigenvalue estimation if ``alpha=None`` eigstol : :obj:`float`, optional Tolerance for eigenvalue estimation if ``alpha=None`` tol : :obj:`float`, optional Tolerance. Stop iterations if difference between inverted model at subsequent iterations is smaller than ``tol`` monitorres : :obj:`bool`, optional Monitor that residual is decreasing returninfo : :obj:`bool`, optional Return info of CG solver show : :obj:`bool`, optional Display iterations log threshkind : :obj:`str`, optional Kind of thresholding ('hard', 'soft', 'half', 'hard-percentile', 'soft-percentile', or 'half-percentile' - 'soft' used as default) perc : :obj:`float`, optional Percentile, as percentage of values to be kept by thresholding (to be provided when thresholding is soft-percentile or half-percentile) callback : :obj:`callable`, optional Function with signature (``callback(x)``) to call after each iteration where ``x`` is the current model vector Returns ------- xinv : :obj:`numpy.ndarray` Inverted model niter : :obj:`int` Number of effective iterations cost : :obj:`numpy.ndarray`, optional History of cost function Raises ------ NotImplementedError If ``threshkind`` is different from hard, soft, half, soft-percentile, or half-percentile ValueError If ``perc=None`` when ``threshkind`` is soft-percentile or half-percentile ValueError If ``monitorres=True`` and residual increases See Also -------- OMP: Orthogonal Matching Pursuit (OMP). FISTA: Fast Iterative Shrinkage-Thresholding Algorithm (FISTA). SPGL1: Spectral Projected-Gradient for L1 norm (SPGL1). SplitBregman: Split Bregman for mixed L2-L1 norms. Notes ----- Solves the following optimization problem for the operator :math:`\mathbf{Op}` and the data :math:`\mathbf{d}`: .. math:: J = ||\mathbf{d} - \mathbf{Op} \mathbf{x}||_2^2 + \epsilon ||\mathbf{x}||_p using the Iterative Shrinkage-Thresholding Algorithms (ISTA) [1]_, where :math:`p=0, 1, 1/2`. This is a very simple iterative algorithm which applies the following step: .. math:: \mathbf{x}^{(i+1)} = T_{(\epsilon \alpha /2, p)} (\mathbf{x}^{(i)} + \alpha \mathbf{Op}^H (\mathbf{d} - \mathbf{Op} \mathbf{x}^{(i)})) where :math:`\epsilon \alpha /2` is the threshold and :math:`T_{(\tau, p)}` is the thresholding rule. The most common variant of ISTA uses the so-called soft-thresholding rule :math:`T(\tau, p=1)`. Alternatively an hard-thresholding rule is used in the case of `p=0` or a half-thresholding rule is used in the case of `p=1/2`. Finally, percentile bases thresholds are also implemented: the damping factor is not used anymore an the threshold changes at every iteration based on the computed percentile. .. [1] Daubechies, I., Defrise, M., and De Mol, C., “An iterative thresholding algorithm for linear inverse problems with a sparsity constraint”, Communications on pure and applied mathematics, vol. 57, pp. 1413-1457. 2004. """ if not threshkind in ['hard', 'soft', 'half', 'hard-percentile', 'soft-percentile', 'half-percentile']: raise NotImplementedError('threshkind should be hard, soft, half,' 'hard-percentile, soft-percentile, ' 'or half-percentile') if threshkind in ['hard-percentile', 'soft-percentile', 'half-percentile'] and perc is None: raise ValueError('Provide a percentile when choosing hard-percentile,' 'soft-percentile, or half-percentile thresholding') # choose thresholding function if threshkind == 'soft': threshf = _softthreshold elif threshkind == 'hard': threshf = _hardthreshold elif threshkind == 'half': threshf = _halfthreshold elif threshkind == 'hard-percentile': threshf = _hardthreshold_percentile elif threshkind == 'soft-percentile': threshf = _softthreshold_percentile else: threshf = _halfthreshold_percentile # identify backend to use ncp = get_array_module(data) if show: tstart = time.time() print('ISTA optimization (%s thresholding)\n' '-----------------------------------------------------------\n' 'The Operator Op has %d rows and %d cols\n' 'eps = %10e\ttol = %10e\tniter = %d' % (threshkind, Op.shape[0], Op.shape[1], eps, tol, niter)) # step size if alpha is None: if not isinstance(Op, LinearOperator): Op = LinearOperator(Op, explicit=False) # compute largest eigenvalues of Op^H * Op Op1 = LinearOperator(Op.H * Op, explicit=False) if get_module_name(ncp) == 'numpy': maxeig = np.abs(Op1.eigs(neigs=1, symmetric=True, niter=eigsiter, **dict(tol=eigstol, which='LM'))[0]) else: maxeig = np.abs(power_iteration(Op1, niter=eigsiter, tol=eigstol, dtype=Op1.dtype, backend='cupy')[0]) alpha = 1./maxeig # define threshold thresh = eps * alpha * 0.5 if show: if perc is None: print('alpha = %10e\tthresh = %10e' % (alpha, thresh)) else: print('alpha = %10e\tperc = %.1f' % (alpha, perc)) print('-----------------------------------------------------------\n') head1 = ' Itn x[0] r2norm r12norm xupdate' print(head1) # initialize model and cost function xinv = ncp.zeros(int(Op.shape[1]), dtype=Op.dtype) if monitorres: normresold = np.inf if returninfo: cost = np.zeros(niter+1) # iterate for iiter in range(niter): xinvold = xinv.copy() # compute residual res = data - Op.matvec(xinv) if monitorres: normres = np.linalg.norm(res) if normres > normresold: raise ValueError('ISTA stopped at iteration %d due to ' 'residual increasing, consider modifying ' 'eps and/or alpha...' % iiter) else: normresold = normres # compute gradient grad = alpha * Op.rmatvec(res) # update inverted model xinv_unthesh = xinv + grad if perc is None: xinv = threshf(xinv_unthesh, thresh) else: xinv = threshf(xinv_unthesh, 100 - perc) # model update xupdate = np.linalg.norm(xinv - xinvold) if returninfo or show: costdata = 0.5 * np.linalg.norm(res) ** 2 costreg = eps * np.linalg.norm(xinv, ord=1) if returninfo: cost[iiter] = costdata + costreg # run callback if callback is not None: callback(xinv) if show: if iiter < 10 or niter - iiter < 10 or iiter % 10 == 0: msg = '%6g %12.5e %10.3e %9.3e %10.3e' % \ (iiter+1, to_numpy(xinv[:2])[0], costdata, costdata + costreg, xupdate) print(msg) # check tolerance if xupdate < tol: logging.warning('update smaller that tolerance for ' 'iteration %d' % iiter) niter = iiter break # get values pre-threshold at locations where xinv is different from zero # xinv = np.where(xinv != 0, xinv_unthesh, xinv) if show: print('\nIterations = %d Total time (s) = %.2f' % (niter, time.time() - tstart)) print('---------------------------------------------------------\n') if returninfo: return xinv, niter, cost[:niter] else: return xinv, niter
def dottest( Op, nr=None, nc=None, tol=1e-6, complexflag=0, raiseerror=True, verb=False, backend="numpy", ): r"""Dot test. Generate random vectors :math:`\mathbf{u}` and :math:`\mathbf{v}` and perform dot-test to verify the validity of forward and adjoint operators. This test can help to detect errors in the operator implementation. Parameters ---------- Op : :obj:`pylops.LinearOperator` Linear operator to test. nr : :obj:`int` Number of rows of operator (i.e., elements in data) nc : :obj:`int` Number of columns of operator (i.e., elements in model) tol : :obj:`float`, optional Dottest tolerance complexflag : :obj:`bool`, optional Generate random vectors with * ``0``: Real entries for model and data * ``1``: Complex entries for model and real entries for data * ``2``: Real entries for model and complex entries for data * ``3``: Complex entries for model and data raiseerror : :obj:`bool`, optional Raise error or simply return ``False`` when dottest fails verb : :obj:`bool`, optional Verbosity backend : :obj:`str`, optional Backend used for dot test computations (``numpy`` or ``cupy``). This parameter will be used to choose how to create the random vectors. Raises ------ ValueError If dot-test is not verified within chosen tolerance. Notes ----- A dot-test is mathematical tool used in the development of numerical linear operators. More specifically, a correct implementation of forward and adjoint for a linear operator should verify the following *equality* within a numerical tolerance: .. math:: (\mathbf{Op}\,\mathbf{u})^H\mathbf{v} = \mathbf{u}^H(\mathbf{Op}^H\mathbf{v}) """ ncp = get_module(backend) if nr is None: nr = Op.shape[0] if nc is None: nc = Op.shape[1] assert (nr, nc) == Op.shape, "Provided nr and nc do not match operator shape" # make u and v vectors if complexflag != 0: rdtype = np.real(np.ones(1, Op.dtype)).dtype if complexflag in (0, 2): u = ncp.random.randn(nc).astype(Op.dtype) else: u = ncp.random.randn(nc).astype( rdtype) + 1j * ncp.random.randn(nc).astype(rdtype) if complexflag in (0, 1): v = ncp.random.randn(nr).astype(Op.dtype) else: v = ncp.random.randn(nr).astype( rdtype) + 1j * ncp.random.randn(nr).astype(rdtype) y = Op.matvec(u) # Op * u x = Op.rmatvec(v) # Op'* v if getattr(Op, "clinear", True): yy = ncp.vdot(y, v) # (Op * u)' * v xx = ncp.vdot(u, x) # u' * (Op' * v) else: # Op is only R-linear, so treat complex numbers as elements of R^2 yy = ncp.dot(y.real, v.real) + ncp.dot(y.imag, v.imag) xx = ncp.dot(u.real, x.real) + ncp.dot(u.imag, x.imag) # convert back to numpy (in case cupy arrays were used), make into a numpy # array and extract the first element. This is ugly but allows to handle # complex numbers in subsequent prints also when using cupy arrays. xx, yy = np.array([to_numpy(xx)])[0], np.array([to_numpy(yy)])[0] # evaluate if dot test is passed if complexflag == 0: if np.abs((yy - xx) / ((yy + xx + 1e-15) / 2)) < tol: if verb: print("Dot test passed, v^T(Opu)=%f - u^T(Op^Tv)=%f" % (yy, xx)) return True else: if raiseerror: raise ValueError( "Dot test failed, v^T(Opu)=%f - u^T(Op^Tv)=%f" % (yy, xx)) if verb: print("Dot test failed, v^T(Opu)=%f - u^T(Op^Tv)=%f" % (yy, xx)) return False else: # Check both real and imag parts checkreal = (np.abs((np.real(yy) - np.real(xx)) / ((np.real(yy) + np.real(xx) + 1e-15) / 2)) < tol) checkimag = (np.abs((np.imag(yy) - np.imag(xx)) / ((np.imag(yy) + np.imag(xx) + 1e-15) / 2)) < tol) if checkreal and checkimag: if verb: print("Dot test passed, v^T(Opu)=%f%+fi - u^T(Op^Tv)=%f%+fi" % (yy.real, yy.imag, xx.real, xx.imag)) return True else: if raiseerror: raise ValueError("Dot test failed, v^H(Opu)=%f%+fi " "- u^H(Op^Hv)=%f%+fi" % (yy.real, yy.imag, xx.real, xx.imag)) if verb: print("Dot test failed, v^H(Opu)=%f%+fi - u^H(Op^Hv)=%f%+fi" % (yy.real, yy.imag, xx.real, xx.imag)) return False