def __init__(self, M, b, Ds=[], hypers=[]): M = lo.aslinearoperator(M) Ds = [lo.aslinearoperator(D) for D in Ds] self.M = M self.b = b self.Ds = Ds self.hypers = hypers
def rls(A, b, Ds=[], hypers=[], optimizer=spl.cgs, **kargs): """Regularized Least Square Inputs: ------- M : model matrix, (needs matvec and rmatvec methods) Dx : priors, (need matvec and rmatvec methods) b : input vector hypers: hyperparameters (scalars) **kargs : parameters of of the least square optimizer Outputs: -------- x : solution conv : convergence status """ verbose = getattr(kargs, 'verbose', True) callback = getattr(kargs, 'callback', None) # normalize hyperparameters if callback is None: kargs['callback'] = CallbackFactory(verbose=verbose) A = lo.aslinearoperator(A) X = A.T * A for h, D in zip(hypers, Ds): D = lo.aslinearoperator(D) X += h * D.T * D out, info = optimizer(X, A.T * b, **kargs) return out
def rls(A, b, Ds=[], hypers=[], optimizer=spl.cgs, **kargs): """Regularized Least Square Inputs: ------- M : model matrix, (needs matvec and rmatvec methods) Dx : priors, (need matvec and rmatvec methods) b : input vector hypers: hyperparameters (scalars) **kargs : parameters of of the least square optimizer Outputs: -------- x : solution conv : convergence status """ verbose = getattr(kargs, 'verbose', True) callback = getattr(kargs, 'callback', None) if callback is None: kargs['callback'] = CallbackFactory(verbose=verbose) A = lo.aslinearoperator(A) X = A.T * A for h, D in zip(hypers, Ds): D = lo.aslinearoperator(D) X += h * D.T * D out, info = optimizer(X, A.T * b, **kargs) return out
def __init__(self, M, b, Ds=[], hypers=[], norms=[], dnorms=[], **kargs): M = lo.aslinearoperator(M) Ds = [lo.aslinearoperator(D) for D in Ds] self.M = M self.b = b self.Ds = Ds self.hypers = hypers self.norms = norms self.dnorms = dnorms
def rirls(M, y, Ds=[], tol1=1e-5, maxiter1=10, p=1, optimizer=spl.cgs, **kargs): """ Regularized Iteratively Reweighted Least Square """ M = lo.aslinearoperator(M) Ds = [lo.aslinearoperator(D) for D in Ds] x0 = M.T * y x = copy(x0) r = M * x - y tol1 /= np.sum(np.abs(r) ** p) for i in xrange(maxiter1): print("\n outer loop " + str(i + 1) + "\n") A = M.T * M for D in Ds: rd = D * x w = np.abs(rd) ** (p - 2) w[np.where(1 - np.isfinite(w))] = 0 # inf A += D.T * lo.diag(w) * D x, t = optimizer(A, x0, **kargs) if np.sum(np.abs(y - M * x) ** p) < tol1: break r = M * x - y return x
def irls(A0, x0, tol1=1e-5, maxiter1=10, p=1, optimizer=spl.cgs, **kargs): """ Iteratively Reweighted Least Square """ A0 = lo.aslinearoperator(A0) out = A0.T * x0 tol1 /= np.sum(np.abs(x0 - A0 * out) ** p) for i in xrange(maxiter1): print("\n outer loop " + str(i + 1) + "\n") w = np.abs(x0 - A0 * out) ** (p - 2) A = A0.T * lo.diag(w) * A0 x = A0.T * (w * x0) out, t = optimizer(A, x, **kargs) if np.sum(np.abs(x0 - A0 * out) ** p) < tol1: break return out
def gacg(M, y, Ds=[], hypers=[], norms=[], dnorms=[], tol=1e-6, x0=None, maxiter=None, callback=None, **kwargs): """Generalized approximate conjugate gradient Approximate conjugate gradient is a gradient method with a polak ribiere update. It is generalized since it does not assume quadratic norm. Inputs: ------- M : model matrix, (needs matvec and rmatvec methods) Ds : priors, (need matvec and rmatvec methods) norms : norms of the likelihood and the priors dnorms : derivation of the norm b : input vector hypers: hyperparameters (scalars) Outputs: -------- x : solution """ if callback is None: callback = CallbackFactory(verbose=True, criterion=True) # ensure linear operators are passed M = lo.aslinearoperator(M) Ds = [lo.aslinearoperator(D) for D in Ds] # first guess if x0 is None: x = M.T * y else: x = copy(x0) # normalize hyperparameters hypers = normalize_hyper(hypers, y, x) # tolerance r = M * x - y rd = [D * x for D in Ds] J = criterion(hypers=hypers, norms=norms, Ds=Ds, r=r, rd=rd) Jnorm = copy(J) resid = 2 * tol # maxiter if maxiter is None: maxiter = x.size iter_ = 0 # main loop while iter_ < maxiter and resid > tol: iter_ += 1 # gradient g, ng = gradient(hypers=hypers, dnorms=dnorms, M=M, Ds=Ds, r=r, rd=rd) # descent direction if (iter_ % 10) == 1: d = -g else: b = ng / ng0 d = -g + b * d g0 = copy(g) ng0 = copy(ng) # step if norms[0] == norm2: a = quadratic_optimal_step(d, g, M, hypers, Ds) else: a = backtracking_line_search(d, g, M, hypers, Ds, x, norms=norms, y=y, f0=J) # update x += a * d # residual r = M * x - y rd = [D * x for D in Ds] # criterion J_old = copy(J) J = criterion(hypers=hypers, norms=norms, Ds=Ds, r=r, rd=rd) #resid = J / Jnorm resid = (J_old - J) / Jnorm callback(x) # define output if resid > tol: info = resid else: info = 0 return x #, info
# deglitch uctod = uncompress(ctod, C, factor) uctod.mask = tod.mask uctod = tm.filter_median(uctod, length=deglitch_filter_length) uctod_mask = tm.deglitch_l2mad(uctod, projection) uctod = uncompress(ctod, C, factor) uctod.mask = uctod_mask uctod = tm.interpolate_linear(uctod) ctod = compress(uctod, C, factor) # filtering ctod = tm.filter_median(ctod, length=filter_length / factor) cov = noise_covariance(ctod, obs) S = cov.aslinearoperator() # model with compression M = lo.aslinearoperator(model.aslinearoperator()) M = C * M # noise covariance cov = noise_covariance(ctod, obs) S = lo.aslinearoperator(cov.aslinearoperator()) #S = C * S * C.T #S = None # backprojection backmap = (M.T * ctod.flatten()).reshape(projection.shapein) # priors Ds = [lo.diff(backmap.shape, axis=0, dtype=np.float64),] Ds.append(lo.diff(backmap.shape, axis=1, dtype=np.float64)) # weights weights = projection.transpose(tod.ones(tod.shape)) # masking the map MM = lo.mask(weights == 0)
def gacg(M, y, Ds=[], hypers=[], norms=[], dnorms=[], tol=1e-6, x0=None, maxiter=None, callback=None, **kwargs): """Generalized approximate conjugate gradient Approximate conjugate gradient is a gradient method with a polak ribiere update. It is generalized since it does not assume quadratic norm. Inputs: ------- M : model matrix, (needs matvec and rmatvec methods) Ds : priors, (need matvec and rmatvec methods) norms : norms of the likelihood and the priors dnorms : derivation of the norm b : input vector hypers: hyperparameters (scalars) Outputs: -------- x : solution """ if callback is None: callback = CallbackFactory(verbose=True) # ensure linear operators are passed M = lo.aslinearoperator(M) Ds = [lo.aslinearoperator(D) for D in Ds] # first guess if x0 is None: x = M.T * y else: x = copy(x0) # tolerance r = M * x - y rd = [D * x for D in Ds] J = criterion(hypers=hypers, norms=norms, Ds=Ds, r=r, rd=rd) Jnorm = copy(J) resid = 2 * tol # maxiter if maxiter is None: maxiter = x.size iter_ = 0 # main loop while iter_ < maxiter and resid > tol: iter_ += 1 # gradient g, ng = gradient(hypers=hypers, dnorms=dnorms, M=M, Ds=Ds, r=r, rd=rd) # descent direction if (iter_ % 10) == 1: d = - g else: b = ng / ng0 d = - g + b * d g0 = copy(g) ng0 = copy(ng) # step if norms[0] == norm2: a = quadratic_optimal_step(d, g, M, hypers, Ds) else: a = backtracking_line_search(d, g, M, hypers, Ds, x, norms, f0=J) # update x += a * d # residual r = M * x - y rd = [D * x for D in Ds] # criterion J = criterion(hypers=hypers, norms=norms, Ds=Ds, r=r, rd=rd) resid = J / Jnorm callback(x) # define output if resid > tol: info = resid else: info = 0 return x#, info
detector_policy='remove') tod = obs.get_tod() header = obs.get_map_header() header.update('CDELT1', resolution / 3600) header.update('CDELT2', resolution / 3600) npix = 5 good_npix = False projection = tm.Projection(obs, header=header, resolution=resolution, oversampling=False, npixels_per_sample=npix) model = projection #C = csh.averaging(tod.shape, factor=factor) compression_shape = [tod.size / factor, tod.size, ] C = tm.CompressionAverage(compression_factor=factor).aslinearoperator(compression_shape) C = lo.aslinearoperator(C) # compress data ctod = compress(tod, C, factor) # uncompress for preprocessing uctod = uncompress(ctod, C, factor) # XXX need mask ... uctod.mask = tod.mask # deglitching uctod = tm.filter_median(uctod, length=10) uctod.mask = tm.deglitch_l2mad(uctod, projection) masking = tm.Masking(uctod.mask) model = masking * projection # filtering ctod = tm.filter_median(ctod, length=filter_length / factor) cov = noise_covariance(ctod, obs) S = cov.aslinearoperator()
"scan_angle":0., "scan_nlegs":1, "scan_length":4., "compression_factor":1, "scan_speed":60.0 } pointing = tm.pacs_create_scan(ra0, dec0, **pointing_params) # create obs object obs = tm.PacsSimulation(pointing, "red", policy_bad_detector="keep") # to define my compression matrix I need only 8 frames # so I mask the others : obs.pointing.removed[:] = True obs.pointing.removed[201:209] = False # create projector projection = tm.Projection(obs) P = lo.aslinearoperator(projection) # define masking of coefficients # ------------------------------ # number of coefficients to keep factor = 8. nk = P.shape[0] / factor # coverage map w = P.T * np.ones(P.shape[0]) # sort coverage map coefficients ws = np.sort(w) # find the nk largest coverage elements threshold = ws[-nk] mask = w < threshold # define decimation Linear Operator according to coverage map coef # selection.
tod = pacs.get_tod() # projector projection = tm.Projection(pacs, resolution=3.2, oversampling=True, npixels_per_sample=6) masking = tm.Masking(tod.mask) compression = tm.CompressionAverage(pacs.compression_factor) model = masking * compression * projection # naive map naive = model.transpose(tod) # coverage map coverage = model.transpose(tod.ones(tod.shape)) # noise covariance length = 2**np.ceil(np.log2(np.array(tod.nsamples) + 200)) invNtt = tm.InvNtt(length, pacs.get_filter_uncorrelated()) fft = tm.Fft(length) padding = tm.Padding(left=invNtt.ncorrelations, right=length - tod.nsamples - invNtt.ncorrelations) weight = padding.T * fft.T * invNtt * fft * padding W = lo.aslinearoperator(weight.aslinearoperator()) # transform to lo P = lo.aslinearoperator(model.aslinearoperator()) # priors Ds = [lo.diff(naive.shape, axis=axis) for axis in xrange(naive.ndim)] # inversion hypers = [1e6, 1e6, ] y = tod.flatten() M = P.T * W * P + np.sum([h * D.T * D for h, D in zip(hypers, Ds)]) x, conv = spl.cgs(M, P.T * W * y, callback=lo.CallbackFactory(verbose=True)) sol = naive.zeros(naive.shape) sol[:] = x.reshape(sol.shape)
#uctod[:] = (C.T * ctod.flatten()).reshape(tod.shape) # deglitching projection = tm.Projection(pacs, header=header, resolution=3., npixels_per_sample=6) uctod.mask = tm.deglitch_l2mad(uctod, projection) #ctod = compression.direct(uctod) # model masking = tm.Masking(uctod.mask) model = masking * projection # remove drift #ctod = tm.filter_median(ctod, length=3000 / 8.) # first map M = C * lo.aslinearoperator(model.aslinearoperator()) #P = lo.aslinearoperator(projection.aslinearoperator()) #C = csh.averaging(tod.shape, factor=8) #I = lo.mask(uctod.mask) #M = C * I.T * I * P #M = C * P backmap = (M.T * ctod.flatten()).reshape(projection.shapein) #weights = (M.T * np.ones(ctod.size)).reshape(projection.shapein) weights = projection.transpose(tod.ones(tod.shape)) MM = lo.mask(weights == 0) M = M * MM.T # define algo # priors Dx = lo.diff(backmap.shape, axis=0, dtype=np.float64) * MM.T Dy = lo.diff(backmap.shape, axis=1, dtype=np.float64) * MM.T #Dw = lo.pywt_lo.wavedec2(backmap.shape, "haar", level=3)
naive = backmap / coverage # mask according to coverage (everything that is covered by less than 10.) mask = Masking(coverage < 10.) # The model is the masking of the sky map then the projection # This is basically matrix multiplication model = projection * mask # Performing inversion # --------------------- # with TAMASIS x_tm = mapper_rls(tod, model, hyper=1e-1, tol=1e-10, maxiter=100) # with lo routines # transform to lo H = lo.aslinearoperator(model * mask) # smoothness priors Ds = [lo.diff(backmap.shape, axis=axis) for axis in (0, 1)] # inversion y = tod.ravel() # requires 1d input x_lo = lo.acg(H, y, Ds, 1e-1 * np.ones(3), tol=1e-10, maxiter=100) x_lo.resize(backmap.shape) # output is 1d so need reshaping # with sparsity assumptions (using Huber algorithm) x_h = lo.hacg(H, y, Ds, 1e1 * np.ones(3), np.asarray((None, 1e-6, 1e-6, 1e-6)), x0=x_lo.flatten(), tol=1e-7,
header = pacs.get_map_header() #header['NAXIS1'] = 192 #header['NAXIS2'] = 192 #header['CRPIX1'] = 96 #header['CRPIX2'] = 96 header.update('CDELT1', resolution / 3600) header.update('CDELT2', resolution / 3600) # data tod = pacs.get_tod() y = tod.flatten() # remove bad pixels (by updating mask !) #tod = remove_bad_pixels(tod) # compress data factor = 8 compression = tm.CompressionAverage(factor) C = lo.aslinearoperator(compression.aslinearoperator(shape=(tod.size / factor, tod.size))) ctod = compression.direct(tod) # uncompress for deglitching uctod = tod.copy(tod.shape) y0, t = lo.spl.cgs(C.T * C, C.T * ctod.flatten()) uctod[:] = y0.reshape(tod.shape) # deglitching projection = tm.Projection(pacs, header=header, resolution=3., npixels_per_sample=5) uctod.mask = tm.deglitch_l2mad(uctod, projection) ctod = compression.direct(uctod) # model masking = tm.Masking(uctod.mask) model = compression * masking * projection # remove drift #ctod = tm.filter_median(ctod, length=3000 / 8.) # first map
coverage = projection.T(np.ones(tod.shape)) # naive map naive = backmap / coverage # mask according to coverage (everything that is covered by less than 10.) mask = Masking(coverage < 10.) # The model is the masking of the sky map then the projection # This is basically matrix multiplication model = projection * mask # Performing inversion # --------------------- # with TAMASIS x_tm = mapper_rls(tod, model, hyper=1e-1, tol=1e-10, maxiter=100) # with lo routines # transform to lo H = lo.aslinearoperator(model * mask) # smoothness priors Ds = [lo.diff(backmap.shape, axis=axis) for axis in (0, 1)] # inversion y = tod.ravel() # requires 1d input x_lo = lo.acg(H, y, Ds, 1e-1 * np.ones(3), tol=1e-10, maxiter=100) x_lo.resize(backmap.shape) # output is 1d so need reshaping # with sparsity assumptions (using Huber algorithm) x_h = lo.hacg(H, y, Ds, 1e1 * np.ones(3), np.asarray((None, 1e-6, 1e-6, 1e-6)), x0=x_lo.flatten(), tol=1e-7, maxiter=200) x_h.resize(backmap.shape)
header = pacs.get_map_header() header['NAXIS1'] = 192 header['NAXIS2'] = 192 header['CRPIX1'] = 96 header['CRPIX2'] = 96 # data tod = pacs.get_tod() # remove bad pixels (by updating mask !) #tod = remove_bad_pixels(tod) # deglitching projection = Projection(pacs, header=header, resolution=3., npixels_per_sample=6) deglitch_l2mad(tod, projection) # model masking = Masking(tod.mask) model = masking * projection P = lo.aslinearoperator(model.aslinearoperator()) # derive filter #tod = filter_median(tod, length=9999) #kernel = filt.kernel_from_tod(tod, length=1000) kernel = (1 + (10. / np.arange(500)) ** .25) kernel = np.concatenate((kernel[::-1], kernel)) #kern = np.mean(kernel, axis=0) N = filt.kernels_convolve(tod.shape, 1 / np.sqrt(kernel)) # apply to data yn = N * tod.flatten() # apply to model M = N * P # first map backmap = model.transpose(tod) # define algo # priors
npix = 5 good_npix = False projection = tm.Projection(obs, header=header, resolution=resolution, oversampling=False, npixels_per_sample=npix) model = projection #C = csh.averaging(tod.shape, factor=factor) compression_shape = [ tod.size / factor, tod.size, ] C = tm.CompressionAverage( compression_factor=factor).aslinearoperator(compression_shape) C = lo.aslinearoperator(C) # compress data ctod = compress(tod, C, factor) # uncompress for preprocessing uctod = uncompress(ctod, C, factor) # XXX need mask ... uctod.mask = tod.mask # deglitching uctod = tm.filter_median(uctod, length=10) uctod.mask = tm.deglitch_l2mad(uctod, projection) masking = tm.Masking(uctod.mask) model = masking * projection # filtering ctod = tm.filter_median(ctod, length=filter_length / factor) cov = noise_covariance(ctod, obs) S = cov.aslinearoperator()
'cam_angle': 0., "scan_angle": 0., "scan_nlegs": 1, "scan_length": 400., "scan_speed": 60.0, "compression_factor": 1 # we will compress later } pointing = tm.pacs_create_scan(ra0, dec0, **pointing_params) # create obs object obs = tm.PacsSimulation(pointing, "red", policy_bad_detector="keep") # we dont need 200 first and last frames obs.pointing.removed[200:] = True obs.pointing.removed[-201:] = True # create projector projection = tm.Projection(obs, npixels_per_sample=4) P = lo.aslinearoperator(projection) # simulate data x0 = tm.gaussian(projection.shapein, 3) # map with gaussian source tod0 = projection(x0) # data n = np.random.randn(*tod0.shape) # noise nsr = 1e-2 tod = tod0 + nsr * n # noisy data y = tod.ravel() # as 1d array # load compression matrix filename = os.path.join(os.getenv("HOME"), "data", "pacs", "mmc_cam_angle_0_scan_angle_0_speed60.fits") c = fa.FitsArray(file=filename).astype(np.float64) cmm = csh.compression.AnyOperator(c) C = cmm((projection.shapeout[0], projection.shapeout[1][0]), ) # compress
import lo.pywt_lo as pywt_lo # data pacs = PacsObservation(filename=tamasis_dir + 'tests/frames_blue.fits', fine_sampling_factor=1, keep_bad_detectors=False) tod = pacs.get_tod() # projector projection = Projection(pacs, resolution=3.2, oversampling=False, npixels_per_sample=6) model = projection # naive map backmap = model.transpose(tod) # mask M = lo.mask(backmap < 0).T #M = lo.identity(2 * (backmap.size,)) # transform to lo P = lo.aslinearoperator(model.aslinearoperator()) * M # prior Dx = lo.diff(backmap.shape, axis=0, dtype=np.float64) * M Dy = lo.diff(backmap.shape, axis=1, dtype=np.float64) * M Dw = pywt_lo.wavelet2(backmap.shape, "haar") * M # inversion y = tod.flatten() callback = lo.CallbackFactory(verbose=True) x = it.rirls(P, (Dx, Dy, Dw), y, p=1.5, maxiter=100, callback=callback) sol = backmap.zeros(backmap.shape) sol[:] = (M * x).reshape(sol.shape)
header = pacs.get_map_header() #header['NAXIS1'] = 192 #header['NAXIS2'] = 192 #header['CRPIX1'] = 96 #header['CRPIX2'] = 96 header.update('CDELT1', resolution / 3600) header.update('CDELT2', resolution / 3600) # data tod = pacs.get_tod() y = tod.flatten() # remove bad pixels (by updating mask !) #tod = remove_bad_pixels(tod) # compress data factor = 8 compression = tm.CompressionAverage(factor) C = lo.aslinearoperator( compression.aslinearoperator(shape=(tod.size / factor, tod.size))) ctod = compression.direct(tod) # uncompress for deglitching uctod = tod.copy(tod.shape) y0, t = lo.spl.cgs(C.T * C, C.T * ctod.flatten()) uctod[:] = y0.reshape(tod.shape) # deglitching projection = tm.Projection(pacs, header=header, resolution=3., npixels_per_sample=5) uctod.mask = tm.deglitch_l2mad(uctod, projection) ctod = compression.direct(uctod) # model masking = tm.Masking(uctod.mask) model = compression * masking * projection