def test_collection(): with setting(config, 'GC_NBYTES_THRESHOLD', 8000): memory.garbage_collect() counter = 0 for i in range(10): data = np.arange(100) counter += data.nbytes op = DiagonalOperator(data) op.delete() if i < 9: assert memory._gc_nbytes_counter == counter assert memory._gc_nbytes_counter == 0
def test_masking(): mask = MaskOperator(0) assert isinstance(mask, IdentityOperator) mask = MaskOperator(0, shapein=(32, 32), dtype=np.float32) assert isinstance(mask, IdentityOperator) assert mask.shapein == (32, 32) assert mask.dtype == np.float32 mask = MaskOperator(1) assert isinstance(mask, ZeroOperator) mask = MaskOperator(1, shapein=(32, 32), dtype=np.float32) assert isinstance(mask, ZeroOperator) assert mask.shapein == (32, 32) assert mask.dtype == np.float32 b = np.array([3., 4., 1., 0., 3., 2.]) c = np.array([3., 4., 0., 0., 3., 0.]) mask = MaskOperator(np.array([0, 0., 1., 1., 0., 1], dtype=np.int8)) assert np.all(mask(b) == c) mask = DiagonalOperator(np.array([1, 1., 0., 0., 1., 0])) assert np.all(mask(b) == c) mask = MaskOperator(np.array([False, False, True, True, False, True])) assert np.all(mask(b) == c) b = np.array([[3., 4.], [1., 0.], [3., 2.]]) c = np.array([[3., 4.], [0., 0.], [3., 0.]]) mask = MaskOperator(np.array([[0, 0.], [1., 1.], [0., 1.]], dtype='int8')) assert np.all(mask(b) == c) mask = DiagonalOperator(np.array([[1, 1.], [0., 0.], [1., 0.]])) assert np.all(mask(b) == c) mask = MaskOperator(np.array([[False, False], [True, True], [False, True]])) assert np.all(mask(b) == c) b = np.array([[[3, 4.], [1., 0.]], [[3., 2], [-1, 9]]]) c = np.array([[[3, 4.], [0., 0.]], [[3., 0], [0, 0]]]) mask = MaskOperator(np.array([[[0, 0.], [1., 1.]], [[0., 1], [1, 1]]], int)) assert np.all(mask(b) == c) mask = DiagonalOperator(np.array([[[1, 1], [0., 0]], [[1, 0], [0, 0]]])) assert np.all(mask(b) == c) mask = MaskOperator(np.array([[[False, False], [True, True]], [[False, True], [True, True]]])) assert np.all(mask(b) == c) c = mask(b, b) assert id(b) == id(c)
def get_polarizer_operator(self, sampling, scene): """ Return operator for the polarizer grid. When the polarizer is not present a transmission of 1 is assumed for the detectors on the first focal plane and of 0 for the other. Otherwise, the signal is split onto the focal planes. """ nd = len(self) nt = len(sampling) grid = self.detector.quadrant // 4 if scene.kind == 'I': if self.optics.polarizer: return HomothetyOperator(1 / 2) # 1 for the first detector grid and 0 for the second one return DiagonalOperator(1 - grid, shapein=(nd, nt), broadcast='rightward') if not self.optics.polarizer: raise NotImplementedError( 'Polarized input is not handled without the polarizer grid.') z = np.zeros(nd) data = np.array([z + 0.5, 0.5 - grid, z]).T[:, None, None, :] return ReshapeOperator((nd, nt, 1), (nd, nt)) * \ DenseBlockDiagonalOperator(data, shapein=(nd, nt, 3))
def get_transmission_operator(self): """ Return the operator that multiplies by the cumulative instrumental transmission. """ return DiagonalOperator( np.product(self.optics.components['transmission']) * self.detector.efficiency, broadcast='rightward')
def get_preconditioner(self, cov): if cov is not None: cov_inv = 1 / cov cov_inv[np.isinf(cov_inv)] = 0. preconditioner = DiagonalOperator(cov_inv, broadcast='rightward') else: preconditioner = None return preconditioner
def get_preconditioner(self, cov): if cov is not None: cov += np.ones(cov.shape) * cov.max() * 0.001 cov_inv = np.array([1. / cov[(self.qubic.nus > mi) * \ (self.qubic.nus < ma)].mean(axis=0) \ for (mi, ma) in self.qubic.bands]) return DiagonalOperator(cov_inv.flatten(), broadcast='rightward') else: return None
def __new__(cls, fft_filter=None, ncorrelations=None, nsamples=None): if fft_filter is None: return Operator.__new__(cls) filter_length = fft_filter.shape[-1] invntt = DiagonalOperator(fft_filter) fft = FftHalfComplexOperator(filter_length) padding = PadOperator(left=ncorrelations, right=filter_length - \ nsamples - ncorrelations) return padding.T * fft.T * invntt * fft * padding
def func(obs, tod): masking = MaskOperator(tod.mask) proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=header_ref_global, commin=comm_map) model = masking * proj m = mapper_ls(tod, model, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator(1/cov_ref_global)) check_map_global(m)
def get_preconditioner(self, cov): if cov is not None: cov_inv = np.array([1. / cov[(self.nus > mi) * (self.nus < ma)].mean(axis=0) \ for (mi, ma) in self.bands]) cov_inv[np.isinf(cov_inv)] = 0. # cov_inv = np.nan_to_num(cov_inv) return BlockDiagonalOperator(\ [DiagonalOperator(ci, broadcast='rightward') for ci in cov_inv], new_axisin=0) else: return None
def test3(): comm_map = MPI.COMM_WORLD for obs, tod in ((obs1,tod1), (obs2,tod2)): masking = MaskOperator(tod.mask) proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=header_ref_global, commin=comm_map) model = masking * proj m = mapper_ls(tod, model, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator(1/cov_ref_local)) yield check_map_local, m
def _get_detector_integration_operator(position, area, secondary_beam): """ Integrate flux density in detector solid angles and take into account the secondary beam transmission. """ theta = np.arctan2(np.sqrt(np.sum(position[..., :2]**2, axis=-1)), position[..., 2]) phi = np.arctan2(position[..., 1], position[..., 0]) sr_det = -area / position[..., 2]**2 * np.cos(theta)**3 sr_beam = secondary_beam.solid_angle sec = secondary_beam(theta, phi) return DiagonalOperator(sr_det / sr_beam * sec, broadcast='rightward')
def test_diagonal2(): ops = (DiagonalOperator([1., 2], broadcast='rightward'), DiagonalOperator([[2., 3, 4], [5, 6, 7]], broadcast='rightward'), DiagonalOperator([1., 2, 3, 4, 5], broadcast='leftward'), DiagonalOperator(np.arange(20).reshape(4, 5), broadcast='leftward'), DiagonalOperator(np.arange(120.).reshape(2, 3, 4, 5)), HomothetyOperator(7.), IdentityOperator()) x = np.arange(120.).reshape(2, 3, 4, 5) / 2 def func(cls, d1, d2): op = {AdditionOperator: operator.add, CompositionOperator: operator.mul, MultiplicationOperator: operator.mul}[cls] d = cls([d1, d2]) if type(d1) is DiagonalOperator: assert_is_type(d, DiagonalOperator) elif type(d1) is HomothetyOperator: assert_is_type(d, HomothetyOperator) elif op is CompositionOperator: assert_is_type(d, IdentityOperator) else: assert_is_type(d, HomothetyOperator) data = op(d1.data.T, d2.data.T).T \ if 'rightward' in (d1.broadcast, d2.broadcast) \ else op(d1.data, d2.data) assert_same(d.data, data) if cls is CompositionOperator: assert_same(d(x), d1(d2(x))) else: assert_same(d(x), op(d1(x), d2(x))) for op in (AdditionOperator, CompositionOperator):#, MultiplicationOperator): for d1, d2 in itertools.combinations(ops, 2): if set((d1.broadcast, d2.broadcast)) == \ set(('leftward', 'rightward')): continue yield func, op, d1, d2
def test_partition1(): o1 = HomothetyOperator(1, shapein=1) o2 = HomothetyOperator(2, shapein=2) o3 = HomothetyOperator(3, shapein=3) r = DiagonalOperator([1, 2, 2, 3, 3, 3]).todense() def func(ops, p): op = BlockDiagonalOperator(ops, partitionin=p, axisin=0) assert_eq(op.todense(6), r, str(op)) for ops, p in zip( ((o1, o2, o3), (I, o2, o3), (o1, 2 * I, o3), (o1, o2, 3 * I)), (None, (1, 2, 3), (1, 2, 3), (1, 2, 3))): yield func, ops, p
def get_preconditioner(self, cov): print('This is the new preconditionner') if cov is not None: cov_av = np.array([cov[(self.nus > mi) * (self.nus < ma)].mean(axis=0) \ for (mi, ma) in self.bands]) cov_inv = np.zeros_like(cov_av) seenpix = cov_av != 0 cov_inv[seenpix] = 1. / cov_av[seenpix] return BlockDiagonalOperator( \ [DiagonalOperator(ci, broadcast='rightward') for ci in cov_inv], new_axisin=0) else: return None
def test_rule_manager_none(): op1 = DiagonalOperator([1, 2, 3]) op2 = 2 def func(cls, none): with rule_manager(none=none): op = cls([op1, op2]) if none: assert_is_instance(op, cls) else: assert_is_instance(op, DiagonalOperator) for cls in AdditionOperator, CompositionOperator, MultiplicationOperator: for none in False, True: yield func, cls, none
def get_invntt_operator(self): sigma_detector = self.instrument.detector.nep / \ np.sqrt(2*self.sampling.period) if self.photon_noise: sigma_photon = self.instrument._get_noise_photon_nep(self.scene) /\ np.sqrt(2 * self.sampling.period) else: sigma_photon = 0 out = DiagonalOperator(1 / (sigma_detector**2 + sigma_photon**2), broadcast='rightward', shapein=(len(self.instrument), len(self.sampling))) if self.effective_duration is not None: nsamplings = self.comm.allreduce(len(self.sampling)) out /= (nsamplings * self.sampling.period / (self.effective_duration * 31557600)) return out
def _rule_pTp(selfT, self): if not isinstance( self.matrix, (FSRMatrix, FSRRotation2dMatrix, FSRRotation3dMatrix)) or \ self.matrix.ncolmax != 1 or self.shapein is None: return di = self.matrix.data.index.dtype dr = self.matrix.dtype dv = self.dtype if di not in (np.int32, np.int64) or \ dr not in (np.float32, np.float64) or \ dv not in (np.float32, np.float64): return f = 'fsc_{0}_ncolmax1_i{1}_r{2}_v{3}'.format(self.matrix._flib_id, di.itemsize, dr.itemsize, dv.itemsize) try: func = getattr(fsp, f) except AttributeError: return m = self.matrix.data.ravel().view(np.int8) data = np.zeros(self.shapein, dtype=self.dtype) func(m, data.ravel()) return DiagonalOperator(data)
def get_invntt_operator(self): return DiagonalOperator(1 / self.sigma**2, broadcast='leftward', shapein=self.scene.shape)
def func(d, e): op = DiagonalOperator(d) if all(_ in (-1, 1) for _ in op.data.flat): assert op.flags.involutary assert_is_type(op, e)
def get_operator(self): return DiagonalOperator(self.mask.astype(np.int), broadcast='rightward', shapein=self.scene.shape)
W = acq.get_hwp_operator() H = W * P coverage = P.pT1() tod = H(input_map) tods[kind] = tod pTxs[kind] = H.T(tod)[coverage > 0] if kind != 'QU': pTx_pT1 = P.pTx_pT1(tod) pTx_pT1s[kind] = pTx_pT1[0][coverage > 0], pTx_pT1[1][coverage > 0] cbik = P.canonical_basis_in_kernel() mask = coverage > 10 P = P.restrict(mask, inplace=True) unpack = UnpackOperator(mask, broadcast='rightward') x0 = unpack.T(input_map) M = DiagonalOperator(1 / coverage[mask], broadcast='rightward') H = W * P solution = pcg(H.T * H, H.T(tod), M=M, disp=True, tol=1e-5) pT1s[kind] = coverage cbiks[kind] = cbik outputs[kind] = solution['x'] def test_sky(): assert_same(tods['I'], tods['IQU'][..., 0].astype(np.float32)) assert_same(tods['QU'], tods['IQU'][..., 1:].astype(np.float32)) assert_same(pTxs['I'], pTxs['IQU'][..., 0].astype(np.float32)) assert_same(pTxs['QU'], pTxs['IQU'][..., 1:].astype(np.float32))
def get_invntt_operator(self): """ Return the inverse time-time noise correlation matrix as an Operator. The input Power Spectrum Density can either be fully specified by using the 'bandwidth' and 'psd' keywords, or by providing the parameters of the gaussian distribution: psd = sigma**2 * (1 + (fknee/f)**fslope) / B where B is the sampling bandwidth equal to sampling_frequency / N. Parameters ---------- sampling : Sampling The temporal sampling. psd : array-like, optional The one-sided or two-sided power spectrum density [signal unit/sqrt Hz]. bandwidth : float, optional The PSD frequency increment [Hz]. twosided : boolean, optional Whether or not the input psd is one-sided (only positive frequencies) or two-sided (positive and negative frequencies). sigma : float Standard deviation of the white noise component. detector_fknee : float The 1/f noise knee frequency [Hz]. detector_fslope : float The 1/f noise slope. sampling_frequency : float The sampling frequency [Hz]. detector_ncorr : int The correlation length of the time-time noise correlation matrix. fftw_flag : string, optional The flags FFTW_ESTIMATE, FFTW_MEASURE, FFTW_PATIENT and FFTW_EXHAUSTIVE can be used to describe the increasing amount of effort spent during the planning stage to create the fastest possible transform. Usually, FFTW_MEASURE is a good compromise and is the default. nthreads : int, optional Tells how many threads to use when invoking FFTW or MKL. Default is the number of cores. """ fftw_flag = 'FFTW_MEASURE' nthreads = None if self.bandwidth is None and self.psd is not None or self.bandwidth is not None and self.psd is None: raise ValueError('The bandwidth or the PSD is not specified.') if self.instrument.detector.nep is not None: self.sigma = self.instrument.detector.nep / np.sqrt( 2 * self.sampling.period) if self.photon_noise: sigma_photon = self.instrument._get_noise_photon_nep( self.scene) / np.sqrt(2 * self.sampling.period) self.sigma = np.sqrt(self.sigma**2 + sigma_photon**2) else: sigma_photon = 0 if self.bandwidth is None and self.psd is None and self.sigma is None: raise ValueError('The noise model is not specified.') shapein = (len(self.instrument), len(self.sampling)) if self.bandwidth is None and self.detector_fknee == 0: print('diagonal case') out = DiagonalOperator(1 / self.sigma**2, broadcast='rightward', shapein=(len(self.instrument), len(self.sampling))) if self.effective_duration is not None: nsamplings = self.comm.allreduce(len(self.sampling)) out /= (nsamplings * self.sampling.period / (self.effective_duration * 31557600)) return out sampling_frequency = 1 / self.sampling.period nsamples_max = len(self.sampling) fftsize = 2 while fftsize < nsamples_max: fftsize *= 2 new_bandwidth = sampling_frequency / fftsize if self.bandwidth is not None and self.psd is not None: if self.twosided: self.psd = _fold_psd(self.psd) f = np.arange(fftsize // 2 + 1, dtype=float) * new_bandwidth p = _unfold_psd(_logloginterp_psd(f, self.bandwidth, self.psd)) else: p = _gaussian_psd_1f(fftsize, sampling_frequency, self.sigma, self.detector_fknee, self.detector_fslope, twosided=True) p[..., 0] = p[..., 1] invntt = _psd2invntt(p, new_bandwidth, self.detector_ncorr, fftw_flag=fftw_flag) print('non diagonal case') if self.effective_duration is not None: nsamplings = self.comm.allreduce(len(self.sampling)) invntt /= (nsamplings * self.sampling.period / (self.effective_duration * 31557600)) return SymmetricBandToeplitzOperator(shapein, invntt, fftw_flag=fftw_flag, nthreads=nthreads)
def test_madcap1(): assert all_eq(map_naive, map_ref) map_naive_1d = mapper_naive(tod, projection) map_naive_2d = packing.T(map_naive_1d) map_naive_2d = packing.T(projection.T(tod) / projection.T(Tod.ones(tod.shape))) map_naive_2d[obs.info.mapmask] = np.nan def test_madcap2(): assert all_eq(map_naive_2d, map_ref) M = DiagonalOperator(packing(1 / map_naive.coverage)) assert np.all(np.isfinite(M.data)) class Callback: def __init__(self): self.niterations = 0 def __call__(self, x): self.niterations += 1 callback = Callback() #callback = None invntt = InvNttOperator(obs)
model = masking_tod * projection * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x): self.niterations += 1 invntt = DiagonalOperator(1 / obs.get_detector_stddev(100)**2, broadcast='rightward') norm = tamasis.linalg.norm2_ellipsoid(invntt) precond = 1. / map_naive.coverage precond[precond > 1] = 0 map_nl = mapper_nl(tod, model, tol=1.e-6, maxiter=1000, norms=[norm], M=DiagonalOperator(precond), callback=Callback()) print('Elapsed time:', map_nl.header['TIME']) if map_nl.header['NITER'] > 21: raise TestFailure()
def _tod2map(acq, tod, coverage_threshold, max_nbytes, callback, disp_pcg, maxiter, tol, criterion, full_output, save_map, hyper): # coverage normalization: # sum coverage = #detectors x #samplings for a uniform secondary beam H = acq.get_operator() coverage = H.T(ones(H.shapeout)) if acq.scene.kind == 'IQU': coverage = coverage[..., 0] elif acq.scene.kind == 'QU': raise NotImplementedError() theta, phi = acq.instrument.detector.theta, acq.instrument.detector.phi ndetectors = acq.instrument.detector.comm.allreduce( np.sum(acq.instrument.secondary_beam(theta, phi))) nsamplings = acq.sampling.comm.allreduce(len(acq.sampling)) coverage *= ndetectors * nsamplings / np.sum(coverage) cov = coverage[coverage > 0] i = np.argsort(cov) cdf = np.cumsum(cov[i]) j = np.argmax(cdf >= coverage_threshold * cdf[-1]) threshold = cov[i[j]] mask = coverage >= threshold rejected = 1 - np.sum(mask) / cov.size if acq.comm.rank == 0 and coverage_threshold > 0: print('Total coverage:', cdf[-1]) print('Threshold coverage set to:', threshold) print('Fraction of rejected observed pixels:', rejected) header = OrderedDict() coverage = coverage.view(ndarraywrap) coverage.header = header header['thresrel'] = coverage_threshold, 'Relative coverage threshold' header['thresabs'] = threshold, 'Absolute coverage threshold' header['fracrej'] = rejected, 'Fraction of rejected observed pixels' acq_restricted = acq[..., mask] H = acq_restricted.get_operator() invNtt = acq_restricted.get_invntt_operator() M = (H.T * H * np.ones(H.shapein))[..., 0] preconditioner = DiagonalOperator(1 / M, broadcast='rightward') # preconditioner = DiagonalOperator(1/coverage[mask], broadcast='rightward') nsamplings = acq.comm.allreduce(len(acq.sampling)) npixels = np.sum(mask) A = H.T * invNtt * H / nsamplings if hyper != 0: L = HealpixLaplacianOperator(acq.scene.nside) L = L.restrict(mask, inplace=True).corestrict(mask, inplace=True) A = A - hyper / npixels / 4e5 * L if criterion: def f(x): Hx_y = H(x) Hx_y -= tod out = [np.dot(Hx_y.ravel(), invNtt(Hx_y).ravel()) / nsamplings] if hyper != 0: out += [ -np.dot(x.ravel(), hyper / npixels / 4e5 * L(x).ravel()) ] else: out += [0.] return out def callback(self): criteria = f(self.x) if len(criteria) == 1: details = '' else: fmt = ', '.join(len(criteria) * ['{:e}']) details = ' (' + fmt.format(*criteria) + ')' print('{:4}: {:e} {:e}{}'.format(self.niterations, self.error, sum(criteria), details)) if not hasattr(self, 'history'): self.history = {} self.history['criterion'] = [] self.history['error'] = [] self.history['iteration'] = [] self.history['criterion'].append(criteria) self.history['error'].append(self.error) self.history['iteration'].append(self.niterations) if save_map is not None and self.niterations in save_map: if not hasattr(self, 'xs'): self.xs = {} self.xs[self.niterations] = self.x.copy() solution = pcg(A, H.T(invNtt(tod)) / nsamplings, M=preconditioner, callback=callback, disp=disp_pcg, maxiter=maxiter, tol=tol) output = acq_restricted.scene.unpack(solution['x']), coverage if full_output: algo = solution['algorithm'] algo.H = H if criterion: pack = PackOperator(mask, broadcast='rightward') algo.f = asoperator(f, shapeout=2)(pack) output += (algo, ) return output
profile = None#'test_ls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * \ obs_ref.get_projection_operator(downsampling=True, npixels_per_sample=6, commin=comm_map) map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2') map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator( 1/map_naive_ref.coverage)) cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape, attrin={'header':header_ref_global}) map_ref_local = tolocal(map_ref_global) cov_ref_local = tolocal(map_ref_global.coverage) mask_ref_local = tolocal(mask_ref_global) def check_map_global(m): assert_all_eq(m.magnitude, map_ref_global.magnitude, mtol) assert_all_eq(m.coverage, cov_ref_global, mtol) def check_map_local(m): assert_all_eq(m.magnitude, map_ref_local.magnitude, mtol)
def get_preconditioner(self, H): M = (H.T * H * np.ones(H.shapein))[..., 0] preconditioner = DiagonalOperator(1 / M, broadcast='rightward') return preconditioner
coverage = H.T(np.ones(H.shapeout)) mask = coverage > 0 # restrict the scene to the observed pixels acq_restricted = acq[..., mask] H_restricted = acq_restricted.get_operator() x0_restricted = x0[mask] y = H_restricted(x0_restricted) invntt = acq_restricted.get_invntt_operator() # solve for x A = H_restricted.T * invntt * H_restricted b = H_restricted.T(invntt(y)) solution = pcg(A, b, M=DiagonalOperator(1 / coverage[mask]), disp=True, tol=1e-4) x = np.zeros_like(x0) x[mask] = solution['x'] # some display def display(x, title, lim=200): x = x.copy() x[~mask] = np.nan hp.gnomview(x, rot=center_gal, reso=5, xsize=600, min=-lim,
telescope = IdentityOperator() projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6) compression = CompressionAverageOperator(obs.slice.compression_factor) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * telescope * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, restricting oneself to observed map pixels unpacking = UnpackOperator(projection.get_mask()) old_settings = np.seterr(divide='ignore') M = DiagonalOperator(unpacking.T(1./map_naive.coverage)) np.seterr(**old_settings) #map_iter1 = mapper_ls(tod, model * unpacking, tol=1.e-4, M=M) #if map_iter1.header['NITER'] > 11: # raise TestFailure() # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x): self.niterations += 1 callback = Callback() #callback=None map_iter2 = mapper_ls(tod, model,