def test_detector_policy(): map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits') obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs.pointing.chop[:] = 0 projection = obs.get_projection_operator(header=map_naive_ref.header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() masking = MaskOperator(tod.mask) model = masking * projection map_naive = mapper_naive(tod, model) assert all_eq(map_naive, map_naive_ref, tol) obs_rem = PacsObservation(data_dir + 'frames_blue.fits', policy_bad_detector='remove', reject_bad_line=False) obs_rem.pointing.chop[:] = 0 projection_rem = obs_rem.get_projection_operator(header=map_naive.header, downsampling=True, npixels_per_sample=7) tod_rem = obs_rem.get_tod() masking_rem = MaskOperator(tod_rem.mask) model_rem = masking_rem * projection_rem map_naive_rem = mapper_naive(tod_rem, model_rem) assert all_eq(map_naive, map_naive_rem, tol)
def test_masking(): mask = MaskOperator(0) assert isinstance(mask, IdentityOperator) mask = MaskOperator(0, shapein=(32, 32), dtype=np.float32) assert isinstance(mask, IdentityOperator) assert mask.shapein == (32, 32) assert mask.dtype == np.float32 mask = MaskOperator(1) assert isinstance(mask, ZeroOperator) mask = MaskOperator(1, shapein=(32, 32), dtype=np.float32) assert isinstance(mask, ZeroOperator) assert mask.shapein == (32, 32) assert mask.dtype == np.float32 b = np.array([3., 4., 1., 0., 3., 2.]) c = np.array([3., 4., 0., 0., 3., 0.]) mask = MaskOperator(np.array([0, 0., 1., 1., 0., 1], dtype=np.int8)) assert np.all(mask(b) == c) mask = DiagonalOperator(np.array([1, 1., 0., 0., 1., 0])) assert np.all(mask(b) == c) mask = MaskOperator(np.array([False, False, True, True, False, True])) assert np.all(mask(b) == c) b = np.array([[3., 4.], [1., 0.], [3., 2.]]) c = np.array([[3., 4.], [0., 0.], [3., 0.]]) mask = MaskOperator(np.array([[0, 0.], [1., 1.], [0., 1.]], dtype='int8')) assert np.all(mask(b) == c) mask = DiagonalOperator(np.array([[1, 1.], [0., 0.], [1., 0.]])) assert np.all(mask(b) == c) mask = MaskOperator(np.array([[False, False], [True, True], [False, True]])) assert np.all(mask(b) == c) b = np.array([[[3, 4.], [1., 0.]], [[3., 2], [-1, 9]]]) c = np.array([[[3, 4.], [0., 0.]], [[3., 0], [0, 0]]]) mask = MaskOperator(np.array([[[0, 0.], [1., 1.]], [[0., 1], [1, 1]]], int)) assert np.all(mask(b) == c) mask = DiagonalOperator(np.array([[[1, 1], [0., 0]], [[1, 0], [0, 0]]])) assert np.all(mask(b) == c) mask = MaskOperator(np.array([[[False, False], [True, True]], [[False, True], [True, True]]])) assert np.all(mask(b) == c) c = mask(b, b) assert id(b) == id(c)
def test_header(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.chop = 0 header = obs.get_map_header() projection = obs.get_projection_operator(header=header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() map_naive = mapper_naive(tod, projection) header2 = header.copy() header2['NAXIS1'] += 500 header2['CRPIX1'] += 250 projection2 = obs.get_projection_operator(header=header2, downsampling=True) map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2) map_naive2.inunit('Jy/arcsec^2') map_naive3 = map_naive2[:,250:header['NAXIS1']+250] assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7) # test compatibility with photproject tod = obs.get_tod() map_naive4 = mapper_naive(tod, projection) hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1] map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel') std_naive = np.std(map_naive4[40:60,40:60]) std_ref = np.std(map_ref[40:60,40:60]) assert abs(std_naive-std_ref) / std_ref < 0.025
def func(valid, x, shape, broadcast): p = PackOperator(valid, broadcast=broadcast) masking = MaskOperator(~valid, broadcast=broadcast) if broadcast == 'leftward': x_ = np.empty(shape + x.shape) x_[...] = x expected_ = np.empty(shape + (expected.size,)) expected_[...] = expected else: x_ = np.empty(x.shape + shape) x_.reshape((x.size, -1))[...] = x.ravel()[..., None] expected_ = np.empty((expected.size,) + shape) expected_.reshape((expected.size, -1))[...] = expected[..., None] if broadcast == 'disabled' and shape != (): assert_raises(ValueError, p, x_) return assert_equal(p(x_), expected_) assert_is_type(p.T, UnpackOperator) assert_equal(p.T.broadcast, p.broadcast) assert_equal(p.T(expected_), masking(x_)) u = UnpackOperator(valid, broadcast=broadcast) assert_is_type(u.T, PackOperator) assert_equal(u.T.broadcast, u.broadcast) assert_equal(u(expected_), masking(x_)) assert_equal(u.T(x_), expected_)
def test_pTx_pT1(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) obs1.pointing.chop = 0 obs2.pointing.chop = 0 header = obs1.get_map_header() tod = obs1.get_tod() model1 = obs1.get_projection_operator(downsampling=True, header=header) ref = mapper_naive(tod, model1, unit='Jy/arcsec^2') model1.apply_mask(tod.mask) tod.inunit('Jy/arcsec^2') b1, w1 = model1.get_pTx_pT1(tod) m1 = b1 / w1 assert all_eq(ref, m1, tol) model2 = obs2.get_projection_operator(downsampling=True, header=header) model2.apply_mask(tod.mask) b2, w2 = model2.get_pTx_pT1(tod) m2 = b2 / w2 assert all_eq(ref, m2, tol) model3 = obs2.get_projection_operator(downsampling=True, header=header, storage='on fly') MaskOperator(tod.mask)(tod, tod) b3, w3 = model3.get_pTx_pT1(tod) m3 = b3 / w3 assert all_eq(ref, m3, tol)
def func(obs, tod): masking = MaskOperator(tod.mask) proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=header_ref_global, commin=comm_map) model = masking * proj m = mapper_ls(tod, model, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator(1/cov_ref_global)) check_map_global(m)
def test3(): comm_map = MPI.COMM_WORLD for obs, tod in ((obs1,tod1), (obs2,tod2)): masking = MaskOperator(tod.mask) proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=header_ref_global, commin=comm_map) model = masking * proj m = mapper_ls(tod, model, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator(1/cov_ref_local)) yield check_map_local, m
def test1(): comm_map = MPI.COMM_SELF for obs, tod in ((obs1, tod1), (obs2, tod2)): masking = MaskOperator(tod.mask) proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=header_ref_global, commin=comm_map) model = masking * proj m = mapper_rls(tod, model, tol=tol, maxiter=maxiter, solver=solver, hyper=hyper) yield check_map_global, m
def func(cls, itype, ftype, inplace): proj_ref = _get_projection[cls](itype, ftype) proj_ = _get_projection[cls](itype, ftype) proj = proj_.restrict(restriction, inplace=inplace) if inplace: assert_is_type(proj_, DeletedOperator) if cls is not FSRMatrix: def pack(v): return v[restriction, :] else: pack = PackOperator(restriction) masking = MaskOperator(~restriction, broadcast='rightward') block_size = proj_ref.matrix.block_shape[1] shape = (5,) + ((block_size,) if block_size > 1 else ()) x = np.arange(5*block_size).reshape(shape) + 1 assert_equal(proj_ref(masking(x)), proj(pack(x))) pack = PackOperator(restriction) assert_equal(pack(kernel), proj.canonical_basis_in_kernel())
def apply_mask(self, mask): mask = np.asarray(mask, np.bool8) dest = 0 if mask.shape != self.shapeout: raise ValueError( "The mask shape '{0}' is incompatible with tha" "t of the projection operator '{1}'.".format( mask.shape, self.shapeout)) if any( isinstance(p, ProjectionOnFlyOperator) for p in self.operands): blocks = self.copy() self.__class__ = CompositionOperator self.__init__([MaskOperator(mask), blocks]) return for p in self.operands: n = p.matrix.shape[1] p.apply_mask(mask[..., dest:dest + n]) dest += n
def mapper_naive(tod, model, unit=None): """ Returns a naive map, i.e.: map = model.T(tod) / model.T(1) This equation is valid for a map and a Time Ordered Data (TOD) expressed as a surface brightness, so when the TOD does not meet this requirement and is a quantity per detector, a unit conversion is attempted. Parameters ---------- tod : Tod The input Time Ordered Data model : Operator The instrument model such as tod = model(map) unit : string Output map unit. By default, the output map unit is chosen to be compatible with the model (usually pixel^-1) """ # apply mask if hasattr(tod, 'mask') and tod.mask is not None: mask = MaskOperator(tod.mask) else: mask = IdentityOperator() tod = mask(tod) # get tod units if not hasattr(tod, '_unit') or len(tod._unit) == 0: attr = {'_unit': {'?': 1.}} model.propagate_attributes(None, attr) u = getattr(attr, '_unit', {}) if 'detector' in u and u['detector'] == -1: u = {'detector': -1.} elif u == {'?': 1.}: u = {} elif len(u) > 1: raise ValueError( 'The timeline units are not known and cannot be in' 'ferred from the model.') tod_du = getattr(attr, '_derived_units', {}) tod = Tod(tod.magnitude, unit=u, derived_units=tod_du, copy=False) else: attr = {'_unit': {'?': 1}} model.T.propagate_attributes(None, attr) u = attr['_unit'] if 'detector' not in tod._unit and 'detector' in u and u[ 'detector'] == 1: raise ValueError("The model is incompatible with input units '{0}'"\ .format(tod.unit)) tod_unit = tod._unit tod_du = tod._derived_units # make sure the input is a surface brightness if 'detector' in tod._unit: tod.inunit(tod.unit + ' detector / arcsec^2') elif 'detector_reference' in tod._unit: tod.inunit(tod.unit + ' detector_reference / arcsec^2') # compute model.T(tod)/model.T(one) mymap = model.T(tod.magnitude) tod[...] = 1 mask(tod, tod) map_weights = model.T(tod.magnitude) old_settings = np.seterr(divide='ignore', invalid='ignore') mymap /= map_weights mymap.unit = tod.unit np.seterr(**old_settings) mymap.coverage = Map(map_weights.magnitude, header=mymap.header, copy=False) if unit is not None: mymap.inunit(unit) return mymap # set map units according to model attr = {'_unit': tod_unit, '_derived_units': tod_du} model.T.propagate_attributes(None, attr) if '_derived_units' in attr: mymap.derived_units = attr['_derived_units'] if '_unit' in attr: mymap.inunit(attr['_unit']) return mymap
def test_masking2(): m = MaskOperator([True, False, True]) assert_eq(m * m, m)
tol = 1.e-2 mtol = 1.e-10 maxiter = 100 rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None#'test_ls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * \ obs_ref.get_projection_operator(downsampling=True, npixels_per_sample=6, commin=comm_map) map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2') map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator( 1/map_naive_ref.coverage)) cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape, attrin={'header':header_ref_global}) map_ref_local = tolocal(map_ref_global) cov_ref_local = tolocal(map_ref_global.coverage) mask_ref_local = tolocal(mask_ref_global)
nfp_x = int(np.sqrt(NPOINT_FOCAL_PLANE)) a = np.r_[FOCAL_PLANE_LIMITS[0]:FOCAL_PLANE_LIMITS[1]:nfp_x * 1j] fp_x, fp_y = np.meshgrid(a, a) fp = np.dstack([fp_x, fp_y, np.full_like(fp_x, -qubic.optics.focal_length)]) fp_spacing = (FOCAL_PLANE_LIMITS[1] - FOCAL_PLANE_LIMITS[0]) / nfp_x ############ # DETECTORS ############ header = create_fitsheader((nfp_x, nfp_x), cdelt=fp_spacing, crval=(0, 0), ctype=['X---CAR', 'Y---CAR'], cunit=['m', 'm']) focal_plane = SceneGrid.fromfits(header) integ = MaskOperator(qubic.detector.all.removed) * \ focal_plane.get_integration_operator( focal_plane.topixel(qubic.detector.all.vertex[..., :2])) ############### # COMPUTATIONS ############### E = qubic._get_response(SOURCE_THETA, SOURCE_PHI, SOURCE_POWER, fp, fp_spacing**2, qubic.filter.nu, qubic.horn, qubic.primary_beam, qubic.secondary_beam) I = np.abs(E)**2 D = integ(I) ########## # DISPLAY ##########
projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector = 5 plot_tod((tod40Hz + drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask) model40Hz = masking * projection map_naive40Hz = mapper_naive(tod40Hz, model40Hz) map_naive40Hz.imshow() # clim doesn't work anymore with AnnotatedImage #clim(-0.00002,0.00002)
def _rule_right_unpack(op, self): if self.mask.shape != op.mask.shape: return if np.any(self.mask != op.mask): return return MaskOperator(self.mask)
UnpackOperator, mapper_ls, mapper_naive) pyoperators.memory.verbose = False profile = None#'test_ls.png' solver = scipy.sparse.linalg.bicgstab tol = 1.e-6 if profile else 1.e-4 maxiter = 10 data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) tod = obs.get_tod() telescope = IdentityOperator() projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6) compression = CompressionAverageOperator(obs.slice.compression_factor) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * telescope * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, restricting oneself to observed map pixels unpacking = UnpackOperator(projection.get_mask()) old_settings = np.seterr(divide='ignore') M = DiagonalOperator(unpacking.T(1./map_naive.coverage)) np.seterr(**old_settings) #map_iter1 = mapper_ls(tod, model * unpacking, tol=1.e-4, M=M) #if map_iter1.header['NITER'] > 11: # raise TestFailure()
def _validate_tod(tod): # make sure that the tod is masked if hasattr(tod, 'mask') and tod.mask is not None: return MaskOperator(tod.mask)(tod) return tod.copy()
hyper = 1 rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None #'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) header_ref_global = create_fitsheader( (97, 108), cdelt=3.2 / 3600, crval=(245.998427916727, 61.5147650744551)) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * obs_ref.get_projection_operator( downsampling=True, npixels_per_sample=6, commin=comm_map, header=header_ref_global) map_ref_global = mapper_rls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, hyper=hyper) header_ref_global = map_ref_global.header cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={'header': header_ref_global})
from pyoperators import BlockColumnOperator, MaskOperator from pyoperators.iterative.algorithms import StopCondition from pyoperators.iterative.dli import DoubleLoopAlgorithm from tamasis import PacsObservation, DiscreteDifferenceOperator, mapper_naive pyoperators.memory.verbose = False tamasis.var.verbose = True data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(filename=data_dir + 'frames_blue.fits') obs.pointing.chop[:] = 0 tod = obs.get_tod() projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection naive = mapper_naive(tod, model) naive[np.isnan(naive)] = 0 prior = BlockColumnOperator( [DiscreteDifferenceOperator(axis, shapein=(103, 97)) for axis in (0, 1)], new_axisout=0) stop_condition = StopCondition(maxiter=2) dli = DoubleLoopAlgorithm(model, tod, prior, stop_condition=stop_condition, lanczos={'maxiter': 5},