def test_detector_policy(): map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits') obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs.pointing.chop[:] = 0 projection = obs.get_projection_operator(header=map_naive_ref.header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() masking = MaskOperator(tod.mask) model = masking * projection map_naive = mapper_naive(tod, model) assert all_eq(map_naive, map_naive_ref, tol) obs_rem = PacsObservation(data_dir + 'frames_blue.fits', policy_bad_detector='remove', reject_bad_line=False) obs_rem.pointing.chop[:] = 0 projection_rem = obs_rem.get_projection_operator(header=map_naive.header, downsampling=True, npixels_per_sample=7) tod_rem = obs_rem.get_tod() masking_rem = MaskOperator(tod_rem.mask) model_rem = masking_rem * projection_rem map_naive_rem = mapper_naive(tod_rem, model_rem) assert all_eq(map_naive, map_naive_rem, tol)
def test_header(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.chop = 0 header = obs.get_map_header() projection = obs.get_projection_operator(header=header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() map_naive = mapper_naive(tod, projection) header2 = header.copy() header2['NAXIS1'] += 500 header2['CRPIX1'] += 250 projection2 = obs.get_projection_operator(header=header2, downsampling=True) map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2) map_naive2.inunit('Jy/arcsec^2') map_naive3 = map_naive2[:,250:header['NAXIS1']+250] assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7) # test compatibility with photproject tod = obs.get_tod() map_naive4 = mapper_naive(tod, projection) hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1] map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel') std_naive = np.std(map_naive4[40:60,40:60]) std_ref = np.std(map_ref[40:60,40:60]) assert abs(std_naive-std_ref) / std_ref < 0.025
def test_pTx_pT1(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) obs1.pointing.chop = 0 obs2.pointing.chop = 0 header = obs1.get_map_header() tod = obs1.get_tod() model1 = obs1.get_projection_operator(downsampling=True, header=header) ref = mapper_naive(tod, model1, unit='Jy/arcsec^2') model1.apply_mask(tod.mask) tod.inunit('Jy/arcsec^2') b1, w1 = model1.get_pTx_pT1(tod) m1 = b1 / w1 assert all_eq(ref, m1, tol) model2 = obs2.get_projection_operator(downsampling=True, header=header) model2.apply_mask(tod.mask) b2, w2 = model2.get_pTx_pT1(tod) m2 = b2 / w2 assert all_eq(ref, m2, tol) model3 = obs2.get_projection_operator(downsampling=True, header=header, storage='on fly') MaskOperator(tod.mask)(tod, tod) b3, w3 = model3.get_pTx_pT1(tod) m3 = b3 / w3 assert all_eq(ref, m3, tol)
def test2(): comm_map = MPI.COMM_SELF for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator( downsampling=True, npixels_per_sample=6, header=map_ref_global.header, packed=True, commin=comm_map ) proj.apply_mask(tod.mask) m = mapper_naive(tod, proj) yield check_map_global, m
def test4(): for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator( downsampling=True, npixels_per_sample=6, header=map_ref_local.header, commin=MPI.COMM_WORLD ) proj.apply_mask(tod.mask) model = proj * tolocal m = mapper_naive(tod, model) yield check_map_global, m
def test2(): comm_map = MPI.COMM_SELF for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=map_ref_global.header, packed=True, commin=comm_map) proj.apply_mask(tod.mask) m = mapper_naive(tod, proj) yield check_map_global, m
def test4(): for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=map_ref_local.header, commin=MPI.COMM_WORLD) proj.apply_mask(tod.mask) model = proj * tolocal m = mapper_naive(tod, model) yield check_map_global, m
def pipeline_photproject(filenames, output_file, keywords): """ Perform regularized least-square inversion of state of the art PACS model. The PACS model includes tod mask, compression, response, projection and noise covariance (invntt). Processing steps are as follows : - define PacsObservation instance - mask first scanlines to avoid drift - get Time Ordered Data (get_tod) - 2nd level deglitching (with standard projector and tod median filtering with a narrow window) - median filtering - define projection - perform inversion on model - save file Arguments --------- filenames: list of strings List of data filenames. output_file : string Name of the output fits file. keywords: dict Dictionary containing options for all steps as dictionary. Returns ------- Returns nothing. Save result as a fits file. """ from scipy.sparse.linalg import cgs import tamasis as tm # define observation obs = tm.PacsObservation(filenames, **keywords["PacsObservation"]) # extra masking tm.step_scanline_masking(obs, **keywords["scanline_masking"]) # get data tod = obs.get_tod(**keywords["get_tod"]) # deglitching tm.step_deglitching(obs, tod, **keywords["deglitching"]) # median filtering tod = tm.filter_median(tod, **keywords["filter_median"]) # define projector projection = obs.get_projection_operator(downsampling=True, **keywords["Projection"]) # build instrument model masking = tm.MaskOperator(tod.mask) model = masking * projection # set tod masked values to zero tod = masking(tod) # perform map-making map_naive = tm.mapper_naive(tod, model, **keywords["mapper_naive"]) # save map_naive.save(output_file)
def test3(): comm_map = MPI.COMM_WORLD for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=map_ref_global.header, commin=comm_map) proj.apply_mask(tod.mask) m = mapper_naive(tod, proj) yield check_map_local, m if size == 1: continue mlocal = m mlocal[:] = rank + 1 mlocalpacked = proj.operands[1](mlocal) mglobal = UnpackOperator(proj.operands[1].mask)(mlocalpacked) def func(s, irank): assert np.all((mglobal[s] == 0) | (mglobal[s] == irank+1)) for irank in range(size): s = distribute_slice(map_ref_global.shape[0], rank=irank) yield func, s, irank
def test3(): comm_map = MPI.COMM_WORLD for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator( downsampling=True, npixels_per_sample=6, header=map_ref_global.header, commin=comm_map ) proj.apply_mask(tod.mask) m = mapper_naive(tod, proj) yield check_map_local, m if size == 1: continue mlocal = m mlocal[:] = rank + 1 mlocalpacked = proj.operands[1](mlocal) mglobal = UnpackOperator(proj.operands[1].mask)(mlocalpacked) def func(s, irank): assert np.all((mglobal[s] == 0) | (mglobal[s] == irank + 1)) for irank in range(size): s = distribute_slice(map_ref_global.shape[0], rank=irank) yield func, s, irank
npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector=5 plot_tod((tod40Hz+drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask) model40Hz = masking * projection map_naive40Hz = mapper_naive(tod40Hz, model40Hz) map_naive40Hz.imshow() # clim doesn't work anymore with AnnotatedImage #clim(-0.00002,0.00002)
rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None#'test_ls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * \ obs_ref.get_projection_operator(downsampling=True, npixels_per_sample=6, commin=comm_map) map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2') map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator( 1/map_naive_ref.coverage)) cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape, attrin={'header':header_ref_global}) map_ref_local = tolocal(map_ref_global) cov_ref_local = tolocal(map_ref_global.coverage) mask_ref_local = tolocal(mask_ref_global) def check_map_global(m): assert_all_eq(m.magnitude, map_ref_global.magnitude, mtol) assert_all_eq(m.coverage, cov_ref_global, mtol)
projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector = 5 plot_tod((tod40Hz + drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask) model40Hz = masking * projection map_naive40Hz = mapper_naive(tod40Hz, model40Hz) map_naive40Hz.imshow() # clim doesn't work anymore with AnnotatedImage #clim(-0.00002,0.00002)
profile = None #'test_madcap.png' path = os.path.abspath(os.path.dirname(__file__)) + '/data/madmap1/' obs = MadMap1Observation(path + 'todSpirePsw_be', path + 'invnttSpirePsw_be', path + 'madmapSpirePsw.fits[coverage]', 'big_endian', 135, missing_value=np.nan) obs.instrument.name = 'SPIRE/PSW' tod = obs.get_tod(unit='Jy/beam') projection = obs.get_projection_operator() packing = PackOperator(obs.info.mapmask) model = projection * packing map_naive = mapper_naive(tod, model) map_ref = pyfits.open(path + 'naivemapSpirePsw.fits')['image'].data def test_madcap1(): assert all_eq(map_naive, map_ref) map_naive_1d = mapper_naive(tod, projection) map_naive_2d = packing.T(map_naive_1d) map_naive_2d = packing.T(projection.T(tod) / projection.T(Tod.ones(tod.shape))) map_naive_2d[obs.info.mapmask] = np.nan def test_madcap2(): assert all_eq(map_naive_2d, map_ref)
maxiter = 10 data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) tod = obs.get_tod() telescope = IdentityOperator() projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6) compression = CompressionAverageOperator(obs.slice.compression_factor) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * telescope * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, restricting oneself to observed map pixels unpacking = UnpackOperator(projection.get_mask()) old_settings = np.seterr(divide='ignore') M = DiagonalOperator(unpacking.T(1./map_naive.coverage)) np.seterr(**old_settings) #map_iter1 = mapper_ls(tod, model * unpacking, tol=1.e-4, M=M) #if map_iter1.header['NITER'] > 11: # raise TestFailure() # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x):
from tamasis.utils import all_eq tamasis.var.verbose = False profile=None#'test_madcap.png' path = os.path.abspath(os.path.dirname(__file__)) + '/data/madmap1/' obs = MadMap1Observation(path+'todSpirePsw_be', path+'invnttSpirePsw_be', path+'madmapSpirePsw.fits[coverage]', 'big_endian', 135, missing_value=np.nan) obs.instrument.name = 'SPIRE/PSW' tod = obs.get_tod(unit='Jy/beam') projection = obs.get_projection_operator() packing = PackOperator(obs.info.mapmask) model = projection*packing map_naive = mapper_naive(tod, model) map_ref = pyfits.open(path+'naivemapSpirePsw.fits')['image'].data def test_madcap1(): assert all_eq(map_naive, map_ref) map_naive_1d = mapper_naive(tod, projection) map_naive_2d = packing.T(map_naive_1d) map_naive_2d = packing.T(projection.T(tod)/projection.T(Tod.ones(tod.shape))) map_naive_2d[obs.info.mapmask] = np.nan def test_madcap2(): assert all_eq(map_naive_2d, map_ref) M = DiagonalOperator(packing(1/map_naive.coverage)) assert np.all(np.isfinite(M.data))