def test_header(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.chop = 0 header = obs.get_map_header() projection = obs.get_projection_operator(header=header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() map_naive = mapper_naive(tod, projection) header2 = header.copy() header2['NAXIS1'] += 500 header2['CRPIX1'] += 250 projection2 = obs.get_projection_operator(header=header2, downsampling=True) map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2) map_naive2.inunit('Jy/arcsec^2') map_naive3 = map_naive2[:,250:header['NAXIS1']+250] assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7) # test compatibility with photproject tod = obs.get_tod() map_naive4 = mapper_naive(tod, projection) hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1] map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel') std_naive = np.std(map_naive4[40:60,40:60]) std_ref = np.std(map_ref[40:60,40:60]) assert abs(std_naive-std_ref) / std_ref < 0.025
def test_detector_policy(): map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits') obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs.pointing.chop[:] = 0 projection = obs.get_projection_operator(header=map_naive_ref.header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() masking = MaskOperator(tod.mask) model = masking * projection map_naive = mapper_naive(tod, model) assert all_eq(map_naive, map_naive_ref, tol) obs_rem = PacsObservation(data_dir + 'frames_blue.fits', policy_bad_detector='remove', reject_bad_line=False) obs_rem.pointing.chop[:] = 0 projection_rem = obs_rem.get_projection_operator(header=map_naive.header, downsampling=True, npixels_per_sample=7) tod_rem = obs_rem.get_tod() masking_rem = MaskOperator(tod_rem.mask) model_rem = masking_rem * projection_rem map_naive_rem = mapper_naive(tod_rem, model_rem) assert all_eq(map_naive, map_naive_rem, tol)
def test_slice1(): obs = PacsObservation(data_dir+'frames_blue.fits[11:20]') tod = obs.get_tod() filename = 'obs-' + uuid + '.fits' obs.save(filename, tod) obs2 = PacsObservation(filename) tod2 = obs2.get_tod() assert all_eq(obs.status[10:20], obs2.status) assert all_eq(tod, tod2)
def test_pTx_pT1(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) obs1.pointing.chop = 0 obs2.pointing.chop = 0 header = obs1.get_map_header() tod = obs1.get_tod() model1 = obs1.get_projection_operator(downsampling=True, header=header) ref = mapper_naive(tod, model1, unit='Jy/arcsec^2') model1.apply_mask(tod.mask) tod.inunit('Jy/arcsec^2') b1, w1 = model1.get_pTx_pT1(tod) m1 = b1 / w1 assert all_eq(ref, m1, tol) model2 = obs2.get_projection_operator(downsampling=True, header=header) model2.apply_mask(tod.mask) b2, w2 = model2.get_pTx_pT1(tod) m2 = b2 / w2 assert all_eq(ref, m2, tol) model3 = obs2.get_projection_operator(downsampling=True, header=header, storage='on fly') MaskOperator(tod.mask)(tod, tod) b3, w3 = model3.get_pTx_pT1(tod) m3 = b3 / w3 assert all_eq(ref, m3, tol)
def test_save(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.ra += 0.1 obs.pointing.dec -= 0.1 obs.pointing.pa += 20 obs.pointing.chop = 0 tod = obs.get_tod() filename = 'obs-' + uuid + '.fits' obs.save(filename, tod) obs2 = PacsObservation(filename, reject_bad_line=False) tod2 = obs2.get_tod() assert all_eq(obs.pointing, obs2.pointing) obs.status.RaArray = obs.pointing.ra obs.status.DecArray = obs.pointing.dec obs.status.PaArray = obs.pointing.pa obs.status.CHOPFPUANGLE = obs.pointing.chop assert all_eq(obs.status, obs2.status) assert all_eq(tod, tod2)
def test_slice2(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) assert all_eq(obs1.pointing, obs2.pointing[~obs2.pointing.removed]) obs1.pointing.chop = 0 obs2.pointing.chop = 0 tod1 = obs1.get_tod() tod2 = obs2.get_tod() assert all_eq(tod1, tod2) header = obs1.get_map_header() proj1 = obs1.get_projection_operator(header=header) proj2 = obs2.get_projection_operator(header=header) proj3 = obs2.get_projection_operator(header=header, storage='on fly') assert all_eq(proj1.get_mask(), proj2.get_mask()) assert all_eq(proj1.get_mask(), proj3.get_mask()) assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \ proj2.operands], axis=1)) assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \ proj3.operands], axis=1)) model1 = CompressionAverageOperator(obs1.slice.compression_factor) * proj1 model2 = CompressionAverageOperator(obs2.slice.compression_factor) * proj2 model3 = CompressionAverageOperator(obs2.slice.compression_factor) * proj3 m1 = model1.T(tod1) m2 = model2.T(tod2) m3 = model3.T(tod2) assert all_eq(m1, m2, tol) assert all_eq(m1, m3, tol) assert all_eq(model1(m1), model2(m1)) assert all_eq(model1(m1), model3(m1))
def test1(): # creation of the sky map msize = 50 mymap = gaussian(2*(msize*2+1,), 10, unit='Jy/pixel') cd = np.array([[-1., 0.],[0., 1.]]) / 3600. header = create_fitsheader(fromdata=mymap, crval=[53.,27.], cd=cd) mymap.header = header # creation of the simulation scan = PacsObservation.create_scan((header['CRVAL1'], header['CRVAL2']), instrument_angle=0., length=60, nlegs=1, angle=20.) simul = PacsSimulation(scan, 'red', policy_bad_detector='keep', policy_other='keep') # build the acquisition model model = CompressionAverageOperator(simul.slice.compression_factor) * \ simul.get_projection_operator(header=header, npixels_per_sample=49) # get the noiseless tod tod = model(mymap) filename = 'simul-'+str(uuid1())+'.fits' try: simul.save(filename, tod) simul2 = PacsObservation(filename, policy_bad_detector='keep', policy_other='keep') status2 = simul2.status tod2 = simul2.get_tod() finally: try: os.remove(filename) except: pass for field in simul.status.dtype.names: if field == 'BAND': continue assert_eq(simul.status[field], status2[field]) assert_eq(tod, tod2) fields = [x for x in simul.slice.dtype.names if x not in ('filename','unit')] for field in fields: if getattr(simul.slice[0], field) != getattr(simul2.slice[0], field): msg = "Field '" + field + "'" if field == 'scan_step': print(msg + ' not implemented.') else: assert False
exit(0) pacs = PacsObservation([datafile[0]+'[6065:20000]', datafile[1]+'[6066:20001]'], fine_sampling_factor=1, calblock_extension_time=0.) telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector=5 plot_tod((tod40Hz+drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask)
solver = tamasis.solvers.cg tol = 1.e-2 mtol = 1.e-10 maxiter = 100 rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None#'test_ls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * \ obs_ref.get_projection_operator(downsampling=True, npixels_per_sample=6, commin=comm_map) map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2') map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator( 1/map_naive_ref.coverage)) cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape, attrin={'header':header_ref_global}) map_ref_local = tolocal(map_ref_global) cov_ref_local = tolocal(map_ref_global.coverage)
import pyoperators import os import scipy from pyoperators import DiagonalOperator, IdentityOperator, MaskOperator from tamasis import (PacsObservation, CompressionAverageOperator, UnpackOperator, mapper_ls, mapper_naive) pyoperators.memory.verbose = False profile = None#'test_ls.png' solver = scipy.sparse.linalg.bicgstab tol = 1.e-6 if profile else 1.e-4 maxiter = 10 data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) tod = obs.get_tod() telescope = IdentityOperator() projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6) compression = CompressionAverageOperator(obs.slice.compression_factor) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * telescope * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, restricting oneself to observed map pixels unpacking = UnpackOperator(projection.get_mask()) old_settings = np.seterr(divide='ignore')
def check_map_global(m): assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10) assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_global.tounit('Jy/arcsec^2'), 1e-10) def check_map_local(m): assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_local.coverage, 1e-10) assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_local.tounit('Jy/arcsec^2'), 1e-10) obs1 = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs1.pointing.chop = 0 tod1 = obs1.get_tod() obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:176]', data_dir + 'frames_blue.fits[177:360]'], reject_bad_line=False) obs2.pointing.chop = 0 tod2 = obs2.get_tod() tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={'header':header_ref_global}) # non-distributed map, distributed TOD def test1(): comm_map = MPI.COMM_SELF for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator(downsampling=True, npixels_per_sample=6, header=map_ref_global.header,
from tamasis.utils import all_eq pyoperators.memory.verbose = False tamasis.var.verbose = False profile = None#'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\ 'dPixelMask_FM_v5.fits' PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\ 'sponsivity_FM_v5.fits' obs = PacsObservation(filename=data_dir+'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) obs.pointing.chop[:] = 0 tod = obs.get_tod(subtraction_mean=True) projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x): self.niterations += 1 map_rls = mapper_rls(tod, model, hyper=1., tol=1.e-4, profile=profile, callback=None if tamasis.var.verbose else Callback(),
def check_map_global(m): assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10) assert_all_eq(m.tounit("Jy/arcsec^2"), map_ref_global.tounit("Jy/arcsec^2"), 1e-10) def check_map_local(m): assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_local.coverage, 1e-10) assert_all_eq(m.tounit("Jy/arcsec^2"), map_ref_local.tounit("Jy/arcsec^2"), 1e-10) obs1 = PacsObservation(data_dir + "frames_blue.fits", reject_bad_line=False) obs1.pointing.chop = 0 tod1 = obs1.get_tod() obs2 = PacsObservation( [data_dir + "frames_blue.fits[1:176]", data_dir + "frames_blue.fits[177:360]"], reject_bad_line=False ) obs2.pointing.chop = 0 tod2 = obs2.get_tod() tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={"header": header_ref_global}) # non-distributed map, distributed TOD def test1(): comm_map = MPI.COMM_SELF for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator( downsampling=True, npixels_per_sample=6, header=map_ref_global.header, commin=comm_map )
pacs = PacsObservation( [datafile[0] + '[6065:20000]', datafile[1] + '[6066:20001]'], fine_sampling_factor=1, calblock_extension_time=0.) telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector = 5 plot_tod((tod40Hz + drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask)
pyoperators.memory.verbose = False tamasis.var.verbose = False profile = None #'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\ 'dPixelMask_FM_v5.fits' PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\ 'sponsivity_FM_v5.fits' obs = PacsObservation(filename=data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) obs.pointing.chop[:] = 0 tod = obs.get_tod(subtraction_mean=True) projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x): self.niterations += 1