def test_header(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.chop = 0 header = obs.get_map_header() projection = obs.get_projection_operator(header=header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() map_naive = mapper_naive(tod, projection) header2 = header.copy() header2['NAXIS1'] += 500 header2['CRPIX1'] += 250 projection2 = obs.get_projection_operator(header=header2, downsampling=True) map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2) map_naive2.inunit('Jy/arcsec^2') map_naive3 = map_naive2[:,250:header['NAXIS1']+250] assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7) # test compatibility with photproject tod = obs.get_tod() map_naive4 = mapper_naive(tod, projection) hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1] map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel') std_naive = np.std(map_naive4[40:60,40:60]) std_ref = np.std(map_ref[40:60,40:60]) assert abs(std_naive-std_ref) / std_ref < 0.025
def test_detector_policy(): map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits') obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs.pointing.chop[:] = 0 projection = obs.get_projection_operator(header=map_naive_ref.header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() masking = MaskOperator(tod.mask) model = masking * projection map_naive = mapper_naive(tod, model) assert all_eq(map_naive, map_naive_ref, tol) obs_rem = PacsObservation(data_dir + 'frames_blue.fits', policy_bad_detector='remove', reject_bad_line=False) obs_rem.pointing.chop[:] = 0 projection_rem = obs_rem.get_projection_operator(header=map_naive.header, downsampling=True, npixels_per_sample=7) tod_rem = obs_rem.get_tod() masking_rem = MaskOperator(tod_rem.mask) model_rem = masking_rem * projection_rem map_naive_rem = mapper_naive(tod_rem, model_rem) assert all_eq(map_naive, map_naive_rem, tol)
def test_pTx_pT1(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) obs1.pointing.chop = 0 obs2.pointing.chop = 0 header = obs1.get_map_header() tod = obs1.get_tod() model1 = obs1.get_projection_operator(downsampling=True, header=header) ref = mapper_naive(tod, model1, unit='Jy/arcsec^2') model1.apply_mask(tod.mask) tod.inunit('Jy/arcsec^2') b1, w1 = model1.get_pTx_pT1(tod) m1 = b1 / w1 assert all_eq(ref, m1, tol) model2 = obs2.get_projection_operator(downsampling=True, header=header) model2.apply_mask(tod.mask) b2, w2 = model2.get_pTx_pT1(tod) m2 = b2 / w2 assert all_eq(ref, m2, tol) model3 = obs2.get_projection_operator(downsampling=True, header=header, storage='on fly') MaskOperator(tod.mask)(tod, tod) b3, w3 = model3.get_pTx_pT1(tod) m3 = b3 / w3 assert all_eq(ref, m3, tol)
def test_npixels_per_sample_is_zero(): obs = PacsObservation(data_dir + 'frames_blue.fits') header = obs.get_map_header() header['crval1'] += 1 proj2 = obs.get_projection_operator(header=header) assert proj2.matrix.shape[-1] == 0 t = proj2(np.ones((header['NAXIS2'],header['NAXIS1']))) assert all_eq(minmax(t), [0,0]) t[:] = 1 assert all_eq(minmax(proj2.T(t)), [0,0])
def test_slice2(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) assert all_eq(obs1.pointing, obs2.pointing[~obs2.pointing.removed]) obs1.pointing.chop = 0 obs2.pointing.chop = 0 tod1 = obs1.get_tod() tod2 = obs2.get_tod() assert all_eq(tod1, tod2) header = obs1.get_map_header() proj1 = obs1.get_projection_operator(header=header) proj2 = obs2.get_projection_operator(header=header) proj3 = obs2.get_projection_operator(header=header, storage='on fly') assert all_eq(proj1.get_mask(), proj2.get_mask()) assert all_eq(proj1.get_mask(), proj3.get_mask()) assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \ proj2.operands], axis=1)) assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \ proj3.operands], axis=1)) model1 = CompressionAverageOperator(obs1.slice.compression_factor) * proj1 model2 = CompressionAverageOperator(obs2.slice.compression_factor) * proj2 model3 = CompressionAverageOperator(obs2.slice.compression_factor) * proj3 m1 = model1.T(tod1) m2 = model2.T(tod2) m3 = model3.T(tod2) assert all_eq(m1, m2, tol) assert all_eq(m1, m3, tol) assert all_eq(model1(m1), model2(m1)) assert all_eq(model1(m1), model3(m1))
deglitch_l2mad, filter_median, filter_polynomial, mapper_naive) datadir = os.getenv('PACS_DATA', '')+'/transpScan/' datafile = [datadir+'1342184598_blue_PreparedFrames.fits', datadir+'1342184599_blue_PreparedFrames.fits'] if not all(map(os.path.exists, datafile)): print('The data files are not found: ' + ', '.join(datafile)) exit(0) pacs = PacsObservation([datafile[0]+'[6065:20000]', datafile[1]+'[6066:20001]'], fine_sampling_factor=1, calblock_extension_time=0.) telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000)
maxiter = 100 rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None#'test_ls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * \ obs_ref.get_projection_operator(downsampling=True, npixels_per_sample=6, commin=comm_map) map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2') map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator( 1/map_naive_ref.coverage)) cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape, attrin={'header':header_ref_global}) map_ref_local = tolocal(map_ref_global) cov_ref_local = tolocal(map_ref_global.coverage) mask_ref_local = tolocal(mask_ref_global) def check_map_global(m):
from pyoperators import DiagonalOperator, IdentityOperator, MaskOperator from tamasis import (PacsObservation, CompressionAverageOperator, UnpackOperator, mapper_ls, mapper_naive) pyoperators.memory.verbose = False profile = None#'test_ls.png' solver = scipy.sparse.linalg.bicgstab tol = 1.e-6 if profile else 1.e-4 maxiter = 10 data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) tod = obs.get_tod() telescope = IdentityOperator() projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6) compression = CompressionAverageOperator(obs.slice.compression_factor) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * telescope * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, restricting oneself to observed map pixels unpacking = UnpackOperator(projection.get_mask()) old_settings = np.seterr(divide='ignore') M = DiagonalOperator(unpacking.T(1./map_naive.coverage)) np.seterr(**old_settings) #map_iter1 = mapper_ls(tod, model * unpacking, tol=1.e-4, M=M)
import tamasis from pyoperators import BlockColumnOperator, MaskOperator from pyoperators.iterative.algorithms import StopCondition from pyoperators.iterative.dli import DoubleLoopAlgorithm from tamasis import PacsObservation, DiscreteDifferenceOperator, mapper_naive pyoperators.memory.verbose = False tamasis.var.verbose = True data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(filename=data_dir + 'frames_blue.fits') obs.pointing.chop[:] = 0 tod = obs.get_tod() projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection naive = mapper_naive(tod, model) naive[np.isnan(naive)] = 0 prior = BlockColumnOperator( [DiscreteDifferenceOperator(axis, shapein=(103, 97)) for axis in (0, 1)], new_axisout=0) stop_condition = StopCondition(maxiter=2) dli = DoubleLoopAlgorithm(model, tod, prior,
pyoperators.memory.verbose = False tamasis.var.verbose = False profile = None#'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\ 'dPixelMask_FM_v5.fits' PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\ 'sponsivity_FM_v5.fits' obs = PacsObservation(filename=data_dir+'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) obs.pointing.chop[:] = 0 tod = obs.get_tod(subtraction_mean=True) projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x): self.niterations += 1 map_rls = mapper_rls(tod, model, hyper=1., tol=1.e-4, profile=profile, callback=None if tamasis.var.verbose else Callback(), solver=cgs) if profile is None:
datafile = [ datadir + '1342184598_blue_PreparedFrames.fits', datadir + '1342184599_blue_PreparedFrames.fits' ] if not all(map(os.path.exists, datafile)): print('The data files are not found: ' + ', '.join(datafile)) exit(0) pacs = PacsObservation( [datafile[0] + '[6065:20000]', datafile[1] + '[6066:20001]'], fine_sampling_factor=1, calblock_extension_time=0.) telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000)
rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None#'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) header_ref_global = create_fitsheader((97,108), cdelt=3.2/3600, crval=(245.998427916727,61.5147650744551)) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * obs_ref.get_projection_operator( downsampling=True, npixels_per_sample=6, commin=comm_map, header=header_ref_global) map_ref_global = mapper_rls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, hyper=hyper) header_ref_global = map_ref_global.header cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={'header':header_ref_global}) map_ref_local = tolocal(map_ref_global) cov_ref_local = tolocal(map_ref_global.coverage) mask_ref_local = tolocal(mask_ref_global) if rank == 0: map_ref_global.save('map_ref_global_v4.fits', comm=MPI.COMM_SELF)
rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None #'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) header_ref_global = create_fitsheader( (97, 108), cdelt=3.2 / 3600, crval=(245.998427916727, 61.5147650744551)) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * obs_ref.get_projection_operator( downsampling=True, npixels_per_sample=6, commin=comm_map, header=header_ref_global) map_ref_global = mapper_rls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, hyper=hyper) header_ref_global = map_ref_global.header cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={'header': header_ref_global}) map_ref_local = tolocal(map_ref_global) cov_ref_local = tolocal(map_ref_global.coverage)