def test_detector_policy(): map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits') obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs.pointing.chop[:] = 0 projection = obs.get_projection_operator(header=map_naive_ref.header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() masking = MaskOperator(tod.mask) model = masking * projection map_naive = mapper_naive(tod, model) assert all_eq(map_naive, map_naive_ref, tol) obs_rem = PacsObservation(data_dir + 'frames_blue.fits', policy_bad_detector='remove', reject_bad_line=False) obs_rem.pointing.chop[:] = 0 projection_rem = obs_rem.get_projection_operator(header=map_naive.header, downsampling=True, npixels_per_sample=7) tod_rem = obs_rem.get_tod() masking_rem = MaskOperator(tod_rem.mask) model_rem = masking_rem * projection_rem map_naive_rem = mapper_naive(tod_rem, model_rem) assert all_eq(map_naive, map_naive_rem, tol)
def test_pTx_pT1(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) obs1.pointing.chop = 0 obs2.pointing.chop = 0 header = obs1.get_map_header() tod = obs1.get_tod() model1 = obs1.get_projection_operator(downsampling=True, header=header) ref = mapper_naive(tod, model1, unit='Jy/arcsec^2') model1.apply_mask(tod.mask) tod.inunit('Jy/arcsec^2') b1, w1 = model1.get_pTx_pT1(tod) m1 = b1 / w1 assert all_eq(ref, m1, tol) model2 = obs2.get_projection_operator(downsampling=True, header=header) model2.apply_mask(tod.mask) b2, w2 = model2.get_pTx_pT1(tod) m2 = b2 / w2 assert all_eq(ref, m2, tol) model3 = obs2.get_projection_operator(downsampling=True, header=header, storage='on fly') MaskOperator(tod.mask)(tod, tod) b3, w3 = model3.get_pTx_pT1(tod) m3 = b3 / w3 assert all_eq(ref, m3, tol)
def test_slice1(): obs = PacsObservation(data_dir+'frames_blue.fits[11:20]') tod = obs.get_tod() filename = 'obs-' + uuid + '.fits' obs.save(filename, tod) obs2 = PacsObservation(filename) tod2 = obs2.get_tod() assert all_eq(obs.status[10:20], obs2.status) assert all_eq(tod, tod2)
def test_header(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.chop = 0 header = obs.get_map_header() projection = obs.get_projection_operator(header=header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() map_naive = mapper_naive(tod, projection) header2 = header.copy() header2['NAXIS1'] += 500 header2['CRPIX1'] += 250 projection2 = obs.get_projection_operator(header=header2, downsampling=True) map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2) map_naive2.inunit('Jy/arcsec^2') map_naive3 = map_naive2[:,250:header['NAXIS1']+250] assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7) # test compatibility with photproject tod = obs.get_tod() map_naive4 = mapper_naive(tod, projection) hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1] map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel') std_naive = np.std(map_naive4[40:60,40:60]) std_ref = np.std(map_ref[40:60,40:60]) assert abs(std_naive-std_ref) / std_ref < 0.025
def test2(): # test data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(filename=data_dir+'frames_blue.fits') pointing = obs.pointing.copy() pointing.header = obs.pointing.header.copy() # check mode time = pointing.time / 4 result = { (1,'blue'): 'calibration', (2,'blue'):'calibration', (4,'blue'):'prime', (8,'blue'):'parallel', (1,'green'): 'calibration', (2,'green'):'calibration', (4,'green'):'prime', (8,'green'):'parallel', (1,'red'): 'calibration', (2,'red'):'calibration', (4,'red'):'prime', (8,'red'):'calibration', } def func1(c, b): pointing.time = time * c simul = PacsSimulation(pointing, b) assert simul.slice[0].mode == result[(c,b)] assert simul.slice[0].compression_factor == c for c in (1,2,4,8): for b in ('blue', 'green', 'red'): yield func1, c, b ok = [ (1,'blue','transparent'), (1,'green','transparent'), (1,'red','transparent'), (4,'blue','prime'), (4,'green','prime'), (4,'red','prime'), (4,'red','parallel'), (8,'blue','parallel'), (8,'green','parallel'), ] def func2(c, b, m): pointing.time = time * c if (c,b,m) in ok: simul = PacsSimulation(pointing, b, mode=m) assert simul.slice[0].compression_factor == c assert simul.slice[0].mode == m else: try: simul = PacsSimulation(pointing, b, mode=m) except ValueError: pass else: print c,b,m assert False for c in (1,2,4,8): for b in ('blue', 'green', 'red'): for m in ('prime', 'parallel', 'transparent'): yield func2, c, b, m def func3(c, b): pointing.time = time * c simul = PacsSimulation(pointing, b, mode='calibration') assert simul.slice[0].compression_factor == c for c in (1,2,4,8): for b in ('blue', 'green', 'red'): yield func3, c, b
def test_npixels_per_sample_is_zero(): obs = PacsObservation(data_dir + 'frames_blue.fits') header = obs.get_map_header() header['crval1'] += 1 proj2 = obs.get_projection_operator(header=header) assert proj2.matrix.shape[-1] == 0 t = proj2(np.ones((header['NAXIS2'],header['NAXIS1']))) assert all_eq(minmax(t), [0,0]) t[:] = 1 assert all_eq(minmax(proj2.T(t)), [0,0])
def test_save(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.ra += 0.1 obs.pointing.dec -= 0.1 obs.pointing.pa += 20 obs.pointing.chop = 0 tod = obs.get_tod() filename = 'obs-' + uuid + '.fits' obs.save(filename, tod) obs2 = PacsObservation(filename, reject_bad_line=False) tod2 = obs2.get_tod() assert all_eq(obs.pointing, obs2.pointing) obs.status.RaArray = obs.pointing.ra obs.status.DecArray = obs.pointing.dec obs.status.PaArray = obs.pointing.pa obs.status.CHOPFPUANGLE = obs.pointing.chop assert all_eq(obs.status, obs2.status) assert all_eq(tod, tod2)
def test_slice2(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) assert all_eq(obs1.pointing, obs2.pointing[~obs2.pointing.removed]) obs1.pointing.chop = 0 obs2.pointing.chop = 0 tod1 = obs1.get_tod() tod2 = obs2.get_tod() assert all_eq(tod1, tod2) header = obs1.get_map_header() proj1 = obs1.get_projection_operator(header=header) proj2 = obs2.get_projection_operator(header=header) proj3 = obs2.get_projection_operator(header=header, storage='on fly') assert all_eq(proj1.get_mask(), proj2.get_mask()) assert all_eq(proj1.get_mask(), proj3.get_mask()) assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \ proj2.operands], axis=1)) assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \ proj3.operands], axis=1)) model1 = CompressionAverageOperator(obs1.slice.compression_factor) * proj1 model2 = CompressionAverageOperator(obs2.slice.compression_factor) * proj2 model3 = CompressionAverageOperator(obs2.slice.compression_factor) * proj3 m1 = model1.T(tod1) m2 = model2.T(tod2) m3 = model3.T(tod2) assert all_eq(m1, m2, tol) assert all_eq(m1, m3, tol) assert all_eq(model1(m1), model2(m1)) assert all_eq(model1(m1), model3(m1))
def test1(): # creation of the sky map msize = 50 mymap = gaussian(2*(msize*2+1,), 10, unit='Jy/pixel') cd = np.array([[-1., 0.],[0., 1.]]) / 3600. header = create_fitsheader(fromdata=mymap, crval=[53.,27.], cd=cd) mymap.header = header # creation of the simulation scan = PacsObservation.create_scan((header['CRVAL1'], header['CRVAL2']), instrument_angle=0., length=60, nlegs=1, angle=20.) simul = PacsSimulation(scan, 'red', policy_bad_detector='keep', policy_other='keep') # build the acquisition model model = CompressionAverageOperator(simul.slice.compression_factor) * \ simul.get_projection_operator(header=header, npixels_per_sample=49) # get the noiseless tod tod = model(mymap) filename = 'simul-'+str(uuid1())+'.fits' try: simul.save(filename, tod) simul2 = PacsObservation(filename, policy_bad_detector='keep', policy_other='keep') status2 = simul2.status tod2 = simul2.get_tod() finally: try: os.remove(filename) except: pass for field in simul.status.dtype.names: if field == 'BAND': continue assert_eq(simul.status[field], status2[field]) assert_eq(tod, tod2) fields = [x for x in simul.slice.dtype.names if x not in ('filename','unit')] for field in fields: if getattr(simul.slice[0], field) != getattr(simul2.slice[0], field): msg = "Field '" + field + "'" if field == 'scan_step': print(msg + ' not implemented.') else: assert False
#solver = scipy.sparse.linalg.bicgstab solver = tamasis.solvers.cg tol = 1.e-2 mtol = 1.e-10 maxiter = 100 rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None#'test_ls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * \ obs_ref.get_projection_operator(downsampling=True, npixels_per_sample=6, commin=comm_map) map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2') map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator( 1/map_naive_ref.coverage)) cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape, attrin={'header':header_ref_global})
import numpy as np import pyoperators import os import scipy from pyoperators import DiagonalOperator, IdentityOperator, MaskOperator from tamasis import (PacsObservation, CompressionAverageOperator, UnpackOperator, mapper_ls, mapper_naive) pyoperators.memory.verbose = False profile = None#'test_ls.png' solver = scipy.sparse.linalg.bicgstab tol = 1.e-6 if profile else 1.e-4 maxiter = 10 data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) tod = obs.get_tod() telescope = IdentityOperator() projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6) compression = CompressionAverageOperator(obs.slice.compression_factor) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * telescope * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, restricting oneself to observed map pixels unpacking = UnpackOperator(projection.get_mask())
mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header def check_map_global(m): assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10) assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_global.tounit('Jy/arcsec^2'), 1e-10) def check_map_local(m): assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_local.coverage, 1e-10) assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_local.tounit('Jy/arcsec^2'), 1e-10) obs1 = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs1.pointing.chop = 0 tod1 = obs1.get_tod() obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:176]', data_dir + 'frames_blue.fits[177:360]'], reject_bad_line=False) obs2.pointing.chop = 0 tod2 = obs2.get_tod() tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={'header':header_ref_global}) # non-distributed map, distributed TOD def test1(): comm_map = MPI.COMM_SELF for obs, tod in ((obs1, tod1), (obs2, tod2)):
import numpy as np import os import pyoperators import tamasis from pyoperators import BlockColumnOperator, MaskOperator from pyoperators.iterative.algorithms import StopCondition from pyoperators.iterative.dli import DoubleLoopAlgorithm from tamasis import PacsObservation, DiscreteDifferenceOperator, mapper_naive pyoperators.memory.verbose = False tamasis.var.verbose = True data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(filename=data_dir + 'frames_blue.fits') obs.pointing.chop[:] = 0 tod = obs.get_tod() projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection naive = mapper_naive(tod, model) naive[np.isnan(naive)] = 0 prior = BlockColumnOperator( [DiscreteDifferenceOperator(axis, shapein=(103, 97)) for axis in (0, 1)], new_axisout=0) stop_condition = StopCondition(maxiter=2)
import os import tamasis from pyoperators import DiagonalOperator, MaskOperator from tamasis import PacsObservation, mapper_naive, mapper_nl class TestFailure(Exception): pass data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1) tod = obs.get_tod() projection = obs.get_projection_operator(downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x):
from tamasis import (PacsObservation, CompressionAverageOperator, deglitch_l2mad, filter_median, filter_polynomial, mapper_naive) datadir = os.getenv('PACS_DATA', '') + '/transpScan/' datafile = [ datadir + '1342184598_blue_PreparedFrames.fits', datadir + '1342184599_blue_PreparedFrames.fits' ] if not all(map(os.path.exists, datafile)): print('The data files are not found: ' + ', '.join(datafile)) exit(0) pacs = PacsObservation( [datafile[0] + '[6065:20000]', datafile[1] + '[6066:20001]'], fine_sampling_factor=1, calblock_extension_time=0.) telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6)