def test_npixels_per_sample_is_zero(): obs = PacsObservation(data_dir + 'frames_blue.fits') header = obs.get_map_header() header['crval1'] += 1 proj2 = obs.get_projection_operator(header=header) assert proj2.matrix.shape[-1] == 0 t = proj2(np.ones((header['NAXIS2'],header['NAXIS1']))) assert all_eq(minmax(t), [0,0]) t[:] = 1 assert all_eq(minmax(proj2.T(t)), [0,0])
def test_pTx_pT1(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) obs1.pointing.chop = 0 obs2.pointing.chop = 0 header = obs1.get_map_header() tod = obs1.get_tod() model1 = obs1.get_projection_operator(downsampling=True, header=header) ref = mapper_naive(tod, model1, unit='Jy/arcsec^2') model1.apply_mask(tod.mask) tod.inunit('Jy/arcsec^2') b1, w1 = model1.get_pTx_pT1(tod) m1 = b1 / w1 assert all_eq(ref, m1, tol) model2 = obs2.get_projection_operator(downsampling=True, header=header) model2.apply_mask(tod.mask) b2, w2 = model2.get_pTx_pT1(tod) m2 = b2 / w2 assert all_eq(ref, m2, tol) model3 = obs2.get_projection_operator(downsampling=True, header=header, storage='on fly') MaskOperator(tod.mask)(tod, tod) b3, w3 = model3.get_pTx_pT1(tod) m3 = b3 / w3 assert all_eq(ref, m3, tol)
def test_detector_policy(): map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits') obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs.pointing.chop[:] = 0 projection = obs.get_projection_operator(header=map_naive_ref.header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() masking = MaskOperator(tod.mask) model = masking * projection map_naive = mapper_naive(tod, model) assert all_eq(map_naive, map_naive_ref, tol) obs_rem = PacsObservation(data_dir + 'frames_blue.fits', policy_bad_detector='remove', reject_bad_line=False) obs_rem.pointing.chop[:] = 0 projection_rem = obs_rem.get_projection_operator(header=map_naive.header, downsampling=True, npixels_per_sample=7) tod_rem = obs_rem.get_tod() masking_rem = MaskOperator(tod_rem.mask) model_rem = masking_rem * projection_rem map_naive_rem = mapper_naive(tod_rem, model_rem) assert all_eq(map_naive, map_naive_rem, tol)
def test1(): # creation of the sky map msize = 50 mymap = gaussian(2*(msize*2+1,), 10, unit='Jy/pixel') cd = np.array([[-1., 0.],[0., 1.]]) / 3600. header = create_fitsheader(fromdata=mymap, crval=[53.,27.], cd=cd) mymap.header = header # creation of the simulation scan = PacsObservation.create_scan((header['CRVAL1'], header['CRVAL2']), instrument_angle=0., length=60, nlegs=1, angle=20.) simul = PacsSimulation(scan, 'red', policy_bad_detector='keep', policy_other='keep') # build the acquisition model model = CompressionAverageOperator(simul.slice.compression_factor) * \ simul.get_projection_operator(header=header, npixels_per_sample=49) # get the noiseless tod tod = model(mymap) filename = 'simul-'+str(uuid1())+'.fits' try: simul.save(filename, tod) simul2 = PacsObservation(filename, policy_bad_detector='keep', policy_other='keep') status2 = simul2.status tod2 = simul2.get_tod() finally: try: os.remove(filename) except: pass for field in simul.status.dtype.names: if field == 'BAND': continue assert_eq(simul.status[field], status2[field]) assert_eq(tod, tod2) fields = [x for x in simul.slice.dtype.names if x not in ('filename','unit')] for field in fields: if getattr(simul.slice[0], field) != getattr(simul2.slice[0], field): msg = "Field '" + field + "'" if field == 'scan_step': print(msg + ' not implemented.') else: assert False
def test2(): # test data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(filename=data_dir+'frames_blue.fits') pointing = obs.pointing.copy() pointing.header = obs.pointing.header.copy() # check mode time = pointing.time / 4 result = { (1,'blue'): 'calibration', (2,'blue'):'calibration', (4,'blue'):'prime', (8,'blue'):'parallel', (1,'green'): 'calibration', (2,'green'):'calibration', (4,'green'):'prime', (8,'green'):'parallel', (1,'red'): 'calibration', (2,'red'):'calibration', (4,'red'):'prime', (8,'red'):'calibration', } def func1(c, b): pointing.time = time * c simul = PacsSimulation(pointing, b) assert simul.slice[0].mode == result[(c,b)] assert simul.slice[0].compression_factor == c for c in (1,2,4,8): for b in ('blue', 'green', 'red'): yield func1, c, b ok = [ (1,'blue','transparent'), (1,'green','transparent'), (1,'red','transparent'), (4,'blue','prime'), (4,'green','prime'), (4,'red','prime'), (4,'red','parallel'), (8,'blue','parallel'), (8,'green','parallel'), ] def func2(c, b, m): pointing.time = time * c if (c,b,m) in ok: simul = PacsSimulation(pointing, b, mode=m) assert simul.slice[0].compression_factor == c assert simul.slice[0].mode == m else: try: simul = PacsSimulation(pointing, b, mode=m) except ValueError: pass else: print c,b,m assert False for c in (1,2,4,8): for b in ('blue', 'green', 'red'): for m in ('prime', 'parallel', 'transparent'): yield func2, c, b, m def func3(c, b): pointing.time = time * c simul = PacsSimulation(pointing, b, mode='calibration') assert simul.slice[0].compression_factor == c for c in (1,2,4,8): for b in ('blue', 'green', 'red'): yield func3, c, b
def test_multiple_pointings(): ra = (23, 24) dec = (50, 51) instrument_angle=(10, 11) scan_angle = (0, -90) scan_length = (10, 20) scan_nlegs = (2, 3) scan_step = (147, 149) scan_speed = (20, 60) compression_factor = (4, 4) acc = PacsSimulation.ACCELERATION pointings = [PacsObservation.create_scan((r,d),sl,sst,None,ssp,acc,sn,sa,ia, cf,False) for r,d,ia,sa,sl,sn,sst,ssp,cf in zip(ra,dec, instrument_angle,scan_angle,scan_length,scan_nlegs,scan_step, scan_speed,compression_factor)] simul = PacsSimulation(pointings, 'blue') s = simul.slice assert len(s) == 2 assert_eq(s.ra, ra) assert_eq(s.dec, dec) assert_eq(s.instrument_angle, instrument_angle) assert_eq(s.scan_angle, scan_angle) assert_eq(s.scan_length, scan_length) assert_eq(s.scan_nlegs, scan_nlegs) assert_eq(s.scan_step, scan_step) assert_eq(s.scan_speed, scan_speed) assert_eq(s.compression_factor, compression_factor)
def test_slice1(): obs = PacsObservation(data_dir+'frames_blue.fits[11:20]') tod = obs.get_tod() filename = 'obs-' + uuid + '.fits' obs.save(filename, tod) obs2 = PacsObservation(filename) tod2 = obs2.get_tod() assert all_eq(obs.status[10:20], obs2.status) assert all_eq(tod, tod2)
def test_header(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.chop = 0 header = obs.get_map_header() projection = obs.get_projection_operator(header=header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() map_naive = mapper_naive(tod, projection) header2 = header.copy() header2['NAXIS1'] += 500 header2['CRPIX1'] += 250 projection2 = obs.get_projection_operator(header=header2, downsampling=True) map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2) map_naive2.inunit('Jy/arcsec^2') map_naive3 = map_naive2[:,250:header['NAXIS1']+250] assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7) # test compatibility with photproject tod = obs.get_tod() map_naive4 = mapper_naive(tod, projection) hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1] map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel') std_naive = np.std(map_naive4[40:60,40:60]) std_ref = np.std(map_ref[40:60,40:60]) assert abs(std_naive-std_ref) / std_ref < 0.025
def test_slice2(): obs1 = PacsObservation(data_dir + 'frames_blue.fits') obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]', data_dir + 'frames_blue.fits[42:43]', data_dir + 'frames_blue.fits[44:360]']) assert all_eq(obs1.pointing, obs2.pointing[~obs2.pointing.removed]) obs1.pointing.chop = 0 obs2.pointing.chop = 0 tod1 = obs1.get_tod() tod2 = obs2.get_tod() assert all_eq(tod1, tod2) header = obs1.get_map_header() proj1 = obs1.get_projection_operator(header=header) proj2 = obs2.get_projection_operator(header=header) proj3 = obs2.get_projection_operator(header=header, storage='on fly') assert all_eq(proj1.get_mask(), proj2.get_mask()) assert all_eq(proj1.get_mask(), proj3.get_mask()) assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \ proj2.operands], axis=1)) assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \ proj3.operands], axis=1)) model1 = CompressionAverageOperator(obs1.slice.compression_factor) * proj1 model2 = CompressionAverageOperator(obs2.slice.compression_factor) * proj2 model3 = CompressionAverageOperator(obs2.slice.compression_factor) * proj3 m1 = model1.T(tod1) m2 = model2.T(tod2) m3 = model3.T(tod2) assert all_eq(m1, m2, tol) assert all_eq(m1, m3, tol) assert all_eq(model1(m1), model2(m1)) assert all_eq(model1(m1), model3(m1))
def test_save(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.ra += 0.1 obs.pointing.dec -= 0.1 obs.pointing.pa += 20 obs.pointing.chop = 0 tod = obs.get_tod() filename = 'obs-' + uuid + '.fits' obs.save(filename, tod) obs2 = PacsObservation(filename, reject_bad_line=False) tod2 = obs2.get_tod() assert all_eq(obs.pointing, obs2.pointing) obs.status.RaArray = obs.pointing.ra obs.status.DecArray = obs.pointing.dec obs.status.PaArray = obs.pointing.pa obs.status.CHOPFPUANGLE = obs.pointing.chop assert all_eq(obs.status, obs2.status) assert all_eq(tod, tod2)
import os import tamasis from pyoperators import DiagonalOperator, MaskOperator from tamasis import PacsObservation, mapper_naive, mapper_nl class TestFailure(Exception): pass data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1) tod = obs.get_tod() projection = obs.get_projection_operator(downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x):
header_ref_global = map_ref_global.header def check_map_global(m): assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10) assert_all_eq(m.tounit("Jy/arcsec^2"), map_ref_global.tounit("Jy/arcsec^2"), 1e-10) def check_map_local(m): assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_local.coverage, 1e-10) assert_all_eq(m.tounit("Jy/arcsec^2"), map_ref_local.tounit("Jy/arcsec^2"), 1e-10) obs1 = PacsObservation(data_dir + "frames_blue.fits", reject_bad_line=False) obs1.pointing.chop = 0 tod1 = obs1.get_tod() obs2 = PacsObservation( [data_dir + "frames_blue.fits[1:176]", data_dir + "frames_blue.fits[177:360]"], reject_bad_line=False ) obs2.pointing.chop = 0 tod2 = obs2.get_tod() tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={"header": header_ref_global}) # non-distributed map, distributed TOD def test1(): comm_map = MPI.COMM_SELF for obs, tod in ((obs1, tod1), (obs2, tod2)): proj = obs.get_projection_operator(
from tamasis import (PacsObservation, CompressionAverageOperator, deglitch_l2mad, filter_median, filter_polynomial, mapper_naive) datadir = os.getenv('PACS_DATA', '') + '/transpScan/' datafile = [ datadir + '1342184598_blue_PreparedFrames.fits', datadir + '1342184599_blue_PreparedFrames.fits' ] if not all(map(os.path.exists, datafile)): print('The data files are not found: ' + ', '.join(datafile)) exit(0) pacs = PacsObservation( [datafile[0] + '[6065:20000]', datafile[1] + '[6066:20001]'], fine_sampling_factor=1, calblock_extension_time=0.) telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6)
from pysimulators import Map from scipy.sparse.linalg import cgs from tamasis import PacsInstrument, PacsObservation, mapper_rls from tamasis.utils import all_eq pyoperators.memory.verbose = False tamasis.var.verbose = False profile = None#'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\ 'dPixelMask_FM_v5.fits' PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\ 'sponsivity_FM_v5.fits' obs = PacsObservation(filename=data_dir+'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) obs.pointing.chop[:] = 0 tod = obs.get_tod(subtraction_mean=True) projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x): self.niterations += 1
import os import tamasis from pyoperators import DiagonalOperator, MaskOperator from tamasis import PacsObservation, mapper_naive, mapper_nl class TestFailure(Exception): pass data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir+'frames_blue.fits', fine_sampling_factor=1) tod = obs.get_tod() projection = obs.get_projection_operator(downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0 def __call__(self, x): self.niterations += 1 invntt = DiagonalOperator(1/obs.get_detector_stddev(100)**2, broadcast='rightward') norm = tamasis.linalg.norm2_ellipsoid(invntt)
import numpy as np import pyoperators import os import scipy from pyoperators import DiagonalOperator, IdentityOperator, MaskOperator from tamasis import (PacsObservation, CompressionAverageOperator, UnpackOperator, mapper_ls, mapper_naive) pyoperators.memory.verbose = False profile = None#'test_ls.png' solver = scipy.sparse.linalg.bicgstab tol = 1.e-6 if profile else 1.e-4 maxiter = 10 data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) tod = obs.get_tod() telescope = IdentityOperator() projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6) compression = CompressionAverageOperator(obs.slice.compression_factor) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * telescope * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, restricting oneself to observed map pixels unpacking = UnpackOperator(projection.get_mask())
tol = 1.e-2 mtol = 1.e-10 maxiter = 100 hyper = 1 rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None#'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) header_ref_global = create_fitsheader((97,108), cdelt=3.2/3600, crval=(245.998427916727,61.5147650744551)) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * obs_ref.get_projection_operator( downsampling=True, npixels_per_sample=6, commin=comm_map, header=header_ref_global) map_ref_global = mapper_rls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, hyper=hyper) header_ref_global = map_ref_global.header cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={'header':header_ref_global}) map_ref_local = tolocal(map_ref_global) cov_ref_local = tolocal(map_ref_global.coverage) mask_ref_local = tolocal(mask_ref_global)
#solver = scipy.sparse.linalg.bicgstab solver = tamasis.solvers.cg tol = 1.e-2 mtol = 1.e-10 maxiter = 100 rank = MPI.COMM_WORLD.rank tamasis.var.verbose = True profile = None#'test_ls.png' data_dir = os.path.dirname(__file__) + '/data/' # reference map (no communication) comm_tod = MPI.COMM_SELF comm_map = MPI.COMM_SELF obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod) obs_ref.pointing.chop = 0 tod_ref = obs_ref.get_tod() model_ref = MaskOperator(tod_ref.mask) * \ obs_ref.get_projection_operator(downsampling=True, npixels_per_sample=6, commin=comm_map) map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2') map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter, solver=solver, M=DiagonalOperator( 1/map_naive_ref.coverage)) cov_ref_global = map_ref_global.coverage mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape, attrin={'header':header_ref_global})
import numpy as np import os import pyoperators import tamasis from pyoperators import BlockColumnOperator, MaskOperator from pyoperators.iterative.algorithms import StopCondition from pyoperators.iterative.dli import DoubleLoopAlgorithm from tamasis import PacsObservation, DiscreteDifferenceOperator, mapper_naive pyoperators.memory.verbose = False tamasis.var.verbose = True data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(filename=data_dir + 'frames_blue.fits') obs.pointing.chop[:] = 0 tod = obs.get_tod() projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection naive = mapper_naive(tod, model) naive[np.isnan(naive)] = 0 prior = BlockColumnOperator( [DiscreteDifferenceOperator(axis, shapein=(103, 97)) for axis in (0, 1)], new_axisout=0) stop_condition = StopCondition(maxiter=2)
mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header def check_map_global(m): assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10) assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_global.tounit('Jy/arcsec^2'), 1e-10) def check_map_local(m): assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_local.coverage, 1e-10) assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_local.tounit('Jy/arcsec^2'), 1e-10) obs1 = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs1.pointing.chop = 0 tod1 = obs1.get_tod() obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:176]', data_dir + 'frames_blue.fits[177:360]'], reject_bad_line=False) obs2.pointing.chop = 0 tod2 = obs2.get_tod() tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={'header':header_ref_global}) # non-distributed map, distributed TOD def test1(): comm_map = MPI.COMM_SELF for obs, tod in ((obs1, tod1), (obs2, tod2)):
from matplotlib.pyplot import plot from pyoperators import IdentityOperator, MaskOperator from pysimulators import plot_tod from tamasis import (PacsObservation, CompressionAverageOperator, deglitch_l2mad, filter_median, filter_polynomial, mapper_naive) datadir = os.getenv('PACS_DATA', '')+'/transpScan/' datafile = [datadir+'1342184598_blue_PreparedFrames.fits', datadir+'1342184599_blue_PreparedFrames.fits'] if not all(map(os.path.exists, datafile)): print('The data files are not found: ' + ', '.join(datafile)) exit(0) pacs = PacsObservation([datafile[0]+'[6065:20000]', datafile[1]+'[6066:20001]'], fine_sampling_factor=1, calblock_extension_time=0.) telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6)
from scipy.sparse.linalg import cgs from tamasis import PacsInstrument, PacsObservation, mapper_rls from tamasis.utils import all_eq pyoperators.memory.verbose = False tamasis.var.verbose = False profile = None #'test_rls.png' data_dir = os.path.dirname(__file__) + '/data/' PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\ 'dPixelMask_FM_v5.fits' PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\ 'sponsivity_FM_v5.fits' obs = PacsObservation(filename=data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) obs.pointing.chop[:] = 0 tod = obs.get_tod(subtraction_mean=True) projection = obs.get_projection_operator(resolution=3.2, downsampling=True, npixels_per_sample=6) masking_tod = MaskOperator(tod.mask) model = masking_tod * projection # iterative map, taking all map pixels class Callback(): def __init__(self): self.niterations = 0