def test_header():
obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False)
obs.pointing.chop = 0
header = obs.get_map_header()
projection = obs.get_projection_operator(header=header, downsampling=True,
npixels_per_sample=6)
tod = obs.get_tod()
map_naive = mapper_naive(tod, projection)
header2 = header.copy()
header2['NAXIS1'] += 500
header2['CRPIX1'] += 250
projection2 = obs.get_projection_operator(header=header2, downsampling=True)
map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2)
map_naive2.inunit('Jy/arcsec^2')
map_naive3 = map_naive2[:,250:header['NAXIS1']+250]
assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7)
# test compatibility with photproject
tod = obs.get_tod()
map_naive4 = mapper_naive(tod, projection)
hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1]
map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel')
std_naive = np.std(map_naive4[40:60,40:60])
std_ref = np.std(map_ref[40:60,40:60])
assert abs(std_naive-std_ref) / std_ref < 0.025
def test_detector_policy():
map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits')
obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False)
obs.pointing.chop[:] = 0
projection = obs.get_projection_operator(header=map_naive_ref.header,
downsampling=True,
npixels_per_sample=6)
tod = obs.get_tod()
masking = MaskOperator(tod.mask)
model = masking * projection
map_naive = mapper_naive(tod, model)
assert all_eq(map_naive, map_naive_ref, tol)
obs_rem = PacsObservation(data_dir + 'frames_blue.fits',
policy_bad_detector='remove',
reject_bad_line=False)
obs_rem.pointing.chop[:] = 0
projection_rem = obs_rem.get_projection_operator(header=map_naive.header,
downsampling=True,
npixels_per_sample=7)
tod_rem = obs_rem.get_tod()
masking_rem = MaskOperator(tod_rem.mask)
model_rem = masking_rem * projection_rem
map_naive_rem = mapper_naive(tod_rem, model_rem)
assert all_eq(map_naive, map_naive_rem, tol)
def test_slice1():
obs = PacsObservation(data_dir+'frames_blue.fits[11:20]')
tod = obs.get_tod()
filename = 'obs-' + uuid + '.fits'
obs.save(filename, tod)
obs2 = PacsObservation(filename)
tod2 = obs2.get_tod()
assert all_eq(obs.status[10:20], obs2.status)
assert all_eq(tod, tod2)
def test_pTx_pT1():
obs1 = PacsObservation(data_dir + 'frames_blue.fits')
obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]',
data_dir + 'frames_blue.fits[42:43]',
data_dir + 'frames_blue.fits[44:360]'])
obs1.pointing.chop = 0
obs2.pointing.chop = 0
header = obs1.get_map_header()
tod = obs1.get_tod()
model1 = obs1.get_projection_operator(downsampling=True, header=header)
ref = mapper_naive(tod, model1, unit='Jy/arcsec^2')
model1.apply_mask(tod.mask)
tod.inunit('Jy/arcsec^2')
b1, w1 = model1.get_pTx_pT1(tod)
m1 = b1 / w1
assert all_eq(ref, m1, tol)
model2 = obs2.get_projection_operator(downsampling=True, header=header)
model2.apply_mask(tod.mask)
b2, w2 = model2.get_pTx_pT1(tod)
m2 = b2 / w2
assert all_eq(ref, m2, tol)
model3 = obs2.get_projection_operator(downsampling=True, header=header,
storage='on fly')
MaskOperator(tod.mask)(tod, tod)
b3, w3 = model3.get_pTx_pT1(tod)
m3 = b3 / w3
assert all_eq(ref, m3, tol)
def test_save():
obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False)
obs.pointing.ra += 0.1
obs.pointing.dec -= 0.1
obs.pointing.pa += 20
obs.pointing.chop = 0
tod = obs.get_tod()
filename = 'obs-' + uuid + '.fits'
obs.save(filename, tod)
obs2 = PacsObservation(filename, reject_bad_line=False)
tod2 = obs2.get_tod()
assert all_eq(obs.pointing, obs2.pointing)
obs.status.RaArray = obs.pointing.ra
obs.status.DecArray = obs.pointing.dec
obs.status.PaArray = obs.pointing.pa
obs.status.CHOPFPUANGLE = obs.pointing.chop
assert all_eq(obs.status, obs2.status)
assert all_eq(tod, tod2)
def test_slice2():
obs1 = PacsObservation(data_dir + 'frames_blue.fits')
obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]',
data_dir + 'frames_blue.fits[42:43]',
data_dir + 'frames_blue.fits[44:360]'])
assert all_eq(obs1.pointing, obs2.pointing[~obs2.pointing.removed])
obs1.pointing.chop = 0
obs2.pointing.chop = 0
tod1 = obs1.get_tod()
tod2 = obs2.get_tod()
assert all_eq(tod1, tod2)
header = obs1.get_map_header()
proj1 = obs1.get_projection_operator(header=header)
proj2 = obs2.get_projection_operator(header=header)
proj3 = obs2.get_projection_operator(header=header, storage='on fly')
assert all_eq(proj1.get_mask(), proj2.get_mask())
assert all_eq(proj1.get_mask(), proj3.get_mask())
assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \
proj2.operands], axis=1))
assert all_eq(proj1.matrix, np.concatenate([p.matrix for p in \
proj3.operands], axis=1))
model1 = CompressionAverageOperator(obs1.slice.compression_factor) * proj1
model2 = CompressionAverageOperator(obs2.slice.compression_factor) * proj2
model3 = CompressionAverageOperator(obs2.slice.compression_factor) * proj3
m1 = model1.T(tod1)
m2 = model2.T(tod2)
m3 = model3.T(tod2)
assert all_eq(m1, m2, tol)
assert all_eq(m1, m3, tol)
assert all_eq(model1(m1), model2(m1))
assert all_eq(model1(m1), model3(m1))
def test1():
# creation of the sky map
msize = 50
mymap = gaussian(2*(msize*2+1,), 10, unit='Jy/pixel')
cd = np.array([[-1., 0.],[0., 1.]]) / 3600.
header = create_fitsheader(fromdata=mymap, crval=[53.,27.], cd=cd)
mymap.header = header
# creation of the simulation
scan = PacsObservation.create_scan((header['CRVAL1'], header['CRVAL2']),
instrument_angle=0., length=60, nlegs=1, angle=20.)
simul = PacsSimulation(scan, 'red', policy_bad_detector='keep',
policy_other='keep')
# build the acquisition model
model = CompressionAverageOperator(simul.slice.compression_factor) * \
simul.get_projection_operator(header=header, npixels_per_sample=49)
# get the noiseless tod
tod = model(mymap)
filename = 'simul-'+str(uuid1())+'.fits'
try:
simul.save(filename, tod)
simul2 = PacsObservation(filename, policy_bad_detector='keep',
policy_other='keep')
status2 = simul2.status
tod2 = simul2.get_tod()
finally:
try:
os.remove(filename)
except:
pass
for field in simul.status.dtype.names:
if field == 'BAND': continue
assert_eq(simul.status[field], status2[field])
assert_eq(tod, tod2)
fields = [x for x in simul.slice.dtype.names if x not in ('filename','unit')]
for field in fields:
if getattr(simul.slice[0], field) != getattr(simul2.slice[0], field):
msg = "Field '" + field + "'"
if field == 'scan_step':
print(msg + ' not implemented.')
else:
assert False
exit(0)
pacs = PacsObservation([datafile[0]+'[6065:20000]', datafile[1]+'[6066:20001]'],
fine_sampling_factor=1, calblock_extension_time=0.)
telescope = IdentityOperator()
projection = pacs.get_projection_operator(resolution=3.2,
npixels_per_sample=5)
multiplexing = CompressionAverageOperator(1)
crosstalk = IdentityOperator()
compression = CompressionAverageOperator(4)
model = compression * crosstalk * multiplexing * projection * telescope
# read the Tod off the disk
tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True)
# remove drift
tod40Hz_filtered = filter_polynomial(tod40Hz, 6)
drift = tod40Hz - tod40Hz_filtered
tod40Hz = filter_median(tod40Hz_filtered, 10000)
# second level deglitching
tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5)
idetector=5
plot_tod((tod40Hz+drift)[idetector])
plot(drift[idetector], 'r')
masking = MaskOperator(tod40Hz.mask)
solver = tamasis.solvers.cg
tol = 1.e-2
mtol = 1.e-10
maxiter = 100
rank = MPI.COMM_WORLD.rank
tamasis.var.verbose = True
profile = None#'test_ls.png'
data_dir = os.path.dirname(__file__) + '/data/'
# reference map (no communication)
comm_tod = MPI.COMM_SELF
comm_map = MPI.COMM_SELF
obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod)
obs_ref.pointing.chop = 0
tod_ref = obs_ref.get_tod()
model_ref = MaskOperator(tod_ref.mask) * \
obs_ref.get_projection_operator(downsampling=True,
npixels_per_sample=6,
commin=comm_map)
map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2')
map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter,
solver=solver, M=DiagonalOperator(
1/map_naive_ref.coverage))
cov_ref_global = map_ref_global.coverage
mask_ref_global = map_ref_global.coverage == 0
header_ref_global = map_ref_global.header
tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape,
attrin={'header':header_ref_global})
map_ref_local = tolocal(map_ref_global)
cov_ref_local = tolocal(map_ref_global.coverage)
import pyoperators
import os
import scipy
from pyoperators import DiagonalOperator, IdentityOperator, MaskOperator
from tamasis import (PacsObservation, CompressionAverageOperator,
UnpackOperator, mapper_ls, mapper_naive)
pyoperators.memory.verbose = False
profile = None#'test_ls.png'
solver = scipy.sparse.linalg.bicgstab
tol = 1.e-6 if profile else 1.e-4
maxiter = 10
data_dir = os.path.dirname(__file__) + '/data/'
obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1,
reject_bad_line=False)
tod = obs.get_tod()
telescope = IdentityOperator()
projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6)
compression = CompressionAverageOperator(obs.slice.compression_factor)
masking_tod = MaskOperator(tod.mask)
masking_map = MaskOperator(projection.get_mask())
model = masking_tod * projection * telescope * masking_map
# naive map
map_naive = mapper_naive(tod, model)
# iterative map, restricting oneself to observed map pixels
unpacking = UnpackOperator(projection.get_mask())
old_settings = np.seterr(divide='ignore')
def check_map_global(m):
assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10)
assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10)
assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_global.tounit('Jy/arcsec^2'),
1e-10)
def check_map_local(m):
assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10)
assert_all_eq(m.coverage, map_ref_local.coverage, 1e-10)
assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_local.tounit('Jy/arcsec^2'),
1e-10)
obs1 = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False)
obs1.pointing.chop = 0
tod1 = obs1.get_tod()
obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:176]',
data_dir + 'frames_blue.fits[177:360]'],
reject_bad_line=False)
obs2.pointing.chop = 0
tod2 = obs2.get_tod()
tolocal = MPIDistributionGlobalOperator(map_ref_global.shape,
attrin={'header':header_ref_global})
# non-distributed map, distributed TOD
def test1():
comm_map = MPI.COMM_SELF
for obs, tod in ((obs1, tod1), (obs2, tod2)):
proj = obs.get_projection_operator(downsampling=True,
npixels_per_sample=6, header=map_ref_global.header,
from tamasis.utils import all_eq
pyoperators.memory.verbose = False
tamasis.var.verbose = False
profile = None#'test_rls.png'
data_dir = os.path.dirname(__file__) + '/data/'
PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\
'dPixelMask_FM_v5.fits'
PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\
'sponsivity_FM_v5.fits'
obs = PacsObservation(filename=data_dir+'frames_blue.fits',
fine_sampling_factor=1, reject_bad_line=False)
obs.pointing.chop[:] = 0
tod = obs.get_tod(subtraction_mean=True)
projection = obs.get_projection_operator(resolution=3.2, downsampling=True,
npixels_per_sample=6)
masking_tod = MaskOperator(tod.mask)
model = masking_tod * projection
# iterative map, taking all map pixels
class Callback():
def __init__(self):
self.niterations = 0
def __call__(self, x):
self.niterations += 1
map_rls = mapper_rls(tod, model, hyper=1., tol=1.e-4, profile=profile,
callback=None if tamasis.var.verbose else Callback(),
def check_map_global(m):
assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10)
assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10)
assert_all_eq(m.tounit("Jy/arcsec^2"), map_ref_global.tounit("Jy/arcsec^2"), 1e-10)
def check_map_local(m):
assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10)
assert_all_eq(m.coverage, map_ref_local.coverage, 1e-10)
assert_all_eq(m.tounit("Jy/arcsec^2"), map_ref_local.tounit("Jy/arcsec^2"), 1e-10)
obs1 = PacsObservation(data_dir + "frames_blue.fits", reject_bad_line=False)
obs1.pointing.chop = 0
tod1 = obs1.get_tod()
obs2 = PacsObservation(
[data_dir + "frames_blue.fits[1:176]", data_dir + "frames_blue.fits[177:360]"], reject_bad_line=False
)
obs2.pointing.chop = 0
tod2 = obs2.get_tod()
tolocal = MPIDistributionGlobalOperator(map_ref_global.shape, attrin={"header": header_ref_global})
# non-distributed map, distributed TOD
def test1():
comm_map = MPI.COMM_SELF
for obs, tod in ((obs1, tod1), (obs2, tod2)):
proj = obs.get_projection_operator(
downsampling=True, npixels_per_sample=6, header=map_ref_global.header, commin=comm_map
)
pacs = PacsObservation(
[datafile[0] + '[6065:20000]', datafile[1] + '[6066:20001]'],
fine_sampling_factor=1,
calblock_extension_time=0.)
telescope = IdentityOperator()
projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5)
multiplexing = CompressionAverageOperator(1)
crosstalk = IdentityOperator()
compression = CompressionAverageOperator(4)
model = compression * crosstalk * multiplexing * projection * telescope
# read the Tod off the disk
tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True)
# remove drift
tod40Hz_filtered = filter_polynomial(tod40Hz, 6)
drift = tod40Hz - tod40Hz_filtered
tod40Hz = filter_median(tod40Hz_filtered, 10000)
# second level deglitching
tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5)
idetector = 5
plot_tod((tod40Hz + drift)[idetector])
plot(drift[idetector], 'r')
masking = MaskOperator(tod40Hz.mask)
pyoperators.memory.verbose = False
tamasis.var.verbose = False
profile = None #'test_rls.png'
data_dir = os.path.dirname(__file__) + '/data/'
PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\
'dPixelMask_FM_v5.fits'
PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\
'sponsivity_FM_v5.fits'
obs = PacsObservation(filename=data_dir + 'frames_blue.fits',
fine_sampling_factor=1,
reject_bad_line=False)
obs.pointing.chop[:] = 0
tod = obs.get_tod(subtraction_mean=True)
projection = obs.get_projection_operator(resolution=3.2,
downsampling=True,
npixels_per_sample=6)
masking_tod = MaskOperator(tod.mask)
model = masking_tod * projection
# iterative map, taking all map pixels
class Callback():
def __init__(self):
self.niterations = 0
def __call__(self, x):
self.niterations += 1
