def test_detector_policy():
map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits')
obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False)
obs.pointing.chop[:] = 0
projection = obs.get_projection_operator(header=map_naive_ref.header,
downsampling=True,
npixels_per_sample=6)
tod = obs.get_tod()
masking = MaskOperator(tod.mask)
model = masking * projection
map_naive = mapper_naive(tod, model)
assert all_eq(map_naive, map_naive_ref, tol)
obs_rem = PacsObservation(data_dir + 'frames_blue.fits',
policy_bad_detector='remove',
reject_bad_line=False)
obs_rem.pointing.chop[:] = 0
projection_rem = obs_rem.get_projection_operator(header=map_naive.header,
downsampling=True,
npixels_per_sample=7)
tod_rem = obs_rem.get_tod()
masking_rem = MaskOperator(tod_rem.mask)
model_rem = masking_rem * projection_rem
map_naive_rem = mapper_naive(tod_rem, model_rem)
assert all_eq(map_naive, map_naive_rem, tol)
def test_header():
obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False)
obs.pointing.chop = 0
header = obs.get_map_header()
projection = obs.get_projection_operator(header=header, downsampling=True,
npixels_per_sample=6)
tod = obs.get_tod()
map_naive = mapper_naive(tod, projection)
header2 = header.copy()
header2['NAXIS1'] += 500
header2['CRPIX1'] += 250
projection2 = obs.get_projection_operator(header=header2, downsampling=True)
map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2)
map_naive2.inunit('Jy/arcsec^2')
map_naive3 = map_naive2[:,250:header['NAXIS1']+250]
assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7)
# test compatibility with photproject
tod = obs.get_tod()
map_naive4 = mapper_naive(tod, projection)
hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1]
map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel')
std_naive = np.std(map_naive4[40:60,40:60])
std_ref = np.std(map_ref[40:60,40:60])
assert abs(std_naive-std_ref) / std_ref < 0.025
def test_pTx_pT1():
obs1 = PacsObservation(data_dir + 'frames_blue.fits')
obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]',
data_dir + 'frames_blue.fits[42:43]',
data_dir + 'frames_blue.fits[44:360]'])
obs1.pointing.chop = 0
obs2.pointing.chop = 0
header = obs1.get_map_header()
tod = obs1.get_tod()
model1 = obs1.get_projection_operator(downsampling=True, header=header)
ref = mapper_naive(tod, model1, unit='Jy/arcsec^2')
model1.apply_mask(tod.mask)
tod.inunit('Jy/arcsec^2')
b1, w1 = model1.get_pTx_pT1(tod)
m1 = b1 / w1
assert all_eq(ref, m1, tol)
model2 = obs2.get_projection_operator(downsampling=True, header=header)
model2.apply_mask(tod.mask)
b2, w2 = model2.get_pTx_pT1(tod)
m2 = b2 / w2
assert all_eq(ref, m2, tol)
model3 = obs2.get_projection_operator(downsampling=True, header=header,
storage='on fly')
MaskOperator(tod.mask)(tod, tod)
b3, w3 = model3.get_pTx_pT1(tod)
m3 = b3 / w3
assert all_eq(ref, m3, tol)
def test2():
comm_map = MPI.COMM_SELF
for obs, tod in ((obs1, tod1), (obs2, tod2)):
proj = obs.get_projection_operator(
downsampling=True, npixels_per_sample=6, header=map_ref_global.header, packed=True, commin=comm_map
)
proj.apply_mask(tod.mask)
m = mapper_naive(tod, proj)
yield check_map_global, m
def test4():
for obs, tod in ((obs1, tod1), (obs2, tod2)):
proj = obs.get_projection_operator(
downsampling=True, npixels_per_sample=6, header=map_ref_local.header, commin=MPI.COMM_WORLD
)
proj.apply_mask(tod.mask)
model = proj * tolocal
m = mapper_naive(tod, model)
yield check_map_global, m
def test2():
comm_map = MPI.COMM_SELF
for obs, tod in ((obs1, tod1), (obs2, tod2)):
proj = obs.get_projection_operator(downsampling=True,
npixels_per_sample=6, header=map_ref_global.header,
packed=True, commin=comm_map)
proj.apply_mask(tod.mask)
m = mapper_naive(tod, proj)
yield check_map_global, m
def test4():
for obs, tod in ((obs1, tod1), (obs2, tod2)):
proj = obs.get_projection_operator(downsampling=True,
npixels_per_sample=6, header=map_ref_local.header,
commin=MPI.COMM_WORLD)
proj.apply_mask(tod.mask)
model = proj * tolocal
m = mapper_naive(tod, model)
yield check_map_global, m
def pipeline_photproject(filenames, output_file, keywords):
"""
Perform regularized least-square inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(downsampling=True,
**keywords["Projection"])
# build instrument model
masking = tm.MaskOperator(tod.mask)
model = masking * projection
# set tod masked values to zero
tod = masking(tod)
# perform map-making
map_naive = tm.mapper_naive(tod, model, **keywords["mapper_naive"])
# save
map_naive.save(output_file)
def pipeline_photproject(filenames, output_file, keywords):
"""
Perform regularized least-square inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(downsampling=True,
**keywords["Projection"])
# build instrument model
masking = tm.MaskOperator(tod.mask)
model = masking * projection
# set tod masked values to zero
tod = masking(tod)
# perform map-making
map_naive = tm.mapper_naive(tod, model, **keywords["mapper_naive"])
# save
map_naive.save(output_file)
def test3():
comm_map = MPI.COMM_WORLD
for obs, tod in ((obs1, tod1), (obs2, tod2)):
proj = obs.get_projection_operator(downsampling=True,
npixels_per_sample=6, header=map_ref_global.header,
commin=comm_map)
proj.apply_mask(tod.mask)
m = mapper_naive(tod, proj)
yield check_map_local, m
if size == 1:
continue
mlocal = m
mlocal[:] = rank + 1
mlocalpacked = proj.operands[1](mlocal)
mglobal = UnpackOperator(proj.operands[1].mask)(mlocalpacked)
def func(s, irank):
assert np.all((mglobal[s] == 0) | (mglobal[s] == irank+1))
for irank in range(size):
s = distribute_slice(map_ref_global.shape[0], rank=irank)
yield func, s, irank
def test3():
comm_map = MPI.COMM_WORLD
for obs, tod in ((obs1, tod1), (obs2, tod2)):
proj = obs.get_projection_operator(
downsampling=True, npixels_per_sample=6, header=map_ref_global.header, commin=comm_map
)
proj.apply_mask(tod.mask)
m = mapper_naive(tod, proj)
yield check_map_local, m
if size == 1:
continue
mlocal = m
mlocal[:] = rank + 1
mlocalpacked = proj.operands[1](mlocal)
mglobal = UnpackOperator(proj.operands[1].mask)(mlocalpacked)
def func(s, irank):
assert np.all((mglobal[s] == 0) | (mglobal[s] == irank + 1))
for irank in range(size):
s = distribute_slice(map_ref_global.shape[0], rank=irank)
yield func, s, irank
npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector=5 plot_tod((tod40Hz+drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask) model40Hz = masking * projection map_naive40Hz = mapper_naive(tod40Hz, model40Hz) map_naive40Hz.imshow() # clim doesn't work anymore with AnnotatedImage #clim(-0.00002,0.00002)
rank = MPI.COMM_WORLD.rank
tamasis.var.verbose = True
profile = None#'test_ls.png'
data_dir = os.path.dirname(__file__) + '/data/'
# reference map (no communication)
comm_tod = MPI.COMM_SELF
comm_map = MPI.COMM_SELF
obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod)
obs_ref.pointing.chop = 0
tod_ref = obs_ref.get_tod()
model_ref = MaskOperator(tod_ref.mask) * \
obs_ref.get_projection_operator(downsampling=True,
npixels_per_sample=6,
commin=comm_map)
map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2')
map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter,
solver=solver, M=DiagonalOperator(
1/map_naive_ref.coverage))
cov_ref_global = map_ref_global.coverage
mask_ref_global = map_ref_global.coverage == 0
header_ref_global = map_ref_global.header
tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape,
attrin={'header':header_ref_global})
map_ref_local = tolocal(map_ref_global)
cov_ref_local = tolocal(map_ref_global.coverage)
mask_ref_local = tolocal(mask_ref_global)
def check_map_global(m):
assert_all_eq(m.magnitude, map_ref_global.magnitude, mtol)
assert_all_eq(m.coverage, cov_ref_global, mtol)
projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector = 5 plot_tod((tod40Hz + drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask) model40Hz = masking * projection map_naive40Hz = mapper_naive(tod40Hz, model40Hz) map_naive40Hz.imshow() # clim doesn't work anymore with AnnotatedImage #clim(-0.00002,0.00002)
profile = None #'test_madcap.png'
path = os.path.abspath(os.path.dirname(__file__)) + '/data/madmap1/'
obs = MadMap1Observation(path + 'todSpirePsw_be',
path + 'invnttSpirePsw_be',
path + 'madmapSpirePsw.fits[coverage]',
'big_endian',
135,
missing_value=np.nan)
obs.instrument.name = 'SPIRE/PSW'
tod = obs.get_tod(unit='Jy/beam')
projection = obs.get_projection_operator()
packing = PackOperator(obs.info.mapmask)
model = projection * packing
map_naive = mapper_naive(tod, model)
map_ref = pyfits.open(path + 'naivemapSpirePsw.fits')['image'].data
def test_madcap1():
assert all_eq(map_naive, map_ref)
map_naive_1d = mapper_naive(tod, projection)
map_naive_2d = packing.T(map_naive_1d)
map_naive_2d = packing.T(projection.T(tod) / projection.T(Tod.ones(tod.shape)))
map_naive_2d[obs.info.mapmask] = np.nan
def test_madcap2():
assert all_eq(map_naive_2d, map_ref)
maxiter = 10
data_dir = os.path.dirname(__file__) + '/data/'
obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1,
reject_bad_line=False)
tod = obs.get_tod()
telescope = IdentityOperator()
projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6)
compression = CompressionAverageOperator(obs.slice.compression_factor)
masking_tod = MaskOperator(tod.mask)
masking_map = MaskOperator(projection.get_mask())
model = masking_tod * projection * telescope * masking_map
# naive map
map_naive = mapper_naive(tod, model)
# iterative map, restricting oneself to observed map pixels
unpacking = UnpackOperator(projection.get_mask())
old_settings = np.seterr(divide='ignore')
M = DiagonalOperator(unpacking.T(1./map_naive.coverage))
np.seterr(**old_settings)
#map_iter1 = mapper_ls(tod, model * unpacking, tol=1.e-4, M=M)
#if map_iter1.header['NITER'] > 11:
# raise TestFailure()
# iterative map, taking all map pixels
class Callback():
def __init__(self):
self.niterations = 0
def __call__(self, x):
from tamasis.utils import all_eq
tamasis.var.verbose = False
profile=None#'test_madcap.png'
path = os.path.abspath(os.path.dirname(__file__)) + '/data/madmap1/'
obs = MadMap1Observation(path+'todSpirePsw_be', path+'invnttSpirePsw_be',
path+'madmapSpirePsw.fits[coverage]', 'big_endian',
135, missing_value=np.nan)
obs.instrument.name = 'SPIRE/PSW'
tod = obs.get_tod(unit='Jy/beam')
projection = obs.get_projection_operator()
packing = PackOperator(obs.info.mapmask)
model = projection*packing
map_naive = mapper_naive(tod, model)
map_ref = pyfits.open(path+'naivemapSpirePsw.fits')['image'].data
def test_madcap1():
assert all_eq(map_naive, map_ref)
map_naive_1d = mapper_naive(tod, projection)
map_naive_2d = packing.T(map_naive_1d)
map_naive_2d = packing.T(projection.T(tod)/projection.T(Tod.ones(tod.shape)))
map_naive_2d[obs.info.mapmask] = np.nan
def test_madcap2():
assert all_eq(map_naive_2d, map_ref)
M = DiagonalOperator(packing(1/map_naive.coverage))
assert np.all(np.isfinite(M.data))
