def test_detector_policy():
map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits')
obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False)
obs.pointing.chop[:] = 0
projection = obs.get_projection_operator(header=map_naive_ref.header,
downsampling=True,
npixels_per_sample=6)
tod = obs.get_tod()
masking = MaskOperator(tod.mask)
model = masking * projection
map_naive = mapper_naive(tod, model)
assert all_eq(map_naive, map_naive_ref, tol)
obs_rem = PacsObservation(data_dir + 'frames_blue.fits',
policy_bad_detector='remove',
reject_bad_line=False)
obs_rem.pointing.chop[:] = 0
projection_rem = obs_rem.get_projection_operator(header=map_naive.header,
downsampling=True,
npixels_per_sample=7)
tod_rem = obs_rem.get_tod()
masking_rem = MaskOperator(tod_rem.mask)
model_rem = masking_rem * projection_rem
map_naive_rem = mapper_naive(tod_rem, model_rem)
assert all_eq(map_naive, map_naive_rem, tol)
def test_masking():
mask = MaskOperator(0)
assert isinstance(mask, IdentityOperator)
mask = MaskOperator(0, shapein=(32, 32), dtype=np.float32)
assert isinstance(mask, IdentityOperator)
assert mask.shapein == (32, 32)
assert mask.dtype == np.float32
mask = MaskOperator(1)
assert isinstance(mask, ZeroOperator)
mask = MaskOperator(1, shapein=(32, 32), dtype=np.float32)
assert isinstance(mask, ZeroOperator)
assert mask.shapein == (32, 32)
assert mask.dtype == np.float32
b = np.array([3., 4., 1., 0., 3., 2.])
c = np.array([3., 4., 0., 0., 3., 0.])
mask = MaskOperator(np.array([0, 0., 1., 1., 0., 1], dtype=np.int8))
assert np.all(mask(b) == c)
mask = DiagonalOperator(np.array([1, 1., 0., 0., 1., 0]))
assert np.all(mask(b) == c)
mask = MaskOperator(np.array([False, False, True, True, False, True]))
assert np.all(mask(b) == c)
b = np.array([[3., 4.], [1., 0.], [3., 2.]])
c = np.array([[3., 4.], [0., 0.], [3., 0.]])
mask = MaskOperator(np.array([[0, 0.], [1., 1.], [0., 1.]], dtype='int8'))
assert np.all(mask(b) == c)
mask = DiagonalOperator(np.array([[1, 1.], [0., 0.], [1., 0.]]))
assert np.all(mask(b) == c)
mask = MaskOperator(np.array([[False, False],
[True, True],
[False, True]]))
assert np.all(mask(b) == c)
b = np.array([[[3, 4.], [1., 0.]], [[3., 2], [-1, 9]]])
c = np.array([[[3, 4.], [0., 0.]], [[3., 0], [0, 0]]])
mask = MaskOperator(np.array([[[0, 0.], [1., 1.]],
[[0., 1], [1, 1]]], int))
assert np.all(mask(b) == c)
mask = DiagonalOperator(np.array([[[1, 1], [0., 0]], [[1, 0], [0, 0]]]))
assert np.all(mask(b) == c)
mask = MaskOperator(np.array([[[False, False], [True, True]],
[[False, True], [True, True]]]))
assert np.all(mask(b) == c)
c = mask(b, b)
assert id(b) == id(c)
def test_header():
obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False)
obs.pointing.chop = 0
header = obs.get_map_header()
projection = obs.get_projection_operator(header=header, downsampling=True,
npixels_per_sample=6)
tod = obs.get_tod()
map_naive = mapper_naive(tod, projection)
header2 = header.copy()
header2['NAXIS1'] += 500
header2['CRPIX1'] += 250
projection2 = obs.get_projection_operator(header=header2, downsampling=True)
map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2)
map_naive2.inunit('Jy/arcsec^2')
map_naive3 = map_naive2[:,250:header['NAXIS1']+250]
assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7)
# test compatibility with photproject
tod = obs.get_tod()
map_naive4 = mapper_naive(tod, projection)
hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1]
map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel')
std_naive = np.std(map_naive4[40:60,40:60])
std_ref = np.std(map_ref[40:60,40:60])
assert abs(std_naive-std_ref) / std_ref < 0.025
def func(valid, x, shape, broadcast):
p = PackOperator(valid, broadcast=broadcast)
masking = MaskOperator(~valid, broadcast=broadcast)
if broadcast == 'leftward':
x_ = np.empty(shape + x.shape)
x_[...] = x
expected_ = np.empty(shape + (expected.size,))
expected_[...] = expected
else:
x_ = np.empty(x.shape + shape)
x_.reshape((x.size, -1))[...] = x.ravel()[..., None]
expected_ = np.empty((expected.size,) + shape)
expected_.reshape((expected.size, -1))[...] = expected[..., None]
if broadcast == 'disabled' and shape != ():
assert_raises(ValueError, p, x_)
return
assert_equal(p(x_), expected_)
assert_is_type(p.T, UnpackOperator)
assert_equal(p.T.broadcast, p.broadcast)
assert_equal(p.T(expected_), masking(x_))
u = UnpackOperator(valid, broadcast=broadcast)
assert_is_type(u.T, PackOperator)
assert_equal(u.T.broadcast, u.broadcast)
assert_equal(u(expected_), masking(x_))
assert_equal(u.T(x_), expected_)
def test_pTx_pT1():
obs1 = PacsObservation(data_dir + 'frames_blue.fits')
obs2 = PacsObservation([data_dir + 'frames_blue.fits[1:41]',
data_dir + 'frames_blue.fits[42:43]',
data_dir + 'frames_blue.fits[44:360]'])
obs1.pointing.chop = 0
obs2.pointing.chop = 0
header = obs1.get_map_header()
tod = obs1.get_tod()
model1 = obs1.get_projection_operator(downsampling=True, header=header)
ref = mapper_naive(tod, model1, unit='Jy/arcsec^2')
model1.apply_mask(tod.mask)
tod.inunit('Jy/arcsec^2')
b1, w1 = model1.get_pTx_pT1(tod)
m1 = b1 / w1
assert all_eq(ref, m1, tol)
model2 = obs2.get_projection_operator(downsampling=True, header=header)
model2.apply_mask(tod.mask)
b2, w2 = model2.get_pTx_pT1(tod)
m2 = b2 / w2
assert all_eq(ref, m2, tol)
model3 = obs2.get_projection_operator(downsampling=True, header=header,
storage='on fly')
MaskOperator(tod.mask)(tod, tod)
b3, w3 = model3.get_pTx_pT1(tod)
m3 = b3 / w3
assert all_eq(ref, m3, tol)
def func(obs, tod):
masking = MaskOperator(tod.mask)
proj = obs.get_projection_operator(downsampling=True,
npixels_per_sample=6,
header=header_ref_global,
commin=comm_map)
model = masking * proj
m = mapper_ls(tod, model, tol=tol, maxiter=maxiter, solver=solver,
M=DiagonalOperator(1/cov_ref_global))
check_map_global(m)
def test3():
comm_map = MPI.COMM_WORLD
for obs, tod in ((obs1,tod1), (obs2,tod2)):
masking = MaskOperator(tod.mask)
proj = obs.get_projection_operator(downsampling=True,
npixels_per_sample=6,
header=header_ref_global,
commin=comm_map)
model = masking * proj
m = mapper_ls(tod, model, tol=tol, maxiter=maxiter, solver=solver,
M=DiagonalOperator(1/cov_ref_local))
yield check_map_local, m
def test1():
comm_map = MPI.COMM_SELF
for obs, tod in ((obs1, tod1), (obs2, tod2)):
masking = MaskOperator(tod.mask)
proj = obs.get_projection_operator(downsampling=True,
npixels_per_sample=6,
header=header_ref_global,
commin=comm_map)
model = masking * proj
m = mapper_rls(tod,
model,
tol=tol,
maxiter=maxiter,
solver=solver,
hyper=hyper)
yield check_map_global, m
def func(cls, itype, ftype, inplace):
proj_ref = _get_projection[cls](itype, ftype)
proj_ = _get_projection[cls](itype, ftype)
proj = proj_.restrict(restriction, inplace=inplace)
if inplace:
assert_is_type(proj_, DeletedOperator)
if cls is not FSRMatrix:
def pack(v):
return v[restriction, :]
else:
pack = PackOperator(restriction)
masking = MaskOperator(~restriction, broadcast='rightward')
block_size = proj_ref.matrix.block_shape[1]
shape = (5,) + ((block_size,) if block_size > 1 else ())
x = np.arange(5*block_size).reshape(shape) + 1
assert_equal(proj_ref(masking(x)), proj(pack(x)))
pack = PackOperator(restriction)
assert_equal(pack(kernel), proj.canonical_basis_in_kernel())
def apply_mask(self, mask):
mask = np.asarray(mask, np.bool8)
dest = 0
if mask.shape != self.shapeout:
raise ValueError(
"The mask shape '{0}' is incompatible with tha"
"t of the projection operator '{1}'.".format(
mask.shape, self.shapeout))
if any(
isinstance(p, ProjectionOnFlyOperator)
for p in self.operands):
blocks = self.copy()
self.__class__ = CompositionOperator
self.__init__([MaskOperator(mask), blocks])
return
for p in self.operands:
n = p.matrix.shape[1]
p.apply_mask(mask[..., dest:dest + n])
dest += n
def mapper_naive(tod, model, unit=None):
"""
Returns a naive map, i.e.: map = model.T(tod) / model.T(1)
This equation is valid for a map and a Time Ordered Data (TOD) expressed as
a surface brightness, so when the TOD does not meet this requirement and
is a quantity per detector, a unit conversion is attempted.
Parameters
----------
tod : Tod
The input Time Ordered Data
model : Operator
The instrument model such as tod = model(map)
unit : string
Output map unit. By default, the output map unit is chosen to be
compatible with the model (usually pixel^-1)
"""
# apply mask
if hasattr(tod, 'mask') and tod.mask is not None:
mask = MaskOperator(tod.mask)
else:
mask = IdentityOperator()
tod = mask(tod)
# get tod units
if not hasattr(tod, '_unit') or len(tod._unit) == 0:
attr = {'_unit': {'?': 1.}}
model.propagate_attributes(None, attr)
u = getattr(attr, '_unit', {})
if 'detector' in u and u['detector'] == -1:
u = {'detector': -1.}
elif u == {'?': 1.}:
u = {}
elif len(u) > 1:
raise ValueError(
'The timeline units are not known and cannot be in'
'ferred from the model.')
tod_du = getattr(attr, '_derived_units', {})
tod = Tod(tod.magnitude, unit=u, derived_units=tod_du, copy=False)
else:
attr = {'_unit': {'?': 1}}
model.T.propagate_attributes(None, attr)
u = attr['_unit']
if 'detector' not in tod._unit and 'detector' in u and u[
'detector'] == 1:
raise ValueError("The model is incompatible with input units '{0}'"\
.format(tod.unit))
tod_unit = tod._unit
tod_du = tod._derived_units
# make sure the input is a surface brightness
if 'detector' in tod._unit:
tod.inunit(tod.unit + ' detector / arcsec^2')
elif 'detector_reference' in tod._unit:
tod.inunit(tod.unit + ' detector_reference / arcsec^2')
# compute model.T(tod)/model.T(one)
mymap = model.T(tod.magnitude)
tod[...] = 1
mask(tod, tod)
map_weights = model.T(tod.magnitude)
old_settings = np.seterr(divide='ignore', invalid='ignore')
mymap /= map_weights
mymap.unit = tod.unit
np.seterr(**old_settings)
mymap.coverage = Map(map_weights.magnitude,
header=mymap.header,
copy=False)
if unit is not None:
mymap.inunit(unit)
return mymap
# set map units according to model
attr = {'_unit': tod_unit, '_derived_units': tod_du}
model.T.propagate_attributes(None, attr)
if '_derived_units' in attr:
mymap.derived_units = attr['_derived_units']
if '_unit' in attr:
mymap.inunit(attr['_unit'])
return mymap
def test_masking2():
m = MaskOperator([True, False, True])
assert_eq(m * m, m)
tol = 1.e-2
mtol = 1.e-10
maxiter = 100
rank = MPI.COMM_WORLD.rank
tamasis.var.verbose = True
profile = None#'test_ls.png'
data_dir = os.path.dirname(__file__) + '/data/'
# reference map (no communication)
comm_tod = MPI.COMM_SELF
comm_map = MPI.COMM_SELF
obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod)
obs_ref.pointing.chop = 0
tod_ref = obs_ref.get_tod()
model_ref = MaskOperator(tod_ref.mask) * \
obs_ref.get_projection_operator(downsampling=True,
npixels_per_sample=6,
commin=comm_map)
map_naive_ref = mapper_naive(tod_ref, model_ref, unit='Jy/arcsec^2')
map_ref_global = mapper_ls(tod_ref, model_ref, tol=tol, maxiter=maxiter,
solver=solver, M=DiagonalOperator(
1/map_naive_ref.coverage))
cov_ref_global = map_ref_global.coverage
mask_ref_global = map_ref_global.coverage == 0
header_ref_global = map_ref_global.header
tolocal = MPIDistributionGlobalOperator(map_naive_ref.shape,
attrin={'header':header_ref_global})
map_ref_local = tolocal(map_ref_global)
cov_ref_local = tolocal(map_ref_global.coverage)
mask_ref_local = tolocal(mask_ref_global)
nfp_x = int(np.sqrt(NPOINT_FOCAL_PLANE))
a = np.r_[FOCAL_PLANE_LIMITS[0]:FOCAL_PLANE_LIMITS[1]:nfp_x * 1j]
fp_x, fp_y = np.meshgrid(a, a)
fp = np.dstack([fp_x, fp_y, np.full_like(fp_x, -qubic.optics.focal_length)])
fp_spacing = (FOCAL_PLANE_LIMITS[1] - FOCAL_PLANE_LIMITS[0]) / nfp_x
############
# DETECTORS
############
header = create_fitsheader((nfp_x, nfp_x),
cdelt=fp_spacing,
crval=(0, 0),
ctype=['X---CAR', 'Y---CAR'],
cunit=['m', 'm'])
focal_plane = SceneGrid.fromfits(header)
integ = MaskOperator(qubic.detector.all.removed) * \
focal_plane.get_integration_operator(
focal_plane.topixel(qubic.detector.all.vertex[..., :2]))
###############
# COMPUTATIONS
###############
E = qubic._get_response(SOURCE_THETA, SOURCE_PHI, SOURCE_POWER, fp,
fp_spacing**2, qubic.filter.nu, qubic.horn,
qubic.primary_beam, qubic.secondary_beam)
I = np.abs(E)**2
D = integ(I)
##########
# DISPLAY
##########
projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector = 5 plot_tod((tod40Hz + drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask) model40Hz = masking * projection map_naive40Hz = mapper_naive(tod40Hz, model40Hz) map_naive40Hz.imshow() # clim doesn't work anymore with AnnotatedImage #clim(-0.00002,0.00002)
def _rule_right_unpack(op, self):
if self.mask.shape != op.mask.shape:
return
if np.any(self.mask != op.mask):
return
return MaskOperator(self.mask)
UnpackOperator, mapper_ls, mapper_naive)
pyoperators.memory.verbose = False
profile = None#'test_ls.png'
solver = scipy.sparse.linalg.bicgstab
tol = 1.e-6 if profile else 1.e-4
maxiter = 10
data_dir = os.path.dirname(__file__) + '/data/'
obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1,
reject_bad_line=False)
tod = obs.get_tod()
telescope = IdentityOperator()
projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6)
compression = CompressionAverageOperator(obs.slice.compression_factor)
masking_tod = MaskOperator(tod.mask)
masking_map = MaskOperator(projection.get_mask())
model = masking_tod * projection * telescope * masking_map
# naive map
map_naive = mapper_naive(tod, model)
# iterative map, restricting oneself to observed map pixels
unpacking = UnpackOperator(projection.get_mask())
old_settings = np.seterr(divide='ignore')
M = DiagonalOperator(unpacking.T(1./map_naive.coverage))
np.seterr(**old_settings)
#map_iter1 = mapper_ls(tod, model * unpacking, tol=1.e-4, M=M)
#if map_iter1.header['NITER'] > 11:
# raise TestFailure()
def _validate_tod(tod):
# make sure that the tod is masked
if hasattr(tod, 'mask') and tod.mask is not None:
return MaskOperator(tod.mask)(tod)
return tod.copy()
hyper = 1
rank = MPI.COMM_WORLD.rank
tamasis.var.verbose = True
profile = None #'test_rls.png'
data_dir = os.path.dirname(__file__) + '/data/'
# reference map (no communication)
header_ref_global = create_fitsheader(
(97, 108), cdelt=3.2 / 3600, crval=(245.998427916727, 61.5147650744551))
comm_tod = MPI.COMM_SELF
comm_map = MPI.COMM_SELF
obs_ref = PacsObservation(data_dir + 'frames_blue.fits', comm=comm_tod)
obs_ref.pointing.chop = 0
tod_ref = obs_ref.get_tod()
model_ref = MaskOperator(tod_ref.mask) * obs_ref.get_projection_operator(
downsampling=True,
npixels_per_sample=6,
commin=comm_map,
header=header_ref_global)
map_ref_global = mapper_rls(tod_ref,
model_ref,
tol=tol,
maxiter=maxiter,
solver=solver,
hyper=hyper)
header_ref_global = map_ref_global.header
cov_ref_global = map_ref_global.coverage
mask_ref_global = map_ref_global.coverage == 0
tolocal = MPIDistributionGlobalOperator(map_ref_global.shape,
attrin={'header': header_ref_global})
from pyoperators import BlockColumnOperator, MaskOperator
from pyoperators.iterative.algorithms import StopCondition
from pyoperators.iterative.dli import DoubleLoopAlgorithm
from tamasis import PacsObservation, DiscreteDifferenceOperator, mapper_naive
pyoperators.memory.verbose = False
tamasis.var.verbose = True
data_dir = os.path.dirname(__file__) + '/data/'
obs = PacsObservation(filename=data_dir + 'frames_blue.fits')
obs.pointing.chop[:] = 0
tod = obs.get_tod()
projection = obs.get_projection_operator(resolution=3.2,
downsampling=True,
npixels_per_sample=6)
masking_tod = MaskOperator(tod.mask)
model = masking_tod * projection
naive = mapper_naive(tod, model)
naive[np.isnan(naive)] = 0
prior = BlockColumnOperator(
[DiscreteDifferenceOperator(axis, shapein=(103, 97)) for axis in (0, 1)],
new_axisout=0)
stop_condition = StopCondition(maxiter=2)
dli = DoubleLoopAlgorithm(model,
tod,
prior,
stop_condition=stop_condition,
lanczos={'maxiter': 5},
