def _get_unit_conversion_operator(self, nu):
"""
Convert sky temperature into W / m^2 / Hz / pixel.
If the scene has been initialised with the 'absolute' keyword, the
scene is assumed to include the CMB background and the fluctuations
(in Kelvin) and the operator follows the non-linear Planck law.
Otherwise, the scene only includes the fluctuations (in microKelvin)
and the operator is linear (i.e. the output also corresponds to power
fluctuations).
"""
if not self.temperature:
return IdentityOperator()
# solid angle of a sky pixel
omega = 4 * np.pi / self.shape[0]
a = 2 * omega * h * nu**3 / c**2
if self.absolute:
hnu_k = h * nu / k
return a / asoperator(np.expm1)(hnu_k * ReciprocalOperator())
T = self.T_cmb
hnu_kT = h * nu / (k * T)
val = 1e-6 * a * hnu_kT * np.exp(hnu_kT) / (np.expm1(hnu_kT)**2 * T)
return asoperator(val)
def __init__(self, shape, topixel=None, ndim=None, dtype=float,
**keywords):
"""
Parameters
----------
shape : tuple of integers
The surface shape.
ndim : int, optional
The number of splittable (indexable) dimensions. It is the actual
number of dimensions of the layout. It can be lower than that
specified by the layout shape, in which case the extra dimensions
are instructed not to be split.
topixel : Operator, optional
World-to-pixel coordinate transform.
"""
PackedTable.__init__(self, shape, ndim=ndim)
if topixel is None:
topixel = IdentityOperator(self.shape[:self.ndim])
else:
topixel = asoperator(topixel)
self.dtype = np.dtype(dtype)
self.topixel = topixel
self.toworld = topixel.I
for k, v in keywords.items():
setattr(self, k, v)
def test_rule_right():
ids = (IdentityOperator(classout=ndarray2, attrout=attr2),
IdentityOperator(shapein=4, classout=ndarray2, attrout=attr2))
def func(id_, op_):
op = id_(op_)
assert_is_type(op, type(op_))
attr = {}
assert_is(op.classout, id_.classout)
attr.update(op_.attrout)
attr.update(id_.attrout)
assert_eq(op.attrout, attr)
assert_eq(op.flags.linear, op_.flags.linear)
assert_eq(op.flags.contiguous_input, op_.flags.contiguous_input)
for id_ in ids:
for op_ in ops:
yield func, id_, op_
def get_filter_operator(self):
"""
Return the filter operator.
Convert units from W/Hz to W.
"""
if self.filter.bandwidth == 0:
return IdentityOperator()
return HomothetyOperator(self.filter.bandwidth)
def func(opout, opin, idin):
if opin is not None and idin is not None and opin != idin:
return
p = Op(shapeout=opout, shapein=opin) * IdentityOperator(shapein=idin)
if idin is None:
idin = opin
assert_is_instance(p, Op)
assert_eq(p.shapein, idin)
assert_eq(p.shapeout, opout)
def get_hwp_operator(self, sampling, scene):
"""
Return the rotation matrix for the half-wave plate.
"""
shape = (len(self), len(sampling))
if scene.kind == 'I':
return IdentityOperator(shapein=shape)
if scene.kind == 'QU':
return Rotation2dOperator(-4 * sampling.angle_hwp,
degrees=True, shapein=shape + (2,))
return Rotation3dOperator('X', -4 * sampling.angle_hwp,
degrees=True, shapein=shape + (3,))
def get_detector_response_operator(self, sampling, tau=None):
"""
Return the operator for the bolometer responses.
"""
if tau is None:
tau = self.detector.tau
sampling_period = sampling.period
shapein = len(self), len(sampling)
if sampling_period == 0:
return IdentityOperator(shapein)
return ConvolutionTruncatedExponentialOperator(tau / sampling_period,
shapein=shapein)
def test_diagonal2():
ops = (DiagonalOperator([1., 2], broadcast='rightward'),
DiagonalOperator([[2., 3, 4], [5, 6, 7]], broadcast='rightward'),
DiagonalOperator([1., 2, 3, 4, 5], broadcast='leftward'),
DiagonalOperator(np.arange(20).reshape(4, 5), broadcast='leftward'),
DiagonalOperator(np.arange(120.).reshape(2, 3, 4, 5)),
HomothetyOperator(7.),
IdentityOperator())
x = np.arange(120.).reshape(2, 3, 4, 5) / 2
def func(cls, d1, d2):
op = {AdditionOperator: operator.add,
CompositionOperator: operator.mul,
MultiplicationOperator: operator.mul}[cls]
d = cls([d1, d2])
if type(d1) is DiagonalOperator:
assert_is_type(d, DiagonalOperator)
elif type(d1) is HomothetyOperator:
assert_is_type(d, HomothetyOperator)
elif op is CompositionOperator:
assert_is_type(d, IdentityOperator)
else:
assert_is_type(d, HomothetyOperator)
data = op(d1.data.T, d2.data.T).T \
if 'rightward' in (d1.broadcast, d2.broadcast) \
else op(d1.data, d2.data)
assert_same(d.data, data)
if cls is CompositionOperator:
assert_same(d(x), d1(d2(x)))
else:
assert_same(d(x), op(d1(x), d2(x)))
for op in (AdditionOperator, CompositionOperator):#, MultiplicationOperator):
for d1, d2 in itertools.combinations(ops, 2):
if set((d1.broadcast, d2.broadcast)) == \
set(('leftward', 'rightward')):
continue
yield func, op, d1, d2
def __init__(self,
band,
scene,
true_sky=None,
factor=1,
fwhm=0,
mask=None,
convolution_operator=None):
"""
Parameters
----------
band : int
The band 150 or 220.
scene : Scene
The acquisition scene.
true_sky : array of shape (npixel,) or (npixel, 3)
The true CMB sky (temperature or polarized). The Planck observation
will be this true sky plus a random independent gaussian noise
realization.
factor : 1 or 3 floats, optional
The factor by which the Planck standard deviation is multiplied.
fwhm : float, optional, !not used!
The fwhm of the Gaussian used to smooth the map [radians].
mask : array of shape (npixel, ) or None
The boolean mask, which is equal True at pixels where we need Planck,
that is outside the QUBIC field.
"""
if band not in (150, 220):
raise ValueError("Invalid band '{}'.".format(band))
if true_sky is None:
raise ValueError('The Planck Q & U maps are not released yet.')
if scene.kind == 'IQU' and true_sky.shape[-1] != 3:
raise TypeError('The Planck sky shape is not (npix, 3).')
true_sky = np.array(hp.ud_grade(true_sky.T, nside_out=scene.nside),
copy=False).T
if scene.kind == 'IQU' and true_sky.shape[-1] != 3:
raise TypeError('The Planck sky shape is not (npix, 3).')
self.scene = scene
self.fwhm = fwhm
self._true_sky = true_sky
if mask is not None:
self.mask = mask
else:
self.mask = np.ones(scene.npixel, dtype=np.bool)
if band == 150:
filename = 'Variance_Planck143GHz_Kcmb2_ns256.fits'
else:
filename = 'Variance_Planck217GHz_Kcmb2_ns256.fits'
sigma = 1e6 * factor * np.sqrt(FitsArray(PATH + filename))
if scene.kind == 'I':
sigma = sigma[:, 0]
elif scene.kind == 'QU':
sigma = sigma[:, :2]
if self.scene.nside != 256:
sigma = np.array(hp.ud_grade(sigma.T, self.scene.nside, power=2),
copy=False).T
self.sigma = sigma
if convolution_operator is None:
self.C = IdentityOperator()
else:
self.C = convolution_operator
def test_block_row1():
I2 = IdentityOperator(2)
I3 = IdentityOperator(3)
assert_raises(ValueError, BlockRowOperator, [I2, 2 * I3], axisin=0)
assert_raises(ValueError, BlockRowOperator, [I2, 2 * I3], new_axisin=0)
Operator, AdditionOperator, CompositionOperator, DiagonalOperator,
HomothetyOperator, IdentityOperator, MultiplicationOperator,
PyOperatorsWarning)
from pyoperators.flags import linear
from pyoperators.rules import BinaryRule, UnaryRule, RuleManager, rule_manager
from pyoperators.utils import ndarraywrap
from pyoperators.utils.testing import (
assert_eq, assert_is, assert_is_none, assert_is_not_none,
assert_is_instance)
from .common import OPS, ndarray2, attr2
op = Operator()
ops = [OP() for OP in OPS]
ids_left = (IdentityOperator(classout=ndarray2, attrout=attr2),
IdentityOperator(shapein=4, classout=ndarray2, attrout=attr2))
ids_right = (IdentityOperator(classout=ndarray2, attrout=attr2),
IdentityOperator(shapein=3, classout=ndarray2, attrout=attr2))
class Operator1(Operator):
pass
class Operator2(Operator):
pass
class Operator3(Operator):
pass
def func(s, v):
solution = s(IdentityOperator(shapein=v.shape), v, x0=v)
assert_same(solution['nit'], 0)
assert_same(solution['x'], v)
import scipy
from pyoperators import DiagonalOperator, IdentityOperator, MaskOperator
from tamasis import (PacsObservation, CompressionAverageOperator,
UnpackOperator, mapper_ls, mapper_naive)
pyoperators.memory.verbose = False
profile = None#'test_ls.png'
solver = scipy.sparse.linalg.bicgstab
tol = 1.e-6 if profile else 1.e-4
maxiter = 10
data_dir = os.path.dirname(__file__) + '/data/'
obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1,
reject_bad_line=False)
tod = obs.get_tod()
telescope = IdentityOperator()
projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6)
compression = CompressionAverageOperator(obs.slice.compression_factor)
masking_tod = MaskOperator(tod.mask)
masking_map = MaskOperator(projection.get_mask())
model = masking_tod * projection * telescope * masking_map
# naive map
map_naive = mapper_naive(tod, model)
# iterative map, restricting oneself to observed map pixels
unpacking = UnpackOperator(projection.get_mask())
old_settings = np.seterr(divide='ignore')
M = DiagonalOperator(unpacking.T(1./map_naive.coverage))
np.seterr(**old_settings)
def mapper_rls(y,
H,
invntt=None,
unpacking=None,
hyper=1.0,
x0=None,
tol=1.e-5,
maxiter=300,
M=None,
solver=None,
verbose=True,
callback=None,
criterion=True,
profile=None):
"""
Solve the linear equation
y = H(x)
where H is an Operator representing the acquisition, and y is the observed
time series.
x is the regularised least square solution as given by:
x = argmin (Hx-y)^T N^-1 (Hx-y) + h ||D(x)||^2
or:
x = (H^T N^-1 H + h D1^T D1 + h D2^T D2)^-1 H^T N^-1 y
"""
comm_map = H.commin or MPI.COMM_WORLD
comm_tod = H.commout or comm_map
if solver is None:
solver = cg
new_solver = solver in (pcg, )
tod = _validate_tod(y)
ntods_ = int(np.sum(~tod.mask)) if getattr(tod, 'mask', None) is not None \
else tod.size
nmaps_ = unpacking.shape[1] if unpacking is not None else None
if nmaps_ is None:
nmaps_ = H.shape[1]
if nmaps_ is None:
raise ValueError('The model H has not an explicit input shape.')
ntods = comm_tod.allreduce(ntods_)
nmaps = comm_map.allreduce(nmaps_)
# get A
if invntt is None:
invntt = IdentityOperator()
A = H.T * invntt * H
npriors = len(H.shapein)
priors = [
DiscreteDifferenceOperator(axis=axis,
shapein=H.shapein,
commin=comm_map) for axis in range(npriors)
]
if comm_map.rank == 0 or comm_map.size > 1:
A += sum((hyper * ntods / nmaps) * p.T * p for p in priors)
# get b
b = (H.T * invntt)(tod)
if not np.all(np.isfinite(b)):
raise ValueError('RHS contains non-finite values.')
if b.shape != A.shapein:
raise ValueError("Incompatible size for RHS: '" + str(b.size) +
"' instead of '" + str(A.shape[1]) + "'.")
if np.min(b) == np.max(b) == 0:
print('Warning: in equation Ax=b, b is zero.')
# unpack input
if unpacking is None:
unpacking = IdentityOperator()
A = unpacking.T * A * unpacking
b = unpacking.T(b)
if x0 is not None:
x0 = unpacking.T(x0)
if not new_solver and solver is not cg:
b = b.ravel()
if x0 is not None:
x0 = x0.ravel()
if M is not None:
if isinstance(M, DiagonalOperator):
filter_nonfinite(M.data, out=M.data)
M = unpacking.T * M * unpacking
H_ = H * unpacking
priors = [p * unpacking for p in priors]
# criterion
if hyper != 0:
hc = np.hstack([1, npriors * [hyper]]) / ntods
else:
hc = [1 / ntods]
norms = [norm2_ellipsoid(invntt)] + npriors * [norm2]
comms = [comm_tod] + npriors * [comm_map]
def criter(x):
rs = [H_.matvec(x) - tod.view(np.ndarray).ravel()
] + [p.matvec(x) for p in priors]
Js = [h * n(r, comm=c) for h, n, r, c in zip(hc, norms, rs, comms)]
return Js
if callback is None:
if new_solver:
callback = NewCgCallback(disp=verbose, comm=comm_map)
else:
callback = CgCallback(verbose=verbose,
objfunc=criter if criterion else None)
if (verbose or profile) and comm_map.rank == 0:
print('')
print('H.T * N^-1 * H:')
print(repr(A))
if M is not None:
print('Preconditioner:')
print(M)
time0 = time.time()
if profile is not None:
def run():
result = solver(A, b, x0=x0, M=M, tol=tol, maxiter=maxiter)
if new_solver:
solution = result['x']
if not result['success']:
print('Solver failure: ' + result['message'])
else:
solution, info = result
if info != 0:
print('Solver failure: info=' + str(info))
cProfile.runctx('run()', globals(), locals(), profile + '.prof')
print('Profile time: ' + str(time.time() - time0))
os.system('python -m gprof2dot -f pstats -o ' + profile + '.dot ' +
profile + '.prof')
os.system('dot -Tpng ' + profile + '.dot > ' + profile)
os.system('rm -f ' + profile + '.prof' + ' ' + profile + '.dot')
return None
result = solver(A,
b,
x0=x0,
M=M,
tol=tol,
maxiter=maxiter,
callback=callback)
time0 = time.time() - time0
if new_solver:
solution = result['x']
if not result['success']:
print('Solver failure: ' + result['message'])
niterations = result['niterations']
error = result['error']
else:
solution, info = result
if info < 0:
raise RuntimeError('Solver failure (code=' + str(info) +
' after ' + str(callback.niterations) +
' iterations).')
if info > 0 and comm_map.rank == 0:
print(
'Warning: Solver reached maximum number of iterations without'
' reaching specified tolerance.')
niterations = getattr(callback, 'niterations', None)
error = getattr(callback, 'residual', None)
Js = criter(solution)
if isinstance(unpacking, IdentityOperator):
solution = solution.reshape(H.shapein)
else:
solution = unpacking(solution)
tod[...] = 1
coverage = H.T(tod)
unit = getattr(coverage, 'unit', None)
derived_units = getattr(coverage, 'derived_units', None)
coverage = coverage.view(Map)
header = getattr(coverage, 'header', None)
if header is None:
header = create_fitsheader(fromdata=coverage)
header.update('likeliho', Js[0])
header.update('criter', sum(Js))
header.update('hyper', hyper)
header.update('nsamples', ntods)
header.update('npixels', nmaps)
header.update('time', time0)
if niterations is not None:
header.update('niter', niterations)
header.update('maxiter', maxiter)
if error is not None:
header.update('error', error)
header.update('tol', tol)
header.update('solver', solver.__name__)
output = Map(solution,
header=header,
coverage=coverage,
unit=unit,
derived_units=derived_units,
copy=False)
return output
def mapper_naive(tod, model, unit=None):
"""
Returns a naive map, i.e.: map = model.T(tod) / model.T(1)
This equation is valid for a map and a Time Ordered Data (TOD) expressed as
a surface brightness, so when the TOD does not meet this requirement and
is a quantity per detector, a unit conversion is attempted.
Parameters
----------
tod : Tod
The input Time Ordered Data
model : Operator
The instrument model such as tod = model(map)
unit : string
Output map unit. By default, the output map unit is chosen to be
compatible with the model (usually pixel^-1)
"""
# apply mask
if hasattr(tod, 'mask') and tod.mask is not None:
mask = MaskOperator(tod.mask)
else:
mask = IdentityOperator()
tod = mask(tod)
# get tod units
if not hasattr(tod, '_unit') or len(tod._unit) == 0:
attr = {'_unit': {'?': 1.}}
model.propagate_attributes(None, attr)
u = getattr(attr, '_unit', {})
if 'detector' in u and u['detector'] == -1:
u = {'detector': -1.}
elif u == {'?': 1.}:
u = {}
elif len(u) > 1:
raise ValueError(
'The timeline units are not known and cannot be in'
'ferred from the model.')
tod_du = getattr(attr, '_derived_units', {})
tod = Tod(tod.magnitude, unit=u, derived_units=tod_du, copy=False)
else:
attr = {'_unit': {'?': 1}}
model.T.propagate_attributes(None, attr)
u = attr['_unit']
if 'detector' not in tod._unit and 'detector' in u and u[
'detector'] == 1:
raise ValueError("The model is incompatible with input units '{0}'"\
.format(tod.unit))
tod_unit = tod._unit
tod_du = tod._derived_units
# make sure the input is a surface brightness
if 'detector' in tod._unit:
tod.inunit(tod.unit + ' detector / arcsec^2')
elif 'detector_reference' in tod._unit:
tod.inunit(tod.unit + ' detector_reference / arcsec^2')
# compute model.T(tod)/model.T(one)
mymap = model.T(tod.magnitude)
tod[...] = 1
mask(tod, tod)
map_weights = model.T(tod.magnitude)
old_settings = np.seterr(divide='ignore', invalid='ignore')
mymap /= map_weights
mymap.unit = tod.unit
np.seterr(**old_settings)
mymap.coverage = Map(map_weights.magnitude,
header=mymap.header,
copy=False)
if unit is not None:
mymap.inunit(unit)
return mymap
# set map units according to model
attr = {'_unit': tod_unit, '_derived_units': tod_du}
model.T.propagate_attributes(None, attr)
if '_derived_units' in attr:
mymap.derived_units = attr['_derived_units']
if '_unit' in attr:
mymap.inunit(attr['_unit'])
return mymap
def test_block_column1():
I2 = IdentityOperator(2)
I3 = IdentityOperator(3)
assert_raises(ValueError, BlockColumnOperator, [I2, 2 * I3], axisout=0)
assert_raises(ValueError, BlockColumnOperator, [I2, 2 * I3], new_axisout=0)
def cg(A,
b,
x0=None,
tol=1.e-5,
maxiter=300,
M=None,
disp=False,
callback=None):
""" OpenMPI/MPI hybrid conjugate gradient solver with preconditioning. """
A = asoperator(A)
comm = A.commin or MPI.COMM_WORLD
if M is None:
M = IdentityOperator()
M = asoperator(M)
maxRelError = tol**2
shape = b.shape
x = np.empty(shape)
d = np.empty(shape)
q = np.empty(shape)
r = np.empty(shape)
s = np.empty(shape)
xfinal = np.zeros(shape)
if x0 is None:
x[...] = 0
else:
x[...] = x0
norm = norm2(b, comm=comm)
if norm == 0:
return xfinal, 0
r[...] = b
r -= A(x)
epsilon = norm2(r, comm=comm) / norm
minEpsilon = epsilon
M(r, d)
delta0 = dot(r, d, comm=comm)
deltaNew = delta0
for iter_ in xrange(maxiter):
if epsilon <= maxRelError:
break
A(d, q)
alpha = deltaNew / dot(d, q, comm=comm)
x += alpha * d
r -= alpha * q
epsilon = norm2(r, comm=comm) / norm
if disp:
print '{0:4}: {1}'.format(iter_ + 1, np.sqrt(epsilon))
if callback is not None:
resid = np.sqrt(epsilon)
callback(x)
if epsilon < minEpsilon:
xfinal[...] = x
minEpsilon = epsilon
M(r, s)
deltaOld = deltaNew
deltaNew = dot(r, s, comm=comm)
beta = deltaNew / deltaOld
d *= beta
d += s
minEpsilon = np.sqrt(minEpsilon)
maxRelError = np.sqrt(maxRelError)
return xfinal.reshape(b.shape), int(minEpsilon > maxRelError)
def _rule_left_unpack(self, op):
if self.mask.shape != op.mask.shape:
return
if np.any(self.mask != op.mask):
return
return IdentityOperator()
datadir = os.getenv('PACS_DATA', '') + '/transpScan/'
datafile = [
datadir + '1342184598_blue_PreparedFrames.fits',
datadir + '1342184599_blue_PreparedFrames.fits'
]
if not all(map(os.path.exists, datafile)):
print('The data files are not found: ' + ', '.join(datafile))
exit(0)
pacs = PacsObservation(
[datafile[0] + '[6065:20000]', datafile[1] + '[6066:20001]'],
fine_sampling_factor=1,
calblock_extension_time=0.)
telescope = IdentityOperator()
projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5)
multiplexing = CompressionAverageOperator(1)
crosstalk = IdentityOperator()
compression = CompressionAverageOperator(4)
model = compression * crosstalk * multiplexing * projection * telescope
# read the Tod off the disk
tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True)
# remove drift
tod40Hz_filtered = filter_polynomial(tod40Hz, 6)
drift = tod40Hz - tod40Hz_filtered
tod40Hz = filter_median(tod40Hz_filtered, 10000)
def _rule_identity(o1, o2):
if o1.nside == o2.nside and o1.nest == o2.nest:
return IdentityOperator()
