def func(d, e):
m = Map(np.array([1, 2, 3], dtype=d), coverage=np.array([0, 1, 0],
dtype=d), error=np.array([2, 2, 2], dtype=d))
m2 = m.astype(e)
assert m2.dtype == e
assert m2.coverage.dtype == e
assert m2.error.dtype == e
def test():
assert map_rls.header['NITER'] < 49
ref = Map(data_dir + 'frames_blue_map_rls_cgs_tol1e-6.fits')
ref.derived_units = map_rls.derived_units
cov = ref.coverage > 80
assert all_eq(ref[cov], map_rls[cov], 1.e-1)
cov = ref.coverage > 125
assert all_eq(ref[cov], map_rls[cov], 1.e-2)
def func(d, e):
m = Map(np.array([1, 2, 3], dtype=d),
coverage=np.array([0, 1, 0], dtype=d),
error=np.array([2, 2, 2], dtype=d))
m2 = m.astype(e)
assert m2.dtype == e
assert m2.coverage.dtype == e
assert m2.error.dtype == e
def test():
assert map_rls.header['NITER'] < 49
ref = Map(data_dir + 'frames_blue_map_rls_cgs_tol1e-6.fits')
ref.derived_units = map_rls.derived_units
cov = ref.coverage > 80
assert all_eq(ref[cov], map_rls[cov], 1.e-1)
cov = ref.coverage > 125
assert all_eq(ref[cov], map_rls[cov], 1.e-2)
def test_detector_policy():
map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits')
obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False)
obs.pointing.chop[:] = 0
projection = obs.get_projection_operator(header=map_naive_ref.header,
downsampling=True,
npixels_per_sample=6)
tod = obs.get_tod()
masking = MaskOperator(tod.mask)
model = masking * projection
map_naive = mapper_naive(tod, model)
assert all_eq(map_naive, map_naive_ref, tol)
obs_rem = PacsObservation(data_dir + 'frames_blue.fits',
policy_bad_detector='remove',
reject_bad_line=False)
obs_rem.pointing.chop[:] = 0
projection_rem = obs_rem.get_projection_operator(header=map_naive.header,
downsampling=True,
npixels_per_sample=7)
tod_rem = obs_rem.get_tod()
masking_rem = MaskOperator(tod_rem.mask)
model_rem = masking_rem * projection_rem
map_naive_rem = mapper_naive(tod_rem, model_rem)
assert all_eq(map_naive, map_naive_rem, tol)
def test_header():
obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False)
obs.pointing.chop = 0
header = obs.get_map_header()
projection = obs.get_projection_operator(header=header, downsampling=True,
npixels_per_sample=6)
tod = obs.get_tod()
map_naive = mapper_naive(tod, projection)
header2 = header.copy()
header2['NAXIS1'] += 500
header2['CRPIX1'] += 250
projection2 = obs.get_projection_operator(header=header2, downsampling=True)
map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2)
map_naive2.inunit('Jy/arcsec^2')
map_naive3 = map_naive2[:,250:header['NAXIS1']+250]
assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7)
# test compatibility with photproject
tod = obs.get_tod()
map_naive4 = mapper_naive(tod, projection)
hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1]
map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel')
std_naive = np.std(map_naive4[40:60,40:60])
std_ref = np.std(map_ref[40:60,40:60])
assert abs(std_naive-std_ref) / std_ref < 0.025
def test_ls():
H = projection
m = pcg(H.T * invntt * H, (H.T*invntt)(tod), M=M, tol=1e-7)
map_ls_packed = Map(m['x'])
map_ls_packed.header['TIME'] = m['time']
#print('Elapsed time:', map_ls_packed.header['TIME'])
#from tamasis import mapper_ls
#map_ls_packed = mapper_ls(tod, projection, invntt=invntt, tol=1e-7, M=M,
# callback=callback, criterion=False,
# profile=profile)
if profile:
return
print('Elapsed time:', map_ls_packed.header['TIME'])
assert m['nit'] < 50
ref = packing(Map(path + 'madmapSpirePSW.fits'))
assert_allclose(map_ls_packed, ref, atol=1e-5)
from tamasis import (PacsInstrument, PacsObservation, UnpackOperator,
mapper_naive)
from tamasis.utils import assert_all_eq
data_dir = os.path.dirname(__file__) + '/data/'
rank = MPI.COMM_WORLD.rank
size = MPI.COMM_WORLD.size
tamasis.var.verbose = True
PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\
'dPixelMask_FM_v5.fits'
PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\
'sponsivity_FM_v5.fits'
map_ref_local = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits')
mask_ref_local = map_ref_local.coverage == 0
header_ref_local = map_ref_local.header
map_ref_global = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits', comm=MPI.COMM_SELF)
mask_ref_global = map_ref_global.coverage == 0
header_ref_global = map_ref_global.header
def check_map_global(m):
assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10)
assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10)
assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_global.tounit('Jy/arcsec^2'),
1e-10)
def check_map_local(m):
assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10)
def mapper_nl(tod,
model,
unpacking=None,
priors=[],
hypers=[],
norms=[],
comms=[],
x0=None,
tol=1.e-6,
maxiter=300,
solver=None,
linesearch=None,
descent_method='pr',
M=None,
verbose=True,
callback=None,
profile=None):
comm_map = model.commin or MPI.COMM_WORLD
comm_tod = model.commout or comm_map
tod = _validate_tod(tod)
if len(priors) == 0 and len(hypers) != 0:
npriors = len(model.shapein)
priors = [
DiscreteDifferenceOperator(axis=axis,
shapein=model.shapein,
commin=comm_map)
for axis in range(npriors)
]
else:
npriors = len(priors)
if np.isscalar(hypers):
hypers = npriors * [hypers]
elif npriors != len(hypers):
raise ValueError('There should be one hyperparameter per prior.')
if len(norms) == 0:
norms = (npriors + 1) * [norm2]
if len(comms) == 0:
comms = [comm_tod] + npriors * [comm_map]
if isinstance(M, DiagonalOperator):
filter_nonfinite(M.data, out=M.data)
hypers = np.asarray(hypers, dtype=var.FLOAT_DTYPE)
ntods = int(np.sum(~tod.mask)) if getattr(tod, 'mask', None) is not None \
else tod.size
ntods = comm_tod.allreduce(ntods)
nmaps = comm_map.allreduce(model.shape[1])
hypers /= nmaps
hypers = np.hstack([1 / ntods, hypers])
y = tod.view(np.ndarray).ravel()
class ObjectiveFunction(Function):
def __init__(self):
pass
def __call__(self, x, inwork=None, outwork=None, out=None):
outwork = self.set_outwork(outwork, self.get_work(x))
return self.set_out(out, [ h * n(r,comm=c) for h, n, r, c in \
zip(hypers, norms, outwork, comms) ])
def D(self, x, inwork=None, outwork=None, out=None):
inwork = self.get_work(x, inwork)
return self.set_out(
out,
sum([
h * M.T * n.D(r, comm=c)
for h, M, n, r, c in zip(hypers, [model] +
priors, norms, inwork, comms)
]))
def get_work(self, x, work=None):
if work is not None:
return work
if unpacking is not None:
x = unpacking.matvec(x)
return [model * x - y] + [p * x for p in priors]
objfunc = ObjectiveFunction()
if linesearch is None:
linesearch = QuadraticStep(hypers, norms, [model] + priors, comms,
unpacking, comm_map)
if callback is None:
callback = CgCallback(verbose=verbose)
time0 = time.time()
solution = nlcg(objfunc,
model.shape[1],
M=M,
maxiter=maxiter,
tol=tol,
descent_method=descent_method,
linesearch=linesearch,
callback=callback,
comm=comm_map)
time0 = time.time() - time0
Js = objfunc(solution)
solution = unpacking(solution)
tod[...] = 1
coverage = model.T(tod)
header = getattr(coverage, 'header', None)
if header is None:
header = create_fitsheader(fromdata=coverage)
unit = getattr(coverage, 'unit', None)
derived_units = getattr(coverage, 'derived_units', None)
coverage = Map(coverage.magnitude, header=header, copy=False)
header.update('likeliho', Js[0])
header.update('criter', sum(Js))
header.update('hyper', str(hypers))
header.update('nsamples', ntods)
header.update('npixels', nmaps)
header.update('time', time0)
if hasattr(callback, 'niterations'):
header.update('niter', callback.niterations)
header.update('maxiter', maxiter)
if hasattr(callback, 'residual'):
header.update('residual', callback.residual)
header.update('tol', tol)
header.update('solver', 'nlcg')
output = Map(solution.reshape(model.shapein),
header=header,
coverage=coverage,
unit=unit,
derived_units=derived_units,
copy=False)
return output
def mapper_naive(tod, model, unit=None):
"""
Returns a naive map, i.e.: map = model.T(tod) / model.T(1)
This equation is valid for a map and a Time Ordered Data (TOD) expressed as
a surface brightness, so when the TOD does not meet this requirement and
is a quantity per detector, a unit conversion is attempted.
Parameters
----------
tod : Tod
The input Time Ordered Data
model : Operator
The instrument model such as tod = model(map)
unit : string
Output map unit. By default, the output map unit is chosen to be
compatible with the model (usually pixel^-1)
"""
# apply mask
if hasattr(tod, 'mask') and tod.mask is not None:
mask = MaskOperator(tod.mask)
else:
mask = IdentityOperator()
tod = mask(tod)
# get tod units
if not hasattr(tod, '_unit') or len(tod._unit) == 0:
attr = {'_unit': {'?': 1.}}
model.propagate_attributes(None, attr)
u = getattr(attr, '_unit', {})
if 'detector' in u and u['detector'] == -1:
u = {'detector': -1.}
elif u == {'?': 1.}:
u = {}
elif len(u) > 1:
raise ValueError(
'The timeline units are not known and cannot be in'
'ferred from the model.')
tod_du = getattr(attr, '_derived_units', {})
tod = Tod(tod.magnitude, unit=u, derived_units=tod_du, copy=False)
else:
attr = {'_unit': {'?': 1}}
model.T.propagate_attributes(None, attr)
u = attr['_unit']
if 'detector' not in tod._unit and 'detector' in u and u[
'detector'] == 1:
raise ValueError("The model is incompatible with input units '{0}'"\
.format(tod.unit))
tod_unit = tod._unit
tod_du = tod._derived_units
# make sure the input is a surface brightness
if 'detector' in tod._unit:
tod.inunit(tod.unit + ' detector / arcsec^2')
elif 'detector_reference' in tod._unit:
tod.inunit(tod.unit + ' detector_reference / arcsec^2')
# compute model.T(tod)/model.T(one)
mymap = model.T(tod.magnitude)
tod[...] = 1
mask(tod, tod)
map_weights = model.T(tod.magnitude)
old_settings = np.seterr(divide='ignore', invalid='ignore')
mymap /= map_weights
mymap.unit = tod.unit
np.seterr(**old_settings)
mymap.coverage = Map(map_weights.magnitude,
header=mymap.header,
copy=False)
if unit is not None:
mymap.inunit(unit)
return mymap
# set map units according to model
attr = {'_unit': tod_unit, '_derived_units': tod_du}
model.T.propagate_attributes(None, attr)
if '_derived_units' in attr:
mymap.derived_units = attr['_derived_units']
if '_unit' in attr:
mymap.inunit(attr['_unit'])
return mymap
def mapper_rls(y,
H,
invntt=None,
unpacking=None,
hyper=1.0,
x0=None,
tol=1.e-5,
maxiter=300,
M=None,
solver=None,
verbose=True,
callback=None,
criterion=True,
profile=None):
"""
Solve the linear equation
y = H(x)
where H is an Operator representing the acquisition, and y is the observed
time series.
x is the regularised least square solution as given by:
x = argmin (Hx-y)^T N^-1 (Hx-y) + h ||D(x)||^2
or:
x = (H^T N^-1 H + h D1^T D1 + h D2^T D2)^-1 H^T N^-1 y
"""
comm_map = H.commin or MPI.COMM_WORLD
comm_tod = H.commout or comm_map
if solver is None:
solver = cg
new_solver = solver in (pcg, )
tod = _validate_tod(y)
ntods_ = int(np.sum(~tod.mask)) if getattr(tod, 'mask', None) is not None \
else tod.size
nmaps_ = unpacking.shape[1] if unpacking is not None else None
if nmaps_ is None:
nmaps_ = H.shape[1]
if nmaps_ is None:
raise ValueError('The model H has not an explicit input shape.')
ntods = comm_tod.allreduce(ntods_)
nmaps = comm_map.allreduce(nmaps_)
# get A
if invntt is None:
invntt = IdentityOperator()
A = H.T * invntt * H
npriors = len(H.shapein)
priors = [
DiscreteDifferenceOperator(axis=axis,
shapein=H.shapein,
commin=comm_map) for axis in range(npriors)
]
if comm_map.rank == 0 or comm_map.size > 1:
A += sum((hyper * ntods / nmaps) * p.T * p for p in priors)
# get b
b = (H.T * invntt)(tod)
if not np.all(np.isfinite(b)):
raise ValueError('RHS contains non-finite values.')
if b.shape != A.shapein:
raise ValueError("Incompatible size for RHS: '" + str(b.size) +
"' instead of '" + str(A.shape[1]) + "'.")
if np.min(b) == np.max(b) == 0:
print('Warning: in equation Ax=b, b is zero.')
# unpack input
if unpacking is None:
unpacking = IdentityOperator()
A = unpacking.T * A * unpacking
b = unpacking.T(b)
if x0 is not None:
x0 = unpacking.T(x0)
if not new_solver and solver is not cg:
b = b.ravel()
if x0 is not None:
x0 = x0.ravel()
if M is not None:
if isinstance(M, DiagonalOperator):
filter_nonfinite(M.data, out=M.data)
M = unpacking.T * M * unpacking
H_ = H * unpacking
priors = [p * unpacking for p in priors]
# criterion
if hyper != 0:
hc = np.hstack([1, npriors * [hyper]]) / ntods
else:
hc = [1 / ntods]
norms = [norm2_ellipsoid(invntt)] + npriors * [norm2]
comms = [comm_tod] + npriors * [comm_map]
def criter(x):
rs = [H_.matvec(x) - tod.view(np.ndarray).ravel()
] + [p.matvec(x) for p in priors]
Js = [h * n(r, comm=c) for h, n, r, c in zip(hc, norms, rs, comms)]
return Js
if callback is None:
if new_solver:
callback = NewCgCallback(disp=verbose, comm=comm_map)
else:
callback = CgCallback(verbose=verbose,
objfunc=criter if criterion else None)
if (verbose or profile) and comm_map.rank == 0:
print('')
print('H.T * N^-1 * H:')
print(repr(A))
if M is not None:
print('Preconditioner:')
print(M)
time0 = time.time()
if profile is not None:
def run():
result = solver(A, b, x0=x0, M=M, tol=tol, maxiter=maxiter)
if new_solver:
solution = result['x']
if not result['success']:
print('Solver failure: ' + result['message'])
else:
solution, info = result
if info != 0:
print('Solver failure: info=' + str(info))
cProfile.runctx('run()', globals(), locals(), profile + '.prof')
print('Profile time: ' + str(time.time() - time0))
os.system('python -m gprof2dot -f pstats -o ' + profile + '.dot ' +
profile + '.prof')
os.system('dot -Tpng ' + profile + '.dot > ' + profile)
os.system('rm -f ' + profile + '.prof' + ' ' + profile + '.dot')
return None
result = solver(A,
b,
x0=x0,
M=M,
tol=tol,
maxiter=maxiter,
callback=callback)
time0 = time.time() - time0
if new_solver:
solution = result['x']
if not result['success']:
print('Solver failure: ' + result['message'])
niterations = result['niterations']
error = result['error']
else:
solution, info = result
if info < 0:
raise RuntimeError('Solver failure (code=' + str(info) +
' after ' + str(callback.niterations) +
' iterations).')
if info > 0 and comm_map.rank == 0:
print(
'Warning: Solver reached maximum number of iterations without'
' reaching specified tolerance.')
niterations = getattr(callback, 'niterations', None)
error = getattr(callback, 'residual', None)
Js = criter(solution)
if isinstance(unpacking, IdentityOperator):
solution = solution.reshape(H.shapein)
else:
solution = unpacking(solution)
tod[...] = 1
coverage = H.T(tod)
unit = getattr(coverage, 'unit', None)
derived_units = getattr(coverage, 'derived_units', None)
coverage = coverage.view(Map)
header = getattr(coverage, 'header', None)
if header is None:
header = create_fitsheader(fromdata=coverage)
header.update('likeliho', Js[0])
header.update('criter', sum(Js))
header.update('hyper', hyper)
header.update('nsamples', ntods)
header.update('npixels', nmaps)
header.update('time', time0)
if niterations is not None:
header.update('niter', niterations)
header.update('maxiter', maxiter)
if error is not None:
header.update('error', error)
header.update('tol', tol)
header.update('solver', solver.__name__)
output = Map(solution,
header=header,
coverage=coverage,
unit=unit,
derived_units=derived_units,
copy=False)
return output
def ProjectionOperator(input, method=None, header=None, resolution=None,
npixels_per_sample=0, units=None, derived_units=None,
downsampling=False, packed=False, commin=MPI.COMM_WORLD,
commout=MPI.COMM_WORLD, onfly_func=None, onfly_ids=None,
onfly_shapeouts=None, **keywords):
"""
Projection operator factory, to handle operations by one or more pointing
matrices.
It is assumed that each pointing matrix row has at most 'npixels_per_sample'
non-null elements, or in other words, an output element can only intersect
a fixed number of input elements.
Given an input vector x and an output vector y, y = P(x) translates into:
y[i] = sum(P.matrix[i,:].value * x[P.matrix[i,:].index])
If the input is not MPI-distributed unlike the output, the projection
operator is automatically multiplied by the operator MPIDistributionIdenti-
tyOperator, to enable MPI reductions.
If the input is MPI-distributed, this operator is automatically packed (see
below) and multiplied by the operator MPIDistributionLocalOperator, which
takes the local input as argument.
Arguments
---------
input : pointing matrix (or sequence of) or observation (deprecated)
method : deprecated
header : deprecated
resolution : deprecated
npixels_per_sample : deprecated
downsampling : deprecated
packed deprecated
"""
# check if there is only one pointing matrix
isonfly = input is None
isobservation = hasattr(input, 'get_pointing_matrix')
if isobservation:
if hasattr(input, 'slice'):
nmatrices = len(input.slice)
else:
nmatrices = 1
commout = input.instrument.comm
elif isonfly:
nmatrices = len(onfly_ids)
else:
if isinstance(input, PointingMatrix):
input = (input,)
if any(not isinstance(i, PointingMatrix) for i in input):
raise TypeError('The input is not a PointingMatrix, (nor a sequence'
' of).')
nmatrices = len(input)
ismapdistributed = commin.size > 1
istoddistributed = commout.size > 1
# get the pointing matrix from the input observation
if isobservation:
if header is None:
if not hasattr(input, 'get_map_header'):
raise AttributeError("No map header has been specified and "
"the observation has no 'get_map_header' method.")
header_global = input.get_map_header(resolution=resolution,
downsampling=downsampling)
else:
if isinstance(header, str):
header = str2fitsheader(header)
header_global = gather_fitsheader_if_needed(header, comm=commin)
#XXX we should hand over the local header
input = input.get_pointing_matrix(header_global, npixels_per_sample,
method=method, downsampling=downsampling, comm=commin)
if isinstance(input, PointingMatrix):
input = (input,)
elif isonfly:
header_global = header
else:
header_global = input[0].info['header']
# check shapein
if not isonfly:
shapeins = [i.shape_input for i in input]
if any(s != shapeins[0] for s in shapeins):
raise ValueError('The pointing matrices do not have the same input '
"shape: {0}.".format(', '.join(str(shapeins))))
shapein = shapeins[0]
else:
shapein = fitsheader2shape(header_global)
# the output is simply a ProjectionOperator instance
if nmatrices == 1 and not ismapdistributed and not istoddistributed \
and not packed:
return ProjectionInMemoryOperator(input[0], units=units, derived_units=
derived_units, commin=commin, commout=commout, **keywords)
if packed or ismapdistributed:
if isonfly:
raise NotImplementedError()
# compute the map mask before this information is lost while packing
mask_global = Map.ones(shapein, dtype=np.bool8, header=header_global)
for i in input:
i.get_mask(out=mask_global)
for i in input:
i.pack(mask_global)
if ismapdistributed:
shapes = distribute_shapes(mask_global.shape, comm=commin)
shape = shapes[commin.rank]
mask = np.empty(shape, bool)
commin.Reduce_scatter([mask_global, MPI.BYTE], [mask, MPI.BYTE],
[product(s) for s in shapes], op=MPI.BAND)
else:
mask = mask_global
if isonfly:
place_holder = {}
operands = [ProjectionOnFlyOperator(place_holder, id, onfly_func,
shapein=shapein, shapeout=shapeout,
units=units,
derived_units=derived_units)
for id, shapeout in zip(onfly_ids, onfly_shapeouts)]
else:
operands = [ProjectionInMemoryOperator(i, units=units, derived_units=
derived_units, commout=commout, **keywords) for i in input]
result = BlockColumnOperator(operands, axisout=-1)
if nmatrices > 1:
def apply_mask(self, mask):
mask = np.asarray(mask, np.bool8)
dest = 0
if mask.shape != self.shapeout:
raise ValueError("The mask shape '{0}' is incompatible with tha"
"t of the projection operator '{1}'.".format(mask.shape,
self.shapeout))
if any(isinstance(p, ProjectionOnFlyOperator)
for p in self.operands):
blocks = self.copy()
self.__class__ = CompositionOperator
self.__init__([MaskOperator(mask), blocks])
return
for p in self.operands:
n = p.matrix.shape[1]
p.apply_mask(mask[...,dest:dest+n])
dest += n
def get_mask(self, out=None):
for p in self.operands:
out = p.get_mask(out=out)
return out
def get_pTp(self, out=None):
for p in self.operands:
out = p.get_pTp(out=out)
return out
def get_pTx_pT1(self, x, out=None, mask=None):
dest = 0
for p in self.operands:
n = p.matrix.shape[1]
out = p.get_pTx_pT1(x[...,dest:dest+n], out=out, mask=mask)
dest += n
return out
def intersects(self, out=None):
raise NotImplementedError('email-me')
result.apply_mask = apply_mask.__get__(result)
result.get_mask = get_mask.__get__(result)
result.get_pTp = get_pTp.__get__(result)
result.get_pTx_pT1 = get_pTx_pT1.__get__(result)
result.intersects = intersects.__get__(result)
if not istoddistributed and not ismapdistributed and not packed:
return result
if packed or ismapdistributed:
def get_mask(self, out=None):
if out is not None:
out &= mask
else:
out = mask
return out
if ismapdistributed:
header = scatter_fitsheader(header_global, comm=commin)
result *= MPIDistributionLocalOperator(mask_global, commin=commin,
attrin={'header':header})
elif packed:
result *= PackOperator(mask)
result.get_mask = get_mask.__get__(result)
if istoddistributed and not ismapdistributed:
def get_mask(self, out=None):
out = self.operands[0].get_mask(out=out)
commout.Allreduce(MPI.IN_PLACE, [out, MPI.BYTE], op=MPI.BAND)
return out
result *= MPIDistributionIdentityOperator(commout=commout)
result.get_mask = get_mask.__get__(result)
def not_implemented(out=None):
raise NotImplementedError('email-me')
def apply_mask(self, mask):
self.operands[0].apply_mask(mask)
result.apply_mask = apply_mask.__get__(result)
result.get_pTp = not_implemented
result.get_pTx_pT1 = not_implemented
result.intersects = not_implemented
return result
def ProjectionOperator(input,
method=None,
header=None,
resolution=None,
npixels_per_sample=0,
units=None,
derived_units=None,
downsampling=False,
packed=False,
commin=MPI.COMM_WORLD,
commout=MPI.COMM_WORLD,
onfly_func=None,
onfly_ids=None,
onfly_shapeouts=None,
**keywords):
"""
Projection operator factory, to handle operations by one or more pointing
matrices.
It is assumed that each pointing matrix row has at most 'npixels_per_sample'
non-null elements, or in other words, an output element can only intersect
a fixed number of input elements.
Given an input vector x and an output vector y, y = P(x) translates into:
y[i] = sum(P.matrix[i,:].value * x[P.matrix[i,:].index])
If the input is not MPI-distributed unlike the output, the projection
operator is automatically multiplied by the operator MPIDistributionIdenti-
tyOperator, to enable MPI reductions.
If the input is MPI-distributed, this operator is automatically packed (see
below) and multiplied by the operator MPIDistributionLocalOperator, which
takes the local input as argument.
Arguments
---------
input : pointing matrix (or sequence of) or observation (deprecated)
method : deprecated
header : deprecated
resolution : deprecated
npixels_per_sample : deprecated
downsampling : deprecated
packed deprecated
"""
# check if there is only one pointing matrix
isonfly = input is None
isobservation = hasattr(input, 'get_pointing_matrix')
if isobservation:
if hasattr(input, 'slice'):
nmatrices = len(input.slice)
else:
nmatrices = 1
commout = input.instrument.comm
elif isonfly:
nmatrices = len(onfly_ids)
else:
if isinstance(input, PointingMatrix):
input = (input, )
if any(not isinstance(i, PointingMatrix) for i in input):
raise TypeError(
'The input is not a PointingMatrix, (nor a sequence'
' of).')
nmatrices = len(input)
ismapdistributed = commin.size > 1
istoddistributed = commout.size > 1
# get the pointing matrix from the input observation
if isobservation:
if header is None:
if not hasattr(input, 'get_map_header'):
raise AttributeError(
"No map header has been specified and "
"the observation has no 'get_map_header' method.")
header_global = input.get_map_header(resolution=resolution,
downsampling=downsampling)
else:
if isinstance(header, str):
header = str2fitsheader(header)
header_global = gather_fitsheader_if_needed(header, comm=commin)
#XXX we should hand over the local header
input = input.get_pointing_matrix(header_global,
npixels_per_sample,
method=method,
downsampling=downsampling,
comm=commin)
if isinstance(input, PointingMatrix):
input = (input, )
elif isonfly:
header_global = header
else:
header_global = input[0].info['header']
# check shapein
if not isonfly:
shapeins = [i.shape_input for i in input]
if any(s != shapeins[0] for s in shapeins):
raise ValueError(
'The pointing matrices do not have the same input '
"shape: {0}.".format(', '.join(str(shapeins))))
shapein = shapeins[0]
else:
shapein = fitsheader2shape(header_global)
# the output is simply a ProjectionOperator instance
if nmatrices == 1 and not ismapdistributed and not istoddistributed \
and not packed:
return ProjectionInMemoryOperator(input[0],
units=units,
derived_units=derived_units,
commin=commin,
commout=commout,
**keywords)
if packed or ismapdistributed:
if isonfly:
raise NotImplementedError()
# compute the map mask before this information is lost while packing
mask_global = Map.ones(shapein, dtype=np.bool8, header=header_global)
for i in input:
i.get_mask(out=mask_global)
for i in input:
i.pack(mask_global)
if ismapdistributed:
shapes = distribute_shapes(mask_global.shape, comm=commin)
shape = shapes[commin.rank]
mask = np.empty(shape, bool)
commin.Reduce_scatter([mask_global, MPI.BYTE], [mask, MPI.BYTE],
[product(s) for s in shapes],
op=MPI.BAND)
else:
mask = mask_global
if isonfly:
place_holder = {}
operands = [
ProjectionOnFlyOperator(place_holder,
id,
onfly_func,
shapein=shapein,
shapeout=shapeout,
units=units,
derived_units=derived_units)
for id, shapeout in zip(onfly_ids, onfly_shapeouts)
]
else:
operands = [
ProjectionInMemoryOperator(i,
units=units,
derived_units=derived_units,
commout=commout,
**keywords) for i in input
]
result = BlockColumnOperator(operands, axisout=-1)
if nmatrices > 1:
def apply_mask(self, mask):
mask = np.asarray(mask, np.bool8)
dest = 0
if mask.shape != self.shapeout:
raise ValueError(
"The mask shape '{0}' is incompatible with tha"
"t of the projection operator '{1}'.".format(
mask.shape, self.shapeout))
if any(
isinstance(p, ProjectionOnFlyOperator)
for p in self.operands):
blocks = self.copy()
self.__class__ = CompositionOperator
self.__init__([MaskOperator(mask), blocks])
return
for p in self.operands:
n = p.matrix.shape[1]
p.apply_mask(mask[..., dest:dest + n])
dest += n
def get_mask(self, out=None):
for p in self.operands:
out = p.get_mask(out=out)
return out
def get_pTp(self, out=None):
for p in self.operands:
out = p.get_pTp(out=out)
return out
def get_pTx_pT1(self, x, out=None, mask=None):
dest = 0
for p in self.operands:
n = p.matrix.shape[1]
out = p.get_pTx_pT1(x[..., dest:dest + n], out=out, mask=mask)
dest += n
return out
def intersects(self, out=None):
raise NotImplementedError('email-me')
result.apply_mask = apply_mask.__get__(result)
result.get_mask = get_mask.__get__(result)
result.get_pTp = get_pTp.__get__(result)
result.get_pTx_pT1 = get_pTx_pT1.__get__(result)
result.intersects = intersects.__get__(result)
if not istoddistributed and not ismapdistributed and not packed:
return result
if packed or ismapdistributed:
def get_mask(self, out=None):
if out is not None:
out &= mask
else:
out = mask
return out
if ismapdistributed:
header = scatter_fitsheader(header_global, comm=commin)
result *= MPIDistributionLocalOperator(mask_global,
commin=commin,
attrin={'header': header})
elif packed:
result *= PackOperator(mask)
result.get_mask = get_mask.__get__(result)
if istoddistributed and not ismapdistributed:
def get_mask(self, out=None):
out = self.operands[0].get_mask(out=out)
commout.Allreduce(MPI.IN_PLACE, [out, MPI.BYTE], op=MPI.BAND)
return out
result *= MPIDistributionIdentityOperator(commout=commout)
result.get_mask = get_mask.__get__(result)
def not_implemented(out=None):
raise NotImplementedError('email-me')
def apply_mask(self, mask):
self.operands[0].apply_mask(mask)
result.apply_mask = apply_mask.__get__(result)
result.get_pTp = not_implemented
result.get_pTx_pT1 = not_implemented
result.intersects = not_implemented
return result
keywords_types = (keywords_q, keywords_f, keywords_m, keywords_t)
dtypes = (bool, int, np.float32, np.float64, np.complex64, np.complex128)
def teardown():
for file in glob.glob(filename + '*'):
os.remove(file)
a = np.ones((4, 3))
a[1, 2] = 4
q = Quantity(a, unit='myunit', derived_units={'myunit': Quantity(2., 'Jy')})
header = create_fitsheader(fromdata=q, cdelt=0.5, crval=(4., 8.))
header['BUNIT'] = 'myunit'
f = FitsArray(q, header=header)
m = Map(f, origin='upper', error=a * 2, coverage=a * 3)
mask = np.zeros((4, 3), np.bool8)
mask[0, 2] = True
t = Tod(f, mask=mask)
del mask
def test_copy_false_subok_true():
def func(obj1, t):
obj2 = t(obj1, copy=False, subok=True)
if isinstance(obj, t):
assert obj1 is obj2
else:
assert obj1 is not obj2
assert_equal(obj1, obj2)
