def pipeline_dli(filename, output_file, keywords, verbose=False):
import numpy as np
import linear_operators as lo
import tamasis as tm
# verbosity
tm.var.verbose = verbose
# define observation
obs = tm.PacsObservation(filename, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(**keywords["Projection"])
# build instrument model
response = tm.ResponseTruncatedExponentialOperator(
obs.pack(obs.instrument.detector.time_constant) /
obs.instrument.SAMPLING_PERIOD)
compression = tm.CompressionAverageOperator(obs.slice.compression_factor)
masking = tm.MaskOperator(tod.mask)
model = masking * compression * response * projection
# set tod masked values to zero
tod = masking(tod)
# N^-1 operator
invntt = tm.InvNttOperator(obs)
# for the dli algorithm
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = lo.aslinearoperator(model) * M.T
N = lo.aslinearoperator(invntt)
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [tm.DiscreteDifferenceOperator(axis=i, shapein=projection.shapein) \
for i in (0, 1)]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
D = lo.concatenate(Ds)
# handle tau which needs to be an ndarray
keywords["dli"]["tau"] *= np.ones(D.shape[0])
algo = lo.DoubleLoopAlgorithm(H,
y,
D,
noise_covariance=N,
fmin_args=keywords["fmin_args"],
lanczos=keywords["lanczos"],
**keywords["dli"])
# optimize
xe = algo()
# reshape
xe = (M.T * xe).reshape(projection.shapein)
# recast as tamasis map
xe = tm.Map(xe)
# save
xe.save(output_file)
def pipeline_dli(filename, output_file, keywords, verbose=False):
import numpy as np
import linear_operators as lo
import tamasis as tm
# verbosity
tm.var.verbose = verbose
# define observation
obs = tm.PacsObservation(filename, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(**keywords["Projection"])
# build instrument model
response = tm.ResponseTruncatedExponentialOperator(obs.pack(
obs.instrument.detector.time_constant) /
obs.instrument.SAMPLING_PERIOD)
compression = tm.CompressionAverageOperator(obs.slice.compression_factor)
masking = tm.MaskOperator(tod.mask)
model = masking * compression * response * projection
# set tod masked values to zero
tod = masking(tod)
# N^-1 operator
invntt = tm.InvNttOperator(obs)
# for the dli algorithm
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = lo.aslinearoperator(model) * M.T
N = lo.aslinearoperator(invntt)
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [tm.DiscreteDifferenceOperator(axis=i, shapein=projection.shapein) \
for i in (0, 1)]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
D = lo.concatenate(Ds)
# handle tau which needs to be an ndarray
keywords["dli"]["tau"] *= np.ones(D.shape[0])
algo = lo.DoubleLoopAlgorithm(H, y, D, noise_covariance=N,
fmin_args=keywords["fmin_args"],
lanczos=keywords["lanczos"],
**keywords["dli"])
# optimize
xe = algo()
# reshape
xe = (M.T * xe).reshape(projection.shapein)
# recast as tamasis map
xe = tm.Map(xe)
# save
xe.save(output_file)
def generate_model(ra0, dec0, pointing_params, repeats=1, cross_scan=False,
span_angles=False, band="red", map_header=None, npixels_per_sample=0):
"""
Generate a PACS projector.
"""
pointing1 = tm.pacs_create_scan(ra0, dec0, **pointing_params)
# create obs object
obs1 = tm.PacsSimulation(pointing1, band)
if map_header is None:
map_header = obs1.get_map_header()
# create projector
projection1 = tm.Projection(obs1, header=map_header,
npixels_per_sample=npixels_per_sample)
P = lo.aslinearoperator(projection1)
# cross scan
if cross_scan:
pointing_params["scan_angle"] += 90.
pointing2 = tm.pacs_create_scan(ra0, dec0, **pointing_params)
obs2 = tm.PacsSimulation(pointing2, band)
projection2 = tm.Projection(obs2, header=map_header,
npixels_per_sample=npixels_per_sample)
P2 = lo.aslinearoperator(projection2)
P = lo.concatenate((P, P2))
# repeats
if repeats > 1:
P = lo.concatenate((P, ) * repeats)
if span_angles:
if cross_scan:
raise ValueError("Span_angles and cross_scan are incompatible.")
# equally spaced but exclude 0 and 90
angles = np.linspace(0, 90, repeats + 2)[1:-1]
for a in angles:
pointing_params["scan_angle"] = a
pointing2 = tm.pacs_create_scan(ra0, dec0, **pointing_params)
obs2 = tm.PacsSimulation(pointing2, band)
projection2 = tm.Projection(obs2, header=map_header,
npixels_per_sample=npixels_per_sample)
P2 = lo.aslinearoperator(projection2)
P = lo.concatenate((P, P2))
# noise
N = generate_noise_filter(obs1, P, band)
# covered area
map_shape = map_header['NAXIS2'], map_header['NAXIS1']
coverage = (P.T * np.ones(P.shape[0])).reshape(map_shape)
seen = coverage != 0
M = lo.decimate(coverage < 10)
# global model
H = N * P * M.T
return H
def __call__(self, data, state):
mymap = state.get('map')
model = state['model']
M = lo.aslinearoperator(model.aslinearoperator())
data = data.ravel()
mymap = np.ravel(mymap)
state['lo_data_shape'] = model.shapeout
state['map_shape'] = model.shapein
state['model'] = M
state['data'] = data
state['map'] = mymap
return data
def generate_noise_filter(obs1, P, band):
# data
if band == "red":
shape0 = 512.
if band == "green" or band == "blue":
shape0 = 2048.
nsamples = P.shape[0] / shape0
# 1/f noise model
length = 2 ** np.ceil(np.log2(np.array(nsamples)))
# square root in Fourier !!!
filt = obs1.get_filter_uncorrelated()
fft0 = lo.fft2(filt.shape, axes=(1,))
filt2 = np.real(fft0.T(np.sqrt(fft0(filt))))
invNtt = tm.InvNtt(length, filt2)
fft = tm.FftHalfComplex(length)
padding = tm.Padding(left=invNtt.ncorrelations,
right=length - nsamples - invNtt.ncorrelations)
cov = padding.T * fft.T * invNtt * fft * padding
N12 = lo.aslinearoperator(cov)
return N12
def pipeline_huber(filenames, output_file, keywords, verbose=False):
"""
Perform huber regularized inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
import linear_operators as lo
# verbosity
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
# need to adapt degltiching to any compression model
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, **keywords["Projection"])
P = lo.aslinearoperator(projection)
# build instrument model
masking = tm.Masking(tod.mask)
Mt = lo.aslinearoperator(masking)
# compression
mode = compressions[keywords["compression"].pop("mode")]
factor = keywords["compression"].pop("factor")
C = mode(data, factor, **keywords["compression"])
# define map mask
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = Mt * C * P * M
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [tm.DiscreteDifference(axis=i, shapein=projection.shapein) for i in (0, 1)]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
# set tod masked values to zero
tod = masking(tod)
# perform map-making inversion
hypers = (keywords["hacg"].pop("hyper"),) * len(Ds)
deltas = (keywords["hacg"].pop("delta"),) * (len(Ds) + 1)
map_huber = lo.hacg(H, tod.ravel(), priors=Ds, hypers=hypers, deltas=deltas, **keywords["hacg"])
# save
map_huber = (M.T * map_huber).reshape(projection.shapein)
map_huber = tm.Map(map_huber)
map_huber.save(output_file)
def pipeline_huber(filenames, output_file, keywords, verbose=False):
"""
Perform huber regularized inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
import linear_operators as lo
# verbosity
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
# need to adapt degltiching to any compression model
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, **keywords["Projection"])
P = lo.aslinearoperator(projection)
# build instrument model
masking = tm.Masking(tod.mask)
Mt = lo.aslinearoperator(masking)
# compression
mode = compressions[keywords["compression"].pop("mode")]
factor = keywords["compression"].pop("factor")
C = mode(data, factor, **keywords["compression"])
# define map mask
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = Mt * C * P * M
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [
tm.DiscreteDifference(axis=i, shapein=projection.shapein)
for i in (0, 1)
]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
# set tod masked values to zero
tod = masking(tod)
# perform map-making inversion
hypers = (keywords["hacg"].pop("hyper"), ) * len(Ds)
deltas = (keywords["hacg"].pop("delta"), ) * (len(Ds) + 1)
map_huber = lo.hacg(H,
tod.ravel(),
priors=Ds,
hypers=hypers,
deltas=deltas,
**keywords["hacg"])
# save
map_huber = (M.T * map_huber).reshape(projection.shapein)
map_huber = tm.Map(map_huber)
map_huber.save(output_file)
def __call__(self, data, state):
obs = state['obs']
cov = noise_covariance(data, obs)
W = lo.aslinearoperator(cov.aslinearoperator())
state['noise_model'] = W
return data
#!/usr/bin/env python
"""
Testing of the lo package
"""
import nose
from numpy.testing import *
import numpy as np
import linear_operators as lo
# collection of linear operators to test
mat16 = lo.aslinearoperator(np.random.rand(16, 16))
lo_list = [mat16, ]
# collection of vectors
ones16 = np.ones(16)
v_list = [ones16, ]
def check_matvec(A, x):
A * x
def test_matvec():
for A in lo_list:
for x in v_list:
yield check_matvec, A, x
if __name__ == "__main__":
nose.run(argv=['', __file__])
def __init__(self, mat):
self.mat = lo.aslinearoperator(mat)
#!/usr/bin/env python
"""
Testing of the lo.iterative module
"""
import nose
from numpy.testing import *
import numpy as np
import linear_operators as lo
from linear_operators import iterative
# collection of linear operators to test
mat16 = lo.aslinearoperator(np.random.rand(16, 16))
id16 = lo.identity((16, 16))
diag16 = lo.diag(np.random.rand(16))
conv16 = lo.convolve(16, np.random.rand(4), mode="same")
lo_list = [mat16, id16, diag16, conv16]
# collection of vectors
ones16 = np.ones(16)
zeros16 = np.zeros(16)
rand16 = np.random.rand(16)
v_list = [ones16, zeros16, rand16]
# collection of methods
methods = [
iterative.acg,
]
def __init__(self, mat):
self.mat = lo.aslinearoperator(mat)