def pipeline_dli(filename, output_file, keywords, verbose=False):
import numpy as np
import linear_operators as lo
import tamasis as tm
# verbosity
tm.var.verbose = verbose
# define observation
obs = tm.PacsObservation(filename, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(**keywords["Projection"])
# build instrument model
response = tm.ResponseTruncatedExponentialOperator(
obs.pack(obs.instrument.detector.time_constant) /
obs.instrument.SAMPLING_PERIOD)
compression = tm.CompressionAverageOperator(obs.slice.compression_factor)
masking = tm.MaskOperator(tod.mask)
model = masking * compression * response * projection
# set tod masked values to zero
tod = masking(tod)
# N^-1 operator
invntt = tm.InvNttOperator(obs)
# for the dli algorithm
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = lo.aslinearoperator(model) * M.T
N = lo.aslinearoperator(invntt)
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [tm.DiscreteDifferenceOperator(axis=i, shapein=projection.shapein) \
for i in (0, 1)]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
D = lo.concatenate(Ds)
# handle tau which needs to be an ndarray
keywords["dli"]["tau"] *= np.ones(D.shape[0])
algo = lo.DoubleLoopAlgorithm(H,
y,
D,
noise_covariance=N,
fmin_args=keywords["fmin_args"],
lanczos=keywords["lanczos"],
**keywords["dli"])
# optimize
xe = algo()
# reshape
xe = (M.T * xe).reshape(projection.shapein)
# recast as tamasis map
xe = tm.Map(xe)
# save
xe.save(output_file)
def pipeline_dli(filename, output_file, keywords, verbose=False):
import numpy as np
import linear_operators as lo
import tamasis as tm
# verbosity
tm.var.verbose = verbose
# define observation
obs = tm.PacsObservation(filename, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(**keywords["Projection"])
# build instrument model
response = tm.ResponseTruncatedExponentialOperator(obs.pack(
obs.instrument.detector.time_constant) /
obs.instrument.SAMPLING_PERIOD)
compression = tm.CompressionAverageOperator(obs.slice.compression_factor)
masking = tm.MaskOperator(tod.mask)
model = masking * compression * response * projection
# set tod masked values to zero
tod = masking(tod)
# N^-1 operator
invntt = tm.InvNttOperator(obs)
# for the dli algorithm
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = lo.aslinearoperator(model) * M.T
N = lo.aslinearoperator(invntt)
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [tm.DiscreteDifferenceOperator(axis=i, shapein=projection.shapein) \
for i in (0, 1)]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
D = lo.concatenate(Ds)
# handle tau which needs to be an ndarray
keywords["dli"]["tau"] *= np.ones(D.shape[0])
algo = lo.DoubleLoopAlgorithm(H, y, D, noise_covariance=N,
fmin_args=keywords["fmin_args"],
lanczos=keywords["lanczos"],
**keywords["dli"])
# optimize
xe = algo()
# reshape
xe = (M.T * xe).reshape(projection.shapein)
# recast as tamasis map
xe = tm.Map(xe)
# save
xe.save(output_file)
def pipeline_photproject(filenames, output_file, keywords):
"""
Perform regularized least-square inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(downsampling=True,
**keywords["Projection"])
# build instrument model
masking = tm.MaskOperator(tod.mask)
model = masking * projection
# set tod masked values to zero
tod = masking(tod)
# perform map-making
map_naive = tm.mapper_naive(tod, model, **keywords["mapper_naive"])
# save
map_naive.save(output_file)
def pipeline_photproject(filenames, output_file, keywords):
"""
Perform regularized least-square inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(downsampling=True,
**keywords["Projection"])
# build instrument model
masking = tm.MaskOperator(tod.mask)
model = masking * projection
# set tod masked values to zero
tod = masking(tod)
# perform map-making
map_naive = tm.mapper_naive(tod, model, **keywords["mapper_naive"])
# save
map_naive.save(output_file)
def __call__(self, data, state):
# get parameters
C = state.get('compression')
factor = state.get('compression_factor', 1)
# decompress
uc_data = uncompress(data, C, factor)
uc_data.mask = state['mask']
# filter
uc_data = tm.filter_median(uc_data, length=self.filter_length)
tm.deglitch_l2mad(uc_data, state['model'])
# define mask
masking = tm.Masking(uc_data.mask)
# update model
state['deglitching_mask'] = uc_data.mask
state['deglitching_mask_lo'] = masking
state['model'] = masking * state['model']
state['deglitching_filter_length'] = self.filter_length
return data
projection = pacs.get_projection_operator(resolution=3.2,
npixels_per_sample=5)
multiplexing = CompressionAverageOperator(1)
crosstalk = IdentityOperator()
compression = CompressionAverageOperator(4)
model = compression * crosstalk * multiplexing * projection * telescope
# read the Tod off the disk
tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True)
# remove drift
tod40Hz_filtered = filter_polynomial(tod40Hz, 6)
drift = tod40Hz - tod40Hz_filtered
tod40Hz = filter_median(tod40Hz_filtered, 10000)
# second level deglitching
tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5)
idetector=5
plot_tod((tod40Hz+drift)[idetector])
plot(drift[idetector], 'r')
masking = MaskOperator(tod40Hz.mask)
model40Hz = masking * projection
map_naive40Hz = mapper_naive(tod40Hz, model40Hz)
map_naive40Hz.imshow()
# clim doesn't work anymore with AnnotatedImage
#clim(-0.00002,0.00002)
def pipeline_huber(filenames, output_file, keywords, verbose=False):
"""
Perform huber regularized inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
import linear_operators as lo
# verbosity
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
# need to adapt degltiching to any compression model
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, **keywords["Projection"])
P = lo.aslinearoperator(projection)
# build instrument model
masking = tm.Masking(tod.mask)
Mt = lo.aslinearoperator(masking)
# compression
mode = compressions[keywords["compression"].pop("mode")]
factor = keywords["compression"].pop("factor")
C = mode(data, factor, **keywords["compression"])
# define map mask
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = Mt * C * P * M
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [tm.DiscreteDifference(axis=i, shapein=projection.shapein) for i in (0, 1)]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
# set tod masked values to zero
tod = masking(tod)
# perform map-making inversion
hypers = (keywords["hacg"].pop("hyper"),) * len(Ds)
deltas = (keywords["hacg"].pop("delta"),) * (len(Ds) + 1)
map_huber = lo.hacg(H, tod.ravel(), priors=Ds, hypers=hypers, deltas=deltas, **keywords["hacg"])
# save
map_huber = (M.T * map_huber).reshape(projection.shapein)
map_huber = tm.Map(map_huber)
map_huber.save(output_file)
def pipeline_huber(filenames, output_file, keywords, verbose=False):
"""
Perform huber regularized inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
import linear_operators as lo
# verbosity
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
# need to adapt degltiching to any compression model
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, **keywords["Projection"])
P = lo.aslinearoperator(projection)
# build instrument model
masking = tm.Masking(tod.mask)
Mt = lo.aslinearoperator(masking)
# compression
mode = compressions[keywords["compression"].pop("mode")]
factor = keywords["compression"].pop("factor")
C = mode(data, factor, **keywords["compression"])
# define map mask
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = Mt * C * P * M
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [
tm.DiscreteDifference(axis=i, shapein=projection.shapein)
for i in (0, 1)
]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
# set tod masked values to zero
tod = masking(tod)
# perform map-making inversion
hypers = (keywords["hacg"].pop("hyper"), ) * len(Ds)
deltas = (keywords["hacg"].pop("delta"), ) * (len(Ds) + 1)
map_huber = lo.hacg(H,
tod.ravel(),
priors=Ds,
hypers=hypers,
deltas=deltas,
**keywords["hacg"])
# save
map_huber = (M.T * map_huber).reshape(projection.shapein)
map_huber = tm.Map(map_huber)
map_huber.save(output_file)
tod = obs.get_tod()
header = obs.get_map_header()
header.update('CDELT1', resolution / 3600)
header.update('CDELT2', resolution / 3600)
npix = 5
good_npix = False
projection = tm.Projection(obs, header=header,
resolution=resolution,
oversampling=False,
npixels_per_sample=npix)
model = projection
#C = csh.averaging(tod.shape, factor=factor)
compression_shape = [tod.size / factor, tod.size, ]
C = tm.CompressionAverage(compression_factor=factor).aslinearoperator(compression_shape)
C = lo.aslinearoperator(C)
# compress data
ctod = compress(tod, C, factor)
# uncompress for preprocessing
uctod = uncompress(ctod, C, factor)
# XXX need mask ...
uctod.mask = tod.mask
# deglitching
uctod = tm.filter_median(uctod, length=10)
uctod.mask = tm.deglitch_l2mad(uctod, projection)
masking = tm.Masking(uctod.mask)
model = masking * projection
# filtering
ctod = tm.filter_median(ctod, length=filter_length / factor)
cov = noise_covariance(ctod, obs)
S = cov.aslinearoperator()
#header['NAXIS2'] = 192 #header['CRPIX1'] = 96 #header['CRPIX2'] = 96 # data tod = pacs.get_tod() # remove bad pixels (by updating mask !) #tod = remove_bad_pixels(tod) # deglitching #projection = tm.Projection(pacs, header=header, resolution=3., npixels_per_sample=6) projection = tm.Projection(pacs, resolution=3., npixels_per_sample=6) tm.deglitch_l2mad(tod, projection) # model masking = tm.Masking(tod.mask) model = masking * projection # remove drift tod = tm.filter_median(tod, length=999) model = masking * projection # naive map backmap = model.transpose(tod) # coverage map weights = model.transpose(tod.ones(tod.shape)) # mask on map mask = weights == 0 M = lo.mask(mask) # preconditionner iweights = 1 / weights iweights[np.where(np.isfinite(iweights) == 0)] = 0. M0 = lo.diag(iweights.flatten()) # transform to lo P = lo.aslinearoperator(model.aslinearoperator())
def pipeline_wrls(filenames, output_file, keywords, verbose=False):
"""
Perform regularized least-square inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# verbosity
keywords["mapper_rls"]["verbose"] = verbose
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, **keywords["Projection"])
# build instrument model
response = tm.ResponseTruncatedExponential(obs.pack(
obs.instrument.detector.time_constant) / obs.SAMPLING_PERIOD)
compression = tm.CompressionAverage(obs.slice.compression_factor)
masking = tm.Masking(tod.mask)
model = masking * compression * response * projection
# set tod masked values to zero
tod = masking(tod)
# N^-1 operator
invntt = tm.InvNtt(obs)
# perform map-making inversion
map_rls = tm.mapper_rls(tod, model, invntt=invntt, solver=cgs,
**keywords["mapper_rls"])
# save
map_rls.save(output_file)
def pipeline_wrls(filenames, output_file, keywords, verbose=False):
"""
Perform regularized least-square inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# verbosity
keywords["mapper_rls"]["verbose"] = verbose
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = obs.get_projection_operator(**keywords["Projection"])
# build instrument model
response = tm.ResponseTruncatedExponentialOperator(
obs.pack(obs.instrument.detector.time_constant) /
obs.instrument.SAMPLING_PERIOD)
compression = tm.CompressionAverageOperator(obs.slice.compression_factor)
masking = tm.MaskOperator(tod.mask)
model = masking * compression * response * projection
# set tod masked values to zero
tod = masking(tod)
# N^-1 operator
invntt = tm.InvNttOperator(obs)
# perform map-making inversion
map_rls = tm.mapper_rls(tod,
model,
invntt=invntt,
solver=cgs,
**keywords["mapper_rls"])
# save
map_rls.save(output_file)
telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000) # second level deglitching tod40Hz.mask = deglitch_l2mad(tod40Hz, projection, nsigma=5) idetector = 5 plot_tod((tod40Hz + drift)[idetector]) plot(drift[idetector], 'r') masking = MaskOperator(tod40Hz.mask) model40Hz = masking * projection map_naive40Hz = mapper_naive(tod40Hz, model40Hz) map_naive40Hz.imshow() # clim doesn't work anymore with AnnotatedImage #clim(-0.00002,0.00002)
good_npix = False
projection = tm.Projection(obs,
header=header,
resolution=resolution,
oversampling=False,
npixels_per_sample=npix)
model = projection
#C = csh.averaging(tod.shape, factor=factor)
compression_shape = [
tod.size / factor,
tod.size,
]
C = tm.CompressionAverage(
compression_factor=factor).aslinearoperator(compression_shape)
C = lo.aslinearoperator(C)
# compress data
ctod = compress(tod, C, factor)
# uncompress for preprocessing
uctod = uncompress(ctod, C, factor)
# XXX need mask ...
uctod.mask = tod.mask
# deglitching
uctod = tm.filter_median(uctod, length=10)
uctod.mask = tm.deglitch_l2mad(uctod, projection)
masking = tm.Masking(uctod.mask)
model = masking * projection
# filtering
ctod = tm.filter_median(ctod, length=filter_length / factor)
cov = noise_covariance(ctod, obs)
S = cov.aslinearoperator()
#header['NAXIS2'] = 192 #header['CRPIX1'] = 96 #header['CRPIX2'] = 96 # data tod = pacs.get_tod() # remove bad pixels (by updating mask !) #tod = remove_bad_pixels(tod) # deglitching #projection = tm.Projection(pacs, header=header, resolution=3., npixels_per_sample=6) projection = tm.Projection(pacs, resolution=3., npixels_per_sample=6) tm.deglitch_l2mad(tod, projection) # model masking = tm.Masking(tod.mask) model = masking * projection # remove drift tod = tm.filter_median(tod, length=999) model = masking * projection # naive map backmap = model.transpose(tod) # coverage map weights = model.transpose(tod.ones(tod.shape)) # mask on map mask = weights == 0 M = lo.mask(mask) # preconditionner iweights = 1 / weights iweights[np.where(np.isfinite(iweights) == 0)] = 0. M0 = lo.diag(iweights.flatten()) # transform to lo P = lo.aslinearoperator(model.aslinearoperator())
header.update('CDELT1', resolution / 3600)
header.update('CDELT2', resolution / 3600)
npix = 5
good_npix = False
projection = tm.Projection(obs, header=header,
resolution=resolution,
oversampling=False,
npixels_per_sample=npix)
model = projection
C = csh.averaging(tod.shape, factor=factor)
# compress data
ctod = compress(tod, C, factor)
# deglitch
uctod = uncompress(ctod, C, factor)
uctod.mask = tod.mask
uctod = tm.filter_median(uctod, length=deglitch_filter_length)
uctod_mask = tm.deglitch_l2mad(uctod, projection)
uctod = uncompress(ctod, C, factor)
uctod.mask = uctod_mask
uctod = tm.interpolate_linear(uctod)
ctod = compress(uctod, C, factor)
# filtering
ctod = tm.filter_median(ctod, length=filter_length / factor)
cov = noise_covariance(ctod, obs)
S = cov.aslinearoperator()
# model with compression
M = lo.aslinearoperator(model.aslinearoperator())
M = C * M
# noise covariance
cov = noise_covariance(ctod, obs)
