def generate_model(ra0, dec0, pointing_params, repeats=1, cross_scan=False,
span_angles=False, band="red", map_header=None, npixels_per_sample=0):
"""
Generate a PACS projector.
"""
pointing1 = tm.pacs_create_scan(ra0, dec0, **pointing_params)
# create obs object
obs1 = tm.PacsSimulation(pointing1, band)
if map_header is None:
map_header = obs1.get_map_header()
# create projector
projection1 = tm.Projection(obs1, header=map_header,
npixels_per_sample=npixels_per_sample)
P = lo.aslinearoperator(projection1)
# cross scan
if cross_scan:
pointing_params["scan_angle"] += 90.
pointing2 = tm.pacs_create_scan(ra0, dec0, **pointing_params)
obs2 = tm.PacsSimulation(pointing2, band)
projection2 = tm.Projection(obs2, header=map_header,
npixels_per_sample=npixels_per_sample)
P2 = lo.aslinearoperator(projection2)
P = lo.concatenate((P, P2))
# repeats
if repeats > 1:
P = lo.concatenate((P, ) * repeats)
if span_angles:
if cross_scan:
raise ValueError("Span_angles and cross_scan are incompatible.")
# equally spaced but exclude 0 and 90
angles = np.linspace(0, 90, repeats + 2)[1:-1]
for a in angles:
pointing_params["scan_angle"] = a
pointing2 = tm.pacs_create_scan(ra0, dec0, **pointing_params)
obs2 = tm.PacsSimulation(pointing2, band)
projection2 = tm.Projection(obs2, header=map_header,
npixels_per_sample=npixels_per_sample)
P2 = lo.aslinearoperator(projection2)
P = lo.concatenate((P, P2))
# noise
N = generate_noise_filter(obs1, P, band)
# covered area
map_shape = map_header['NAXIS2'], map_header['NAXIS1']
coverage = (P.T * np.ones(P.shape[0])).reshape(map_shape)
seen = coverage != 0
M = lo.decimate(coverage < 10)
# global model
H = N * P * M.T
return H
def pipeline_dli(filename, output_file, keywords, verbose=False):
import numpy as np
import linear_operators as lo
import tamasis as tm
# verbosity
tm.var.verbose = verbose
# define observation
obs = tm.PacsObservation(filename, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, **keywords["Projection"])
# build instrument model
response = tm.ResponseTruncatedExponential(obs.pack(
obs.instrument.detector.time_constant) / obs.SAMPLING_PERIOD)
compression = tm.CompressionAverage(obs.slice.compression_factor)
masking = tm.Masking(tod.mask)
model = masking * compression * response * projection
# set tod masked values to zero
tod = masking(tod)
# N^-1 operator
invntt = tm.InvNtt(obs)
# for the dli algorithm
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = lo.aslinearoperator(model) * M.T
N = lo.aslinearoperator(invntt)
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [tm.DiscreteDifference(axis=i, shapein=projection.shapein) for i in (0, 1)]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
D = lo.concatenate(Ds)
# handle tau which needs to be an ndarray
keywords["dli"]["tau"] *= np.ones(D.shape[0])
algo = lo.DoubleLoopAlgorithm(H, y, D, noise_covariance=N,
fmin_args=keywords["fmin_args"],
lanczos=keywords["lanczos"],
**keywords["dli"])
# optimize
xe = algo()
# reshape
xe = (M.T * xe).reshape(projection.shapein)
# recast as tamasis map
xe = tm.Map(xe)
# save
xe.save(output_file)
def pipeline_photproject(filenames, output_file, keywords):
"""
Perform regularized least-square inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, oversampling=False, **keywords["Projection"])
# build instrument model
masking = tm.Masking(tod.mask)
model = masking * projection
# set tod masked values to zero
tod = masking(tod)
# perform map-making
map_naive = tm.mapper_naive(tod, model, **keywords["mapper_naive"])
# save
map_naive.save(output_file)
def pipeline_rls(filenames, output_file, keywords, verbose=False):
"""
Perform regularized least-square inversion of a simple model (tod
mask and projection). Processing steps are as follows :
- define PacsObservation instance
- get Time Ordered Data (get_tod)
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# verbosity
keywords["mapper_rls"]["verbose"] = verbose
# define observation
obs_keys = keywords.get("PacsObservation", {})
obs = tm.PacsObservation(filenames, **obs_keys)
# get data
tod_keys = keywords.get("get_tod", {})
tod = obs.get_tod(**tod_keys)
# define projector
proj_keys = keywords.get("Projection", {})
projection = tm.Projection(obs, **proj_keys)
# define mask
masking_tod = tm.Masking(tod.mask)
# define full model
model = masking_tod * projection
# perform map-making inversion
mapper_keys = keywords.get("mapper_rls", {})
map_rls = tm.mapper_rls(tod, model, solver=cgs, **mapper_keys)
# save
map_rls.save(output_file)
def load_data(filename, header=None, resolution=3.):
#mask = np.zeros((32, 64), dtype=np.int8)
obs = tm.PacsObservation(filename=filename,
fine_sampling_factor=1,
#detector_mask=mask
)
tod = obs.get_tod()
if header is None:
header = obs.get_map_header()
header.update('CDELT1', resolution / 3600)
header.update('CDELT2', resolution / 3600)
npix = 5
good_npix = False
while good_npix is False:
try:
projection = tm.Projection(obs, header=header,
resolution=resolution,
oversampling=False,
npixels_per_sample=npix)
good_npix = True
except(RuntimeError):
npix +=1
return tod, projection, header, obs
def pipeline_huber(filenames, output_file, keywords, verbose=False):
"""
Perform huber regularized inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
import linear_operators as lo
# verbosity
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
# need to adapt degltiching to any compression model
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, **keywords["Projection"])
P = lo.aslinearoperator(projection)
# build instrument model
masking = tm.Masking(tod.mask)
Mt = lo.aslinearoperator(masking)
# compression
mode = compressions[keywords["compression"].pop("mode")]
factor = keywords["compression"].pop("factor")
C = mode(data, factor, **keywords["compression"])
# define map mask
M = map_mask(tod, model, **keywords["map_mask"])
# recast model as a new-style LinearOperator from linear_operators package
H = Mt * C * P * M
# vectorize data so it is accepted by LinearOperators
y = tod.ravel()
Ds = [
tm.DiscreteDifference(axis=i, shapein=projection.shapein)
for i in (0, 1)
]
Ds = [lo.aslinearoperator(D) for D in Ds]
Ds = [D * M.T for D in Ds]
# set tod masked values to zero
tod = masking(tod)
# perform map-making inversion
hypers = (keywords["hacg"].pop("hyper"), ) * len(Ds)
deltas = (keywords["hacg"].pop("delta"), ) * (len(Ds) + 1)
map_huber = lo.hacg(H,
tod.ravel(),
priors=Ds,
hypers=hypers,
deltas=deltas,
**keywords["hacg"])
# save
map_huber = (M.T * map_huber).reshape(projection.shapein)
map_huber = tm.Map(map_huber)
map_huber.save(output_file)
#!/usr/bin/env python import numpy as np import tamasis as tm import lo import csh.filter as filt from time import time import scipy.sparse.linalg as spl # data pacs = tm.PacsObservation(filename=tm.tamasis_dir+'tests/frames_blue.fits') tod = pacs.get_tod() # projector model = tm.Projection(pacs, resolution=3.2, oversampling=False, npixels_per_sample=6) # naive map backmap = model.transpose(tod) # transform to lo P = lo.aslinearoperator(model.aslinearoperator()) # derive filter kernel = filt.kernel_from_tod(tod, length=10) #kern = np.mean(kernel, axis=0) N = filt.kernels_convolve(tod.shape, 1 / np.sqrt(kernel)) # apply to data yn = N * tod.flatten() # apply to model M = N * P # priors Ds = [lo.diff(backmap.shape, axis=axis) for axis in xrange(backmap.ndim)] #Ds.append(lo.pywt_lo.wavelet2(backmap.shape, "haar")) # inversion #y = tod.flatten() x, conv = lo.rls(M, Ds, (1e1, 1e1, 1e-1), yn, spl.bicgstab)
pointing_params = {
'cam_angle': 0.,
"scan_angle": 0.,
"scan_nlegs": 1,
"scan_length": 400.,
"scan_speed": 60.0,
"compression_factor": 1 # we will compress later
}
pointing = tm.pacs_create_scan(ra0, dec0, **pointing_params)
# create obs object
obs = tm.PacsSimulation(pointing, "red", policy_bad_detector="keep")
# we dont need 200 first and last frames
obs.pointing.removed[200:] = True
obs.pointing.removed[-201:] = True
# create projector
projection = tm.Projection(obs, npixels_per_sample=4)
P = lo.aslinearoperator(projection)
# simulate data
x0 = tm.gaussian(projection.shapein, 3) # map with gaussian source
tod0 = projection(x0) # data
n = np.random.randn(*tod0.shape) # noise
nsr = 1e-2
tod = tod0 + nsr * n # noisy data
y = tod.ravel() # as 1d array
# load compression matrix
filename = os.path.join(os.getenv("HOME"), "data", "pacs",
"mmc_cam_angle_0_scan_angle_0_speed60.fits")
c = fa.FitsArray(file=filename).astype(np.float64)
cmm = csh.compression.AnyOperator(c)
C = cmm((projection.shapeout[0], projection.shapeout[1][0]), )
y = tod.flatten()
# remove bad pixels (by updating mask !)
#tod = remove_bad_pixels(tod)
# compress data
factor = 8
compression = tm.CompressionAverage(factor)
C = lo.aslinearoperator(
compression.aslinearoperator(shape=(tod.size / factor, tod.size)))
ctod = compression.direct(tod)
# uncompress for deglitching
uctod = tod.copy(tod.shape)
y0, t = lo.spl.cgs(C.T * C, C.T * ctod.flatten())
uctod[:] = y0.reshape(tod.shape)
# deglitching
projection = tm.Projection(pacs,
header=header,
resolution=3.,
npixels_per_sample=5)
uctod.mask = tm.deglitch_l2mad(uctod, projection)
ctod = compression.direct(uctod)
# model
masking = tm.Masking(uctod.mask)
model = compression * masking * projection
# remove drift
#ctod = tm.filter_median(ctod, length=3000 / 8.)
# first map
M = lo.aslinearoperator(model.aslinearoperator())
#P = lo.aslinearoperator(projection.aslinearoperator())
#C = csh.averaging(tod.shape, factor=8)
#I = lo.mask(uctod.mask)
#M = C * I.T * I * P
#M = C * P
factor = 4
ext = ".fits"
pre = "ngc6946_madmap1_"
# find a map for each compression and save it
obs = tm.PacsObservation(filename=filename,
fine_sampling_factor=1,
detector_policy='remove')
tod = obs.get_tod()
header = obs.get_map_header()
header.update('CDELT1', resolution / 3600)
header.update('CDELT2', resolution / 3600)
npix = 5
good_npix = False
projection = tm.Projection(obs,
header=header,
resolution=resolution,
oversampling=False,
npixels_per_sample=npix)
model = projection
#C = csh.averaging(tod.shape, factor=factor)
compression_shape = [
tod.size / factor,
tod.size,
]
C = tm.CompressionAverage(
compression_factor=factor).aslinearoperator(compression_shape)
C = lo.aslinearoperator(C)
# compress data
ctod = compress(tod, C, factor)
# uncompress for preprocessing
uctod = uncompress(ctod, C, factor)
pacs = tm.PacsObservation(filename=filenames,
fine_sampling_factor=1,
keep_bad_detectors=False)
# reset pacs header to have a shape multiple of 4
#header = pacs.get_map_header()
#header['NAXIS1'] = 192
#header['NAXIS2'] = 192
#header['CRPIX1'] = 96
#header['CRPIX2'] = 96
# data
tod = pacs.get_tod()
# remove bad pixels (by updating mask !)
#tod = remove_bad_pixels(tod)
# deglitching
#projection = tm.Projection(pacs, header=header, resolution=3., npixels_per_sample=6)
projection = tm.Projection(pacs, resolution=3., npixels_per_sample=6)
tm.deglitch_l2mad(tod, projection)
# model
masking = tm.Masking(tod.mask)
model = masking * projection
# remove drift
tod = tm.filter_median(tod, length=999)
model = masking * projection
# naive map
backmap = model.transpose(tod)
# coverage map
weights = model.transpose(tod.ones(tod.shape))
# mask on map
mask = weights == 0
M = lo.mask(mask)
def pipeline_wrls(filenames, output_file, keywords, verbose=False):
"""
Perform regularized least-square inversion of state of the art
PACS model. The PACS model includes tod mask, compression,
response, projection and noise covariance (invntt).
Processing steps are as follows :
- define PacsObservation instance
- mask first scanlines to avoid drift
- get Time Ordered Data (get_tod)
- 2nd level deglitching (with standard projector and tod median
filtering with a narrow window)
- median filtering
- define projection
- perform inversion on model
- save file
Arguments
---------
filenames: list of strings
List of data filenames.
output_file : string
Name of the output fits file.
keywords: dict
Dictionary containing options for all steps as dictionary.
verbose: boolean (default False)
Set verbosity.
Returns
-------
Returns nothing. Save result as a fits file.
"""
from scipy.sparse.linalg import cgs
import tamasis as tm
# verbosity
keywords["mapper_rls"]["verbose"] = verbose
# define observation
obs = tm.PacsObservation(filenames, **keywords["PacsObservation"])
# extra masking
tm.step_scanline_masking(obs, **keywords["scanline_masking"])
# get data
tod = obs.get_tod(**keywords["get_tod"])
# deglitching
tm.step_deglitching(obs, tod, **keywords["deglitching"])
# median filtering
tod = tm.filter_median(tod, **keywords["filter_median"])
# define projector
projection = tm.Projection(obs, **keywords["Projection"])
# build instrument model
response = tm.ResponseTruncatedExponential(
obs.pack(obs.instrument.detector.time_constant) / obs.SAMPLING_PERIOD)
compression = tm.CompressionAverage(obs.slice.compression_factor)
masking = tm.Masking(tod.mask)
model = masking * compression * response * projection
# set tod masked values to zero
tod = masking(tod)
# N^-1 operator
invntt = tm.InvNtt(obs)
# perform map-making inversion
map_rls = tm.mapper_rls(tod,
model,
invntt=invntt,
solver=cgs,
**keywords["mapper_rls"])
# save
map_rls.save(output_file)
'cam_angle': 0.,
"scan_angle": 0.,
"scan_nlegs": 1,
"scan_length": 4.,
"compression_factor": 1,
"scan_speed": 60.0
}
pointing = tm.pacs_create_scan(ra0, dec0, **pointing_params)
# create obs object
obs = tm.PacsSimulation(pointing, "red", policy_bad_detector="keep")
# to define my compression matrix I need only 8 frames
# so I mask the others :
obs.pointing.removed[:] = True
obs.pointing.removed[201:209] = False
# create projector
projection = tm.Projection(obs)
P = lo.aslinearoperator(projection)
# define masking of coefficients
# ------------------------------
# number of coefficients to keep
factor = 8.
nk = P.shape[0] / factor
# coverage map
w = P.T * np.ones(P.shape[0])
# sort coverage map coefficients
ws = np.sort(w)
# find the nk largest coverage elements
threshold = ws[-nk]
mask = w < threshold
# define decimation Linear Operator according to coverage map coef
