def __init__(self,
scans,
area,
comm,
cuts=None,
name="main",
ofmt="{name}",
output=True,
ext="fits",
pmat_order=None,
sys=None,
nuisance=False):
Signal.__init__(self, name, ofmt, output, ext)
self.area = area
self.cuts = cuts
self.dof = zipper.ArrayZipper(area, comm=comm)
self.dtype = area.dtype
self.comm = comm
self.data = {
scan: [
pmat.PmatMap(scan, area, order=pmat_order, sys=sys),
pmat.PmatMapMultibeam(scan,
area,
scan.buddy_offs,
scan.buddy_comps,
order=pmat_order,
sys=sys)
]
for scan in scans
}
def __init__(self,
scans,
area,
comm,
cuts=None,
name="main",
ofmt="{name}",
output=True,
ext="fits",
pmat_order=None,
sys=None,
nuisance=False,
data=None):
Signal.__init__(self, name, ofmt, output, ext)
self.area = area
self.cuts = cuts
self.dof = zipper.ArrayZipper(area, comm=comm)
self.dtype = area.dtype
if data is not None:
self.data = data
else:
self.data = {
scan: pmat.PmatMap(scan, area, order=pmat_order, sys=sys)
for scan in scans
}
def __init__(self,
scanpattern,
dets,
map,
noise,
simsys="equ",
cache=False,
seed=0,
noise_scale=1):
# Set up the telescope
self.boresight = scanpattern.boresight
self.sys = scanpattern.sys
self.offsets = dets.offsets
self.comps = dets.comps
self.cut = nocut(self.ndet, self.nsamp)
self.mjd0 = scanpattern.mjd0
# Set up the simulated signal properties
self.map = map.copy()
self.noise = noise
self.seed = seed
self.dets = np.arange(len(self.comps))
self.site = scanpattern.site
self.noise_scale = noise_scale
self.simsys = simsys
self.pmat = pmat.PmatMap(self, self.map, sys=simsys)
def __init__(self,
scans,
subinds,
area,
cuts=None,
name="main",
ofmt="{name}",
output=True,
ext="fits",
pmat_order=None,
sys=None,
nuisance=False,
data=None):
Signal.__init__(self, name, ofmt, output, ext)
self.area = area
self.cuts = cuts
self.dof = dmap.DmapZipper(area)
self.dtype = area.dtype
self.subs = subinds
if data is None:
data = {}
work = area.tile2work()
for scan, subind in zip(scans, subinds):
data[scan] = [
pmat.PmatMap(scan, work[subind], order=pmat_order,
sys=sys), subind
]
self.data = data
def __init__(self,
scans,
subinds,
dmap,
sys=None,
mul=1,
tmul=1,
pmat_order=None):
self.map, self.sys, self.mul, self.tmul = dmap, sys, mul, tmul
self.data = {}
work = dmap.tile2work()
for scan, subind in zip(scans, subinds):
self.data[scan] = [
pmat.PmatMap(scan, work[subind], order=pmat_order, sys=sys),
work[subind]
]
def __init__(self, scan, srcpos, res=0.25*utils.arcmin, rad=20*utils.arcmin, perdet=False, detoff=10*utils.arcmin):
if perdet:
# Offset each detector's pointing so that we produce a grid of images, one per detector.
gside = int(np.ceil(scan.ndet**0.5))
goffs = np.mgrid[:gside,:gside] - (gside-1)/2.0
goffs = goffs.reshape(2,-1).T[:scan.ndet]*detoff
scan.offsets = scan.offsets.copy()
scan.offsets[:,1:] += goffs
rad = rad + np.max(np.abs(goffs))
# Build geometry for each source
shape, wcs = enmap.geometry(pos=[[-rad,-rad],[rad,rad]], res=res, proj="car")
area = enmap.zeros((3,)+shape, wcs, dtype=scan.tod.dtype)
self.pmats = []
for i, pos in enumerate(srcpos.T):
if planet: sys = src_sys
else: sys = ["icrs",[np.array([[pos[0]],[pos[1]],[0],[0]]),False]]
with config.override("pmat_accuracy", 10):
self.pmats.append(pmat.PmatMap(scan, area, sys=sys))
self.shape = (len(srcpos.T),3)+shape
self.wcs = wcs
def map_cuts(entry,
data,
hitmap,
cutmap,
keep_buffer=False,
nocommon_frac=None):
'''
Project hits and hits-cuts onto the sky.
parameters
----------
entry : filedb.data object
data : actdata instance
hitmap : (1, ny, nx) enmap
cutmap : (1, ny, nx) enmap
keep_buffer : bool, optional
Do not remove the 200 sample buffers before and after cuts.
common_frac : float, optional
Do not consider cuts that are at least common to this
fraction of detectors.
'''
scan = actscan.ACTScan(entry, d=data)
pmap = pmat.PmatMap(scan, hitmap)
cut = data.cut
if nocommon_frac is not None:
cut = remove_common(cut, frac=nocommon_frac)
if keep_buffer is False:
cut = remove_buffer(cut)
tod = np.full([data.ndet, data.nsamp], 1.0, dtype)
pmap.backward(tod, hitmap)
sampcut.gapfill_const(cut, tod, 0.0, inplace=True)
pmap.backward(tod, cutmap)
hits = enmap.zeros((3, ) + area.shape[-2:], area.wcs, dtype=dtype)
myinds = np.arange(comm.rank, len(ids), comm.size)
for ind in myinds:
id = ids[ind]
entry = filedb.data[id]
try:
scan = actscan.ACTScan(entry)
if scan.ndet == 0 or scan.nsamp == 0:
raise errors.DataMissing("Tod contains no valid data")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (str(id), e.message))
continue
scan = scan[:, ::config.get("downsample")]
L.debug("Processing %s" % str(id))
pmap = pmat.PmatMap(scan, hits)
pcut = pmat.PmatCut(scan)
tod = np.full([scan.ndet, scan.nsamp], 1, dtype=dtype)
junk = np.zeros(pcut.njunk, dtype=dtype)
pcut.backward(tod, junk)
pmap.backward(tod, hits)
hits = hits[0]
# Collect result
L.info("Reducing")
hits[:] = utils.allreduce(hits, comm)
# Undo effect of downsampling
hits *= config.get("downsample")
# And write it
L.info("Writing")
t.append(time.time())
try:
fields = ["gain", "tconst", "cut", "tod", "boresight", "noise_cut"]
if args.spikecut: fields.append("spikes")
if args.imap: fields += ["polangle", "point_offsets", "site"]
d = data.read(entry, fields)
t.append(time.time())
d = data.calibrate(d)
t.append(time.time())
if args.imap:
# Make a full scan object, so we can perform pointing projection
# operations
d.noise = None
scan = data.ACTScan(entry, d=d)
imap.map = imap.map.astype(d.tod.dtype, copy=False)
pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys)
# Subtract input map from tod inplace
pmap.forward(d.tod, imap.map, tmul=1, mmul=-1)
utils.deslope(d.tod, w=8, inplace=True)
ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5
t.append(time.time())
spikes = d.spikes[:2].T if args.spikecut else None
if model == "old":
noise = nmat_measure.detvecs_old(ft, d.srate, d.dets)
elif model == "jon":
noise = nmat_measure.detvecs_jon(ft,
d.srate,
d.dets,
shared,
cut_bins=spikes)
elif model == "simple":
ft = fft.rfft(model)
freq = fft.rfftfreq(model.shape[-1])*d.srate
flt = 1/(1+(freq/model_fknee)**model_alpha)
ft *= flt
fft.ifft(ft, model, normalize=True)
del ft, flt, freq
with bench.show("atm subtract"):
tod -= model
del model
tod = tod.astype(dtype, copy=False)
# Should now be reasonably clean of correlated noise.
# Proceed to make simple binned map
with bench.show("actscan"):
scan = actscan.ACTScan(entry, d=d)
with bench.show("pmat"):
pmap = pmat.PmatMap(scan, area, sys=sys)
pcut = pmat.PmatCut(scan)
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype)
div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype)
junk = np.zeros(pcut.njunk, dtype)
with bench.show("rhs"):
tod *= ivar[:,None]
pcut.backward(tod, junk)
pmap.backward(tod, rhs)
with bench.show("hits"):
for i in range(ncomp):
div[i,i] = 1
pmap.forward(tod, div[i])
tod *= ivar[:,None]
pcut.backward(tod, junk)
div[i] = 0
try:
object_pos = coordinates.interpol_pos("cel",
"hor",
args.objname,
mjd,
site=scan.site)
except AttributeError as e:
print(
"Unexpected error in interpol_pos for %s. mid time was %.5f. message: %s. skipping"
% (id, mjd[len(mjd) // 2], e))
continue
visible = np.any(object_pos[1] >= -margin)
if not visible:
cut = sampcut.empty(d.ndet, d.nsamp)
else:
pmap = pmat.PmatMap(scan, mask, sys="sidelobe:%s" % args.objname)
# Build a tod to project onto.
tod = np.zeros((d.ndet, d.nsamp), dtype=dtype)
# And project
pmap.forward(tod, mask)
# Any nonzero samples should be cut
tod = tod != 0
cut = sampcut.from_mask(tod)
del tod
progress = 100.0 * (ind - comm.rank * ntod // comm.size) / (
(comm.rank + 1) * ntod // comm.size -
comm.rank * ntod // comm.size)
print("%3d %5.1f %s %6.4f %d %8.3f %8.3f" %
(comm.rank, progress, id, float(cut.sum()) / cut.size, visible,
memory.current() / 1024.**3, memory.max() / 1024.**3))
mystats.append([ind, float(cut.sum()) / cut.size, visible])
srate = 0
speed = 0
site = {}
inspec = np.zeros(nbin)
offsets = np.zeros([ndet_array, 2])
det_hit = np.zeros([ndet_array], dtype=int)
for ind, d in scanutils.scan_iterator(pids,
myinds,
actscan.ACTScan,
filedb.data,
dets=args.dets,
downsample=config.get("downsample")):
id = pids[ind]
with bench.mark("pbuild"):
# Build pointing matrices
pmap = pmat.PmatMap(d, area)
pcut = pmat.PmatCut(d)
with bench.mark("tod"):
# Get tod
tod = d.get_samples()
tod -= np.mean(tod, 1)[:, None]
tod = tod.astype(dtype)
junk = np.zeros(pcut.njunk, dtype=dtype)
with bench.mark("nmat"):
# Build noise model
ft = fft.rfft(tod) * tod.shape[1]**-0.5
nmat = nmat_measure.detvecs_simple(ft, d.srate)
del ft
with bench.mark("rhs"):
# Calc rhs, accumulating into pattern total
nmat.apply(tod)
scan = actscan.ACTScan(entry, verbose=verbose >= 2)
if scan.ndet < 2 or scan.nsamp < 1:
raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print("%s skipped: %s" % (id, e))
continue
# Apply downsampling
scan = scan[:, ::down]
# Prepeare our samples
scan.tod = scan.get_samples()
utils.deslope(scan.tod, w=5, inplace=True)
scan.tod = scan.tod.astype(dtype)
# Background subtraction
if args.sub:
Pmap = pmat.PmatMap(scan, background, sys=sys)
Pmap.forward(scan.tod, background, tmul=-1)
# Build the noise model
N = NmatTot(scan, model="uncorr", window=2.0, filter=highpass)
P = PmatTot(scan, srcpos[:, sids], perdet=True, sys=sys)
# rhs
N.apply(scan.tod)
rhs = P.backward(scan.tod, ncomp=1)
# div
scan.tod[:] = 0
P.forward(scan.tod, rhs * 0 + 1)
N.apply(scan.tod)
div = P.backward(scan.tod, ncomp=1)
def __init__(self, scans, map, sys=None, mul=1, tmul=1, pmat_order=None):
self.map, self.sys, self.mul, self.tmul = map, sys, mul, tmul
self.data = {
scan: pmat.PmatMap(scan, map, order=pmat_order, sys=sys)
for scan in scans
}
raw_bore = raw_bore[:cal_bore.shape[0]]
# And a proper ACTScan
scan = data.ACTScan(entry, subdets=[args.di])
# Detector pointing
det_ipoint = scan.boresight + scan.offsets[0]
# Manual transformation
trf = pmat.pos2pix(scan, None, eqsys)
det_opoint_exact = trf(det_ipoint.T).T
psi_rot_exact = np.arctan2(det_opoint_exact[:, 3], det_opoint_exact[:, 2])
psi_det = np.arctan2(scan.comps[:, 2], scan.comps[:, 1])
psi_exact = (psi_rot_exact + psi_det) / 2 % np.pi
ppix = pmat.PmatMap(scan, area)
# Exact to pixels
det_opix_exact = ppix.transform(det_ipoint.T).T
# Interpolated to pixels
det_opix, det_ophase = ppix.translate(scan.boresight, scan.offsets, scan.comps)
# Get cut status of each sample also
cut = scan.cut.to_mask()[0]
# Ok, output all the information. First a header describing our information
print "# id %s det %d samp_off %d" % (args.id, scan.dets[0], d.sample_offset)
print "# det angle %10.3f" % (psi_det / 2 * 180 / np.pi)
print "# FITS header"
for card in area.wcs.to_header().cards:
print "# " + str(card)
tod = tod.astype(dtype)
# Construct noise model for this tod
winsize = int(config.get("tod_window") * d.srate)
nmat.apply_window(tod, winsize)
d.noise = d.noise.update(tod, d.srate)
L.debug("Noise %s" % id)
# Apply it, to get N"d
d.noise.apply(tod)
nmat.apply_window(tod, winsize)
# Compute the variance per detector. If our noise model
# were correct and our data were pure noise, this would be
# N"<nn'>N" = N". But our noise model isn't totally accurate.
vars = np.var(tod, 1)
# Project each detectors result on the sky
tod[:] = vars[:, None]
pmap = pmat.PmatMap(d, osig)
pcut = pmat.PmatCut(d)
junk = np.zeros(pcut.njunk, dtype=dtype)
pcut.backward(tod, junk)
pmap.backward(tod, osig)
# Also do the fiducial noise model
tod[:] = 1
nmat.apply_window(tod, winsize)
d.noise.white(tod)
nmat.apply_window(tod, winsize)
pcut.backward(tod, junk)
pmap.backward(tod, odiv)
# Collect some statistics
sig_all[ind] = np.sum(vars) * d.nsamp
sig_med[ind] = np.median(vars) * d.ndet * d.nsamp
div_all[ind] = np.sum(tod)
def __init__(self, scan, area, sys=None): self.pmap = pmat.PmatMap(scan, area, sys=sys) self.pcut = pmat.PmatCut(scan) self.err = self.pmap.err
bore_corners = utils.box2corners(bore_box)
scan.entry = d.entry
# Is the source above the horizon? If not, it doesn't matter how close
# it is.
mjd = utils.ctime2mjd(
utils.mjd2ctime(scan.mjd0) + scan.boresight[::100, 0])
object_pos = coordinates.interpol_pos("cel",
"hor",
args.objname,
mjd,
site=scan.site)
visible = np.any(object_pos[1] >= margin)
if not visible:
cut = rangelist.zeros((d.ndet, d.nsamp))
else:
pmap = pmat.PmatMap(scan, mask, sys="hor:%s" % args.objname)
# Build a tod to project onto.
tod = np.zeros((d.ndet, d.nsamp), dtype=dtype)
# And project
pmap.forward(tod, mask)
# Any nonzero samples should be cut
tod = np.rint(tod)
cut = rangelist.Multirange([rangelist.Rangelist(t) for t in tod])
print "%s %6.4f %d" % (id, float(cut.sum()) / cut.size, visible)
mystats.append([ind, float(cut.sum()) / cut.size, visible])
# Write cuts to output directory
if args.persample:
files.write_cut("%s/%s.cuts" % (args.odir, id),
d.dets,
cut,
nrow=d.array_info.nrow,
if args.precompute:
prec_NNmap = {lam: area * 0 for lam in np.unique(cooldown)}
prec_NNjunk = {lam: [] for lam in np.unique(cooldown)}
scans = []
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
entry = filedb.data[id]
try:
scan = actscan.ACTScan(entry)
if scan.ndet == 0 or scan.nsamp == 0:
raise errors.DataMissing("No samples in scan")
if args.ndet:
scan = scan[:args.ndet]
if downsample > 1:
scan = scan[:, ::downsample]
scan.pmap = pmat.PmatMap(scan, area)
scan.pcut = pmat.PmatCut(scan)
# Build the noise model
tod = scan.get_samples()
tod -= np.mean(tod, 1)[:, None]
tod = tod.astype(dtype)
scan.noise = scan.noise.update(tod, scan.srate)
scan.T = np.min(scan.noise.D) * Tscale
scan.noise_bar = nmat.NmatDetvecs(scan.noise.D - scan.T, scan.noise.V,
scan.noise.E, scan.noise.bins,
scan.noise.ebins, scan.noise.dets)
# Set up cuts
scan.cut_range = [njunk_tot, njunk_tot + scan.pcut.njunk]
njunk_tot += scan.pcut.njunk
# Prepare our filtered data. We do this one of two ways.
# Either store Nb"d for each TOD, which can end up taking
