def __init__(self, data, srcpos, ndir=1, sys=sys):
# Build source parameter struct for PmatPtsrc
self.params = np.zeros([srcpos.shape[-1], ndir, 8], np.float)
self.params[:, :, :2] = srcpos[::-1, None, :].T
self.params[:, :, 5:7] = 1
scan = actscan.ACTScan(data.entry, d=data)
self.psrc = pmat.PmatPtsrc(scan, self.params, sys=sys)
self.pcut = pmat.PmatCut(scan)
# Extract basic offset
self.off0 = data.point_correction
self.off = self.off0 * 1
self.el = np.mean(data.boresight[2, ::100])
self.point_template = data.point_template
def __init__(self, data, srcpos, res=0.25*utils.arcmin, rad=20*utils.arcmin, perdet=False, detoff=10*utils.arcmin):
scan = actscan.ACTScan(data.entry, d=data)
if perdet:
# Offset each detector's pointing so that we produce a grid of images, one per detector.
gside = int(np.ceil(data.ndet**0.5))
goffs = np.mgrid[:gside,:gside] - (gside-1)/2.0
goffs = goffs.reshape(2,-1).T[:data.ndet]*detoff
scan.offsets = scan.offsets.copy()
scan.offsets[:,1:] += goffs
rad = rad + np.max(np.abs(goffs))
# Build geometry for each source
shape, wcs = enmap.geometry(pos=[[-rad,-rad],[rad,rad]], res=res, proj="car")
area = enmap.zeros((3,)+shape, wcs, dtype=data.tod.dtype)
self.pmats = []
for i, pos in enumerate(srcpos.T):
if planet: sys = src_sys
else: sys = ["icrs",[np.array([[pos[0]],[pos[1]],[0],[0]]),False]]
with config.override("pmat_accuracy", 10):
self.pmats.append(pmat.PmatMap(scan, area, sys=sys))
self.shape = (len(srcpos.T),3)+shape
self.wcs = wcs
def map_cuts(entry,
data,
hitmap,
cutmap,
keep_buffer=False,
nocommon_frac=None):
'''
Project hits and hits-cuts onto the sky.
parameters
----------
entry : filedb.data object
data : actdata instance
hitmap : (1, ny, nx) enmap
cutmap : (1, ny, nx) enmap
keep_buffer : bool, optional
Do not remove the 200 sample buffers before and after cuts.
common_frac : float, optional
Do not consider cuts that are at least common to this
fraction of detectors.
'''
scan = actscan.ACTScan(entry, d=data)
pmap = pmat.PmatMap(scan, hitmap)
cut = data.cut
if nocommon_frac is not None:
cut = remove_common(cut, frac=nocommon_frac)
if keep_buffer is False:
cut = remove_buffer(cut)
tod = np.full([data.ndet, data.nsamp], 1.0, dtype)
pmap.backward(tod, hitmap)
sampcut.gapfill_const(cut, tod, 0.0, inplace=True)
pmap.backward(tod, cutmap)
print asens
with bench.show("smooth"):
ft = fft.rfft(model)
freq = fft.rfftfreq(model.shape[-1])*d.srate
flt = 1/(1+(freq/model_fknee)**model_alpha)
ft *= flt
fft.ifft(ft, model, normalize=True)
del ft, flt, freq
with bench.show("atm subtract"):
tod -= model
del model
tod = tod.astype(dtype, copy=False)
# Should now be reasonably clean of correlated noise.
# Proceed to make simple binned map
with bench.show("actscan"):
scan = actscan.ACTScan(entry, d=d)
with bench.show("pmat"):
pmap = pmat.PmatMap(scan, area, sys=sys)
pcut = pmat.PmatCut(scan)
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype)
div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype)
junk = np.zeros(pcut.njunk, dtype)
with bench.show("rhs"):
tod *= ivar[:,None]
pcut.backward(tod, junk)
pmap.backward(tod, rhs)
with bench.show("hits"):
for i in range(ncomp):
div[i,i] = 1
pmap.forward(tod, div[i])
tod *= ivar[:,None]
site=d.site,
ncomp=ncomp,
dtype=dtype)
except WorkspaceError as e:
L.debug("Skipped pattern %s (%s)" % (wid, e.message))
continue
print "%-18s %5d %5d" % ((wid, ) + tuple(wgeo.shape[-2:]))
tot_work = Workspace(wgeo)
oname = "%s/%s.hdf" % (args.odir, wid)
# And process the tods that fall within this workspace
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
entry = filedb.data[id]
try:
d = actscan.ACTScan(entry)
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("Tod contains no valid data")
d = d[:, ::downsample]
d = d[:, :]
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (id, e.message))
continue
L.debug("Processing %s" % id)
# Get the actual tod
tod = d.get_samples()
tod -= np.mean(tod, 1)[:, None]
tod = tod.astype(dtype)
# Compute the per-detector spectrum
ft = fft.rfft(tod) * d.nsamp**-0.5
tfilter, binds = measure_inv_noise_spectrum(ft, nbin)
return np.mean((blocks-m[:,None]-A[:,None]*s[None,:])**2,1)
def onlyfinite(a): return a[np.isfinite(a)]
# Process each scan independently
myinds = np.arange(len(filelist))[myid::nproc]
for ind in myinds:
ofile = args.odir + "/%s.hdf" % filelist[ind]
if args.c and os.path.isfile(ofile):
L.info("Already done %s" % filelist[ind])
continue
L.info("Processing %s" % filelist[ind])
try:
d = scan.read_scan(filelist[ind])
except (IOError, OSError):
try:
d = actscan.ACTScan(db[filelist[ind]])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("all samples cut")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], str(e)))
continue
try:
L.debug("Reading samples")
tod = d.get_samples().astype(dtype)
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], str(e)))
continue
# Measure noise
L.debug("Noise")
ivar = 1/np.array([np.median(onlyfinite(get_desloped_var(blockify(t,20)))) for t in tod])
# Set up pmat for this scan
bid = id.replace(":", "_")
root = args.odir + "/src%03d_%s_" % (si, bid)
ofile = root + "main_map.fits"
if os.path.isfile(ofile): continue
tasks.append((si, id))
# Each task processes tasks independently
for ti in range(comm.rank, len(tasks), comm.size):
si, id = tasks[ti]
bid = id.replace(":", "_")
L.info("Processing src %3d id %s" % (si, id))
root = args.odir + "/src%03d_%s_" % (si, bid)
entry = filedb.data[id]
osys = "hor:%.6f_%.6f:cel/0_0:hor" % tuple(srcs[:2, si])
try:
scans = [actscan.ACTScan(entry)]
if scans[0].nsamp == 0 or scans[0].ndet == 0:
raise errors.DataMissing("no data in scan")
except errors.DataMissing as e:
print "Skipping %s: %s" % (id, str(e))
continue
# Signals
signal_cut = mapmaking.SignalCut(scans, dtype=dtype, comm=tcomm)
signal_map = mapmaking.SignalMap(scans, area, comm=tcomm, sys=osys)
# Weights
weights = [mapmaking.FilterWindow(config.get("tod_window"))]
# And equation system
eqsys = mapmaking.Eqsys(scans, [signal_cut, signal_map],
weights=weights,
dtype=dtype,
comm=tcomm)
def get_scans(area, signal, bore, dets, noise, seed=0, real=None, noise_override=None):
scans = []
# Get real scan information if necessary
L.debug("real")
if real:
real_scans = []
filedb.init()
db = filedb.data
ids = fileb.scans[real].ids
for id in ids:
try:
real_scans.append(actscan.ACTScan(db[id]))
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (id, str(e)))
# Dets
L.debug("dets")
sim_dets = []
toks = dets.split(":")
if toks[0] == "scattered":
ngroup, nper, rad = int(toks[1]), int(toks[2]), float(toks[3])
sim_dets = [scansim.dets_scattered(ngroup, nper,rad=rad*np.pi/180/60)]
margin = rad*np.pi/180/60
elif toks[0] == "real":
ndet = int(toks[1])
dslice = slice(0,ndet) if ndet > 0 else slice(None)
sim_dets = [bunch.Bunch(comps=s.comps[dslice], offsets=s.offsets[dslice]) for s in real_scans]
margin = np.max([np.sum(s.offsets**2,1)**0.5 for s in sim_dets])
else: raise ValueError
# Boresight. Determines our number of scans
L.debug("bore")
sim_bore = []
toks = bore.split(":")
if toks[0] == "grid":
nscan, density, short = int(toks[1]), float(toks[2]), float(toks[3])
for i in range(nscan):
tbox = shorten(area.box(),i%2,short)
sim_bore.append(scansim.scan_grid(tbox, density*np.pi/180/60, dir=i, margin=margin))
elif toks[0] == "ces":
nscan = int(toks[1])
azs = [float(w)*utils.degree for w in toks[2].split(",")]
els = [float(w)*utils.degree for w in toks[3].split(",")]
mjd0 = float(toks[4])
dur = float(toks[5])
azrate= float(toks[6]) if len(toks) > 6 else 1.5*utils.degree
srate = float(toks[7]) if len(toks) > 7 else 400
nsamp = utils.nint(dur*srate)
for i in range(nscan):
mjd = mjd0 + dur*(i//(2*len(els)))/(24*3600)
el = els[(i//2)%len(els)]
az1, az2 = azs
if i%2 == 1: az1, az2 = -az2, -az1
box = np.array([[az1,el],[az2,el]])
sim_bore.append(scansim.scan_ceslike(nsamp, box, mjd0=mjd, srate=srate, azrate=azrate))
elif toks[0] == "real":
sim_bore = [bunch.Bunch(boresight=s.boresight, hwp_phase=s.hwp_phase, sys=s.sys, site=s.site, mjd0=s.mjd0) for s in real_scans]
else: raise ValueError
nsim = len(sim_bore)
# Make one det info per scan
sim_dets = sim_dets*(nsim/len(sim_dets))+sim_dets[:nsim%len(sim_dets)]
# Noise
L.debug("noise")
sim_nmat = []
toks = noise.split(":")
nonoise = False
if toks[0] == "1/f":
sigma, alpha, fknee = [float(v) for v in toks[1:4]]
nonoise = sigma < 0
for i in range(nsim):
sim_nmat.append(scansim.oneoverf_noise(sim_dets[i].comps.shape[0], sim_bore[i].boresight.shape[0], sigma=np.abs(sigma), alpha=alpha, fknee=fknee))
elif toks[0] == "detcorr":
sigma, alpha, fknee = [float(v) for v in toks[1:4]]
nonoise = sigma < 0
for i in range(nsim):
sim_nmat.append(scansim.oneoverf_detcorr_noise(sim_dets[i].comps.shape[0], sim_bore[i].boresight.shape[0], sigma=np.abs(sigma), alpha=alpha, fknee=fknee))
elif toks[0] == "real":
scale = 1.0 if len(toks) < 2 else float(toks[1])
for i,s in enumerate(real_scans):
ndet = len(sim_dets[i].offsets)
nmat = s.noise[:ndet]*scale**-2
sim_nmat.append(nmat)
else: raise ValueError
noise_scale = not nonoise if noise_override is None else noise_override
sim_nmat = sim_nmat*(nsim/len(sim_nmat))+sim_nmat[:nsim%len(sim_nmat)]
# Signal
L.debug("signal")
toks = signal.split(":")
if toks[0] == "none":
for i in range(nsim):
scans.append(scansim.SimPlain(sim_bore[i], sim_dets[i], sim_nmat[i], seed=seed+i, noise_scale=noise_scale))
elif toks[0] == "ptsrc":
# This one always operates in the same coordinates as
nsrc, amp, fwhm = int(toks[1]), float(toks[2]), float(toks[3])
np.random.seed(seed)
sim_srcs = scansim.rand_srcs(area.box(), nsrc, amp, abs(fwhm)*np.pi/180/60, rand_fwhm=fwhm<0)
for i in range(nsim):
scans.append(scansim.SimSrcs(sim_bore[i], sim_dets[i], sim_srcs, sim_nmat[i], seed=seed+i, noise_scale=noise_scale))
elif toks[0] == "vsrc":
# Create a single variable source
ra, dec, fwhm = float(toks[1])*np.pi/180, float(toks[2])*np.pi/180, float(toks[3])*np.pi/180/60
amps = [float(t) for t in toks[4].split(",")]
for i in range(nsim):
sim_srcs = bunch.Bunch(pos=np.array([[dec,ra]]),amps=np.array([[amps[i],0,0,0]]), beam=np.array([fwhm/(8*np.log(2)**0.5)]))
scans.append(scansim.SimSrcs(sim_bore[i], sim_dets[i], sim_srcs, sim_nmat[i], seed=seed+i, noise_scale=noise_scale, nsigma=20))
elif toks[0] == "cmb":
np.random.seed(seed)
ps = powspec.read_spectrum(toks[1])
sim_map = enmap.rand_map(area.shape, area.wcs, ps)
for i in range(nsim):
scans.append(scansim.SimMap(sim_bore[i], sim_dets[i], sim_map, sim_nmat[i], seed=seed+i, noise_scale=noise_scale))
else: raise ValueError
return scans
print("%s has 0 srcs: skipping" % id)
continue
try:
nsrc = len(sids)
print("%s has %d srcs: %s" % (id, nsrc, ", ".join(
["%d (%.1f)" % (i, a) for i, a in zip(sids, amps[sids])])))
except TypeError as e:
print("Weird: %s" % e)
print(sids)
print(amps)
continue
# Read the data
entry = filedb.data[id]
try:
scan = actscan.ACTScan(entry, verbose=verbose >= 2)
if scan.ndet < 2 or scan.nsamp < 1:
raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print("%s skipped: %s" % (id, e))
continue
# Apply downsampling
scan = scan[:, ::down]
# Prepeare our samples
scan.tod = scan.get_samples()
utils.deslope(scan.tod, w=5, inplace=True)
scan.tod = scan.tod.astype(dtype)
# Background subtraction
if args.sub:
Pmap = pmat.PmatMap(scan, background, sys=sys)
sshape, swcs = enmap.read_map_geometry(args.mapsub)
pixbox = enmap.pixbox_of(swcs, shape, wcs)
if not use_dmap:
refmap = enmap.read_map(args.mapsub,
pixbox=pixbox).astype(dtype)
else:
refmap = dmap.read_map(args.mapsub,
pixbox=pixbox,
bbox=mybbox,
comm=comm).astype(dtype)
refmap = signal.prepare(refmap)
# Get the frequency and beam for this chunk. We assume that
# this is the same for every member of the chunk, so we only need
# to do this for one scan
scan = actscan.ACTScan(filedb.data[chunk_ids[inds[0]]])
_, dets = actdata.split_detname(scan.dets)
beam = scan.beam
freq = scan.array_info.info.nom_freq[dets[0]]
barea = planet9.calc_beam_area(scan.beam)
# Get the conversion from ref-freq flux to observed amplitude. This includes
# dilution by the beam area
flux2amp = 1 / utils.flux_factor(barea, args.fref * 1e9, utils.T_cmb)
fref2freq = utils.planck(freq * 1e9, args.Tref) / utils.planck(
args.fref * 1e9, args.Tref)
rfact = flux2amp * fref2freq * 1e3 # 1e3 for flux in mJy and amp in uK
# only work will be 3,ny,nx. The rest are scalar. Will copy in-out as necessary
work = signal.work()
rhs = area[0]
div = rhs.copy()
