def update(self, tod, srate):
# If we have noise estimation cuts, we must gapfill these
# before measuring the noise, and restore them afterwards
if self.cut is not None:
vals = self.cut.extract_samples(tod)
gapfill.gapfill(tod, self.cut, inplace=True)
try:
if self.model == "jon":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_jon(ft, srate, cut_bins=self.spikes)
elif self.model == "uncorr":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_simple(ft, srate)
elif self.model == "white":
noise_model = nmat.NoiseMatrix(len(tod))
elif self.model == "scaled":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_scaled(ft, srate)
else:
raise ValueError("Unknown noise model '%s'" % self.model)
except (errors.ModelError, np.linalg.LinAlgError, AssertionError) as e:
print "Warning: Noise model fit failed for tod with shape %s. Assigning zero weight" % str(tod.shape)
noise_model = nmat.NmatNull(np.arange(tod.shape[0]))
if self.cut is not None:
self.cut.insert_samples(tod, vals)
return noise_model
def update(self, tod, srate):
# If we have noise estimation cuts, we must gapfill these
# before measuring the noise, and restore them afterwards
if self.cut is not None:
vals = self.cut.extract_samples(tod)
gapfill.gapfill(tod, self.cut, inplace=True)
try:
if self.model == "jon":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_jon(ft, srate, cut_bins=self.spikes)
elif self.model == "uncorr":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_simple(ft, srate)
elif self.model == "white":
noise_model = nmat.NoiseMatrix(len(tod))
elif self.model == "scaled":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_scaled(ft, srate)
else:
raise ValueError("Unknown noise model '%s'" % self.model)
except (errors.ModelError, np.linalg.LinAlgError, AssertionError) as e:
print("Warning: Noise model fit failed for tod with shape %s. Assigning zero weight" % str(tod.shape))
noise_model = nmat.NmatNull(np.arange(tod.shape[0]))
if self.cut is not None:
self.cut.insert_samples(tod, vals)
return noise_model
def deproject_vecs_smooth(tods, dark, nmode=50, cuts=None, deslope=True, inplace=True): if not inplace: tods=tods.copy() dark = dark.copy() ftod = fft.rfft(tods) fdark = fft.rfft(dark) fdark = todops.smooth_basis_fourier(ftod, fdark) smooth= np.zeros((fdark.shape[0],dark.shape[1]),dtype=dark.dtype) fft.ifft(fdark, smooth, normalize=True) todops.fit_basis(tods, smooth, highpass=nmode, cuts=cuts, clean_tod=True) if deslope: utils.deslope(tods, w=8, inplace=True)
def smooth(tod, srate, fknee=10, alpha=10): ft = fft.rfft(tod) freq = fft.rfftfreq(tod.shape[-1])*srate flt = 1/(1+(freq/fknee)**alpha) ft *= flt fft.ifft(ft, tod, normalize=True) return tod
def __call__(self, poss, amps, taus, dets=None):
if dets is None: dets = np.arange(len(self.rdata.dets))
rmask = np.in1d(self.rdata.detmap, dets)
if np.sum(rmask) == 0:
return np.zeros([0, self.rdata.pos.shape[1]], float)
rpos = self.rdata.pos[rmask]
mask = self.mask[rmask]
detinds = build_detinds(self.rdata.detmap[rmask], dets)
# Evaluate the plain beam
r = np.sum((rpos - poss[detinds][:, None])**2, -1)**0.5
bpix = r / self.dr
model = utils.interpol(self.rdata.beam[1],
bpix[None],
mask_nan=False,
order=1)
# Must mask invalid regions *before* fourier stuff
model *= mask
# Apply the butterworth filter and time constants
fmodel = fft.rfft(model)
tfilters = filters.tconst_filter(self.rdata.freqs[None],
taus[:, None]) * self.rdata.butter
fmodel *= tfilters[detinds]
fft.ifft(fmodel, model, normalize=True)
# Apply the amplitudes
model *= amps[detinds, None]
return model
def calc_cmode_corrfun(ushape, uwcs, offset_upos, sigma, nsigma=10): """Compute the real-space correlation function for the atmospheric common mode in unskewed coordinates. The result has an arbitrary overall scaling.""" res = enmap.zeros(ushape, uwcs) # Generate corrfun around center of map upos = offset_upos + np.mean(res.box(),0)[:,None] # We will work on a smaller cutout to speed things up pad = sigma*nsigma box = np.array([np.min(upos,1)-pad,np.max(upos,1)+pad]) pixbox = res.sky2pix(box.T).T work = res[pixbox[0,0]:pixbox[1,0],pixbox[0,1]:pixbox[1,1]] posmap = work.posmap() # Generate each part of the corrfun as a gaussian in real space. # Could do this in fourier space, but easier to get subpixel precision this way # (not that that is very important, though) for p in upos.T: r2 = np.sum((posmap-p[:,None,None])**2,0) work += np.exp(-0.5*r2/sigma**2) # Convolute with itself mirrored to get the actual correlation function fres = fft.rfft(res, axes=[-2,-1]) fres *= np.conj(fres) fft.ifft(fres, res, axes=[-2,-1]) res /= np.max(res) return res
def smooth(tod, srate): ft = fft.rfft(tod) freq = fft.rfftfreq(tod.shape[-1])*srate flt = 1/(1+(freq/model_fknee)**model_alpha) ft *= flt fft.ifft(ft, tod, normalize=True) return tod
def smooth(tod, srate, fknee=10, alpha=10):
ft = fft.rfft(tod)
freq = fft.rfftfreq(tod.shape[-1]) * srate
flt = 1 / (1 + (freq / fknee)**alpha)
ft *= flt
fft.ifft(ft, tod, normalize=True)
return tod
def calc_model_constrained(tod,
cut,
srate=400,
mask_scale=0.3,
lim=3e-4,
maxiter=50,
verbose=False):
# First do some simple gapfilling to avoid messing up the noise model
tod = sampcut.gapfill_linear(cut, tod, inplace=False)
ft = fft.rfft(tod) * tod.shape[1]**-0.5
iN = nmat_measure.detvecs_jon(ft, srate)
del ft
iV = iN.ivar * mask_scale
def A(x):
x = x.reshape(tod.shape)
Ax = iN.apply(x.copy())
Ax += sampcut.gapfill_const(cut, x * iV[:, None], 0, inplace=True)
return Ax.reshape(-1)
b = sampcut.gapfill_const(cut, tod * iV[:, None], 0,
inplace=True).reshape(-1)
x0 = sampcut.gapfill_linear(cut, tod).reshape(-1)
solver = cg.CG(A, b, x0)
while solver.i < maxiter and solver.err > lim:
solver.step()
if verbose:
print("%5d %15.7e" % (solver.i, solver.err))
res = solver.x.reshape(tod.shape)
res = smooth(res, srate)
return res
def smooth(tod, srate):
ft = fft.rfft(tod)
freq = fft.rfftfreq(tod.shape[-1]) * srate
flt = 1 / (1 + (freq / model_fknee)**model_alpha)
ft *= flt
fft.ifft(ft, tod, normalize=True)
return tod
def calc_cmode_corrfun(ushape, uwcs, offset_upos, sigma, nsigma=10):
"""Compute the real-space correlation function for the atmospheric
common mode in unskewed coordinates. The result has an arbitrary
overall scaling."""
res = enmap.zeros(ushape, uwcs)
# Generate corrfun around center of map
upos = offset_upos + np.mean(res.box(), 0)[:, None]
# We will work on a smaller cutout to speed things up
pad = sigma * nsigma
box = np.array([np.min(upos, 1) - pad, np.max(upos, 1) + pad])
pixbox = res.sky2pix(box.T).T
work = res[pixbox[0, 0]:pixbox[1, 0], pixbox[0, 1]:pixbox[1, 1]]
posmap = work.posmap()
# Generate each part of the corrfun as a gaussian in real space.
# Could do this in fourier space, but easier to get subpixel precision this way
# (not that that is very important, though)
for p in upos.T:
r2 = np.sum((posmap - p[:, None, None])**2, 0)
work += np.exp(-0.5 * r2 / sigma**2)
# Convolute with itself mirrored to get the actual correlation function
fres = fft.rfft(res, axes=[-2, -1])
fres *= np.conj(fres)
fft.ifft(fres, res, axes=[-2, -1])
res /= np.max(res)
return res
def apply(self, arr, inplace=False): # Because of our padding and multiplication by the hitcount # before this, we should be safely apodized, and can assume # periodic boundaries if not inplace: arr = np.array(arr) ft = fft.rfft(arr, axes=[-2]) ft *= self.spec_full[:,None] return fft.ifft(ft, arr, axes=[-2], normalize=True)
def broaden_beam_hor(tod, d, ibeam, obeam): ft = fft.rfft(tod) k = 2*np.pi*fft.rfftfreq(d.nsamp, 1/d.srate) el = np.mean(d.boresight[2,::100]) skyspeed = d.speed*np.cos(el) sigma = (obeam**2-ibeam**2)**0.5 ft *= np.exp(-0.5*(sigma/skyspeed)**2*k**2) fft.ifft(ft, tod, normalize=True)
def apply(self, arr, inplace=False):
# Because of our padding and multiplication by the hitcount
# before this, we should be safely apodized, and can assume
# periodic boundaries
if not inplace: arr = np.array(arr)
ft = fft.rfft(arr, axes=[-2])
ft *= self.spec_full[:, None]
return fft.ifft(ft, arr, axes=[-2], normalize=True)
def estimate_atmosphere(tod, region_cut, srate, fknee, alpha):
model = gapfill.gapfill_joneig(tod, region_cut, inplace=False)
ft = fft.rfft(model)
freq = fft.rfftfreq(model.shape[-1]) * srate
flt = 1 / (1 + (freq / fknee)**alpha)
ft *= flt
fft.ifft(ft, model, normalize=True)
return model
def estimate_atmosphere(tod, region_cut, srate, fknee, alpha): model = gapfill.gapfill_joneig(tod, region_cut, inplace=False) ft = fft.rfft(model) freq = fft.rfftfreq(model.shape[-1])*srate flt = 1/(1+(freq/fknee)**alpha) ft *= flt fft.ifft(ft, model, normalize=True) return model
def broaden_beam_hor(tod, d, ibeam, obeam):
ft = fft.rfft(tod)
k = 2 * np.pi * fft.rfftfreq(d.nsamp, 1 / d.srate)
el = np.mean(d.boresight[2, ::100])
skyspeed = d.speed * np.cos(el)
sigma = (obeam**2 - ibeam**2)**0.5
ft *= np.exp(-0.5 * (sigma / skyspeed)**2 * k**2)
fft.ifft(ft, tod, normalize=True)
def apply(self, tod):
nmat.apply_window(tod, self.window)
ft = fft.rfft(tod)
self.nmat.apply_ft(ft, tod.shape[-1], tod.dtype)
if self.filter is not None: ft *= self.filter
fft.irfft(ft, tod, flags=['FFTW_ESTIMATE','FFTW_DESTROY_INPUT'])
nmat.apply_window(tod, self.window)
return tod
def calibrate_dark_fourier(data): """Fourier deconvolution of dark detectors. Can't do this completely, as we don't have time constants.""" require(data, ["dark_tod", "srate"]) if data.dark_tod.size == 0: return data ft = fft.rfft(data.dark_tod) freqs = np.linspace(0, data.srate/2, ft.shape[-1]) butter = filters.butterworth_filter(freqs) ft /= butter[None] fft.irfft(ft, data.dark_tod, normalize=True) return data
def A(self, x): map = self.dof.unzip(x) res = map*0 for work in self.workspaces: # This is normall P'N"P. In our case wmap = enmap.zeros(work.geometry.shape, work.geometry.lwcs, work.geometry.dtype) work.pmat.forward(wmap, map) #wmap[:] = array_ops.matmul(work.hdiv_norm_sqrt, wmap, [0,1]) wmap *= work.hdiv_norm_sqrt ft = fft.rfft(wmap) ft *= work.wfilter fft.ifft(ft, wmap, normalize=True) wmap *= work.hdiv_norm_sqrt # Noise weighting would go here. No weighting for now #wmap[:] = array_ops.matmul(np.rollaxis(work.hdiv_norm_sqrt,1), wmap, [0,1]) work.pmat.backward(wmap, res) res = utils.allreduce(res, self.comm) return self.dof.zip(res)
def __init__(self,
fwhm,
vel,
dec_ref,
fknee,
alpha,
nsigma=1000,
nsub=50,
order=3):
# Vel is [dra,ddec]/s.
sigma = fwhm / (8 * np.log(2))**0.5
res = sigma / nsub
vel = np.array(vel)
vel[0] *= np.cos(dec_ref)
speed = np.sum(vel**2)**0.5
# Build coordinate system along velocity
npoint = 2 * nsigma * nsub
x = (np.arange(npoint) - npoint / 2) * res
# Build the beam along the velocity
vbeam = np.exp(-0.5 * (x**2 / sigma**2))
# Apply fourier filter. Our angular step size is res radians. This
# corresponds to a time step of res/speed in seconds.
fbeam = fft.rfft(vbeam)
freq = fft.rfftfreq(npoint, res / speed)
fbeam[1:] /= 1 + (freq[1:] / fknee)**-alpha
vbeam = fft.ifft(fbeam, vbeam, normalize=True)
# Beam should be zero at large distances
vbeam -= vbeam[0]
# Prefilter for fast lookups
vbeam = utils.interpol_prefilter(vbeam, npre=0, order=order)
# The total beam will be this beam times a normal one in the
# perpendicular direction.
self.dec_ref = dec_ref
self.e_para = vel / np.sum(vel**2)**0.5
self.e_orto = np.array([-self.e_para[1], self.e_para[0]])
self.sigma = sigma
self.res = res
self.vbeam = vbeam
self.order = order
# Not really necessary to store these
self.fwhm = fwhm
self.vel = vel # physical
self.fknee = fknee
self.alpha = alpha
def A(self, x):
map = self.dof.unzip(x)
res = map * 0
for work in self.workspaces:
# This is normall P'N"P. In our case
wmap = enmap.zeros(work.geometry.shape, work.geometry.lwcs,
work.geometry.dtype)
work.pmat.forward(wmap, map)
#wmap[:] = array_ops.matmul(work.hdiv_norm_sqrt, wmap, [0,1])
wmap *= work.hdiv_norm_sqrt
ft = fft.rfft(wmap)
ft *= work.wfilter
fft.ifft(ft, wmap, normalize=True)
wmap *= work.hdiv_norm_sqrt
# Noise weighting would go here. No weighting for now
#wmap[:] = array_ops.matmul(np.rollaxis(work.hdiv_norm_sqrt,1), wmap, [0,1])
work.pmat.backward(wmap, res)
res = utils.allreduce(res, self.comm)
return self.dof.zip(res)
def calc_model_constrained(tod, cut, srate=400, mask_scale=0.3, lim=3e-4, maxiter=50, verbose=False): # First do some simple gapfilling to avoid messing up the noise model tod = sampcut.gapfill_linear(cut, tod, inplace=False) ft = fft.rfft(tod) * tod.shape[1]**-0.5 iN = nmat_measure.detvecs_jon(ft, srate) del ft iV = iN.ivar*mask_scale def A(x): x = x.reshape(tod.shape) Ax = iN.apply(x.copy()) Ax += sampcut.gapfill_const(cut, x*iV[:,None], 0, inplace=True) return Ax.reshape(-1) b = sampcut.gapfill_const(cut, tod*iV[:,None], 0, inplace=True).reshape(-1) x0 = sampcut.gapfill_linear(cut, tod).reshape(-1) solver = cg.CG(A, b, x0) while solver.i < maxiter and solver.err > lim: solver.step() if verbose: print "%5d %15.7e" % (solver.i, solver.err) return solver.x.reshape(tod.shape)
def calibrate_tod_fourier(data):
"""Deconvolve instrument filters and time constants from TOD"""
require(data, ["tod", "tau", "srate", "speed", "mce_params"])
if data.tod.size == 0: return data
ft = fft.rfft(data.tod)
freqs = np.linspace(0, data.srate/2, ft.shape[-1])
# Deconvolve the butterworth filter
butter = filters.mce_filter(freqs, data.mce_fsamp, data.mce_params)
ft /= butter
# And the time constants
for di in range(len(ft)):
ft[di] /= filters.tconst_filter(freqs, data.tau[di])
## Optinally apply the beam aspect ratio correction
#hbeam, vbeam = np.array(map(float,config.get("fix_beam_aspect").split(":")))*utils.arcmin*utils.fwhm
#if vbeam != hbeam:
# el = np.mean(data.boresight[2,::100])
# k = 2*np.pi*freqs
# skyspeed = data.speed * np.cos(el)
# tsigma = (vbeam**2-hbeam**2)**0.5/skyspeed
# ft *= np.exp(-0.5*tsigma**2*k**2)
fft.irfft(ft, data.tod, normalize=True)
#np.savetxt("test_enki1/tod_detau.txt", data.tod[0])
del ft
return data
def __init__(self, fwhm, vel, dec_ref, fknee, alpha, nsigma=1000, nsub=50, order=3): # Vel is [dra,ddec]/s. sigma = fwhm/(8*np.log(2))**0.5 res = sigma/nsub vel = np.array(vel) vel[0]*= np.cos(dec_ref) speed = np.sum(vel**2)**0.5 # Build coordinate system along velocity npoint = 2*nsigma*nsub x = (np.arange(npoint)-npoint/2)*res # Build the beam along the velocity vbeam = np.exp(-0.5*(x**2/sigma**2)) # Apply fourier filter. Our angular step size is res radians. This # corresponds to a time step of res/speed in seconds. fbeam = fft.rfft(vbeam) freq = fft.rfftfreq(npoint, res/speed) fbeam[1:] /= 1 + (freq[1:]/fknee)**-alpha vbeam = fft.ifft(fbeam, vbeam, normalize=True) # Beam should be zero at large distances vbeam -= vbeam[0] # Prefilter for fast lookups vbeam = utils.interpol_prefilter(vbeam, npre=0, order=order) # The total beam will be this beam times a normal one in the # perpendicular direction. self.dec_ref = dec_ref self.e_para = vel/np.sum(vel**2)**0.5 self.e_orto = np.array([-self.e_para[1],self.e_para[0]]) self.sigma = sigma self.res = res self.vbeam = vbeam self.order = order # Not really necessary to store these self.fwhm = fwhm self.vel = vel # physical self.fknee = fknee self.alpha = alpha
try:
d = actscan.ACTScan(entry)
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("Tod contains no valid data")
d = d[:, ::downsample]
d = d[:, :]
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (id, e.message))
continue
L.debug("Processing %s" % id)
# Get the actual tod
tod = d.get_samples()
tod -= np.mean(tod, 1)[:, None]
tod = tod.astype(dtype)
# Compute the per-detector spectrum
ft = fft.rfft(tod) * d.nsamp**-0.5
tfilter, binds = measure_inv_noise_spectrum(ft, nbin)
# Apply inverse noise weighting to the tod
ft *= tfilter[:, binds]
ft *= d.nsamp**-0.5
fft.ifft(ft, tod)
del ft
my_rhs, my_hdiv, my_yhits = project_tod_on_workspace(d, tod, wgeo)
my_wfilter = project_binned_spec_on_workspace(
tfilter, d.srate, my_yhits, wgeo)
# Add to the totals
tot_work.rhs += my_rhs
tot_work.hdiv += my_hdiv
tot_work.wfilter += my_wfilter
tot_work.ids.append(id)
del my_rhs, my_hdiv, my_yhits, my_wfilter
def get_model(s, area):
pos = area.posmap().reshape(2,-1)[::-1].T
model = np.rollaxis(s.get_model(pos),-1).reshape(-1,area.shape[1],area.shape[2])
return enmap.ndmap(model, area.wcs)[:area.shape[0]]
if args.measure is None:
scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real)
else:
# Build noise model the same way we do for the real data, i.e. based on
# measuring data itself. But do that based on a version with more noise
# than the real one, to simulate realistic S/N ratios without needing
# too many samples
scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real, noise_override=args.measure)
nmats = []
for scan in scans:
ft = fft.rfft(scan.get_samples()) * scan.nsamp**-0.5
nmats.append(nmat_measure.detvecs_jon(ft, 400.0, shared=True))
scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real)
for scan,nmat in zip(scans,nmats):
scan.noise = nmat
enmap.write_map(args.odir + "/area.fits", area)
model = get_model(scans[0], area)
enmap.write_map(args.odir + "/model.fits", model)
with open(args.odir + "/tods.txt", "w") as ofile:
for i, scan in enumerate(scans):
L.info("scan %2d/%d" % (i+1,len(scans)))
enlib.scan.write_scan(args.odir + "/scan%03d.hdf" % i, scan)
ofile.write("%s/scan%03d.hdf\n" % (args.odir, i))
fields = ["gain","tconst","cut","tod","boresight", "noise_cut"]
if args.spikecut: fields.append("spikes")
if args.imap: fields += ["polangle","point_offsets","site"]
d = data.read(entry, fields) ; t.append(time.time())
d = data.calibrate(d) ; t.append(time.time())
if args.imap:
# Make a full scan object, so we can perform pointing projection
# operations
d.noise = None
scan = data.ACTScan(entry, d=d)
imap.map = imap.map.astype(d.tod.dtype, copy=False)
pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys)
# Subtract input map from tod inplace
pmap.forward(d.tod, imap.map, tmul=1, mmul=-1)
utils.deslope(d.tod, w=8, inplace=True)
ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5 ; t.append(time.time())
spikes = d.spikes[:2].T if args.spikecut else None
if model == "old":
noise = nmat_measure.detvecs_old(ft, d.srate, d.dets)
elif model == "jon":
noise = nmat_measure.detvecs_jon(ft, d.srate, d.dets, shared, cut_bins=spikes)
elif model == "simple":
noise = nmat_measure.detvecs_simple(ft, d.srate, d.dets)
elif model == "joint":
noise = nmat_measure.detvecs_joint(ft, d.srate, d.dets, cut_bins=spikes)
t.append(time.time())
with h5py.File("%s/%s.hdf" % (args.odir, id),"w") as hfile:
nmat.write_nmat(hfile, noise) ; t.append(time.time())
if args.covtest:
# Measure full cov per bin
ndet = ft.shape[0]
tod.signal += readout_sim.gen_noise(tod.nsamp) elif args.noise == "replace": tod.signal[:] = readout_sim.gen_noise(tod.nsamp) tod.signal = readout_sim.apply_filters(tod.signal, exp=-1) # Detector measures 1/4 of the signal due to how the interferrometry # is set up. gain = 1.0/4 # Get out tod samples d = tod.signal / gain #dump(pre+"_1", d) # Deconvolve the sample window. Each of our samples # is approximately the integral of the signal inside its # duration. If subsample_num is 1, then we assume the # data has no sample window. if config.subsample_num > 1: fd = fft.rfft(d, axes=[-1]) fd /= np.sinc(np.arange(fd.shape[-1],dtype=float)/d.shape[-1]) d = fft.ifft(fd, d, axes=[-1], normalize=True) # Undo the effect of drift in theta and spin during each stroke d = d.reshape(d.shape[0], ntheta, nspin, ndelay) #d = pixie.fix_drift(d) nt,nspin,ndelay = d.shape[-3:] if args.spin == 1: d = pixie.froll( d.reshape(-1,nt,nspin*ndelay), np.arange(nspin*ndelay)/float(nspin*ndelay), -2).reshape(-1,nt,nspin,ndelay) elif args.spin == 2: d = d.reshape(-1, nt*2, nspin*ndelay/2) x = np.arange(nspin*ndelay/2)/float(nspin*ndelay/2) d1= pixie.froll(d, x, -2)[:,0::2] # even half-spins
for ind in range(comm.rank, nlabel, comm.size):
id = ids[labels == ind]
order = np.argsort([i[-1] for i in id])
id = id[order]
if len(id) != nsub: continue
print "Processing " + "+".join(id)
entries = [filedb.data[i] for i in id]
try:
d = actdata.read(entries)
d = actdata.calibrate(d, exclude=["autocut"])
except errors.DataMissing as e:
print "Skipping %s: %s" % (id, str(e))
continue
tod = d.tod.astype(dtype)
del d.tod
ft = fft.rfft(tod)
del tod
if corr is None:
ndet = d.array_info.ndet
corr = np.zeros((nbin, ndet, ndet), dtype=dtype)
hits = np.zeros((nbin, ndet, ndet), dtype=int)
var = np.zeros((nbin, ndet))
pos = np.zeros((ndet, 2))
doff = actdata.read(entries, fields=["point_offsets"])
pos[doff.dets] = doff.point_template
# Fourier-sample bins
ibins = (fbins / (d.srate / 2) * ft.shape[1]).astype(int)
# Measure each bin
diff = tod[:,1:]-tod[:,:-1]
diff = diff[:,:diff.shape[-1]//csize*csize].reshape(d.ndet,-1,csize)
ivar = 1/(np.median(np.mean(diff**2,-1),-1)/2**0.5)
del diff
# Generate planet cut
with bench.show("planet cut"):
planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site,
args.planet, R)
# Subtract atmospheric model
with bench.show("atm model"):
model= gapfill.gapfill_joneig(tod, planet_cut, inplace=False)
# Estimate noise level
asens = np.sum(ivar)**-0.5 / d.srate**0.5
print(asens)
with bench.show("smooth"):
ft = fft.rfft(model)
freq = fft.rfftfreq(model.shape[-1])*d.srate
flt = 1/(1+(freq/model_fknee)**model_alpha)
ft *= flt
fft.ifft(ft, model, normalize=True)
del ft, flt, freq
with bench.show("atm subtract"):
tod -= model
del model
tod = tod.astype(dtype, copy=False)
# Should now be reasonably clean of correlated noise, so we can from now on use
# a white noise model.
with bench.show("pmat"):
P = PmatTot(scan, srcpos, sys=sys)
N = NmatWhite(ivar)
try:
d = actscan.ACTScan(entry)
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("Tod contains no valid data")
d = d[:,::downsample]
d = d[:,:]
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (id, e.message))
continue
L.debug("Processing %s" % id)
# Get the actual tod
tod = d.get_samples()
tod -= np.mean(tod,1)[:,None]
tod = tod.astype(dtype)
# Compute the per-detector spectrum
ft = fft.rfft(tod) * d.nsamp ** -0.5
tfilter, binds = measure_inv_noise_spectrum(ft, nbin)
# Apply inverse noise weighting to the tod
ft *= tfilter[:,binds]
ft *= d.nsamp ** -0.5
fft.ifft(ft, tod)
del ft
my_rhs, my_hdiv, my_yhits = project_tod_on_workspace(d, tod, wgeo)
my_wfilter = project_binned_spec_on_workspace(tfilter, d.srate, my_yhits, wgeo)
# Add to the totals
tot_work.rhs += my_rhs
tot_work.hdiv += my_hdiv
tot_work.wfilter += my_wfilter
tot_work.ids.append(id)
del my_rhs, my_hdiv, my_yhits, my_wfilter
# Reduce
dets=args.dets,
downsample=config.get("downsample")):
id = pids[ind]
with bench.mark("pbuild"):
# Build pointing matrices
pmap = pmat.PmatMap(d, area)
pcut = pmat.PmatCut(d)
with bench.mark("tod"):
# Get tod
tod = d.get_samples()
tod -= np.mean(tod, 1)[:, None]
tod = tod.astype(dtype)
junk = np.zeros(pcut.njunk, dtype=dtype)
with bench.mark("nmat"):
# Build noise model
ft = fft.rfft(tod) * tod.shape[1]**-0.5
nmat = nmat_measure.detvecs_simple(ft, d.srate)
del ft
with bench.mark("rhs"):
# Calc rhs, accumulating into pattern total
nmat.apply(tod)
pcut.backward(tod, junk)
pmap.backward(tod, rhs)
with bench.mark("hitmap"):
# Calc hitcount map, accumulating into pattern total
myhits = area * 0
tod[:] = 1
pcut.backward(tod, junk)
pmap.backward(tod, myhits)
myhits = myhits[0]
hits += myhits
def hpass(a, n): f = fft.rfft(a) f[...,:n] = 0 return fft.ifft(f,a.copy(),normalize=True)
d, exclude=(["autocut"] if not args.no_autocut else []))
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("empty tod")
except (IOError, OSError, errors.DataMissing) as e:
print "Skipped (%s)" % (str(e))
continue
print "Processing %s" % id
srates[i] = d.srate
mce_fsamps[i] = d.mce_fsamp
mce_params[i] = d.mce_params[:4]
# Compute the power spectrum
d.tod = d.tod.astype(dtype)
nsamp = d.nsamp
srate = d.srate
ifmax = d.srate / 2
ft = fft.rfft(d.tod) / (nsamp * srate)**0.5
nfreq = ft.shape[-1]
del d.tod
ps = np.abs(ft)**2
# Det specs
zoom = int(round(ifmax / args.fmax_zoom))
dets = actdata.split_detname(d.dets)[1]
dspecs[i, dets] = bin(ps, args.nbin_det)
dzooms[i, dets] = bin(ps, args.nbin_zoom, zoom=zoom)
# Aggregate specs. First bin in small bins
dhigh, binds = bin(ps, args.nbin, return_inds=True)
nhits[i] = np.bincount(binds, minlength=args.nbin)
# Then compute quantiles
tspecs[0, i] = np.median(dhigh, 0)
tspecs[1, i] = np.percentile(dhigh, 15.86553, 0)
tspecs[2, i] = np.percentile(dhigh, 84.13447, 0)
#d = d.reshape(-1,nspin,nblock*bsize)
hfile["d3"] = d
hfile["delay"] = delay
def stroke2spec(arr, wcs):
ndelay = arr.shape[-1]
spec = fft.rfft(arr, axes=[-1]).real[..., ::2] * 2 / ndelay
owcs = pixie.wcs_delay2spec(wcs, ndelay / 4)
return spec, owcs
# Try to recover first spectrum
#spec, swcs = pixie.delay2spec(d, dwcs, axis=-1)
spec, swcs = stroke2spec(d, dwcs)
freq = swcs.wcs_pix2world(np.arange(spec.shape[-1]), 0)[0]
hfile["spec"] = spec
hfile["freq"] = freq
# Fourier-decompose spin
fd = fft.rfft(d, axes=[1])
dcomp = np.array([fd[:, 0].real, fd[:, 2].real, fd[:, 2].imag])
hfile["dcomp"] = dcomp
# And spectra from this
#scomp, _ = pixie.delay2spec(dcomp, dwcs, axis=-1)
scomp, _ = stroke2spec(dcomp, dwcs)
hfile["scomp"] = scomp
hfile.close()
def highpass(tod, f, srate=400): tod = tod.copy() ft = fft.rfft(tod) ft[:,:int(f/float(srate)*tod.shape[1])] = 0 fft.ifft(ft, tod, normalize=True) return tod
if args.imap: fields += ["polangle", "point_offsets", "site"]
d = data.read(entry, fields)
t.append(time.time())
d = data.calibrate(d)
t.append(time.time())
if args.imap:
# Make a full scan object, so we can perform pointing projection
# operations
d.noise = None
scan = data.ACTScan(entry, d=d)
imap.map = imap.map.astype(d.tod.dtype, copy=False)
pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys)
# Subtract input map from tod inplace
pmap.forward(d.tod, imap.map, tmul=1, mmul=-1)
utils.deslope(d.tod, w=8, inplace=True)
ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5
t.append(time.time())
spikes = d.spikes[:2].T if args.spikecut else None
if model == "old":
noise = nmat_measure.detvecs_old(ft, d.srate, d.dets)
elif model == "jon":
noise = nmat_measure.detvecs_jon(ft,
d.srate,
d.dets,
shared,
cut_bins=spikes)
elif model == "simple":
noise = nmat_measure.detvecs_simple(ft, d.srate, d.dets)
elif model == "joint":
noise = nmat_measure.detvecs_joint(ft,
d.srate,
ind1 = chunk*chunk_size
ind2 = min(ind1+chunk_size, ntod)
for ind in range(ind1+comm.rank, ind2, comm.size):
id = ids[ind]
entry = filedb.data[id]
try:
d = actdata.read(entry, fields=["gain","tconst","cut","tod","boresight","apex"])
d = actdata.calibrate(d, exclude=["autocut"])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod")
except (IOError, errors.DataMissing) as e:
print("Skipped (%s)" % (e))
continue
# Compute the power spectrum
nsamp = d.nsamp
srate = d.srate
ft = fft.rfft(d.tod)
del d.tod
ps = np.abs(ft)**2/(nsamp*srate)
del ft
# Choose a random set of detectors to be examples.
iex = np.random.permutation(len(ps))[:nex]
examples[ind] = ps[iex]
# Want mean and dev between detectors. These can
# be built from ps and ps**2
stats[:,ind] = calc_bin_stats(ps, bin_freqs, nsamp, srate)
n, a, a2 = calc_bin_stats(ps, bin_freqs_status, nsamp, srate)
del ps
amean = a/n
adev = (a2/n - amean**2)**0.5
print(id + " " + "".join([get_token(me,de) for me,de in zip(amean,adev)]) + " %5.2f" % d.apex.pwv)
tot_stats = utils.allreduce(stats, comm)
d = actdata.read(entry, fields=["array_info", "tags", "site", "mce_filter", "gain","cut","tod","boresight"])
d = actdata.calibrate(d, exclude=["tod_fourier"]+(["autocut"] if not args.no_autocut else []))
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod")
except (IOError, errors.DataMissing) as e:
print "Skipped (%s)" % (e.message)
continue
print "Processing %s" % id
srates[i] = d.srate
mce_fsamps[i] = d.mce_fsamp
mce_params[i] = d.mce_params[:4]
# Compute the power spectrum
d.tod = d.tod.astype(dtype)
nsamp = d.nsamp
srate = d.srate
ifmax = d.srate/2
ft = fft.rfft(d.tod) / (nsamp*srate)**0.5
nfreq = ft.shape[-1]
del d.tod
ps = np.abs(ft)**2
# Det specs
zoom = int(round(ifmax/args.fmax_zoom))
dspecs[i,d.dets] = bin(ps, args.nbin_det)
dzooms[i,d.dets] = bin(ps, args.nbin_zoom, zoom=zoom)
# Aggregate specs. First bin in small bins
dhigh, binds = bin(ps, args.nbin, return_inds=True)
# Then compute quantiles
tspecs[0,i] = np.median(dhigh,0)
tspecs[1,i] = np.percentile(dhigh,15.86553,0)
tspecs[2,i] = np.percentile(dhigh,84.13447,0)
tspecs[3,i] = np.min(dhigh,0)
tspecs[4,i] = np.max(dhigh,0)
def get_model(s, area):
pos = area.posmap().reshape(2,-1)[::-1].T
model = np.rollaxis(s.get_model(pos),-1).reshape(-1,area.shape[1],area.shape[2])
return enmap.ndmap(model, area.wcs)[:area.shape[0]]
if args.measure is None:
scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real)
else:
# Build noise model the same way we do for the real data, i.e. based on
# measuring data itself. But do that based on a version with more noise
# than the real one, to simulate realistic S/N ratios without needing
# too many samples
scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real, noise_override=args.measure)
nmats = []
for scan in scans:
ft = fft.rfft(scan.get_samples()) * scan.nsamp**-0.5
nmats.append(nmat_measure.detvecs_jon(ft, 400.0, shared=True))
scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real)
for scan,nmat in zip(scans,nmats):
scan.noise = nmat
enmap.write_map(args.odir + "/area.fits", area)
model = get_model(scans[0], area)
enmap.write_map(args.odir + "/model.fits", model)
with open(args.odir + "/tods.txt", "w") as ofile:
for i, scan in enumerate(scans):
L.info("scan %2d/%d" % (i+1,len(scans)))
enlib.scan.write_scan(args.odir + "/scan%03d.hdf" % i, scan)
ofile.write("%s/scan%03d.hdf\n" % (args.odir, i))
diff = tod[:,1:]-tod[:,:-1]
diff = diff[:,:diff.shape[-1]/csize*csize].reshape(d.ndet,-1,csize)
ivar = 1/(np.median(np.mean(diff**2,-1),-1)/2**0.5)
del diff
# Generate planet cut
with bench.show("planet cut"):
planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site,
args.planet, R)
# Subtract atmospheric model
with bench.show("atm model"):
model= gapfill.gapfill_joneig(tod, planet_cut, inplace=False)
# Estimate noise level
asens = np.sum(ivar)**-0.5 / d.srate**0.5
print asens
with bench.show("smooth"):
ft = fft.rfft(model)
freq = fft.rfftfreq(model.shape[-1])*d.srate
flt = 1/(1+(freq/model_fknee)**model_alpha)
ft *= flt
fft.ifft(ft, model, normalize=True)
del ft, flt, freq
with bench.show("atm subtract"):
tod -= model
del model
tod = tod.astype(dtype, copy=False)
# Should now be reasonably clean of correlated noise.
# Proceed to make simple binned map
with bench.show("actscan"):
scan = actscan.ACTScan(entry, d=d)
with bench.show("pmat"):
pmap = pmat.PmatMap(scan, area, sys=sys)
for ind in range(comm.rank, nlabel, comm.size): id = ids[labels==ind] order = np.argsort([i[-1] for i in id]) id = id[order] if len(id) != nsub: continue print "Processing " + "+".join(id) entries = [filedb.data[i] for i in id] try: d = actdata.read(entries) d = actdata.calibrate(d, exclude=["autocut"]) except errors.DataMissing as e: print "Skipping %s: %s" % (id,e.message) continue tod = d.tod.astype(dtype) del d.tod ft = fft.rfft(tod) del tod if corr is None: ndet = d.array_info.ndet corr = np.zeros((nbin,ndet,ndet),dtype=dtype) hits = np.zeros((nbin,ndet,ndet),dtype=int) var = np.zeros((nbin,ndet)) pos = np.zeros((ndet,2)) doff = actdata.read(entries, fields=["point_offsets"]) pos[doff.dets] = doff.point_template # Fourier-sample bins ibins = (fbins/(d.srate/2)*ft.shape[1]).astype(int) # Measure each bin
try:
d = actdata.read(
entry,
fields=["gain", "tconst", "cut", "tod", "boresight", "hwp"])
d = actdata.calibrate(d, exclude=["tod_fourier", "autocut"])
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("empty tod")
except (IOError, errors.DataMissing) as e:
print "Skipped (%s)" % (e.message)
continue
print "Read %s" % id
# Filter the HWP signal
print "no hwp filter"
#d.tod = todfilter.filter_poly_jon(d.tod, d.boresight[1], hwp=d.hwp)
ft = fft.rfft(d.tod)
ps = np.abs(ft)**2 / (d.tod.shape[1] * srate)
inds = bins * ps.shape[1] / fmax
bfreqs = np.mean(bins, 1)
#tod_moo = fft.irfft(ft, normalize=True)
#ind = np.where(d.dets==640)[0]
#with h5py.File("foo.hdf", "w") as hfile:
# hfile["moo"] = tod_moo[ind]
# hfile["ps"] = ps[ind]
# hfile["tod"] = d.tod[ind]
rms_raw = np.array([np.mean(ps[:, b[0]:b[1]], 1) for b in inds]).T**0.5
#print rms_raw[ind]
# Compute amount of deconvolution
def stroke2spec(arr, wcs):
ndelay = arr.shape[-1]
spec = fft.rfft(arr, axes=[-1]).real[..., ::2] * 2 / ndelay
owcs = pixie.wcs_delay2spec(wcs, ndelay / 4)
return spec, owcs
