def update(self, tod, srate): # If we have noise estimation cuts, we must gapfill these # before measuring the noise, and restore them afterwards if self.cut is not None: vals = self.cut.extract_samples(tod) gapfill.gapfill(tod, self.cut, inplace=True) try: if self.model == "jon": ft = fft.rfft(tod) * tod.shape[1]**-0.5 noise_model = detvecs_jon(ft, srate, cut_bins=self.spikes) elif self.model == "uncorr": ft = fft.rfft(tod) * tod.shape[1]**-0.5 noise_model = detvecs_simple(ft, srate) elif self.model == "white": noise_model = nmat.NoiseMatrix(len(tod)) elif self.model == "scaled": ft = fft.rfft(tod) * tod.shape[1]**-0.5 noise_model = detvecs_scaled(ft, srate) else: raise ValueError("Unknown noise model '%s'" % self.model) except (errors.ModelError, np.linalg.LinAlgError, AssertionError) as e: print "Warning: Noise model fit failed for tod with shape %s. Assigning zero weight" % str(tod.shape) noise_model = nmat.NmatNull(np.arange(tod.shape[0])) if self.cut is not None: self.cut.insert_samples(tod, vals) return noise_model
def update(self, tod, srate): # If we have noise estimation cuts, we must gapfill these # before measuring the noise, and restore them afterwards if self.cut is not None: vals = self.cut.extract_samples(tod) gapfill.gapfill(tod, self.cut, inplace=True) try: if self.model == "jon": ft = fft.rfft(tod) * tod.shape[1]**-0.5 noise_model = detvecs_jon(ft, srate, cut_bins=self.spikes) elif self.model == "uncorr": ft = fft.rfft(tod) * tod.shape[1]**-0.5 noise_model = detvecs_simple(ft, srate) elif self.model == "white": noise_model = nmat.NoiseMatrix(len(tod)) elif self.model == "scaled": ft = fft.rfft(tod) * tod.shape[1]**-0.5 noise_model = detvecs_scaled(ft, srate) else: raise ValueError("Unknown noise model '%s'" % self.model) except (errors.ModelError, np.linalg.LinAlgError, AssertionError) as e: print("Warning: Noise model fit failed for tod with shape %s. Assigning zero weight" % str(tod.shape)) noise_model = nmat.NmatNull(np.arange(tod.shape[0])) if self.cut is not None: self.cut.insert_samples(tod, vals) return noise_model
def deproject_vecs_smooth(tods, dark, nmode=50, cuts=None, deslope=True, inplace=True): if not inplace: tods=tods.copy() dark = dark.copy() ftod = fft.rfft(tods) fdark = fft.rfft(dark) fdark = todops.smooth_basis_fourier(ftod, fdark) smooth= np.zeros((fdark.shape[0],dark.shape[1]),dtype=dark.dtype) fft.ifft(fdark, smooth, normalize=True) todops.fit_basis(tods, smooth, highpass=nmode, cuts=cuts, clean_tod=True) if deslope: utils.deslope(tods, w=8, inplace=True)
def smooth(tod, srate, fknee=10, alpha=10): ft = fft.rfft(tod) freq = fft.rfftfreq(tod.shape[-1])*srate flt = 1/(1+(freq/fknee)**alpha) ft *= flt fft.ifft(ft, tod, normalize=True) return tod
def __call__(self, poss, amps, taus, dets=None): if dets is None: dets = np.arange(len(self.rdata.dets)) rmask = np.in1d(self.rdata.detmap, dets) if np.sum(rmask) == 0: return np.zeros([0, self.rdata.pos.shape[1]], float) rpos = self.rdata.pos[rmask] mask = self.mask[rmask] detinds = build_detinds(self.rdata.detmap[rmask], dets) # Evaluate the plain beam r = np.sum((rpos - poss[detinds][:, None])**2, -1)**0.5 bpix = r / self.dr model = utils.interpol(self.rdata.beam[1], bpix[None], mask_nan=False, order=1) # Must mask invalid regions *before* fourier stuff model *= mask # Apply the butterworth filter and time constants fmodel = fft.rfft(model) tfilters = filters.tconst_filter(self.rdata.freqs[None], taus[:, None]) * self.rdata.butter fmodel *= tfilters[detinds] fft.ifft(fmodel, model, normalize=True) # Apply the amplitudes model *= amps[detinds, None] return model
def calc_cmode_corrfun(ushape, uwcs, offset_upos, sigma, nsigma=10): """Compute the real-space correlation function for the atmospheric common mode in unskewed coordinates. The result has an arbitrary overall scaling.""" res = enmap.zeros(ushape, uwcs) # Generate corrfun around center of map upos = offset_upos + np.mean(res.box(),0)[:,None] # We will work on a smaller cutout to speed things up pad = sigma*nsigma box = np.array([np.min(upos,1)-pad,np.max(upos,1)+pad]) pixbox = res.sky2pix(box.T).T work = res[pixbox[0,0]:pixbox[1,0],pixbox[0,1]:pixbox[1,1]] posmap = work.posmap() # Generate each part of the corrfun as a gaussian in real space. # Could do this in fourier space, but easier to get subpixel precision this way # (not that that is very important, though) for p in upos.T: r2 = np.sum((posmap-p[:,None,None])**2,0) work += np.exp(-0.5*r2/sigma**2) # Convolute with itself mirrored to get the actual correlation function fres = fft.rfft(res, axes=[-2,-1]) fres *= np.conj(fres) fft.ifft(fres, res, axes=[-2,-1]) res /= np.max(res) return res
def smooth(tod, srate): ft = fft.rfft(tod) freq = fft.rfftfreq(tod.shape[-1])*srate flt = 1/(1+(freq/model_fknee)**model_alpha) ft *= flt fft.ifft(ft, tod, normalize=True) return tod
def smooth(tod, srate, fknee=10, alpha=10): ft = fft.rfft(tod) freq = fft.rfftfreq(tod.shape[-1]) * srate flt = 1 / (1 + (freq / fknee)**alpha) ft *= flt fft.ifft(ft, tod, normalize=True) return tod
def calc_model_constrained(tod, cut, srate=400, mask_scale=0.3, lim=3e-4, maxiter=50, verbose=False): # First do some simple gapfilling to avoid messing up the noise model tod = sampcut.gapfill_linear(cut, tod, inplace=False) ft = fft.rfft(tod) * tod.shape[1]**-0.5 iN = nmat_measure.detvecs_jon(ft, srate) del ft iV = iN.ivar * mask_scale def A(x): x = x.reshape(tod.shape) Ax = iN.apply(x.copy()) Ax += sampcut.gapfill_const(cut, x * iV[:, None], 0, inplace=True) return Ax.reshape(-1) b = sampcut.gapfill_const(cut, tod * iV[:, None], 0, inplace=True).reshape(-1) x0 = sampcut.gapfill_linear(cut, tod).reshape(-1) solver = cg.CG(A, b, x0) while solver.i < maxiter and solver.err > lim: solver.step() if verbose: print("%5d %15.7e" % (solver.i, solver.err)) res = solver.x.reshape(tod.shape) res = smooth(res, srate) return res
def smooth(tod, srate): ft = fft.rfft(tod) freq = fft.rfftfreq(tod.shape[-1]) * srate flt = 1 / (1 + (freq / model_fknee)**model_alpha) ft *= flt fft.ifft(ft, tod, normalize=True) return tod
def calc_cmode_corrfun(ushape, uwcs, offset_upos, sigma, nsigma=10): """Compute the real-space correlation function for the atmospheric common mode in unskewed coordinates. The result has an arbitrary overall scaling.""" res = enmap.zeros(ushape, uwcs) # Generate corrfun around center of map upos = offset_upos + np.mean(res.box(), 0)[:, None] # We will work on a smaller cutout to speed things up pad = sigma * nsigma box = np.array([np.min(upos, 1) - pad, np.max(upos, 1) + pad]) pixbox = res.sky2pix(box.T).T work = res[pixbox[0, 0]:pixbox[1, 0], pixbox[0, 1]:pixbox[1, 1]] posmap = work.posmap() # Generate each part of the corrfun as a gaussian in real space. # Could do this in fourier space, but easier to get subpixel precision this way # (not that that is very important, though) for p in upos.T: r2 = np.sum((posmap - p[:, None, None])**2, 0) work += np.exp(-0.5 * r2 / sigma**2) # Convolute with itself mirrored to get the actual correlation function fres = fft.rfft(res, axes=[-2, -1]) fres *= np.conj(fres) fft.ifft(fres, res, axes=[-2, -1]) res /= np.max(res) return res
def apply(self, arr, inplace=False): # Because of our padding and multiplication by the hitcount # before this, we should be safely apodized, and can assume # periodic boundaries if not inplace: arr = np.array(arr) ft = fft.rfft(arr, axes=[-2]) ft *= self.spec_full[:,None] return fft.ifft(ft, arr, axes=[-2], normalize=True)
def broaden_beam_hor(tod, d, ibeam, obeam): ft = fft.rfft(tod) k = 2*np.pi*fft.rfftfreq(d.nsamp, 1/d.srate) el = np.mean(d.boresight[2,::100]) skyspeed = d.speed*np.cos(el) sigma = (obeam**2-ibeam**2)**0.5 ft *= np.exp(-0.5*(sigma/skyspeed)**2*k**2) fft.ifft(ft, tod, normalize=True)
def apply(self, arr, inplace=False): # Because of our padding and multiplication by the hitcount # before this, we should be safely apodized, and can assume # periodic boundaries if not inplace: arr = np.array(arr) ft = fft.rfft(arr, axes=[-2]) ft *= self.spec_full[:, None] return fft.ifft(ft, arr, axes=[-2], normalize=True)
def estimate_atmosphere(tod, region_cut, srate, fknee, alpha): model = gapfill.gapfill_joneig(tod, region_cut, inplace=False) ft = fft.rfft(model) freq = fft.rfftfreq(model.shape[-1]) * srate flt = 1 / (1 + (freq / fknee)**alpha) ft *= flt fft.ifft(ft, model, normalize=True) return model
def estimate_atmosphere(tod, region_cut, srate, fknee, alpha): model = gapfill.gapfill_joneig(tod, region_cut, inplace=False) ft = fft.rfft(model) freq = fft.rfftfreq(model.shape[-1])*srate flt = 1/(1+(freq/fknee)**alpha) ft *= flt fft.ifft(ft, model, normalize=True) return model
def broaden_beam_hor(tod, d, ibeam, obeam): ft = fft.rfft(tod) k = 2 * np.pi * fft.rfftfreq(d.nsamp, 1 / d.srate) el = np.mean(d.boresight[2, ::100]) skyspeed = d.speed * np.cos(el) sigma = (obeam**2 - ibeam**2)**0.5 ft *= np.exp(-0.5 * (sigma / skyspeed)**2 * k**2) fft.ifft(ft, tod, normalize=True)
def apply(self, tod): nmat.apply_window(tod, self.window) ft = fft.rfft(tod) self.nmat.apply_ft(ft, tod.shape[-1], tod.dtype) if self.filter is not None: ft *= self.filter fft.irfft(ft, tod, flags=['FFTW_ESTIMATE','FFTW_DESTROY_INPUT']) nmat.apply_window(tod, self.window) return tod
def calibrate_dark_fourier(data): """Fourier deconvolution of dark detectors. Can't do this completely, as we don't have time constants.""" require(data, ["dark_tod", "srate"]) if data.dark_tod.size == 0: return data ft = fft.rfft(data.dark_tod) freqs = np.linspace(0, data.srate/2, ft.shape[-1]) butter = filters.butterworth_filter(freqs) ft /= butter[None] fft.irfft(ft, data.dark_tod, normalize=True) return data
def A(self, x): map = self.dof.unzip(x) res = map*0 for work in self.workspaces: # This is normall P'N"P. In our case wmap = enmap.zeros(work.geometry.shape, work.geometry.lwcs, work.geometry.dtype) work.pmat.forward(wmap, map) #wmap[:] = array_ops.matmul(work.hdiv_norm_sqrt, wmap, [0,1]) wmap *= work.hdiv_norm_sqrt ft = fft.rfft(wmap) ft *= work.wfilter fft.ifft(ft, wmap, normalize=True) wmap *= work.hdiv_norm_sqrt # Noise weighting would go here. No weighting for now #wmap[:] = array_ops.matmul(np.rollaxis(work.hdiv_norm_sqrt,1), wmap, [0,1]) work.pmat.backward(wmap, res) res = utils.allreduce(res, self.comm) return self.dof.zip(res)
def __init__(self, fwhm, vel, dec_ref, fknee, alpha, nsigma=1000, nsub=50, order=3): # Vel is [dra,ddec]/s. sigma = fwhm / (8 * np.log(2))**0.5 res = sigma / nsub vel = np.array(vel) vel[0] *= np.cos(dec_ref) speed = np.sum(vel**2)**0.5 # Build coordinate system along velocity npoint = 2 * nsigma * nsub x = (np.arange(npoint) - npoint / 2) * res # Build the beam along the velocity vbeam = np.exp(-0.5 * (x**2 / sigma**2)) # Apply fourier filter. Our angular step size is res radians. This # corresponds to a time step of res/speed in seconds. fbeam = fft.rfft(vbeam) freq = fft.rfftfreq(npoint, res / speed) fbeam[1:] /= 1 + (freq[1:] / fknee)**-alpha vbeam = fft.ifft(fbeam, vbeam, normalize=True) # Beam should be zero at large distances vbeam -= vbeam[0] # Prefilter for fast lookups vbeam = utils.interpol_prefilter(vbeam, npre=0, order=order) # The total beam will be this beam times a normal one in the # perpendicular direction. self.dec_ref = dec_ref self.e_para = vel / np.sum(vel**2)**0.5 self.e_orto = np.array([-self.e_para[1], self.e_para[0]]) self.sigma = sigma self.res = res self.vbeam = vbeam self.order = order # Not really necessary to store these self.fwhm = fwhm self.vel = vel # physical self.fknee = fknee self.alpha = alpha
def A(self, x): map = self.dof.unzip(x) res = map * 0 for work in self.workspaces: # This is normall P'N"P. In our case wmap = enmap.zeros(work.geometry.shape, work.geometry.lwcs, work.geometry.dtype) work.pmat.forward(wmap, map) #wmap[:] = array_ops.matmul(work.hdiv_norm_sqrt, wmap, [0,1]) wmap *= work.hdiv_norm_sqrt ft = fft.rfft(wmap) ft *= work.wfilter fft.ifft(ft, wmap, normalize=True) wmap *= work.hdiv_norm_sqrt # Noise weighting would go here. No weighting for now #wmap[:] = array_ops.matmul(np.rollaxis(work.hdiv_norm_sqrt,1), wmap, [0,1]) work.pmat.backward(wmap, res) res = utils.allreduce(res, self.comm) return self.dof.zip(res)
def calc_model_constrained(tod, cut, srate=400, mask_scale=0.3, lim=3e-4, maxiter=50, verbose=False): # First do some simple gapfilling to avoid messing up the noise model tod = sampcut.gapfill_linear(cut, tod, inplace=False) ft = fft.rfft(tod) * tod.shape[1]**-0.5 iN = nmat_measure.detvecs_jon(ft, srate) del ft iV = iN.ivar*mask_scale def A(x): x = x.reshape(tod.shape) Ax = iN.apply(x.copy()) Ax += sampcut.gapfill_const(cut, x*iV[:,None], 0, inplace=True) return Ax.reshape(-1) b = sampcut.gapfill_const(cut, tod*iV[:,None], 0, inplace=True).reshape(-1) x0 = sampcut.gapfill_linear(cut, tod).reshape(-1) solver = cg.CG(A, b, x0) while solver.i < maxiter and solver.err > lim: solver.step() if verbose: print "%5d %15.7e" % (solver.i, solver.err) return solver.x.reshape(tod.shape)
def calibrate_tod_fourier(data): """Deconvolve instrument filters and time constants from TOD""" require(data, ["tod", "tau", "srate", "speed", "mce_params"]) if data.tod.size == 0: return data ft = fft.rfft(data.tod) freqs = np.linspace(0, data.srate/2, ft.shape[-1]) # Deconvolve the butterworth filter butter = filters.mce_filter(freqs, data.mce_fsamp, data.mce_params) ft /= butter # And the time constants for di in range(len(ft)): ft[di] /= filters.tconst_filter(freqs, data.tau[di]) ## Optinally apply the beam aspect ratio correction #hbeam, vbeam = np.array(map(float,config.get("fix_beam_aspect").split(":")))*utils.arcmin*utils.fwhm #if vbeam != hbeam: # el = np.mean(data.boresight[2,::100]) # k = 2*np.pi*freqs # skyspeed = data.speed * np.cos(el) # tsigma = (vbeam**2-hbeam**2)**0.5/skyspeed # ft *= np.exp(-0.5*tsigma**2*k**2) fft.irfft(ft, data.tod, normalize=True) #np.savetxt("test_enki1/tod_detau.txt", data.tod[0]) del ft return data
def __init__(self, fwhm, vel, dec_ref, fknee, alpha, nsigma=1000, nsub=50, order=3): # Vel is [dra,ddec]/s. sigma = fwhm/(8*np.log(2))**0.5 res = sigma/nsub vel = np.array(vel) vel[0]*= np.cos(dec_ref) speed = np.sum(vel**2)**0.5 # Build coordinate system along velocity npoint = 2*nsigma*nsub x = (np.arange(npoint)-npoint/2)*res # Build the beam along the velocity vbeam = np.exp(-0.5*(x**2/sigma**2)) # Apply fourier filter. Our angular step size is res radians. This # corresponds to a time step of res/speed in seconds. fbeam = fft.rfft(vbeam) freq = fft.rfftfreq(npoint, res/speed) fbeam[1:] /= 1 + (freq[1:]/fknee)**-alpha vbeam = fft.ifft(fbeam, vbeam, normalize=True) # Beam should be zero at large distances vbeam -= vbeam[0] # Prefilter for fast lookups vbeam = utils.interpol_prefilter(vbeam, npre=0, order=order) # The total beam will be this beam times a normal one in the # perpendicular direction. self.dec_ref = dec_ref self.e_para = vel/np.sum(vel**2)**0.5 self.e_orto = np.array([-self.e_para[1],self.e_para[0]]) self.sigma = sigma self.res = res self.vbeam = vbeam self.order = order # Not really necessary to store these self.fwhm = fwhm self.vel = vel # physical self.fknee = fknee self.alpha = alpha
try: d = actscan.ACTScan(entry) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("Tod contains no valid data") d = d[:, ::downsample] d = d[:, :] except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (id, e.message)) continue L.debug("Processing %s" % id) # Get the actual tod tod = d.get_samples() tod -= np.mean(tod, 1)[:, None] tod = tod.astype(dtype) # Compute the per-detector spectrum ft = fft.rfft(tod) * d.nsamp**-0.5 tfilter, binds = measure_inv_noise_spectrum(ft, nbin) # Apply inverse noise weighting to the tod ft *= tfilter[:, binds] ft *= d.nsamp**-0.5 fft.ifft(ft, tod) del ft my_rhs, my_hdiv, my_yhits = project_tod_on_workspace(d, tod, wgeo) my_wfilter = project_binned_spec_on_workspace( tfilter, d.srate, my_yhits, wgeo) # Add to the totals tot_work.rhs += my_rhs tot_work.hdiv += my_hdiv tot_work.wfilter += my_wfilter tot_work.ids.append(id) del my_rhs, my_hdiv, my_yhits, my_wfilter
def get_model(s, area): pos = area.posmap().reshape(2,-1)[::-1].T model = np.rollaxis(s.get_model(pos),-1).reshape(-1,area.shape[1],area.shape[2]) return enmap.ndmap(model, area.wcs)[:area.shape[0]] if args.measure is None: scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real) else: # Build noise model the same way we do for the real data, i.e. based on # measuring data itself. But do that based on a version with more noise # than the real one, to simulate realistic S/N ratios without needing # too many samples scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real, noise_override=args.measure) nmats = [] for scan in scans: ft = fft.rfft(scan.get_samples()) * scan.nsamp**-0.5 nmats.append(nmat_measure.detvecs_jon(ft, 400.0, shared=True)) scans = get_scans(area, args.signal, args.bore, args.dets, args.noise, seed=args.seed, real=args.real) for scan,nmat in zip(scans,nmats): scan.noise = nmat enmap.write_map(args.odir + "/area.fits", area) model = get_model(scans[0], area) enmap.write_map(args.odir + "/model.fits", model) with open(args.odir + "/tods.txt", "w") as ofile: for i, scan in enumerate(scans): L.info("scan %2d/%d" % (i+1,len(scans))) enlib.scan.write_scan(args.odir + "/scan%03d.hdf" % i, scan) ofile.write("%s/scan%03d.hdf\n" % (args.odir, i))
fields = ["gain","tconst","cut","tod","boresight", "noise_cut"] if args.spikecut: fields.append("spikes") if args.imap: fields += ["polangle","point_offsets","site"] d = data.read(entry, fields) ; t.append(time.time()) d = data.calibrate(d) ; t.append(time.time()) if args.imap: # Make a full scan object, so we can perform pointing projection # operations d.noise = None scan = data.ACTScan(entry, d=d) imap.map = imap.map.astype(d.tod.dtype, copy=False) pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys) # Subtract input map from tod inplace pmap.forward(d.tod, imap.map, tmul=1, mmul=-1) utils.deslope(d.tod, w=8, inplace=True) ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5 ; t.append(time.time()) spikes = d.spikes[:2].T if args.spikecut else None if model == "old": noise = nmat_measure.detvecs_old(ft, d.srate, d.dets) elif model == "jon": noise = nmat_measure.detvecs_jon(ft, d.srate, d.dets, shared, cut_bins=spikes) elif model == "simple": noise = nmat_measure.detvecs_simple(ft, d.srate, d.dets) elif model == "joint": noise = nmat_measure.detvecs_joint(ft, d.srate, d.dets, cut_bins=spikes) t.append(time.time()) with h5py.File("%s/%s.hdf" % (args.odir, id),"w") as hfile: nmat.write_nmat(hfile, noise) ; t.append(time.time()) if args.covtest: # Measure full cov per bin ndet = ft.shape[0]
tod.signal += readout_sim.gen_noise(tod.nsamp) elif args.noise == "replace": tod.signal[:] = readout_sim.gen_noise(tod.nsamp) tod.signal = readout_sim.apply_filters(tod.signal, exp=-1) # Detector measures 1/4 of the signal due to how the interferrometry # is set up. gain = 1.0/4 # Get out tod samples d = tod.signal / gain #dump(pre+"_1", d) # Deconvolve the sample window. Each of our samples # is approximately the integral of the signal inside its # duration. If subsample_num is 1, then we assume the # data has no sample window. if config.subsample_num > 1: fd = fft.rfft(d, axes=[-1]) fd /= np.sinc(np.arange(fd.shape[-1],dtype=float)/d.shape[-1]) d = fft.ifft(fd, d, axes=[-1], normalize=True) # Undo the effect of drift in theta and spin during each stroke d = d.reshape(d.shape[0], ntheta, nspin, ndelay) #d = pixie.fix_drift(d) nt,nspin,ndelay = d.shape[-3:] if args.spin == 1: d = pixie.froll( d.reshape(-1,nt,nspin*ndelay), np.arange(nspin*ndelay)/float(nspin*ndelay), -2).reshape(-1,nt,nspin,ndelay) elif args.spin == 2: d = d.reshape(-1, nt*2, nspin*ndelay/2) x = np.arange(nspin*ndelay/2)/float(nspin*ndelay/2) d1= pixie.froll(d, x, -2)[:,0::2] # even half-spins
for ind in range(comm.rank, nlabel, comm.size): id = ids[labels == ind] order = np.argsort([i[-1] for i in id]) id = id[order] if len(id) != nsub: continue print "Processing " + "+".join(id) entries = [filedb.data[i] for i in id] try: d = actdata.read(entries) d = actdata.calibrate(d, exclude=["autocut"]) except errors.DataMissing as e: print "Skipping %s: %s" % (id, str(e)) continue tod = d.tod.astype(dtype) del d.tod ft = fft.rfft(tod) del tod if corr is None: ndet = d.array_info.ndet corr = np.zeros((nbin, ndet, ndet), dtype=dtype) hits = np.zeros((nbin, ndet, ndet), dtype=int) var = np.zeros((nbin, ndet)) pos = np.zeros((ndet, 2)) doff = actdata.read(entries, fields=["point_offsets"]) pos[doff.dets] = doff.point_template # Fourier-sample bins ibins = (fbins / (d.srate / 2) * ft.shape[1]).astype(int) # Measure each bin
diff = tod[:,1:]-tod[:,:-1] diff = diff[:,:diff.shape[-1]//csize*csize].reshape(d.ndet,-1,csize) ivar = 1/(np.median(np.mean(diff**2,-1),-1)/2**0.5) del diff # Generate planet cut with bench.show("planet cut"): planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site, args.planet, R) # Subtract atmospheric model with bench.show("atm model"): model= gapfill.gapfill_joneig(tod, planet_cut, inplace=False) # Estimate noise level asens = np.sum(ivar)**-0.5 / d.srate**0.5 print(asens) with bench.show("smooth"): ft = fft.rfft(model) freq = fft.rfftfreq(model.shape[-1])*d.srate flt = 1/(1+(freq/model_fknee)**model_alpha) ft *= flt fft.ifft(ft, model, normalize=True) del ft, flt, freq with bench.show("atm subtract"): tod -= model del model tod = tod.astype(dtype, copy=False) # Should now be reasonably clean of correlated noise, so we can from now on use # a white noise model. with bench.show("pmat"): P = PmatTot(scan, srcpos, sys=sys) N = NmatWhite(ivar)
try: d = actscan.ACTScan(entry) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("Tod contains no valid data") d = d[:,::downsample] d = d[:,:] except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (id, e.message)) continue L.debug("Processing %s" % id) # Get the actual tod tod = d.get_samples() tod -= np.mean(tod,1)[:,None] tod = tod.astype(dtype) # Compute the per-detector spectrum ft = fft.rfft(tod) * d.nsamp ** -0.5 tfilter, binds = measure_inv_noise_spectrum(ft, nbin) # Apply inverse noise weighting to the tod ft *= tfilter[:,binds] ft *= d.nsamp ** -0.5 fft.ifft(ft, tod) del ft my_rhs, my_hdiv, my_yhits = project_tod_on_workspace(d, tod, wgeo) my_wfilter = project_binned_spec_on_workspace(tfilter, d.srate, my_yhits, wgeo) # Add to the totals tot_work.rhs += my_rhs tot_work.hdiv += my_hdiv tot_work.wfilter += my_wfilter tot_work.ids.append(id) del my_rhs, my_hdiv, my_yhits, my_wfilter # Reduce
dets=args.dets, downsample=config.get("downsample")): id = pids[ind] with bench.mark("pbuild"): # Build pointing matrices pmap = pmat.PmatMap(d, area) pcut = pmat.PmatCut(d) with bench.mark("tod"): # Get tod tod = d.get_samples() tod -= np.mean(tod, 1)[:, None] tod = tod.astype(dtype) junk = np.zeros(pcut.njunk, dtype=dtype) with bench.mark("nmat"): # Build noise model ft = fft.rfft(tod) * tod.shape[1]**-0.5 nmat = nmat_measure.detvecs_simple(ft, d.srate) del ft with bench.mark("rhs"): # Calc rhs, accumulating into pattern total nmat.apply(tod) pcut.backward(tod, junk) pmap.backward(tod, rhs) with bench.mark("hitmap"): # Calc hitcount map, accumulating into pattern total myhits = area * 0 tod[:] = 1 pcut.backward(tod, junk) pmap.backward(tod, myhits) myhits = myhits[0] hits += myhits
def hpass(a, n): f = fft.rfft(a) f[...,:n] = 0 return fft.ifft(f,a.copy(),normalize=True)
d, exclude=(["autocut"] if not args.no_autocut else [])) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, OSError, errors.DataMissing) as e: print "Skipped (%s)" % (str(e)) continue print "Processing %s" % id srates[i] = d.srate mce_fsamps[i] = d.mce_fsamp mce_params[i] = d.mce_params[:4] # Compute the power spectrum d.tod = d.tod.astype(dtype) nsamp = d.nsamp srate = d.srate ifmax = d.srate / 2 ft = fft.rfft(d.tod) / (nsamp * srate)**0.5 nfreq = ft.shape[-1] del d.tod ps = np.abs(ft)**2 # Det specs zoom = int(round(ifmax / args.fmax_zoom)) dets = actdata.split_detname(d.dets)[1] dspecs[i, dets] = bin(ps, args.nbin_det) dzooms[i, dets] = bin(ps, args.nbin_zoom, zoom=zoom) # Aggregate specs. First bin in small bins dhigh, binds = bin(ps, args.nbin, return_inds=True) nhits[i] = np.bincount(binds, minlength=args.nbin) # Then compute quantiles tspecs[0, i] = np.median(dhigh, 0) tspecs[1, i] = np.percentile(dhigh, 15.86553, 0) tspecs[2, i] = np.percentile(dhigh, 84.13447, 0)
#d = d.reshape(-1,nspin,nblock*bsize) hfile["d3"] = d hfile["delay"] = delay def stroke2spec(arr, wcs): ndelay = arr.shape[-1] spec = fft.rfft(arr, axes=[-1]).real[..., ::2] * 2 / ndelay owcs = pixie.wcs_delay2spec(wcs, ndelay / 4) return spec, owcs # Try to recover first spectrum #spec, swcs = pixie.delay2spec(d, dwcs, axis=-1) spec, swcs = stroke2spec(d, dwcs) freq = swcs.wcs_pix2world(np.arange(spec.shape[-1]), 0)[0] hfile["spec"] = spec hfile["freq"] = freq # Fourier-decompose spin fd = fft.rfft(d, axes=[1]) dcomp = np.array([fd[:, 0].real, fd[:, 2].real, fd[:, 2].imag]) hfile["dcomp"] = dcomp # And spectra from this #scomp, _ = pixie.delay2spec(dcomp, dwcs, axis=-1) scomp, _ = stroke2spec(dcomp, dwcs) hfile["scomp"] = scomp hfile.close()
def highpass(tod, f, srate=400): tod = tod.copy() ft = fft.rfft(tod) ft[:,:int(f/float(srate)*tod.shape[1])] = 0 fft.ifft(ft, tod, normalize=True) return tod
if args.imap: fields += ["polangle", "point_offsets", "site"] d = data.read(entry, fields) t.append(time.time()) d = data.calibrate(d) t.append(time.time()) if args.imap: # Make a full scan object, so we can perform pointing projection # operations d.noise = None scan = data.ACTScan(entry, d=d) imap.map = imap.map.astype(d.tod.dtype, copy=False) pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys) # Subtract input map from tod inplace pmap.forward(d.tod, imap.map, tmul=1, mmul=-1) utils.deslope(d.tod, w=8, inplace=True) ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5 t.append(time.time()) spikes = d.spikes[:2].T if args.spikecut else None if model == "old": noise = nmat_measure.detvecs_old(ft, d.srate, d.dets) elif model == "jon": noise = nmat_measure.detvecs_jon(ft, d.srate, d.dets, shared, cut_bins=spikes) elif model == "simple": noise = nmat_measure.detvecs_simple(ft, d.srate, d.dets) elif model == "joint": noise = nmat_measure.detvecs_joint(ft, d.srate,
ind1 = chunk*chunk_size ind2 = min(ind1+chunk_size, ntod) for ind in range(ind1+comm.rank, ind2, comm.size): id = ids[ind] entry = filedb.data[id] try: d = actdata.read(entry, fields=["gain","tconst","cut","tod","boresight","apex"]) d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, errors.DataMissing) as e: print("Skipped (%s)" % (e)) continue # Compute the power spectrum nsamp = d.nsamp srate = d.srate ft = fft.rfft(d.tod) del d.tod ps = np.abs(ft)**2/(nsamp*srate) del ft # Choose a random set of detectors to be examples. iex = np.random.permutation(len(ps))[:nex] examples[ind] = ps[iex] # Want mean and dev between detectors. These can # be built from ps and ps**2 stats[:,ind] = calc_bin_stats(ps, bin_freqs, nsamp, srate) n, a, a2 = calc_bin_stats(ps, bin_freqs_status, nsamp, srate) del ps amean = a/n adev = (a2/n - amean**2)**0.5 print(id + " " + "".join([get_token(me,de) for me,de in zip(amean,adev)]) + " %5.2f" % d.apex.pwv) tot_stats = utils.allreduce(stats, comm)
d = actdata.read(entry, fields=["array_info", "tags", "site", "mce_filter", "gain","cut","tod","boresight"]) d = actdata.calibrate(d, exclude=["tod_fourier"]+(["autocut"] if not args.no_autocut else [])) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, errors.DataMissing) as e: print "Skipped (%s)" % (e.message) continue print "Processing %s" % id srates[i] = d.srate mce_fsamps[i] = d.mce_fsamp mce_params[i] = d.mce_params[:4] # Compute the power spectrum d.tod = d.tod.astype(dtype) nsamp = d.nsamp srate = d.srate ifmax = d.srate/2 ft = fft.rfft(d.tod) / (nsamp*srate)**0.5 nfreq = ft.shape[-1] del d.tod ps = np.abs(ft)**2 # Det specs zoom = int(round(ifmax/args.fmax_zoom)) dspecs[i,d.dets] = bin(ps, args.nbin_det) dzooms[i,d.dets] = bin(ps, args.nbin_zoom, zoom=zoom) # Aggregate specs. First bin in small bins dhigh, binds = bin(ps, args.nbin, return_inds=True) # Then compute quantiles tspecs[0,i] = np.median(dhigh,0) tspecs[1,i] = np.percentile(dhigh,15.86553,0) tspecs[2,i] = np.percentile(dhigh,84.13447,0) tspecs[3,i] = np.min(dhigh,0) tspecs[4,i] = np.max(dhigh,0)
diff = tod[:,1:]-tod[:,:-1] diff = diff[:,:diff.shape[-1]/csize*csize].reshape(d.ndet,-1,csize) ivar = 1/(np.median(np.mean(diff**2,-1),-1)/2**0.5) del diff # Generate planet cut with bench.show("planet cut"): planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site, args.planet, R) # Subtract atmospheric model with bench.show("atm model"): model= gapfill.gapfill_joneig(tod, planet_cut, inplace=False) # Estimate noise level asens = np.sum(ivar)**-0.5 / d.srate**0.5 print asens with bench.show("smooth"): ft = fft.rfft(model) freq = fft.rfftfreq(model.shape[-1])*d.srate flt = 1/(1+(freq/model_fknee)**model_alpha) ft *= flt fft.ifft(ft, model, normalize=True) del ft, flt, freq with bench.show("atm subtract"): tod -= model del model tod = tod.astype(dtype, copy=False) # Should now be reasonably clean of correlated noise. # Proceed to make simple binned map with bench.show("actscan"): scan = actscan.ACTScan(entry, d=d) with bench.show("pmat"): pmap = pmat.PmatMap(scan, area, sys=sys)
for ind in range(comm.rank, nlabel, comm.size): id = ids[labels==ind] order = np.argsort([i[-1] for i in id]) id = id[order] if len(id) != nsub: continue print "Processing " + "+".join(id) entries = [filedb.data[i] for i in id] try: d = actdata.read(entries) d = actdata.calibrate(d, exclude=["autocut"]) except errors.DataMissing as e: print "Skipping %s: %s" % (id,e.message) continue tod = d.tod.astype(dtype) del d.tod ft = fft.rfft(tod) del tod if corr is None: ndet = d.array_info.ndet corr = np.zeros((nbin,ndet,ndet),dtype=dtype) hits = np.zeros((nbin,ndet,ndet),dtype=int) var = np.zeros((nbin,ndet)) pos = np.zeros((ndet,2)) doff = actdata.read(entries, fields=["point_offsets"]) pos[doff.dets] = doff.point_template # Fourier-sample bins ibins = (fbins/(d.srate/2)*ft.shape[1]).astype(int) # Measure each bin
try: d = actdata.read( entry, fields=["gain", "tconst", "cut", "tod", "boresight", "hwp"]) d = actdata.calibrate(d, exclude=["tod_fourier", "autocut"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, errors.DataMissing) as e: print "Skipped (%s)" % (e.message) continue print "Read %s" % id # Filter the HWP signal print "no hwp filter" #d.tod = todfilter.filter_poly_jon(d.tod, d.boresight[1], hwp=d.hwp) ft = fft.rfft(d.tod) ps = np.abs(ft)**2 / (d.tod.shape[1] * srate) inds = bins * ps.shape[1] / fmax bfreqs = np.mean(bins, 1) #tod_moo = fft.irfft(ft, normalize=True) #ind = np.where(d.dets==640)[0] #with h5py.File("foo.hdf", "w") as hfile: # hfile["moo"] = tod_moo[ind] # hfile["ps"] = ps[ind] # hfile["tod"] = d.tod[ind] rms_raw = np.array([np.mean(ps[:, b[0]:b[1]], 1) for b in inds]).T**0.5 #print rms_raw[ind] # Compute amount of deconvolution
def stroke2spec(arr, wcs): ndelay = arr.shape[-1] spec = fft.rfft(arr, axes=[-1]).real[..., ::2] * 2 / ndelay owcs = pixie.wcs_delay2spec(wcs, ndelay / 4) return spec, owcs