def filter_phase_blockwise(tods, blocks, az, daz=None, cuts=None, niter=None, deslope=True, inplace=True, weight="auto"): """Given a tod[ndet,nsamp], fit for a common azimuth phase signal per block in blocks[nblock][dets], and subtract it. The binning size is given in arc minutes.""" # Loop over and filter each block #np.savetxt("moo0.txt", tods[0]) if not inplace: tods = tods.copy() for bi, block in enumerate(blocks): btod = np.ascontiguousarray(tods[block]) bcut = None if cuts is None else cuts[block] phase = todops.fit_phase_flat(btod, az, daz=daz, cuts=bcut, niter=niter, clean_tod=True, weight=weight) tods[block] = btod if deslope: utils.deslope(tods, w=8, inplace=True) #np.savetxt("moo.txt", tods[0]) #1/0 return tods
def filter_common_blockwise(tods, blocks, cuts=None, niter=None, deslope=True, inplace=True, weight="auto", nmin=5): # Loop over and filter each block #np.savetxt("comtod0.txt", tods[0]) if not inplace: tods = tods.copy() for bi, block in enumerate(blocks): if len(block) < nmin: continue btod = np.ascontiguousarray(tods[block]) bcut = None if cuts is None else cuts[block] common = todops.fit_common(btod, cuts=bcut, niter=niter, clean_tod=True, weight=weight) tods[block] = btod #if 0 in block: np.savetxt("comcom.txt", common) if deslope: utils.deslope(tods, w=8, inplace=True) #np.savetxt("comtod1.txt", tods[0]) #1/0 return tods
def deproject_vecs(tods, dark, nmode=50, cuts=None, deslope=True, inplace=True): """Given a tod[ndet,nsamp] and a set of basis modes dark[nmode,nsamp], fit each tod in to the basis modes and subtract them from the tod. The fit ignores the lowest nmode fourier modes, and cut regions are approximately ignored.""" if not inplace: tods=tods.copy() todops.fit_basis(tods, dark, highpass=nmode, cuts=cuts, clean_tod=True) if deslope: utils.deslope(tods, w=8, inplace=True) return tods
def deproject_vecs_smooth(tods, dark, nmode=50, cuts=None, deslope=True, inplace=True): if not inplace: tods=tods.copy() dark = dark.copy() ftod = fft.rfft(tods) fdark = fft.rfft(dark) fdark = todops.smooth_basis_fourier(ftod, fdark) smooth= np.zeros((fdark.shape[0],dark.shape[1]),dtype=dark.dtype) fft.ifft(fdark, smooth, normalize=True) todops.fit_basis(tods, smooth, highpass=nmode, cuts=cuts, clean_tod=True) if deslope: utils.deslope(tods, w=8, inplace=True)
def calibrate_tod_real(data, nthread=None): """Apply gain to tod, fill gaps and deslope. We only gapfill data that's bad enough that it should be excluded when estimating the noise model.""" require(data, ["tod","gain","cut_basic"]) if data.tod.size == 0: raise errors.DataMissing("No tod samples") data.tod = data.tod.astype(np.int32, copy=False) data.tod /= 128 data.tod = data.tod * (data.gain[:,None]*8) gapfill_helper(data.tod, data.cut_basic) utils.deslope(data.tod, w=8, inplace=True) return data
def calibrate_dark_real(data): """Calibrate dark detectors. Mostly desloping.""" #require(data, ["dark_tod","dark_cut"]) require(data, ["dark_tod"]) if data.dark_tod.size == 0: return data data.dark_tod = data.dark_tod * 1.0 dark_cut = todops.find_spikes(data.dark_tod) gapfill_helper(data.dark_tod, dark_cut) utils.deslope(data.dark_tod, w=8, inplace=True) # Add the cuts to the dataset data += dataset.DataField("dark_cut", dark_cut, sample_index=1, samples=data.datafields["dark_tod"].samples) return data
if args.spikecut: fields.append("spikes") if args.imap: fields += ["polangle", "point_offsets", "site"] d = data.read(entry, fields) t.append(time.time()) d = data.calibrate(d) t.append(time.time()) if args.imap: # Make a full scan object, so we can perform pointing projection # operations d.noise = None scan = data.ACTScan(entry, d=d) imap.map = imap.map.astype(d.tod.dtype, copy=False) pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys) # Subtract input map from tod inplace pmap.forward(d.tod, imap.map, tmul=1, mmul=-1) utils.deslope(d.tod, w=8, inplace=True) ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5 t.append(time.time()) spikes = d.spikes[:2].T if args.spikecut else None if model == "old": noise = nmat_measure.detvecs_old(ft, d.srate, d.dets) elif model == "jon": noise = nmat_measure.detvecs_jon(ft, d.srate, d.dets, shared, cut_bins=spikes) elif model == "simple": noise = nmat_measure.detvecs_simple(ft, d.srate, d.dets) elif model == "joint": noise = nmat_measure.detvecs_joint(ft,
if absdets: d.restrict(dets=absdets) if subdets: d.restrict(dets=d.dets[subdets]) if args.calib: d = actdata.calibrate(d, exclude=["autocut"]) elif args.manual_calib: ops = args.manual_calib.split(",") if "safe" in ops: d.boresight[1:] = utils.unwind(d.boresight[1:], period=360) if "rad" in ops: d.boresight[1:] *= np.pi/180 if "bgap" in ops: bad = (d.flags!=0)*(d.flags!=0x10) for b in d.boresight: gapfill.gapfill_linear(b, bad, inplace=True) if "gain" in ops: d.tod *= d.gain[:,None] if "tgap" in ops: gapfiller = {"copy":gapfill.gapfill_copy, "linear":gapfill.gapfill_linear}[config.get("gapfill")] gapfiller(d.tod, d.cut, inplace=True) if "slope" in ops: utils.deslope(d.tod, w=8, inplace=True) if "deconv" in ops: d = actdata.calibrate(d, operations=["tod_fourier"]) if args.bin > 1: d.tod = resample.downsample_bin(d.tod, steps=[args.bin]) d.boresight = resample.downsample_bin(d.boresight, steps=[args.bin]) d.flags = resample.downsample_bin(d.flags, steps=[args.bin]) oname = args.ofile if len(ids) > 1: oname = "%s/%s.hdf" % (args.ofile, id) with h5py.File(oname, "w") as hfile: if "tod" in d: hfile["tod"] = d.tod if "boresight" in d: hfile["az"] = d.boresight[1] hfile["el"] = d.boresight[2] hfile["dets"] = d.dets try:
def filter_poly_jon(tod, az, weights=None, naz=None, nt=None, niter=None, cuts=None, hwp=None, nhwp=None, deslope=True, inplace=True): """Fix naz Legendre polynomials in az and nt other polynomials in t jointly. Then subtract the best fit from the data. The subtraction is inplace, so tod is modified. If naz or nt are negative, they are fit for, but not subtracted. NOTE: This function may leave tod nonperiodic. """ #moomoo = tod[:8].copy() naz = config.get("gfilter_jon_naz", naz) nt = config.get("gfilter_jon_nt", nt) nhwp= config.get("gfilter_jon_nhwp", nhwp) niter = config.get("gfilter_jon_niter", niter) if not inplace: tod = tod.copy() do_gapfill = cuts is not None #print "Mos", naz, nt, nhwp #print hwp # No point in iterating if we aren't gapfilling if not do_gapfill: niter = 1 if hwp is None or np.all(hwp==0): nhwp = 0 naz, asign = np.abs(naz), np.sign(naz) nt, tsign = np.abs(nt), np.sign(nt) nhwp,hsign = np.abs(nhwp),np.sign(nhwp) d = tod.reshape(-1,tod.shape[-1]) if naz == 0 and nt == 0 and nhwp == 0: return tod # Build our set of basis functions. These are shared # across iterations. B = np.zeros([naz+nt+nhwp,d.shape[-1]],dtype=tod.dtype) if naz > 0: # Build azimuth basis as polynomials x = utils.rescale(az,[-1,1]) for i in range(naz): B[i] = x**(i+1) if nt > 0: x = np.linspace(-1,1,d.shape[-1],endpoint=False) for i in range(nt): B[naz+i] = x**i if nhwp > 0: # Use sin and cos to avoid discontinuities c = np.cos(hwp) s = np.sin(hwp) for i in range(nhwp): j = i/2+1 x = np.cos(j*hwp) if i%2 == 0 else np.sin(j*hwp) B[naz+nt+i] = x for it in range(niter): if do_gapfill: gapfill.gapfill(d, cuts, inplace=True) # Solve for the best fit for each detector, [nbasis,ndet] # B[b,n], d[d,n], amps[b,d] if weights is None: try: amps = np.linalg.solve(B.dot(B.T),B.dot(d.T)) except np.linalg.LinAlgError as e: print "LinAlgError in todfilter. Skipping" continue else: w = weights.reshape(-1,weights.shape[-1]) amps = np.zeros([naz+nt+nhwp,d.shape[0]],dtype=tod.dtype) for di in range(len(tod)): try: amps[:,di] = np.linalg.solve((B*w[di]).dot(B.T),B.dot(w[di]*d[di])) except np.linalg.LinAlgError as e: print "LinAlgError in todfilter di %d. Skipping" % di continue #print "amps", amps[:,2] # Subtract the best fit if asign > 0: d -= amps[:naz].T.dot(B[:naz]) if tsign > 0: d -= amps[naz:naz+nt].T.dot(B[naz:naz+nt]) if hsign > 0: d -= amps[naz+nt:naz+nt+nhwp].T.dot(B[naz+nt:naz+nt+nhwp]) # Why was this necessary? if do_gapfill: gapfill.gapfill(d, cuts, inplace=True) if deslope: utils.deslope(tod, w=8, inplace=True) res = d.reshape(tod.shape) return res
continue # Read the data entry = filedb.data[id] try: scan = actscan.ACTScan(entry, verbose=verbose >= 2) if scan.ndet < 2 or scan.nsamp < 1: raise errors.DataMissing("no data in tod") except errors.DataMissing as e: print("%s skipped: %s" % (id, e)) continue # Apply downsampling scan = scan[:, ::down] # Prepeare our samples scan.tod = scan.get_samples() utils.deslope(scan.tod, w=5, inplace=True) scan.tod = scan.tod.astype(dtype) # Background subtraction if args.sub: Pmap = pmat.PmatMap(scan, background, sys=sys) Pmap.forward(scan.tod, background, tmul=-1) # Build the noise model N = NmatTot(scan, model="uncorr", window=2.0, filter=highpass) P = PmatTot(scan, srcpos[:, sids], perdet=True, sys=sys) # rhs N.apply(scan.tod) rhs = P.backward(scan.tod, ncomp=1) # div
def filter_poly_jon(tod, az, weights=None, naz=None, nt=None, niter=None, cuts=None, hwp=None, nhwp=None, deslope=True, inplace=True, use_phase=None): """Fix naz Legendre polynomials in az and nt other polynomials in t jointly. Then subtract the best fit from the data. The subtraction is inplace, so tod is modified. If naz or nt are negative, they are fit for, but not subtracted. NOTE: This function may leave tod nonperiodic. """ naz = config.get("gfilter_jon_naz", naz) nt = config.get("gfilter_jon_nt", nt) nhwp = config.get("gfilter_jon_nhwp", nhwp) niter = config.get("gfilter_jon_niter", niter) use_phase = config.get("gfilter_jon_phase", use_phase) if not inplace: tod = tod.copy() do_gapfill = cuts is not None # No point in iterating if we aren't gapfilling if not do_gapfill: niter = 1 if hwp is None or np.all(hwp == 0): nhwp = 0 naz, asign = np.abs(naz), np.sign(naz) nt, tsign = np.abs(nt), np.sign(nt) nhwp, hsign = np.abs(nhwp), np.sign(nhwp) d = tod.reshape(-1, tod.shape[-1]) nsamp = d.shape[-1] if naz == 0 and nt == 0 and nhwp == 0: return tod B = [] # Set up our time basis if nt > 0: B.append(np.full((1, nsamp), 1.0, d.dtype)) if nt > 1: t = np.linspace(-1, 1, nsamp, endpoint=False) B.append(utils.build_legendre(t, nt - 1)) if naz > 0: if not use_phase: # Set up our azimuth basis B.append(utils.build_legendre(az, naz)) else: # Set up phase basis. Vectors should be periodic # in phase to avoid discontinuities. cossin is good for this. phase = build_phase(az) * np.pi B.append(utils.build_cossin(phase, naz)) if nhwp > 0: B.append(utils.build_cossin(hwp, nhwp)) B = np.concatenate(B, 0) for it in range(niter): if do_gapfill: gapfill.gapfill(d, cuts, inplace=True) # Solve for the best fit for each detector, [nbasis,ndet] # B[b,n], d[d,n], amps[b,d] if weights is None: try: amps = np.linalg.solve(B.dot(B.T), B.dot(d.T)) except np.linalg.LinAlgError as e: print "LinAlgError in todfilter. Skipping" continue else: w = weights.reshape(-1, weights.shape[-1]) amps = np.zeros([naz + nt + nhwp, d.shape[0]], dtype=tod.dtype) for di in range(len(tod)): try: amps[:, di] = np.linalg.solve((B * w[di]).dot(B.T), B.dot(w[di] * d[di])) except np.linalg.LinAlgError as e: print "LinAlgError in todfilter di %d. Skipping" % di continue # Subtract the best fit, but skip some basis functions if requested if tsign < 0: amps[:nt] = 0 if asign < 0: amps[nt:nt + naz] = 0 if hsign < 0: amps[nt + naz:nt + naz + nhwp] = 0 d -= amps.T.dot(B) if do_gapfill: gapfill.gapfill(d, cuts, inplace=True) # This filtering might have casued the tod to become # non-continous, which would mess up future fourier transforms. # So it's safest to deslope here. if deslope: utils.deslope(tod, w=8, inplace=True) res = d.reshape(tod.shape) return res
div = rhs.copy() wrhs = signal.prepare(rhs) wdiv = signal.prepare(div) kvals = np.zeros(len(corr_pos), dtype) freqs, bareas = np.zeros(ntod), np.zeros(ntod) bleh = True and args.inject if bleh: sim_rhs = np.zeros(len(inject_params)) sim_div = np.zeros(len(inject_params)) for si, scan in zip(myinds, myscans): L.debug("Processing %s" % scan.id) # Read the tod tod = scan.get_samples().astype(dtype) tod = utils.deslope(tod) if args.mapsub: # Subtract the reference map. If the reference map is not source free, # then this could reintroduce strong point sources that were cut earlier. # To avoid this we do another round of gapfilling signal.forward(scan, tod, refmap, tmul=1, mmul=-1) gapfill.gapfill_joneig(tod, scan.cut, inplace=True) # Inject simulated signal if requested if args.inject: dmjd = scan.mjd0 - mjd0 earth_pos = -ephemeris.ephem_vec("Sun", scan.mjd0)[:, 0] # Set the position and amplitude uK of each simulated source sim_srcs = np.zeros([len(inject_params), 8]) # TODO: inject and analyze with no displacement to see if interpolation is the