def __init__(self, data, srcpos, ndir=1, sys=sys): # Build source parameter struct for PmatPtsrc self.params = np.zeros([srcpos.shape[-1], ndir, 8], np.float) self.params[:, :, :2] = srcpos[::-1, None, :].T self.params[:, :, 5:7] = 1 scan = actscan.ACTScan(data.entry, d=data) self.psrc = pmat.PmatPtsrc(scan, self.params, sys=sys) self.pcut = pmat.PmatCut(scan) # Extract basic offset self.off0 = data.point_correction self.off = self.off0 * 1 self.el = np.mean(data.boresight[2, ::100]) self.point_template = data.point_template
def __init__(self, data, srcpos, res=0.25*utils.arcmin, rad=20*utils.arcmin, perdet=False, detoff=10*utils.arcmin): scan = actscan.ACTScan(data.entry, d=data) if perdet: # Offset each detector's pointing so that we produce a grid of images, one per detector. gside = int(np.ceil(data.ndet**0.5)) goffs = np.mgrid[:gside,:gside] - (gside-1)/2.0 goffs = goffs.reshape(2,-1).T[:data.ndet]*detoff scan.offsets = scan.offsets.copy() scan.offsets[:,1:] += goffs rad = rad + np.max(np.abs(goffs)) # Build geometry for each source shape, wcs = enmap.geometry(pos=[[-rad,-rad],[rad,rad]], res=res, proj="car") area = enmap.zeros((3,)+shape, wcs, dtype=data.tod.dtype) self.pmats = [] for i, pos in enumerate(srcpos.T): if planet: sys = src_sys else: sys = ["icrs",[np.array([[pos[0]],[pos[1]],[0],[0]]),False]] with config.override("pmat_accuracy", 10): self.pmats.append(pmat.PmatMap(scan, area, sys=sys)) self.shape = (len(srcpos.T),3)+shape self.wcs = wcs
def map_cuts(entry, data, hitmap, cutmap, keep_buffer=False, nocommon_frac=None): ''' Project hits and hits-cuts onto the sky. parameters ---------- entry : filedb.data object data : actdata instance hitmap : (1, ny, nx) enmap cutmap : (1, ny, nx) enmap keep_buffer : bool, optional Do not remove the 200 sample buffers before and after cuts. common_frac : float, optional Do not consider cuts that are at least common to this fraction of detectors. ''' scan = actscan.ACTScan(entry, d=data) pmap = pmat.PmatMap(scan, hitmap) cut = data.cut if nocommon_frac is not None: cut = remove_common(cut, frac=nocommon_frac) if keep_buffer is False: cut = remove_buffer(cut) tod = np.full([data.ndet, data.nsamp], 1.0, dtype) pmap.backward(tod, hitmap) sampcut.gapfill_const(cut, tod, 0.0, inplace=True) pmap.backward(tod, cutmap)
print asens with bench.show("smooth"): ft = fft.rfft(model) freq = fft.rfftfreq(model.shape[-1])*d.srate flt = 1/(1+(freq/model_fknee)**model_alpha) ft *= flt fft.ifft(ft, model, normalize=True) del ft, flt, freq with bench.show("atm subtract"): tod -= model del model tod = tod.astype(dtype, copy=False) # Should now be reasonably clean of correlated noise. # Proceed to make simple binned map with bench.show("actscan"): scan = actscan.ACTScan(entry, d=d) with bench.show("pmat"): pmap = pmat.PmatMap(scan, area, sys=sys) pcut = pmat.PmatCut(scan) rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype) div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype) junk = np.zeros(pcut.njunk, dtype) with bench.show("rhs"): tod *= ivar[:,None] pcut.backward(tod, junk) pmap.backward(tod, rhs) with bench.show("hits"): for i in range(ncomp): div[i,i] = 1 pmap.forward(tod, div[i]) tod *= ivar[:,None]
site=d.site, ncomp=ncomp, dtype=dtype) except WorkspaceError as e: L.debug("Skipped pattern %s (%s)" % (wid, e.message)) continue print "%-18s %5d %5d" % ((wid, ) + tuple(wgeo.shape[-2:])) tot_work = Workspace(wgeo) oname = "%s/%s.hdf" % (args.odir, wid) # And process the tods that fall within this workspace for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] entry = filedb.data[id] try: d = actscan.ACTScan(entry) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("Tod contains no valid data") d = d[:, ::downsample] d = d[:, :] except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (id, e.message)) continue L.debug("Processing %s" % id) # Get the actual tod tod = d.get_samples() tod -= np.mean(tod, 1)[:, None] tod = tod.astype(dtype) # Compute the per-detector spectrum ft = fft.rfft(tod) * d.nsamp**-0.5 tfilter, binds = measure_inv_noise_spectrum(ft, nbin)
return np.mean((blocks-m[:,None]-A[:,None]*s[None,:])**2,1) def onlyfinite(a): return a[np.isfinite(a)] # Process each scan independently myinds = np.arange(len(filelist))[myid::nproc] for ind in myinds: ofile = args.odir + "/%s.hdf" % filelist[ind] if args.c and os.path.isfile(ofile): L.info("Already done %s" % filelist[ind]) continue L.info("Processing %s" % filelist[ind]) try: d = scan.read_scan(filelist[ind]) except (IOError, OSError): try: d = actscan.ACTScan(db[filelist[ind]]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("all samples cut") except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (filelist[ind], str(e))) continue try: L.debug("Reading samples") tod = d.get_samples().astype(dtype) except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (filelist[ind], str(e))) continue # Measure noise L.debug("Noise") ivar = 1/np.array([np.median(onlyfinite(get_desloped_var(blockify(t,20)))) for t in tod]) # Set up pmat for this scan
bid = id.replace(":", "_") root = args.odir + "/src%03d_%s_" % (si, bid) ofile = root + "main_map.fits" if os.path.isfile(ofile): continue tasks.append((si, id)) # Each task processes tasks independently for ti in range(comm.rank, len(tasks), comm.size): si, id = tasks[ti] bid = id.replace(":", "_") L.info("Processing src %3d id %s" % (si, id)) root = args.odir + "/src%03d_%s_" % (si, bid) entry = filedb.data[id] osys = "hor:%.6f_%.6f:cel/0_0:hor" % tuple(srcs[:2, si]) try: scans = [actscan.ACTScan(entry)] if scans[0].nsamp == 0 or scans[0].ndet == 0: raise errors.DataMissing("no data in scan") except errors.DataMissing as e: print "Skipping %s: %s" % (id, str(e)) continue # Signals signal_cut = mapmaking.SignalCut(scans, dtype=dtype, comm=tcomm) signal_map = mapmaking.SignalMap(scans, area, comm=tcomm, sys=osys) # Weights weights = [mapmaking.FilterWindow(config.get("tod_window"))] # And equation system eqsys = mapmaking.Eqsys(scans, [signal_cut, signal_map], weights=weights, dtype=dtype, comm=tcomm)
def get_scans(area, signal, bore, dets, noise, seed=0, real=None, noise_override=None): scans = [] # Get real scan information if necessary L.debug("real") if real: real_scans = [] filedb.init() db = filedb.data ids = fileb.scans[real].ids for id in ids: try: real_scans.append(actscan.ACTScan(db[id])) except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (id, str(e))) # Dets L.debug("dets") sim_dets = [] toks = dets.split(":") if toks[0] == "scattered": ngroup, nper, rad = int(toks[1]), int(toks[2]), float(toks[3]) sim_dets = [scansim.dets_scattered(ngroup, nper,rad=rad*np.pi/180/60)] margin = rad*np.pi/180/60 elif toks[0] == "real": ndet = int(toks[1]) dslice = slice(0,ndet) if ndet > 0 else slice(None) sim_dets = [bunch.Bunch(comps=s.comps[dslice], offsets=s.offsets[dslice]) for s in real_scans] margin = np.max([np.sum(s.offsets**2,1)**0.5 for s in sim_dets]) else: raise ValueError # Boresight. Determines our number of scans L.debug("bore") sim_bore = [] toks = bore.split(":") if toks[0] == "grid": nscan, density, short = int(toks[1]), float(toks[2]), float(toks[3]) for i in range(nscan): tbox = shorten(area.box(),i%2,short) sim_bore.append(scansim.scan_grid(tbox, density*np.pi/180/60, dir=i, margin=margin)) elif toks[0] == "ces": nscan = int(toks[1]) azs = [float(w)*utils.degree for w in toks[2].split(",")] els = [float(w)*utils.degree for w in toks[3].split(",")] mjd0 = float(toks[4]) dur = float(toks[5]) azrate= float(toks[6]) if len(toks) > 6 else 1.5*utils.degree srate = float(toks[7]) if len(toks) > 7 else 400 nsamp = utils.nint(dur*srate) for i in range(nscan): mjd = mjd0 + dur*(i//(2*len(els)))/(24*3600) el = els[(i//2)%len(els)] az1, az2 = azs if i%2 == 1: az1, az2 = -az2, -az1 box = np.array([[az1,el],[az2,el]]) sim_bore.append(scansim.scan_ceslike(nsamp, box, mjd0=mjd, srate=srate, azrate=azrate)) elif toks[0] == "real": sim_bore = [bunch.Bunch(boresight=s.boresight, hwp_phase=s.hwp_phase, sys=s.sys, site=s.site, mjd0=s.mjd0) for s in real_scans] else: raise ValueError nsim = len(sim_bore) # Make one det info per scan sim_dets = sim_dets*(nsim/len(sim_dets))+sim_dets[:nsim%len(sim_dets)] # Noise L.debug("noise") sim_nmat = [] toks = noise.split(":") nonoise = False if toks[0] == "1/f": sigma, alpha, fknee = [float(v) for v in toks[1:4]] nonoise = sigma < 0 for i in range(nsim): sim_nmat.append(scansim.oneoverf_noise(sim_dets[i].comps.shape[0], sim_bore[i].boresight.shape[0], sigma=np.abs(sigma), alpha=alpha, fknee=fknee)) elif toks[0] == "detcorr": sigma, alpha, fknee = [float(v) for v in toks[1:4]] nonoise = sigma < 0 for i in range(nsim): sim_nmat.append(scansim.oneoverf_detcorr_noise(sim_dets[i].comps.shape[0], sim_bore[i].boresight.shape[0], sigma=np.abs(sigma), alpha=alpha, fknee=fknee)) elif toks[0] == "real": scale = 1.0 if len(toks) < 2 else float(toks[1]) for i,s in enumerate(real_scans): ndet = len(sim_dets[i].offsets) nmat = s.noise[:ndet]*scale**-2 sim_nmat.append(nmat) else: raise ValueError noise_scale = not nonoise if noise_override is None else noise_override sim_nmat = sim_nmat*(nsim/len(sim_nmat))+sim_nmat[:nsim%len(sim_nmat)] # Signal L.debug("signal") toks = signal.split(":") if toks[0] == "none": for i in range(nsim): scans.append(scansim.SimPlain(sim_bore[i], sim_dets[i], sim_nmat[i], seed=seed+i, noise_scale=noise_scale)) elif toks[0] == "ptsrc": # This one always operates in the same coordinates as nsrc, amp, fwhm = int(toks[1]), float(toks[2]), float(toks[3]) np.random.seed(seed) sim_srcs = scansim.rand_srcs(area.box(), nsrc, amp, abs(fwhm)*np.pi/180/60, rand_fwhm=fwhm<0) for i in range(nsim): scans.append(scansim.SimSrcs(sim_bore[i], sim_dets[i], sim_srcs, sim_nmat[i], seed=seed+i, noise_scale=noise_scale)) elif toks[0] == "vsrc": # Create a single variable source ra, dec, fwhm = float(toks[1])*np.pi/180, float(toks[2])*np.pi/180, float(toks[3])*np.pi/180/60 amps = [float(t) for t in toks[4].split(",")] for i in range(nsim): sim_srcs = bunch.Bunch(pos=np.array([[dec,ra]]),amps=np.array([[amps[i],0,0,0]]), beam=np.array([fwhm/(8*np.log(2)**0.5)])) scans.append(scansim.SimSrcs(sim_bore[i], sim_dets[i], sim_srcs, sim_nmat[i], seed=seed+i, noise_scale=noise_scale, nsigma=20)) elif toks[0] == "cmb": np.random.seed(seed) ps = powspec.read_spectrum(toks[1]) sim_map = enmap.rand_map(area.shape, area.wcs, ps) for i in range(nsim): scans.append(scansim.SimMap(sim_bore[i], sim_dets[i], sim_map, sim_nmat[i], seed=seed+i, noise_scale=noise_scale)) else: raise ValueError return scans
print("%s has 0 srcs: skipping" % id) continue try: nsrc = len(sids) print("%s has %d srcs: %s" % (id, nsrc, ", ".join( ["%d (%.1f)" % (i, a) for i, a in zip(sids, amps[sids])]))) except TypeError as e: print("Weird: %s" % e) print(sids) print(amps) continue # Read the data entry = filedb.data[id] try: scan = actscan.ACTScan(entry, verbose=verbose >= 2) if scan.ndet < 2 or scan.nsamp < 1: raise errors.DataMissing("no data in tod") except errors.DataMissing as e: print("%s skipped: %s" % (id, e)) continue # Apply downsampling scan = scan[:, ::down] # Prepeare our samples scan.tod = scan.get_samples() utils.deslope(scan.tod, w=5, inplace=True) scan.tod = scan.tod.astype(dtype) # Background subtraction if args.sub: Pmap = pmat.PmatMap(scan, background, sys=sys)
sshape, swcs = enmap.read_map_geometry(args.mapsub) pixbox = enmap.pixbox_of(swcs, shape, wcs) if not use_dmap: refmap = enmap.read_map(args.mapsub, pixbox=pixbox).astype(dtype) else: refmap = dmap.read_map(args.mapsub, pixbox=pixbox, bbox=mybbox, comm=comm).astype(dtype) refmap = signal.prepare(refmap) # Get the frequency and beam for this chunk. We assume that # this is the same for every member of the chunk, so we only need # to do this for one scan scan = actscan.ACTScan(filedb.data[chunk_ids[inds[0]]]) _, dets = actdata.split_detname(scan.dets) beam = scan.beam freq = scan.array_info.info.nom_freq[dets[0]] barea = planet9.calc_beam_area(scan.beam) # Get the conversion from ref-freq flux to observed amplitude. This includes # dilution by the beam area flux2amp = 1 / utils.flux_factor(barea, args.fref * 1e9, utils.T_cmb) fref2freq = utils.planck(freq * 1e9, args.Tref) / utils.planck( args.fref * 1e9, args.Tref) rfact = flux2amp * fref2freq * 1e3 # 1e3 for flux in mJy and amp in uK # only work will be 3,ny,nx. The rest are scalar. Will copy in-out as necessary work = signal.work() rhs = area[0] div = rhs.copy()