def __init__(self, scans, area, comm, cuts=None, name="main", ofmt="{name}", output=True, ext="fits", pmat_order=None, sys=None, nuisance=False): Signal.__init__(self, name, ofmt, output, ext) self.area = area self.cuts = cuts self.dof = zipper.ArrayZipper(area, comm=comm) self.dtype = area.dtype self.comm = comm self.data = { scan: [ pmat.PmatMap(scan, area, order=pmat_order, sys=sys), pmat.PmatMapMultibeam(scan, area, scan.buddy_offs, scan.buddy_comps, order=pmat_order, sys=sys) ] for scan in scans }
def __init__(self, scans, area, comm, cuts=None, name="main", ofmt="{name}", output=True, ext="fits", pmat_order=None, sys=None, nuisance=False, data=None): Signal.__init__(self, name, ofmt, output, ext) self.area = area self.cuts = cuts self.dof = zipper.ArrayZipper(area, comm=comm) self.dtype = area.dtype if data is not None: self.data = data else: self.data = { scan: pmat.PmatMap(scan, area, order=pmat_order, sys=sys) for scan in scans }
def __init__(self, scanpattern, dets, map, noise, simsys="equ", cache=False, seed=0, noise_scale=1): # Set up the telescope self.boresight = scanpattern.boresight self.sys = scanpattern.sys self.offsets = dets.offsets self.comps = dets.comps self.cut = nocut(self.ndet, self.nsamp) self.mjd0 = scanpattern.mjd0 # Set up the simulated signal properties self.map = map.copy() self.noise = noise self.seed = seed self.dets = np.arange(len(self.comps)) self.site = scanpattern.site self.noise_scale = noise_scale self.simsys = simsys self.pmat = pmat.PmatMap(self, self.map, sys=simsys)
def __init__(self, scans, subinds, area, cuts=None, name="main", ofmt="{name}", output=True, ext="fits", pmat_order=None, sys=None, nuisance=False, data=None): Signal.__init__(self, name, ofmt, output, ext) self.area = area self.cuts = cuts self.dof = dmap.DmapZipper(area) self.dtype = area.dtype self.subs = subinds if data is None: data = {} work = area.tile2work() for scan, subind in zip(scans, subinds): data[scan] = [ pmat.PmatMap(scan, work[subind], order=pmat_order, sys=sys), subind ] self.data = data
def __init__(self, scans, subinds, dmap, sys=None, mul=1, tmul=1, pmat_order=None): self.map, self.sys, self.mul, self.tmul = dmap, sys, mul, tmul self.data = {} work = dmap.tile2work() for scan, subind in zip(scans, subinds): self.data[scan] = [ pmat.PmatMap(scan, work[subind], order=pmat_order, sys=sys), work[subind] ]
def __init__(self, scan, srcpos, res=0.25*utils.arcmin, rad=20*utils.arcmin, perdet=False, detoff=10*utils.arcmin): if perdet: # Offset each detector's pointing so that we produce a grid of images, one per detector. gside = int(np.ceil(scan.ndet**0.5)) goffs = np.mgrid[:gside,:gside] - (gside-1)/2.0 goffs = goffs.reshape(2,-1).T[:scan.ndet]*detoff scan.offsets = scan.offsets.copy() scan.offsets[:,1:] += goffs rad = rad + np.max(np.abs(goffs)) # Build geometry for each source shape, wcs = enmap.geometry(pos=[[-rad,-rad],[rad,rad]], res=res, proj="car") area = enmap.zeros((3,)+shape, wcs, dtype=scan.tod.dtype) self.pmats = [] for i, pos in enumerate(srcpos.T): if planet: sys = src_sys else: sys = ["icrs",[np.array([[pos[0]],[pos[1]],[0],[0]]),False]] with config.override("pmat_accuracy", 10): self.pmats.append(pmat.PmatMap(scan, area, sys=sys)) self.shape = (len(srcpos.T),3)+shape self.wcs = wcs
def map_cuts(entry, data, hitmap, cutmap, keep_buffer=False, nocommon_frac=None): ''' Project hits and hits-cuts onto the sky. parameters ---------- entry : filedb.data object data : actdata instance hitmap : (1, ny, nx) enmap cutmap : (1, ny, nx) enmap keep_buffer : bool, optional Do not remove the 200 sample buffers before and after cuts. common_frac : float, optional Do not consider cuts that are at least common to this fraction of detectors. ''' scan = actscan.ACTScan(entry, d=data) pmap = pmat.PmatMap(scan, hitmap) cut = data.cut if nocommon_frac is not None: cut = remove_common(cut, frac=nocommon_frac) if keep_buffer is False: cut = remove_buffer(cut) tod = np.full([data.ndet, data.nsamp], 1.0, dtype) pmap.backward(tod, hitmap) sampcut.gapfill_const(cut, tod, 0.0, inplace=True) pmap.backward(tod, cutmap)
hits = enmap.zeros((3, ) + area.shape[-2:], area.wcs, dtype=dtype) myinds = np.arange(comm.rank, len(ids), comm.size) for ind in myinds: id = ids[ind] entry = filedb.data[id] try: scan = actscan.ACTScan(entry) if scan.ndet == 0 or scan.nsamp == 0: raise errors.DataMissing("Tod contains no valid data") except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (str(id), e.message)) continue scan = scan[:, ::config.get("downsample")] L.debug("Processing %s" % str(id)) pmap = pmat.PmatMap(scan, hits) pcut = pmat.PmatCut(scan) tod = np.full([scan.ndet, scan.nsamp], 1, dtype=dtype) junk = np.zeros(pcut.njunk, dtype=dtype) pcut.backward(tod, junk) pmap.backward(tod, hits) hits = hits[0] # Collect result L.info("Reducing") hits[:] = utils.allreduce(hits, comm) # Undo effect of downsampling hits *= config.get("downsample") # And write it L.info("Writing")
t.append(time.time()) try: fields = ["gain", "tconst", "cut", "tod", "boresight", "noise_cut"] if args.spikecut: fields.append("spikes") if args.imap: fields += ["polangle", "point_offsets", "site"] d = data.read(entry, fields) t.append(time.time()) d = data.calibrate(d) t.append(time.time()) if args.imap: # Make a full scan object, so we can perform pointing projection # operations d.noise = None scan = data.ACTScan(entry, d=d) imap.map = imap.map.astype(d.tod.dtype, copy=False) pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys) # Subtract input map from tod inplace pmap.forward(d.tod, imap.map, tmul=1, mmul=-1) utils.deslope(d.tod, w=8, inplace=True) ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5 t.append(time.time()) spikes = d.spikes[:2].T if args.spikecut else None if model == "old": noise = nmat_measure.detvecs_old(ft, d.srate, d.dets) elif model == "jon": noise = nmat_measure.detvecs_jon(ft, d.srate, d.dets, shared, cut_bins=spikes) elif model == "simple":
ft = fft.rfft(model) freq = fft.rfftfreq(model.shape[-1])*d.srate flt = 1/(1+(freq/model_fknee)**model_alpha) ft *= flt fft.ifft(ft, model, normalize=True) del ft, flt, freq with bench.show("atm subtract"): tod -= model del model tod = tod.astype(dtype, copy=False) # Should now be reasonably clean of correlated noise. # Proceed to make simple binned map with bench.show("actscan"): scan = actscan.ACTScan(entry, d=d) with bench.show("pmat"): pmap = pmat.PmatMap(scan, area, sys=sys) pcut = pmat.PmatCut(scan) rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype) div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype) junk = np.zeros(pcut.njunk, dtype) with bench.show("rhs"): tod *= ivar[:,None] pcut.backward(tod, junk) pmap.backward(tod, rhs) with bench.show("hits"): for i in range(ncomp): div[i,i] = 1 pmap.forward(tod, div[i]) tod *= ivar[:,None] pcut.backward(tod, junk) div[i] = 0
try: object_pos = coordinates.interpol_pos("cel", "hor", args.objname, mjd, site=scan.site) except AttributeError as e: print( "Unexpected error in interpol_pos for %s. mid time was %.5f. message: %s. skipping" % (id, mjd[len(mjd) // 2], e)) continue visible = np.any(object_pos[1] >= -margin) if not visible: cut = sampcut.empty(d.ndet, d.nsamp) else: pmap = pmat.PmatMap(scan, mask, sys="sidelobe:%s" % args.objname) # Build a tod to project onto. tod = np.zeros((d.ndet, d.nsamp), dtype=dtype) # And project pmap.forward(tod, mask) # Any nonzero samples should be cut tod = tod != 0 cut = sampcut.from_mask(tod) del tod progress = 100.0 * (ind - comm.rank * ntod // comm.size) / ( (comm.rank + 1) * ntod // comm.size - comm.rank * ntod // comm.size) print("%3d %5.1f %s %6.4f %d %8.3f %8.3f" % (comm.rank, progress, id, float(cut.sum()) / cut.size, visible, memory.current() / 1024.**3, memory.max() / 1024.**3)) mystats.append([ind, float(cut.sum()) / cut.size, visible])
srate = 0 speed = 0 site = {} inspec = np.zeros(nbin) offsets = np.zeros([ndet_array, 2]) det_hit = np.zeros([ndet_array], dtype=int) for ind, d in scanutils.scan_iterator(pids, myinds, actscan.ACTScan, filedb.data, dets=args.dets, downsample=config.get("downsample")): id = pids[ind] with bench.mark("pbuild"): # Build pointing matrices pmap = pmat.PmatMap(d, area) pcut = pmat.PmatCut(d) with bench.mark("tod"): # Get tod tod = d.get_samples() tod -= np.mean(tod, 1)[:, None] tod = tod.astype(dtype) junk = np.zeros(pcut.njunk, dtype=dtype) with bench.mark("nmat"): # Build noise model ft = fft.rfft(tod) * tod.shape[1]**-0.5 nmat = nmat_measure.detvecs_simple(ft, d.srate) del ft with bench.mark("rhs"): # Calc rhs, accumulating into pattern total nmat.apply(tod)
scan = actscan.ACTScan(entry, verbose=verbose >= 2) if scan.ndet < 2 or scan.nsamp < 1: raise errors.DataMissing("no data in tod") except errors.DataMissing as e: print("%s skipped: %s" % (id, e)) continue # Apply downsampling scan = scan[:, ::down] # Prepeare our samples scan.tod = scan.get_samples() utils.deslope(scan.tod, w=5, inplace=True) scan.tod = scan.tod.astype(dtype) # Background subtraction if args.sub: Pmap = pmat.PmatMap(scan, background, sys=sys) Pmap.forward(scan.tod, background, tmul=-1) # Build the noise model N = NmatTot(scan, model="uncorr", window=2.0, filter=highpass) P = PmatTot(scan, srcpos[:, sids], perdet=True, sys=sys) # rhs N.apply(scan.tod) rhs = P.backward(scan.tod, ncomp=1) # div scan.tod[:] = 0 P.forward(scan.tod, rhs * 0 + 1) N.apply(scan.tod) div = P.backward(scan.tod, ncomp=1)
def __init__(self, scans, map, sys=None, mul=1, tmul=1, pmat_order=None): self.map, self.sys, self.mul, self.tmul = map, sys, mul, tmul self.data = { scan: pmat.PmatMap(scan, map, order=pmat_order, sys=sys) for scan in scans }
raw_bore = raw_bore[:cal_bore.shape[0]] # And a proper ACTScan scan = data.ACTScan(entry, subdets=[args.di]) # Detector pointing det_ipoint = scan.boresight + scan.offsets[0] # Manual transformation trf = pmat.pos2pix(scan, None, eqsys) det_opoint_exact = trf(det_ipoint.T).T psi_rot_exact = np.arctan2(det_opoint_exact[:, 3], det_opoint_exact[:, 2]) psi_det = np.arctan2(scan.comps[:, 2], scan.comps[:, 1]) psi_exact = (psi_rot_exact + psi_det) / 2 % np.pi ppix = pmat.PmatMap(scan, area) # Exact to pixels det_opix_exact = ppix.transform(det_ipoint.T).T # Interpolated to pixels det_opix, det_ophase = ppix.translate(scan.boresight, scan.offsets, scan.comps) # Get cut status of each sample also cut = scan.cut.to_mask()[0] # Ok, output all the information. First a header describing our information print "# id %s det %d samp_off %d" % (args.id, scan.dets[0], d.sample_offset) print "# det angle %10.3f" % (psi_det / 2 * 180 / np.pi) print "# FITS header" for card in area.wcs.to_header().cards: print "# " + str(card)
tod = tod.astype(dtype) # Construct noise model for this tod winsize = int(config.get("tod_window") * d.srate) nmat.apply_window(tod, winsize) d.noise = d.noise.update(tod, d.srate) L.debug("Noise %s" % id) # Apply it, to get N"d d.noise.apply(tod) nmat.apply_window(tod, winsize) # Compute the variance per detector. If our noise model # were correct and our data were pure noise, this would be # N"<nn'>N" = N". But our noise model isn't totally accurate. vars = np.var(tod, 1) # Project each detectors result on the sky tod[:] = vars[:, None] pmap = pmat.PmatMap(d, osig) pcut = pmat.PmatCut(d) junk = np.zeros(pcut.njunk, dtype=dtype) pcut.backward(tod, junk) pmap.backward(tod, osig) # Also do the fiducial noise model tod[:] = 1 nmat.apply_window(tod, winsize) d.noise.white(tod) nmat.apply_window(tod, winsize) pcut.backward(tod, junk) pmap.backward(tod, odiv) # Collect some statistics sig_all[ind] = np.sum(vars) * d.nsamp sig_med[ind] = np.median(vars) * d.ndet * d.nsamp div_all[ind] = np.sum(tod)
def __init__(self, scan, area, sys=None): self.pmap = pmat.PmatMap(scan, area, sys=sys) self.pcut = pmat.PmatCut(scan) self.err = self.pmap.err
bore_corners = utils.box2corners(bore_box) scan.entry = d.entry # Is the source above the horizon? If not, it doesn't matter how close # it is. mjd = utils.ctime2mjd( utils.mjd2ctime(scan.mjd0) + scan.boresight[::100, 0]) object_pos = coordinates.interpol_pos("cel", "hor", args.objname, mjd, site=scan.site) visible = np.any(object_pos[1] >= margin) if not visible: cut = rangelist.zeros((d.ndet, d.nsamp)) else: pmap = pmat.PmatMap(scan, mask, sys="hor:%s" % args.objname) # Build a tod to project onto. tod = np.zeros((d.ndet, d.nsamp), dtype=dtype) # And project pmap.forward(tod, mask) # Any nonzero samples should be cut tod = np.rint(tod) cut = rangelist.Multirange([rangelist.Rangelist(t) for t in tod]) print "%s %6.4f %d" % (id, float(cut.sum()) / cut.size, visible) mystats.append([ind, float(cut.sum()) / cut.size, visible]) # Write cuts to output directory if args.persample: files.write_cut("%s/%s.cuts" % (args.odir, id), d.dets, cut, nrow=d.array_info.nrow,
if args.precompute: prec_NNmap = {lam: area * 0 for lam in np.unique(cooldown)} prec_NNjunk = {lam: [] for lam in np.unique(cooldown)} scans = [] for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] entry = filedb.data[id] try: scan = actscan.ACTScan(entry) if scan.ndet == 0 or scan.nsamp == 0: raise errors.DataMissing("No samples in scan") if args.ndet: scan = scan[:args.ndet] if downsample > 1: scan = scan[:, ::downsample] scan.pmap = pmat.PmatMap(scan, area) scan.pcut = pmat.PmatCut(scan) # Build the noise model tod = scan.get_samples() tod -= np.mean(tod, 1)[:, None] tod = tod.astype(dtype) scan.noise = scan.noise.update(tod, scan.srate) scan.T = np.min(scan.noise.D) * Tscale scan.noise_bar = nmat.NmatDetvecs(scan.noise.D - scan.T, scan.noise.V, scan.noise.E, scan.noise.bins, scan.noise.ebins, scan.noise.dets) # Set up cuts scan.cut_range = [njunk_tot, njunk_tot + scan.pcut.njunk] njunk_tot += scan.pcut.njunk # Prepare our filtered data. We do this one of two ways. # Either store Nb"d for each TOD, which can end up taking