def scan_iterator(filelist, inds, reader, db=None, dets=None, quiet=False, downsample=1, hwp_resample=False): """Given a set of ids/files and a set of indices into that list. Try to read each of these scans. Returns a list of successfully read scans and a list of their indices.""" for ind in inds: try: if isinstance(filelist[ind],list): raise IOError d = enscan.read_scan(filelist[ind]) actdata.read(filedb.data[filelist[ind]]) except IOError: try: if isinstance(filelist[ind],list): entry = [db[id] for id in filelist[ind]] else: entry = db[filelist[ind]] d = reader(entry) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("Tod contains no valid data") except errors.DataMissing as e: if not quiet: L.debug("Skipped %s (%s)" % (str(filelist[ind]), e.message)) continue if dets: if dets.startswith("@"): uids = [int(line.split()[0]) for line in open(dets[1:],"r")] _,det_inds = utils.common_inds([uids,d.dets]) d = d[det_inds] else: d = eval("d[%s]" % dets) hwp_active = np.any(d.hwp_phase[0] != 0) if hwp_resample and hwp_active: mapping = enscan.build_hwp_sample_mapping(d.hwp) d = d.resample(mapping) d = d[:,::downsample] if not quiet: L.debug("Read %s" % str(filelist[ind])) yield ind, d
def get_samples(self, verbose=False): """Return the actual detector samples. Slow! Data is read from disk and calibrated on the fly, so store the result if you need to reuse it.""" # Because we've read the tod_shape field earlier, we know that reading tod # won't cause any additional truncation of the samples or detectors. # tags is only needed here for read_combo support, but that is mostly broken # anyway. t1 = time.time() self.d += actdata.read(self.entry, fields=["tod", "tags"], dets=self.d.dets) #if debug_inject is not None: self.d.tod += debug_inject t2 = time.time() if verbose: print("read %-14s in %6.3f s" % ("tod", t2 - t1)) if config.get("tod_skip_deconv"): ops = ["tod_real"] else: ops = ["tod"] actdata.calibrate(self.d, operations=ops, verbose=verbose) tod = self.d.tod # Remove tod from our local d, so we won't end up hauling it around forever del self.d.tod # HWP resample if needed if self.mapping is not None: tod = np.ascontiguousarray( utils.interpol(tod, self.mapping.oimap[None], order=1, mask_nan=False)) method = config.get("downsample_method") for s in self.sampslices: tod = scan.slice_tod_samps(tod, s, method=method) tod = np.ascontiguousarray(tod) return tod
def get_rangedata(id): entry = filedb.data[id] # Read the tod as usual with show("read"): d = actdata.read(entry) with show("calibrate"): # Don't apply time constant (and hence butterworth) deconvolution since we # will fit these ourselves d = actdata.calibrate(d, exclude=["autocut", "tod_fourier"]) if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod") tod = d.tod del d.tod # Very simple white noise model with show("noise"): ivar = estimate_ivar(tod) asens = np.sum(ivar)**-0.5 / d.srate**0.5 with show("planet mask"): # Generate planet cut planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site, args.planet, R) with show("atmosphere"): # Subtract atmospheric model tod -= estimate_atmosphere(tod, planet_cut, d.srate, model_fknee, model_alpha) tod = tod.astype(dtype, copy=False) with show("extract"): # Should now be reasonably clean of correlated noise. Extract our range data rdata = build_rangedata(tod, planet_cut, d, ivar) return rdata
def get_samples(self, verbose=False): """Return the actual detector samples. Slow! Data is read from disk and calibrated on the fly, so store the result if you need to reuse it.""" # Because we've read the tod_shape field earlier, we know that reading tod # won't cause any additional truncation of the samples or detectors. # tags is only needed here for read_combo support, but that is mostly broken # anyway. t1 = time.time() self.d += actdata.read(self.entry, fields=["tod", "tags"], dets=self.d.dets) t2 = time.time() if verbose: print "read %-14s in %6.3f s" % ("tod", t2-t1) if config.get("tod_skip_deconv"): ops = ["tod_real"] else: ops = ["tod"] actdata.calibrate(self.d, operations=ops, verbose=verbose) tod = self.d.tod # Remove tod from our local d, so we won't end up hauling it around forever del self.d.tod # HWP resample if needed if self.mapping is not None: tod = np.ascontiguousarray(utils.interpol(tod, self.mapping.oimap[None], order=1, mask_nan=False)) method = config.get("downsample_method") for s in self.sampslices: srange = slice(s.start, s.stop, np.sign(s.step) if s.step else None) tod = tod[:,srange] tod = resample.resample(tod, 1.0/np.abs(s.step or 1), method=method) tod = np.ascontiguousarray(tod) return tod
def get_rangedata(id): entry = filedb.data[id] # Read the tod as usual with show("read"): d = actdata.read(entry) with show("calibrate"): # Don't apply time constant (and hence butterworth) deconvolution since we # will fit these ourselves d = actdata.calibrate(d, exclude=["autocut","tod_fourier"]) if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod") tod = d.tod; del d.tod # Very simple white noise model with show("noise"): ivar = estimate_ivar(tod) asens = np.sum(ivar)**-0.5 / d.srate**0.5 with show("planet mask"): # Generate planet cut planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site, args.planet, R) with show("atmosphere"): # Subtract atmospheric model tod -= estimate_atmosphere(tod, planet_cut, d.srate, model_fknee, model_alpha) tod = tod.astype(dtype, copy=False) with show("extract"): # Should now be reasonably clean of correlated noise. Extract our range data rdata = build_rangedata(tod, planet_cut, d, ivar) return rdata
def read_metadata(entry): ''' Parameters ---------- entry : filedb.data object Returns ------- data : enlib.dataset.DataSet instance ''' data = actdata.read(entry, exclude=['tod']) data = actdata.calibrate(data, exclude=['autocut']) if data.ndet == 0 or data.nsamp == 0: raise errors.DataMissing("No data in tod") return data
def get_samples(self, verbose=False): """Return the actual detector samples. Slow! Data is read from disk and calibrated on the fly, so store the result if you need to reuse it.""" # Because we've read the tod_shape field earlier, we know that reading tod # won't cause any additional truncation of the samples or detectors. # tags is only needed here for read_combo support, but that is mostly broken # anyway. t1 = time.time() self.d += actdata.read(self.entry, fields=["tod", "tags"], dets=self.d.dets) t2 = time.time() if verbose: print("read %-14s in %6.3f s" % ("tod", t2 - t1)) if config.get("tod_skip_deconv"): ops = ["tod_real"] else: ops = ["tod"] actdata.calibrate(self.d, operations=ops, verbose=verbose) tod = self.d.tod # Remove tod from our local d, so we won't end up hauling it around forever del self.d.tod # HWP resample if needed if self.mapping is not None: tod = np.ascontiguousarray( utils.interpol(tod, self.mapping.oimap[None], order=1, mask_nan=False)) method = config.get("downsample_method") for s in self.sampslices: srange = slice(s.start, s.stop, np.sign(s.step) if s.step else None) tod = tod[:, srange] # make sure we get exactly the same length the cuts will be expecting step = np.abs(s.step or 1) olen = (tod.shape[1] + step - 1) // step tod = resample.resample(tod, float(olen) / tod.shape[1], method=method) tod = np.ascontiguousarray(tod) return tod
def to_pairdiff(d, pairs): dtod = d.tod[pairs[:,0]]-d.tod[pairs[:,1]] dcuts = {key: d[key][pairs[:,0]]+d[key][pairs[:,1]] for key in ["cut","cut_basic","cut_noiseest"]} d = d.restrict(dets=d.dets[pairs[:,0]]) d.tod = dtod for key in dcuts: d[key] = dcuts[key] return d # Loop through and analyse each tod-group for ind in range(comm.rank, len(groups), comm.size): ids = groups[ind] entries = [filedb.data[id] for id in ids] try: d = actdata.read(entries, verbose=args.verbose) d = actdata.calibrate(d, verbose=args.verbose, exclude=["autocut"]) if args.pairdiff: pairs = find_pairs_blind(d.point_template) d = to_pairdiff(d, pairs) if d.ndet < 2 or d.nsamp < 2: raise errors.DataMissing("No data in tod") except (errors.DataMissing, AssertionError, IndexError) as e: print "Skipping %s (%s)" % (str(ids), e.message) continue print "Processing %s: %4d %6d" % (str(ids), d.ndet, d.nsamp) tod = d.tod del d.tod tod -= np.mean(tod,1)[:,None] tod = tod.astype(dtype) ft = fft.rfft(tod) nfreq= ft.shape[-1]
from enact import actdata, filedb parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc") parser.add_argument("pickup_map") parser.add_argument("template") parser.add_argument("sel_repr") parser.add_argument("el", type=float) parser.add_argument("ofile") args = parser.parse_args() filedb.init() nrow, ncol = 33, 32 # Read our template, which represents the output horizontal coordinates template = enmap.read_map(args.template) # Use our representative selector to get focalplane offsets and polangles entry = filedb.data[filedb.scans[args.sel_repr][0]] d = actdata.read(entry, ["boresight", "point_offsets", "polangle"]) d.boresight[2] = args.el # In degrees, calibrated in next step d = actdata.calibrate(d, exclude=["autocut"]) def reorder(map, nrow, ncol, dets): return enmap.samewcs(map[utils.transpose_inds(dets, nrow, ncol)], map) # Read our map, and give each row a weight pickup = enmap.read_map(args.pickup_map) pickup = reorder(pickup, nrow, ncol, d.dets) weight = np.median((pickup[:, 1:] - pickup[:, :-1])**2, -1) weight[weight > 0] = 1 / weight[weight > 0] # Find the output pixel for each input pixel
srates = np.zeros([nctod], dtype=dtype) mce_fsamps = np.zeros([nctod], dtype=dtype) mce_params = np.zeros([nctod, 4], dtype=dtype) for ind in range(ind1 + comm.rank, ind2, comm.size): i = ind - ind1 id = ids[ind] entry = filedb.data[id] try: # Do not apply time constants. We want raw spectra so that we can # use them to estimate time constants ourselves. fields = [ "array_info", "tags", "site", "mce_filter", "gain", "cut", "tod", "boresight" ] if args.tconst: fields.append("tconst") d = actdata.read(entry, fields=fields) d = actdata.calibrate( d, exclude=(["autocut"] if not args.no_autocut else [])) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, OSError, errors.DataMissing) as e: print "Skipped (%s)" % (str(e)) continue print "Processing %s" % id srates[i] = d.srate mce_fsamps[i] = d.mce_fsamp mce_params[i] = d.mce_params[:4] # Compute the power spectrum d.tod = d.tod.astype(dtype) nsamp = d.nsamp srate = d.srate
if len(sids) == 0: print("%s has 0 srcs: skipping" % id) continue try: nsrc = len(sids) print("%s has %d srcs: %s" % (id,nsrc,", ".join(["%d (%.1f)" % (i,a) for i,a in zip(sids,amps[sids])]))) except TypeError as e: print("Weird: %s" % e) print(sids) print(amps) continue # Read the data entry = filedb.data[id] try: data = actdata.read(entry, exclude=["tod"], verbose=verbose) data+= actdata.read_tod(entry) data = actdata.calibrate(data, verbose=verbose) #print("fixme") # FIXME #data.restrict(dets=data.dets[100:150]) # Avoid planets while building noise model if planet is not None: data.cut_noiseest *= actdata.cuts.avoidance_cut(data.boresight, data.point_offset, data.site, planet, R) if data.ndet < 2 or data.nsamp < 1: raise errors.DataMissing("no data in tod") except errors.DataMissing as e: print("%s skipped: %s" % (id, e)) continue # Prepeare our samples #data.tod -= np.mean(data.tod,1)[:,None] data.tod -= data.tod[:,None,0].copy() data.tod = data.tod.astype(dtype)
filedb.init() min_samps = 20e3 log_level = log.verbosity2level(config.get("verbosity")) L = log.init(level=log_level, rank=comm.rank) tagger = WorkspaceTagger() ids = filedb.scans[args.sel] for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] entry = filedb.data[id] try: # We need the tod and all its dependences to estimate which noise # category the tod falls into. But we don't need all the dets. # Speed things up by only reading 25% of them. d = actdata.read(entry, ["boresight", "point_offsets", "site"]) d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("Tod contains no valid data") if d.nsamp < min_samps: raise errors.DataMissing("Tod is too short") except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (id, e.message)) continue L.debug(id) # Get the scan el and az bounds az1 = np.min(d.boresight[1]) az2 = np.max(d.boresight[1]) el = np.mean(d.boresight[2])
utils.mkdir(args.odir) root = args.odir + "/" + (args.prefix + "_" if args.prefix else "") # Set up logging utils.mkdir(root + "log") logfile = root + "log/log%03d.txt" % comm_world.rank log_level = log.verbosity2level(config.get("verbosity")) L = log.init(level=log_level, file=logfile, rank=comm_world.rank, shared=False) # Run through all tods to determine the scanning patterns L.info("Detecting scanning patterns") boxes = np.zeros([len(ids), 2, 2]) for ind in range(comm_world.rank, len(ids), comm_world.size): id = ids[ind] entry = filedb.data[id] try: d = actdata.read(entry, ["boresight", "tconst", "cut", "cut_noiseest"]) d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("no data") except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (ids[ind], str(e))) continue # Reorder from az,el to el,az boxes[ind] = [ np.min(d.boresight[2:0:-1], 1), np.max(d.boresight[2:0:-1], 1) ] L.info("%5d: %s" % (ind, id)) boxes = utils.allreduce(boxes, comm_world) # Prune null boxes usable = np.all(boxes != 0, (1, 2))
fbins = np.concatenate(fbins, 0) nbin = len(fbins) del freqs, nbins corr = None pos = None for ind in range(comm.rank, nlabel, comm.size): id = ids[labels == ind] order = np.argsort([i[-1] for i in id]) id = id[order] if len(id) != nsub: continue print "Processing " + "+".join(id) entries = [filedb.data[i] for i in id] try: d = actdata.read(entries) d = actdata.calibrate(d, exclude=["autocut"]) except errors.DataMissing as e: print "Skipping %s: %s" % (id, str(e)) continue tod = d.tod.astype(dtype) del d.tod ft = fft.rfft(tod) del tod if corr is None: ndet = d.array_info.ndet corr = np.zeros((nbin, ndet, ndet), dtype=dtype) hits = np.zeros((nbin, ndet, ndet), dtype=int) var = np.zeros((nbin, ndet)) pos = np.zeros((ndet, 2))
help= "det: output will be in det-seconds per square arcmin. arr: arr-seconds per square arcmin" ) args = parser.parse_args() filedb.init() ids = filedb.scans[args.sel] db = filedb.scans.select(ids) comm = mpi.COMM_WORLD shape, wcs = enmap.read_map_geometry(args.template) # We assume that site and pointing offsets are the same for all tods, # so get them based on the first one entry = filedb.data[ids[0]] site = actdata.read(entry, ["site"]).site omap = fastweight.fastweight(shape, wcs, db, weight=args.weight, array_rad=args.rad * utils.degree, comm=comm, dtype=np.float64, daz=args.daz * utils.degree, chunk_size=args.chunk_size, site=site, verbose=True) if comm.rank == 0: enmap.write_map(args.omap, omap)
res[i] = len(np.unique(tod[i])) return res def write_cuts(ofile, cuts, array_info, dets): with open(ofile, "w") as f: f.write("""format = 'TODCuts' format_version = 2 n_det = %d n_samp = %d samp_offset = 0\n""" % (array_info.ndet, cuts.shape[1])) for ind, det in enumerate(dets): rstr = " ".join(["(%d,%d)" % tuple(rn) for rn in cuts[ind].ranges]) f.write("%4d: %s\n" % (det, rstr)) for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] print id entry = filedb.data[id] # Read uncalibrated data d = actdata.read(entry, fields=["tod","array_info"], verbose=True) dmask = np.zeros(d.ndet, int) quant = measure_quant(d.tod) dmask |= quant < 1000 cuts = det_mask_to_cuts(dmask, d.nsamp) #cuts_null = find_null(d.tod) #cuts_jump = find_jumps(d.tod) #cuts = cuts_null # cuts_jump + cuts_null # Write cut file dets = np.arange(cuts.shape[0]) write_cuts(args.odir + "/%s.cuts" % id, cuts, d.array_info, dets)
bin_freqs = parse_bin_freqs(args.bins) bin_freqs_status = parse_bin_freqs(args.bins_status) nbin = len(bin_freqs) stats = np.zeros([3,ntod,nbin]) examples = np.zeros([ntod,nex,nbin]) nchunk= (ntod+chunk_size-1)//chunk_size for chunk in range(nchunk): ind1 = chunk*chunk_size ind2 = min(ind1+chunk_size, ntod) for ind in range(ind1+comm.rank, ind2, comm.size): id = ids[ind] entry = filedb.data[id] try: d = actdata.read(entry, fields=["gain","tconst","cut","tod","boresight","apex"]) d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, errors.DataMissing) as e: print("Skipped (%s)" % (e)) continue # Compute the power spectrum nsamp = d.nsamp srate = d.srate ft = fft.rfft(d.tod) del d.tod ps = np.abs(ft)**2/(nsamp*srate) del ft # Choose a random set of detectors to be examples. iex = np.random.permutation(len(ps))[:nex] examples[ind] = ps[iex]
ind_ownership = match_existing(aset, ctime) block_ownership = get_block_ownership(ind_ownership, block_inds) else: block_ownership = np.full(len(block_inds),-1,int) fixed_blocks = np.where(block_ownership>=0)[0] free_blocks = np.where(block_ownership<0)[0] nfixed = len(fixed_blocks) nfree = len(free_blocks) sys.stderr.write("splitting %d:[%s] tods into %d splits via %d blocks%s" % ( ntod, atolist(nper), nsplit, nblock, (" with %d:%d free:fixed" % (nfree,nfixed)) if nfixed > 0 else "") + "\n") # We assume that site and pointing offsets are the same for all tods, # so get them based on the first one entry = filedb.data[ids[0]] site = actdata.read(entry, ["site"]).site # Determine the bounding box of our selected data bounds = db.data["bounds"].reshape(2,-1).copy() bounds[0] = utils.rewind(bounds[0], bounds[0,0], 360) box = utils.widen_box(utils.bounding_box(bounds.T), 4*args.rad, relative=False) waz, wel = box[1]-box[0] # Use fullsky horizontally if we wrap too far if waz <= 180: shape, wcs = enmap.geometry(pos=box[:,::-1]*utils.degree, res=args.res*utils.degree, proj="car", ref=(0,0)) else: shape, wcs = enmap.fullsky_geometry(res=args.res*utils.degree) y1, y2 = np.sort(enmap.sky2pix(shape, wcs, [box[:,1]*utils.degree,[0,0]])[0].astype(int)) shape, wcs = enmap.slice_geometry(shape, wcs, (slice(y1,y2),slice(None))) sys.stderr.write("using %s workspace with resolution %.2f deg" % (str(shape), args.res) + "\n")
def __init__(self, entry, subdets=None, d=None, verbose=False, dark=False): self.fields = [ "gain", "mce_filter", "tags", "polangle", "tconst", "hwp", "cut", "point_offsets", "boresight", "site", "tod_shape", "array_info", "beam", "pointsrcs", "buddies" ] if dark: self.fields += ["dark"] if config.get("noise_model") == "file": self.fields += ["noise"] else: if config.get("cut_noise_whiteness"): self.fields += ["noise_cut"] if config.get("cut_spikes"): self.fields += ["spikes"] if d is None: d = actdata.read(entry, self.fields, verbose=verbose) d = actdata.calibrate(d, verbose=verbose) if subdets is not None: d.restrict(dets=d.dets[subdets]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("No data in scan") ndet = d.ndet # Necessary components for Scan interface self.mjd0 = utils.ctime2mjd(d.boresight[0, 0]) self.boresight = np.ascontiguousarray( d.boresight.T.copy()) # [nsamp,{t,az,el}] self.boresight[:, 0] -= self.boresight[0, 0] self.offsets = np.zeros([ndet, self.boresight.shape[1]]) self.offsets[:, 1:] = d.point_offset self.cut = d.cut.copy() self.cut_noiseest = d.cut_noiseest.copy() self.comps = np.zeros([ndet, 4]) self.beam = d.beam self.pointsrcs = d.pointsrcs self.comps = d.det_comps self.hwp = d.hwp self.hwp_phase = d.hwp_phase self.dets = d.dets self.dgrid = (d.array_info.nrow, d.array_info.ncol) self.array_info = d.array_info self.sys = config.get("tod_sys", entry.tod_sys if "tod_sys" in entry else None) self.site = d.site self.speed = d.speed if "noise" in d: self.noise = d.noise else: spikes = d.spikes[:2].T if "spikes" in d else None self.noise = nmat_measure.NmatBuildDelayed( model=config.get("noise_model"), spikes=spikes, cut=self.cut_noiseest) if "dark_tod" in d: self.dark_tod = d.dark_tod if "dark_cut" in d: self.dark_cut = d.dark_cut if "buddy_comps" in d: # Expand buddy_offs to {dt,daz,ddec} self.buddy_comps = d.buddy_comps self.buddy_offs = np.concatenate( [d.buddy_offs[..., :1] * 0, d.buddy_offs], -1) self.autocut = d.autocut if "autocut" in d else [] # Implementation details. d is our DataSet, which we keep around in # because we need it to read tod consistently later. It will *not* # take part in any sample slicing operations, as that might make the # delayed tod read inconsistent with the rest. It could take part in # detector slicing as long as calibrate_tod operates on each detector # independently. This is true now, but would not be so if we did stuff # like common mode subtraction there. On the other hand, not doing this # would prevent slicing before reading from giving any speedup or memory # savings. I don't think allowing this should be a serious problem. self.d = d self.entry = entry def fmt_id(entry): if isinstance(entry, list): return "+".join([fmt_id(e) for e in entry]) else: if entry.tag: return entry.id + ":" + entry.tag else: return entry.id self.id = fmt_id(entry) self.sampslices = [] self.mapping = None # FIXME: debug test if config.get("dummy_cut") > 0: nmax = int(config.get("dummy_cut_len")) # Power law between 1 and nmax, with slope -1. # C(w) = log(w)/log(nmax) # P(w) = w**-1/log(nmax) # w(C) = n**C # Mean: (nmax-1)/log(nmax) nmean = (nmax - 1) / np.log(nmax) ncut = int(self.nsamp * config.get("dummy_cut") / nmean) cut_ranges = np.zeros([self.ndet, ncut, 2], int) w = (nmax**np.random.uniform(0, 1, size=[self.ndet, ncut])).astype(int) np.clip(w, 1, nmax) cut_ranges[:, :, 0] = np.random.uniform(0, self.nsamp, size=[self.ndet, ncut]).astype(int) cut_ranges[:, :, 0] = np.sort(cut_ranges[:, :, 0], 1) cut_ranges[:, :, 1] = cut_ranges[:, :, 0] + w np.clip(cut_ranges[:, :, 1], 0, self.nsamp) cut_dummy = sampcut.from_list(cut_ranges, self.nsamp) print(np.mean(w), nmean, nmax, ncut) print("cut fraction before", float(self.cut.sum()) / self.cut.size) self.cut *= cut_dummy print("cut fraction after", float(self.cut.sum()) / self.cut.size)
# We can't allocate the full buffer, since we don't know the # length of each tod a priori ind1 = chunk*chunk_size ind2 = min(ind1+chunk_size,ntod) cntod = ind2-ind1 lens = np.zeros(cntod,int) mystats= [] myinds = [] for ind in range(ind1+comm.rank, ind2, comm.size): i = ind-ind1 id = ids[ind] entry = filedb.data[id] try: # Get completely raw tods. No cut gapfilling. No gains. Output # will be in ADC units. d = actdata.read(entry, fields=["tod"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, errors.DataMissing) as e: print "Skipped %s [%3d/%d] (%s)" % (id, i+1, ind2-ind1, e.message) continue print "Processing %s [%3d/%d]" % (id, i+1, ind2-ind1) # Get rid of non-data bits d.tod /= 128 nsamp = d.nsamp # Find the number of segments in this tod. We only want whole segments nseg = nsamp/seg_size if nseg < 1: print "Skipped %s: To short tod" % id continue tod = d.tod[:,:nseg*seg_size] stat = np.zeros([2,nstat,ndet,nseg],dtype=dtype)
mask = enmap.zeros((3,)+imask.shape[-2:], imask.wcs, dtype) mask[0] = imask.reshape((-1,)+imask.shape[-2:])[0] del imask utils.mkdir(args.odir) # Each mpi tasks opens its own work file wfname = args.odir + "/work_%03d.hdf" % comm.rank mystats = [] with h5py.File(wfname, "w") as hfile: for ind in range(comm.rank*ntod//comm.size, (comm.rank+1)*ntod//comm.size): id = ids[ind] entry = filedb.data[id] # We only need pointing to build this cut try: d = actdata.read(entry, ["point_offsets","boresight","site","array_info"]) d = actdata.calibrate(d, exclude=["autocut"]) except (errors.DataMissing, AttributeError) as e: print("Skipping %s (%s)" % (id, e)) continue # Build a projector between samples and mask. This # requires us to massage d into scan form. It's getting # annoying that scan and data objects aren't compatible. bore = d.boresight.T.copy() bore[:,0] -= bore[0,0] scan = enscan.Scan( boresight = bore, offsets = np.concatenate([np.zeros(d.ndet)[:,None],d.point_offset],1), comps = np.concatenate([np.ones(d.ndet)[:,None],np.zeros((d.ndet,3))],1), mjd0 = utils.ctime2mjd(d.boresight[0,0]), sys = "hor", site = d.site)
fbins = np.concatenate(fbins,0) nbin = len(fbins) del freqs, nbins corr = None pos = None for ind in range(comm.rank, nlabel, comm.size): id = ids[labels==ind] order = np.argsort([i[-1] for i in id]) id = id[order] if len(id) != nsub: continue print "Processing " + "+".join(id) entries = [filedb.data[i] for i in id] try: d = actdata.read(entries) d = actdata.calibrate(d, exclude=["autocut"]) except errors.DataMissing as e: print "Skipping %s: %s" % (id,e.message) continue tod = d.tod.astype(dtype) del d.tod ft = fft.rfft(tod) del tod if corr is None: ndet = d.array_info.ndet corr = np.zeros((nbin,ndet,ndet),dtype=dtype) hits = np.zeros((nbin,ndet,ndet),dtype=int) var = np.zeros((nbin,ndet)) pos = np.zeros((ndet,2))
filedb.init() min_samps = 20e3 log_level = log.verbosity2level(config.get("verbosity")) L = log.init(level=log_level, rank=comm.rank) tagger = WorkspaceTagger() ids = filedb.scans[args.sel] for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] entry = filedb.data[id] try: # We need the tod and all its dependences to estimate which noise # category the tod falls into. But we don't need all the dets. # Speed things up by only reading 25% of them. d = actdata.read(entry, ["boresight","point_offsets","site"]) d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("Tod contains no valid data") if d.nsamp < min_samps: raise errors.DataMissing("Tod is too short") except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (id, e.message)) continue L.debug(id) # Get the scan el and az bounds az1 = np.min(d.boresight[1]) az2 = np.max(d.boresight[1]) el = np.mean(d.boresight[2])
nsigma = (amps[i,0]**2*aicov[i,0])**0.5 msg += " %7.3f %4.1f" % (amps[i,0]/self.amp_unit, nsigma) msg += " %12.5e %7.2f" % (self.chisq0-chisq, t2-t1) print(msg) self.i += 1 return chisq return wrapper for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] bid = id.replace(":","_") entry = filedb.data[id] # Read the tod as usual try: with bench.show("read"): d = actdata.read(entry) with bench.show("calibrate"): d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod") except errors.DataMissing as e: print("Skipping %s (%s)" % (id, e)) continue print("Processing %s" % id) # Very simple white noise model with bench.show("ivar"): tod = d.tod del d.tod tod -= np.mean(tod,1)[:,None] tod = tod.astype(dtype) diff = tod[:,1:]-tod[:,:-1] diff = diff[:,:diff.shape[-1]//csize*csize].reshape(d.ndet,-1,csize)
def build_tod_stats(entry, Naz=8, Nt=2): """Collect summary information for the tod in the given entry, returning it as a bunch. If some information can't be found, then those fields will be set to a placeholder value (usually NaN), but the fields will still all be present.""" # At the very least we need the pointing, so no try catch around this d = actdata.read(entry, ["boresight", "site"]) d += actdata.read_point_offsets(entry, no_correction=True) d = actdata.calibrate(d, exclude=["autocut"]) # Get the array center and radius acenter = np.mean(d.point_offset, 0) arad = np.mean((d.point_offset - acenter)**2, 0)**0.5 t, baz, bel = 0.5 * (np.min(d.boresight, 1) + np.max(d.boresight, 1)) #t, baz, bel = np.mean(d.boresight,1) az = baz + acenter[0] el = bel + acenter[1] dur, waz, wel = np.max(d.boresight, 1) - np.min(d.boresight, 1) if waz > 180 * utils.degree: print("bad waz %8.3f for %s" % (waz / utils.degree, entry.id)) mjd = utils.ctime2mjd(t) hour = t / 3600. % 24 day = hour >= day_range[0] and hour < day_range[1] night = not day jon = (t - jon_ref) / (3600 * 24) ra, dec = coordinates.transform(tsys, "cel", [az, el], mjd, site=d.site) # Get the array center bounds on the sky, assuming constant elevation ts = utils.ctime2mjd(t + dur / 2 * np.linspace(-1, 1, Nt)) azs = az + waz / 2 * np.linspace(-1, 1, Naz) E1 = coordinates.transform(tsys, "cel", [azs, [el] * Naz], time=[ts[0]] * Naz, site=d.site)[:, 1:] E2 = coordinates.transform(tsys, "cel", [[azs[-1]] * Nt, [el] * Nt], time=ts, site=d.site)[:, 1:] E3 = coordinates.transform(tsys, "cel", [azs[::-1], [el] * Naz], time=[ts[-1]] * Naz, site=d.site)[:, 1:] E4 = coordinates.transform(tsys, "cel", [[azs[0]] * Nt, [el] * Nt], time=ts[::-1], site=d.site)[:, 1:] bounds = np.concatenate([E1, E2, E3, E4], 1) bounds[0] = utils.rewind(bounds[0]) ## Grow bounds by array radius #bmid = np.mean(bounds,1) #for i in range(2): # bounds[i,bounds[i]<bmid[i]] -= arad[i] # bounds[i,bounds[i]>bmid[i]] += arad[i] tot_id = entry.id + (":" + entry.tag if entry.tag else "") res = bunch.Bunch(id=tot_id, nsamp=d.nsamp, t=t, mjd=mjd, jon=jon, hour=hour, day=day, night=night, dur=dur, az=az / utils.degree, el=el / utils.degree, baz=baz / utils.degree, bel=bel / utils.degree, waz=waz / utils.degree, wel=wel / utils.degree, ra=ra / utils.degree, dec=dec / utils.degree, bounds=bounds / utils.degree) if "gseason" in entry: res[entry.gseason] = True # Planets for obj in [ "Sun", "Moon", "Mercury", "Venus", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune" ]: res[obj] = coordinates.ephem_pos(obj, utils.ctime2mjd(t)) / utils.degree # Get our weather information, if available try: d += actdata.read(entry, ["apex"]) d = actdata.calibrate_apex(d) res["pwv"] = d.apex.pwv res["wx"] = d.apex.wind[0] res["wy"] = d.apex.wind[1] res["wind_speed"] = d.apex.wind_speed res["T"] = d.apex.temperature except errors.DataMissing: res["pwv"] = np.NaN res["wx"] = np.NaN res["wy"] = np.NaN res["wind_speed"] = np.NaN res["T"] = np.NaN # Try to get our cut info, so that we can select on # number of detectors and cut fraction try: npre = d.nsamp * d.ndet d += actdata.read(entry, ["cut"]) res["ndet"] = d.ndet res["cut"] = 1 - d.nsamp * d.ndet / float(npre) except errors.DataMissing: res["ndet"] = 0 res["cut"] = 1.0 # Try to get hwp info res["hwp"] = False res["hwp_name"] = "none" try: epochs = actdata.try_read(files.read_hwp_epochs, "hwp_epochs", entry.hwp_epochs) t, _, ar = entry.id.split(".") t = float(t) if ar in epochs: for epoch in epochs[ar]: if t >= epoch[0] and t < epoch[1]: res["hwp"] = True res["hwp_name"] = epoch[2] except errors.DataMissing: pass return res
def write_cuts(ofile, cuts, array_info, dets): with open(ofile, "w") as f: f.write("""format = 'TODCuts' format_version = 2 n_det = %d n_samp = %d samp_offset = 0\n""" % (array_info.ndet, cuts.shape[1])) for ind, det in enumerate(dets): rstr = " ".join(["(%d,%d)" % tuple(rn) for rn in cuts[ind].ranges]) f.write("%4d: %s\n" % (det, rstr)) for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] print id entry = filedb.data[id] # Read uncalibrated data d = actdata.read(entry, fields=["tod", "array_info"], verbose=True) dmask = np.zeros(d.ndet, int) quant = measure_quant(d.tod) dmask |= quant < 1000 cuts = det_mask_to_cuts(dmask, d.nsamp) #cuts_null = find_null(d.tod) #cuts_jump = find_jumps(d.tod) #cuts = cuts_null # cuts_jump + cuts_null # Write cut file dets = np.arange(cuts.shape[0]) write_cuts(args.odir + "/%s.cuts" % id, cuts, d.array_info, dets)
# We can't allocate the full buffer, since we don't know the # length of each tod a priori ind1 = chunk * chunk_size ind2 = min(ind1 + chunk_size, ntod) cntod = ind2 - ind1 lens = np.zeros(cntod, int) mystats = [] myinds = [] for ind in range(ind1 + comm.rank, ind2, comm.size): i = ind - ind1 id = ids[ind] entry = filedb.data[id] try: # Get completely raw tods. No cut gapfilling. No gains. Output # will be in ADC units. d = actdata.read(entry, fields=["tod"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, errors.DataMissing) as e: print "Skipped %s [%3d/%d] (%s)" % (id, i + 1, ind2 - ind1, e.message) continue print "Processing %s [%3d/%d]" % (id, i + 1, ind2 - ind1) # Get rid of non-data bits d.tod /= 128 nsamp = d.nsamp # Find the number of segments in this tod. We only want whole segments nseg = nsamp / seg_size if nseg < 1: print "Skipped %s: To short tod" % id continue
return tod abscal_dict_sub = {} # For abscal values on this process. for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] bid = id.replace(":", "_") entry = filedb.data[id] if args.tags: entry.tag = args.tags # Read the tod as usual try: if not args.sim: with bench.show("read"): d = actdata.read(entry) else: sim_id = sim_ids[ind] sim_entry = filedb.data[sim_id] with bench.show("read"): d = actdata.read(entry, ["boresight"]) d += actdata.read(sim_entry, exclude=["boresight"]) # Store the abscal value. abscal_dict_sub[id] = d.gain_correction[entry.tag] abscal = d.gain_correction[entry.tag] if d.gain_mode == 'mce': abscal /= d.mce_gain elif d.gain_mode == 'mce_compat': abscal /= d.mce_gain * 1217.8583043
ft *= np.exp(-0.5*(sigma/skyspeed)**2*k**2) fft.ifft(ft, tod, normalize=True) for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] bid = id.replace(":","_") entry = filedb.data[id] oname = "%s%s_map.fits" % (prefix, bid) if args.cont and os.path.isfile(oname): print "Skipping %s (already done)" % (id) continue # Read the tod as usual try: if not args.sim: with bench.show("read"): d = actdata.read(entry) else: sim_id = sim_ids[ind] sim_entry = filedb.data[sim_id] with bench.show("read"): d = actdata.read(entry, ["boresight"]) d += actdata.read(sim_entry, exclude=["boresight"]) with bench.show("calibrate"): d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod") # Select detectors if needed if dbox is not None: mid = np.mean(utils.minmax(d.point_template, 0), 0) off = d.point_template-mid good = np.all((off > dbox[0])&(off < dbox[1]),-1) d = d.restrict(dets=d.dets[good])
utils.mkdir(args.odir) root = args.odir + "/" + (args.prefix + "_" if args.prefix else "") # Set up logging utils.mkdir(root + "log") logfile = root + "log/log%03d.txt" % comm_world.rank log_level = log.verbosity2level(config.get("verbosity")) L = log.init(level=log_level, file=logfile, rank=comm_world.rank, shared=False) # Run through all tods to determine the scanning patterns L.info("Detecting scanning patterns") boxes = np.zeros([len(ids),2,2]) for ind in range(comm_world.rank, len(ids), comm_world.size): id = ids[ind] entry = filedb.data[id] try: d = actdata.read(entry, ["boresight","tconst","cut","cut_noiseest"]) d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("no data") except errors.DataMissing as e: L.debug("Skipped %s (%s)" % (ids[ind], e.message)) continue # Reorder from az,el to el,az boxes[ind] = [np.min(d.boresight[2:0:-1],1),np.max(d.boresight[2:0:-1],1)] L.info("%5d: %s" % (ind, id)) boxes = utils.allreduce(boxes, comm_world) # Prune null boxes usable = np.all(boxes!=0,(1,2)) moo = ids[usable] cow = boxes[usable]
key: d[key][pairs[:, 0]] + d[key][pairs[:, 1]] for key in ["cut", "cut_basic", "cut_noiseest"] } d = d.restrict(dets=d.dets[pairs[:, 0]]) d.tod = dtod for key in dcuts: d[key] = dcuts[key] return d # Loop through and analyse each tod-group for ind in range(comm.rank, len(groups), comm.size): ids = groups[ind] entries = [filedb.data[id] for id in ids] try: d = actdata.read(entries, verbose=args.verbose) d = actdata.calibrate(d, verbose=args.verbose, exclude=["autocut"]) if args.pairdiff: pairs = find_pairs_blind(d.point_template) d = to_pairdiff(d, pairs) if d.ndet < 2 or d.nsamp < 2: raise errors.DataMissing("No data in tod") except (errors.DataMissing, AssertionError, IndexError) as e: print "Skipping %s (%s)" % (str(ids), str(e)) continue print "Processing %s: %4d %6d" % (str(ids), d.ndet, d.nsamp) tod = d.tod del d.tod tod -= np.mean(tod, 1)[:, None] tod = tod.astype(dtype) ft = fft.rfft(tod)
def __init__(self, entry, subdets=None, d=None, verbose=False, dark=False): self.fields = ["gain","mce_filter","tags","polangle","tconst","hwp","cut","point_offsets","boresight","site","tod_shape","array_info","beam","pointsrcs", "buddies"] if dark: self.fields += ["dark"] if config.get("noise_model") == "file": self.fields += ["noise"] else: if config.get("cut_noise_whiteness"): self.fields += ["noise_cut"] if config.get("cut_spikes"): self.fields += ["spikes"] if d is None: d = actdata.read(entry, self.fields, verbose=verbose) d = actdata.calibrate(d, verbose=verbose) if subdets is not None: d.restrict(dets=d.dets[subdets]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("No data in scan") ndet = d.ndet # Necessary components for Scan interface self.mjd0 = utils.ctime2mjd(d.boresight[0,0]) self.boresight = np.ascontiguousarray(d.boresight.T.copy()) # [nsamp,{t,az,el}] self.boresight[:,0] -= self.boresight[0,0] self.offsets = np.zeros([ndet,self.boresight.shape[1]]) self.offsets[:,1:] = d.point_offset self.cut = d.cut.copy() self.cut_noiseest = d.cut_noiseest.copy() self.comps = np.zeros([ndet,4]) self.beam = d.beam self.pointsrcs = d.pointsrcs self.comps = d.det_comps self.hwp = d.hwp self.hwp_phase = d.hwp_phase self.dets = d.dets self.dgrid = (d.array_info.nrow, d.array_info.ncol) self.array_info = d.array_info self.sys = config.get("tod_sys") self.site = d.site self.speed = d.speed if "noise" in d: self.noise = d.noise else: spikes = d.spikes[:2].T if "spikes" in d else None self.noise = nmat_measure.NmatBuildDelayed(model = config.get("noise_model"), spikes=spikes, cut=self.cut_noiseest) if "dark_tod" in d: self.dark_tod = d.dark_tod if "dark_cut" in d: self.dark_cut = d.dark_cut if "buddy_comps" in d: # Expand buddy_offs to {dt,daz,ddec} self.buddy_comps = d.buddy_comps self.buddy_offs = np.concatenate([d.buddy_offs[...,:1]*0,d.buddy_offs],-1) self.autocut = d.autocut if "autocut" in d else [] # Implementation details. d is our DataSet, which we keep around in # because we need it to read tod consistently later. It will *not* # take part in any sample slicing operations, as that might make the # delayed tod read inconsistent with the rest. It could take part in # detector slicing as long as calibrate_tod operates on each detector # independently. This is true now, but would not be so if we did stuff # like common mode subtraction there. On the other hand, not doing this # would prevent slicing before reading from giving any speedup or memory # savings. I don't think allowing this should be a serious problem. self.d = d self.entry = entry def fmt_id(entry): if isinstance(entry, list): return "+".join([fmt_id(e) for e in entry]) else: if entry.tag: return entry.id + ":" + entry.tag else: return entry.id self.id = fmt_id(entry) self.sampslices = [] self.mapping = None # FIXME: debug test if config.get("dummy_cut") > 0: nmax = int(config.get("dummy_cut_len")) # Power law between 1 and nmax, with slope -1. # C(w) = log(w)/log(nmax) # P(w) = w**-1/log(nmax) # w(C) = n**C # Mean: (nmax-1)/log(nmax) nmean = (nmax-1)/np.log(nmax) ncut = int(self.nsamp * config.get("dummy_cut") / nmean) cut_ranges = np.zeros([self.ndet, ncut, 2],int) w = (nmax**np.random.uniform(0, 1, size=[self.ndet, ncut])).astype(int) np.clip(w, 1, nmax) cut_ranges[:,:,0] = np.random.uniform(0, self.nsamp, size=[self.ndet, ncut]).astype(int) cut_ranges[:,:,0] = np.sort(cut_ranges[:,:,0],1) cut_ranges[:,:,1] = cut_ranges[:,:,0] + w np.clip(cut_ranges[:,:,1], 0, self.nsamp) cut_dummy = sampcut.from_list(cut_ranges, self.nsamp) print np.mean(w), nmean, nmax, ncut print "cut fraction before", float(self.cut.sum())/self.cut.size self.cut *= cut_dummy print "cut fraction after", float(self.cut.sum())/self.cut.size
if len(ids) > 1: # Will process multiple files utils.mkdir(args.ofile) for id in ids: print id entry = filedb.data[id] subdets = None absdets = None if args.absdets is not None: absdets = [int(w) for w in args.absdets.split(",")] elif args.dets is not None: subdets = [int(w) for w in args.dets.split(",")] fields = ["gain","tconst","cut","tod","boresight"] if args.fields: fields = args.fields.split(",") d = actdata.read(entry, fields=fields) if absdets: d.restrict(dets=absdets) if subdets: d.restrict(dets=d.dets[subdets]) if args.calib: d = actdata.calibrate(d, exclude=["autocut"]) elif args.manual_calib: ops = args.manual_calib.split(",") if "safe" in ops: d.boresight[1:] = utils.unwind(d.boresight[1:], period=360) if "rad" in ops: d.boresight[1:] *= np.pi/180 if "bgap" in ops: bad = (d.flags!=0)*(d.flags!=0x10) for b in d.boresight: gapfill.gapfill_linear(b, bad, inplace=True) if "gain" in ops: d.tod *= d.gain[:,None] if "tgap" in ops: gapfiller = {"copy":gapfill.gapfill_copy, "linear":gapfill.gapfill_linear}[config.get("gapfill")] gapfiller(d.tod, d.cut, inplace=True) if "slope" in ops:
nctod = ind2-ind1 dspecs = np.zeros([nctod,ndet,args.nbin_det], dtype=dtype) dzooms = np.zeros([nctod,ndet,args.nbin_zoom],dtype=dtype) tspecs = np.zeros([5,nctod,args.nbin],dtype=dtype) tcorrs = np.zeros([nctod,args.nbin],dtype=dtype) srates = np.zeros([nctod],dtype=dtype) mce_fsamps = np.zeros([nctod],dtype=dtype) mce_params = np.zeros([nctod,4],dtype=dtype) for ind in range(ind1+comm.rank, ind2, comm.size): i = ind-ind1 id = ids[ind] entry = filedb.data[id] try: # Do not apply time constants. We want raw spectra so that we can # use them to estimate time constants ourselves. d = actdata.read(entry, fields=["array_info", "tags", "site", "mce_filter", "gain","cut","tod","boresight"]) d = actdata.calibrate(d, exclude=["tod_fourier"]+(["autocut"] if not args.no_autocut else [])) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, errors.DataMissing) as e: print "Skipped (%s)" % (e.message) continue print "Processing %s" % id srates[i] = d.srate mce_fsamps[i] = d.mce_fsamp mce_params[i] = d.mce_params[:4] # Compute the power spectrum d.tod = d.tod.astype(dtype) nsamp = d.nsamp srate = d.srate ifmax = d.srate/2 ft = fft.rfft(d.tod) / (nsamp*srate)**0.5
imask = enmap.read_map(args.mask) # Expand to 3 components, as the pointing code expects that mask = enmap.zeros((3, ) + imask.shape[-2:], imask.wcs, dtype) mask[0] = imask.reshape((-1, ) + imask.shape[-2:])[0] del imask utils.mkdir(args.odir) myinds = np.arange(comm.rank, len(ids), comm.size) mystats = [] for ind in myinds: id = ids[ind] entry = filedb.data[id] # We only need pointing to build this cut try: d = actdata.read(entry, ["point_offsets", "boresight", "site", "array_info"]) d = actdata.calibrate(d, exclude=["autocut"]) except (errors.DataMissing, AttributeError) as e: print "Skipping %s (%s)" % (id, e.message) continue # Build a projector between samples and mask. This # requires us to massage d into scan form. It's getting # annoying that scan and data objects aren't compatible. bore = d.boresight.T.copy() bore[:, 0] -= bore[0, 0] scan = enscan.Scan( boresight=bore, offsets=np.concatenate([np.zeros(d.ndet)[:, None], d.point_offset], 1), comps=np.concatenate([np.ones(d.ndet)[:, None], np.zeros((d.ndet, 3))], 1), mjd0=utils.ctime2mjd(d.boresight[0, 0]),
model_fknee = 10 model_alpha = 10 sys = "hor:"+args.planet if args.equator: sys += "/0_0" utils.mkdir(args.odir) prefix = args.odir + "/" if args.tag: prefix += args.tag + "_" for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] bid = id.replace(":","_") entry = filedb.data[id] # Read the tod as usual try: with bench.show("read"): d = actdata.read(entry) with bench.show("calibrate"): d = actdata.calibrate(d, exclude=["autocut"]) if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod") except errors.DataMissing as e: print "Skipping %s (%s)" % (id, e.message) continue print "Processing %s" % id # Very simple white noise model with bench.show("ivar"): tod = d.tod del d.tod tod -= np.mean(tod,1)[:,None] tod = tod.astype(dtype) diff = tod[:,1:]-tod[:,:-1] diff = diff[:,:diff.shape[-1]/csize*csize].reshape(d.ndet,-1,csize)
filedb.init() ids = filedb.scans[args.sel] ntod = len(ids) cuts = np.zeros([ntod, ndet], dtype=np.uint8) stats = None if args.full_stats: stats = np.zeros([ntod, ndet, 4]) for si in range(comm.rank, ntod, comm.size): try: id = ids[si] entry = filedb.data[id] ofile = "%s/%s.txt" % (args.odir, id) try: d = actdata.read( entry, fields=["gain", "tconst", "cut", "tod", "boresight", "hwp"]) d = actdata.calibrate(d, exclude=["tod_fourier", "autocut"]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod") except (IOError, errors.DataMissing) as e: print "Skipped (%s)" % (e.message) continue print "Read %s" % id # Filter the HWP signal print "no hwp filter" #d.tod = todfilter.filter_poly_jon(d.tod, d.boresight[1], hwp=d.hwp) ft = fft.rfft(d.tod) ps = np.abs(ft)**2 / (d.tod.shape[1] * srate) inds = bins * ps.shape[1] / fmax
from enact import actdata, filedb parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc") parser.add_argument("pickup_map") parser.add_argument("template") parser.add_argument("sel_repr") parser.add_argument("el", type=float) parser.add_argument("ofile") args = parser.parse_args() filedb.init() nrow, ncol = 33, 32 # Read our template, which represents the output horizontal coordinates template = enmap.read_map(args.template) # Use our representative selector to get focalplane offsets and polangles entry = filedb.data[filedb.scans[args.sel_repr][0]] d = actdata.read(entry, ["boresight", "point_offsets", "polangle"]) d.boresight[2] = args.el # In degrees, calibrated in next step d = actdata.calibrate(d, exclude=["autocut"]) def reorder(map, nrow, ncol, dets): return enmap.samewcs(map[utils.transpose_inds(dets,nrow,ncol)],map) # Read our map, and give each row a weight pickup = enmap.read_map(args.pickup_map) pickup = reorder(pickup, nrow, ncol, d.dets) weight = np.median((pickup[:,1:]-pickup[:,:-1])**2,-1) weight[weight>0] = 1/weight[weight>0] # Find the output pixel for each input pixel baz = pickup[:1].posmap()[1,0] bel = baz*0 + args.el * utils.degree