def get_rangedata(id):
entry = filedb.data[id]
# Read the tod as usual
with show("read"):
d = actdata.read(entry)
with show("calibrate"):
# Don't apply time constant (and hence butterworth) deconvolution since we
# will fit these ourselves
d = actdata.calibrate(d, exclude=["autocut", "tod_fourier"])
if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod")
tod = d.tod
del d.tod
# Very simple white noise model
with show("noise"):
ivar = estimate_ivar(tod)
asens = np.sum(ivar)**-0.5 / d.srate**0.5
with show("planet mask"):
# Generate planet cut
planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site,
args.planet, R)
with show("atmosphere"):
# Subtract atmospheric model
tod -= estimate_atmosphere(tod, planet_cut, d.srate, model_fknee,
model_alpha)
tod = tod.astype(dtype, copy=False)
with show("extract"):
# Should now be reasonably clean of correlated noise. Extract our range data
rdata = build_rangedata(tod, planet_cut, d, ivar)
return rdata
def read_metadata(entry):
'''
Parameters
----------
entry : filedb.data object
Returns
-------
data : enlib.dataset.DataSet instance
'''
data = actdata.read(entry, exclude=['tod'])
data = actdata.calibrate(data, exclude=['autocut'])
if data.ndet == 0 or data.nsamp == 0:
raise errors.DataMissing("No data in tod")
return data
def scan_iterator(filelist,
inds,
reader,
db=None,
dets=None,
quiet=False,
downsample=1,
hwp_resample=False):
"""Given a set of ids/files and a set of indices into that list. Try
to read each of these scans. Returns a list of successfully read scans
and a list of their indices."""
for ind in inds:
try:
if isinstance(filelist[ind], list): raise IOError
d = enscan.read_scan(filelist[ind])
actdata.read(filedb.data[filelist[ind]])
except IOError:
try:
if isinstance(filelist[ind], list):
entry = [db[id] for id in filelist[ind]]
else:
entry = db[filelist[ind]]
d = reader(entry)
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("Tod contains no valid data")
except errors.DataMissing as e:
if not quiet:
L.debug("Skipped %s (%s)" %
(str(filelist[ind]), e.message))
continue
if dets:
if dets.startswith("@"):
uids = [int(w) for w in open(dets[1:], "r")]
_, det_inds = utils.common_inds([uids, d.dets])
d = d[det_inds]
else:
d = eval("d[%s]" % dets)
hwp_active = np.any(d.hwp_phase[0] != 0)
if hwp_resample and hwp_active:
mapping = enscan.build_hwp_sample_mapping(d.hwp)
d = d.resample(mapping)
d = d[:, ::downsample]
if not quiet: L.debug("Read %s" % str(filelist[ind]))
yield ind, d
if args.equator: sys += "/0_0"
utils.mkdir(args.odir)
prefix = args.odir + "/"
if args.tag: prefix += args.tag + "_"
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
bid = id.replace(":","_")
entry = filedb.data[id]
# Read the tod as usual
try:
with bench.show("read"):
d = actdata.read(entry)
with bench.show("calibrate"):
d = actdata.calibrate(d, exclude=["autocut"])
if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print "Skipping %s (%s)" % (id, e.message)
continue
print "Processing %s" % id
# Very simple white noise model
with bench.show("ivar"):
tod = d.tod
del d.tod
tod -= np.mean(tod,1)[:,None]
tod = tod.astype(dtype)
diff = tod[:,1:]-tod[:,:-1]
diff = diff[:,:diff.shape[-1]/csize*csize].reshape(d.ndet,-1,csize)
ivar = 1/(np.median(np.mean(diff**2,-1),-1)/2**0.5)
del diff
# Generate planet cut
abscal = d.gain_correction[entry.tag]
if d.gain_mode == 'mce':
abscal /= d.mce_gain
elif d.gain_mode == 'mce_compat':
abscal /= d.mce_gain * 1217.8583043
else:
raise ValueError('gain_mode {} not understood'.format(d.gain_mode))
with bench.show("calibrate"):
d = actdata.calibrate(d, exclude=["autocut"])
rel_gain = d.gain_raw.copy() # To store later on.
if d.ndet == 0 or d.nsamp < 2:
raise errors.DataMissing("no data in tod")
# Select detectors if needed.
if dbox is not None:
mid = np.mean(utils.minmax(d.point_template, 0), 0)
off = d.point_template - mid
good = np.all((off > dbox[0]) & (off < dbox[1]), -1)
d = d.restrict(dets=d.dets[good])
except errors.DataMissing as e:
print("Skipping %s (%s)" % (id, e.message))
continue
print("Processing %s" % id, d.ndet, d.nsamp)
# Very simple white noise model.
with bench.show("ivar"):
tod = d.tod
del d.tod
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, rank=comm.rank)
tagger = WorkspaceTagger()
ids = filedb.scans[args.sel]
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
entry = filedb.data[id]
try:
# We need the tod and all its dependences to estimate which noise
# category the tod falls into. But we don't need all the dets.
# Speed things up by only reading 25% of them.
d = actdata.read(entry, ["boresight", "point_offsets", "site"])
d = actdata.calibrate(d, exclude=["autocut"])
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("Tod contains no valid data")
if d.nsamp < min_samps:
raise errors.DataMissing("Tod is too short")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (id, e.message))
continue
L.debug(id)
# Get the scan el and az bounds
az1 = np.min(d.boresight[1])
az2 = np.max(d.boresight[1])
el = np.mean(d.boresight[2])
if not valid_az_range(az1, az2):
L.debug("Skipped %s (%s)" % (id, "Azimuth crosses poles"))
continue
def onlyfinite(a): return a[np.isfinite(a)]
# Process each scan independently
myinds = np.arange(len(filelist))[myid::nproc]
for ind in myinds:
ofile = args.odir + "/%s.hdf" % filelist[ind]
if args.c and os.path.isfile(ofile):
L.info("Already done %s" % filelist[ind])
continue
L.info("Processing %s" % filelist[ind])
try:
d = scan.read_scan(filelist[ind])
except (IOError, OSError):
try:
d = actscan.ACTScan(db[filelist[ind]])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("all samples cut")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], str(e)))
continue
try:
L.debug("Reading samples")
tod = d.get_samples().astype(dtype)
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], str(e)))
continue
# Measure noise
L.debug("Noise")
ivar = 1/np.array([np.median(onlyfinite(get_desloped_var(blockify(t,20)))) for t in tod])
# Set up pmat for this scan
L.debug("Pmats")
for ind in range(len(ids)):
id = ids[ind]
ofile = op.join(args.odir, id.replace(":","_")+".png")
# find sources
sids = lib.get_sids_in_tod(id, srcpos[:,base_sids], bounds[...,ind], base_sids, src_sys=sys, pad=poly_pad)
if len(sids) == 0:
print(f"{id} has no sources: skipping")
continue
else:
print(f"found {len(sids)} sources")
# insert source into tod
entry = filedb.data[id]
try:
scan = actscan.ACTScan(entry, verbose=verbose>=2)
if scan.ndet < 2 or scan.nsamp < 1: raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print("%s skipped: %s" % (id, e))
continue
scan = scan[:,::down]
scan.tod = scan.get_samples()
# build source lists
# ra, dec, T, Q, U, omg, phi
t0 = u.mjd2ctime(scan.mjd0)
phi0 = np.mod(phi+t0*omg, 2*np.pi)
srcs = np.array([srcpos[0], srcpos[1], amps, amps*0, amps*0, omg, phi0, D])
# srcs = np.array([srcpos[0], srcpos[1], amps, amps*0, amps*0])
# build pointing matrix
P = lib.PmatTotVar(scan, srcs, perdet=False, sys=sys)
# P = lib.PmatTot(scan, srcpos[:,sids], perdet=False, sys=sys)
# project pulsar into the given tod
logfile = root + "log/log%03d.txt" % comm.rank
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, file=logfile, rank=comm.rank)
L.info("Initialized")
# Loop through each scan, and compute the hits
hits = enmap.zeros((3, ) + area.shape[-2:], area.wcs, dtype=dtype)
myinds = np.arange(comm.rank, len(ids), comm.size)
for ind in myinds:
id = ids[ind]
entry = filedb.data[id]
try:
scan = actscan.ACTScan(entry)
if scan.ndet == 0 or scan.nsamp == 0:
raise errors.DataMissing("Tod contains no valid data")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (str(id), e.message))
continue
scan = scan[:, ::config.get("downsample")]
L.debug("Processing %s" % str(id))
pmap = pmat.PmatMap(scan, hits)
pcut = pmat.PmatCut(scan)
tod = np.full([scan.ndet, scan.nsamp], 1, dtype=dtype)
junk = np.zeros(pcut.njunk, dtype=dtype)
pcut.backward(tod, junk)
pmap.backward(tod, hits)
hits = hits[0]
# Collect result
ofile = root + "main_map.fits"
if os.path.isfile(ofile): continue
tasks.append((si, id))
# Each task processes tasks independently
for ti in range(comm.rank, len(tasks), comm.size):
si, id = tasks[ti]
bid = id.replace(":", "_")
L.info("Processing src %3d id %s" % (si, id))
root = args.odir + "/src%03d_%s_" % (si, bid)
entry = filedb.data[id]
osys = "hor:%.6f_%.6f:cel/0_0:hor" % tuple(srcs[:2, si])
try:
scans = [actscan.ACTScan(entry)]
if scans[0].nsamp == 0 or scans[0].ndet == 0:
raise errors.DataMissing("no data in scan")
except errors.DataMissing as e:
print "Skipping %s: %s" % (id, str(e))
continue
# Signals
signal_cut = mapmaking.SignalCut(scans, dtype=dtype, comm=tcomm)
signal_map = mapmaking.SignalMap(scans, area, comm=tcomm, sys=osys)
# Weights
weights = [mapmaking.FilterWindow(config.get("tod_window"))]
# And equation system
eqsys = mapmaking.Eqsys(scans, [signal_cut, signal_map],
weights=weights,
dtype=dtype,
comm=tcomm)
eqsys.calc_b()
# Precons
print(sids)
print(amps)
continue
# Read the data
entry = filedb.data[id]
try:
data = actdata.read(entry, exclude=["tod"], verbose=verbose)
data+= actdata.read_tod(entry)
data = actdata.calibrate(data, verbose=verbose)
#print("fixme") # FIXME
#data.restrict(dets=data.dets[100:150])
# Avoid planets while building noise model
if planet is not None:
data.cut_noiseest *= actdata.cuts.avoidance_cut(data.boresight, data.point_offset, data.site, planet, R)
if data.ndet < 2 or data.nsamp < 1: raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print("%s skipped: %s" % (id, e))
continue
# Prepeare our samples
#data.tod -= np.mean(data.tod,1)[:,None]
data.tod -= data.tod[:,None,0].copy()
data.tod = data.tod.astype(dtype)
# Set up our likelihood
L = Likelihood(data, srcpos[:,sids], amps[sids], filter=highpass)
# Find out which sources are reliable, so we don't waste time on bad ones
if prune_unreliable_srcs:
_, aicov = L.fit_amp()
good = amps[sids]**2*aicov[:,0,0,0,0] > args.minsn**2
sids = [sid for sid,g in zip(sids,good) if g]
nsrc = len(sids)
# Read my scans
njunk_tot = 0
cg_rhs = area * 0
cg_rjunk = []
if args.precompute:
prec_NNmap = {lam: area * 0 for lam in np.unique(cooldown)}
prec_NNjunk = {lam: [] for lam in np.unique(cooldown)}
scans = []
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
entry = filedb.data[id]
try:
scan = actscan.ACTScan(entry)
if scan.ndet == 0 or scan.nsamp == 0:
raise errors.DataMissing("No samples in scan")
if args.ndet:
scan = scan[:args.ndet]
if downsample > 1:
scan = scan[:, ::downsample]
scan.pmap = pmat.PmatMap(scan, area)
scan.pcut = pmat.PmatCut(scan)
# Build the noise model
tod = scan.get_samples()
tod -= np.mean(tod, 1)[:, None]
tod = tod.astype(dtype)
scan.noise = scan.noise.update(tod, scan.srate)
scan.T = np.min(scan.noise.D) * Tscale
scan.noise_bar = nmat.NmatDetvecs(scan.noise.D - scan.T, scan.noise.V,
scan.noise.E, scan.noise.bins,
scan.noise.ebins, scan.noise.dets)
def __init__(self, entry, subdets=None, d=None, verbose=False, dark=False):
self.fields = [
"gain", "mce_filter", "tags", "polangle", "tconst", "hwp", "cut",
"point_offsets", "boresight", "site", "tod_shape", "array_info",
"beam", "pointsrcs", "buddies"
]
if dark: self.fields += ["dark"]
if config.get("noise_model") == "file":
self.fields += ["noise"]
else:
if config.get("cut_noise_whiteness"):
self.fields += ["noise_cut"]
if config.get("cut_spikes"):
self.fields += ["spikes"]
if d is None:
d = actdata.read(entry, self.fields, verbose=verbose)
d = actdata.calibrate(d, verbose=verbose)
if subdets is not None:
d.restrict(dets=d.dets[subdets])
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("No data in scan")
ndet = d.ndet
# Necessary components for Scan interface
self.mjd0 = utils.ctime2mjd(d.boresight[0, 0])
self.boresight = np.ascontiguousarray(
d.boresight.T.copy()) # [nsamp,{t,az,el}]
self.boresight[:, 0] -= self.boresight[0, 0]
self.offsets = np.zeros([ndet, self.boresight.shape[1]])
self.offsets[:, 1:] = d.point_offset
self.cut = d.cut.copy()
self.cut_noiseest = d.cut_noiseest.copy()
self.comps = np.zeros([ndet, 4])
self.beam = d.beam
self.pointsrcs = d.pointsrcs
self.comps = d.det_comps
self.hwp = d.hwp
self.hwp_phase = d.hwp_phase
self.dets = d.dets
self.dgrid = (d.array_info.nrow, d.array_info.ncol)
self.array_info = d.array_info
self.sys = config.get("tod_sys",
entry.tod_sys if "tod_sys" in entry else None)
self.site = d.site
self.speed = d.speed
if "noise" in d:
self.noise = d.noise
else:
spikes = d.spikes[:2].T if "spikes" in d else None
self.noise = nmat_measure.NmatBuildDelayed(
model=config.get("noise_model"),
spikes=spikes,
cut=self.cut_noiseest)
if "dark_tod" in d:
self.dark_tod = d.dark_tod
if "dark_cut" in d:
self.dark_cut = d.dark_cut
if "buddy_comps" in d:
# Expand buddy_offs to {dt,daz,ddec}
self.buddy_comps = d.buddy_comps
self.buddy_offs = np.concatenate(
[d.buddy_offs[..., :1] * 0, d.buddy_offs], -1)
self.autocut = d.autocut if "autocut" in d else []
# Implementation details. d is our DataSet, which we keep around in
# because we need it to read tod consistently later. It will *not*
# take part in any sample slicing operations, as that might make the
# delayed tod read inconsistent with the rest. It could take part in
# detector slicing as long as calibrate_tod operates on each detector
# independently. This is true now, but would not be so if we did stuff
# like common mode subtraction there. On the other hand, not doing this
# would prevent slicing before reading from giving any speedup or memory
# savings. I don't think allowing this should be a serious problem.
self.d = d
self.entry = entry
def fmt_id(entry):
if isinstance(entry, list):
return "+".join([fmt_id(e) for e in entry])
else:
if entry.tag: return entry.id + ":" + entry.tag
else: return entry.id
self.id = fmt_id(entry)
self.sampslices = []
self.mapping = None
# FIXME: debug test
if config.get("dummy_cut") > 0:
nmax = int(config.get("dummy_cut_len"))
# Power law between 1 and nmax, with slope -1.
# C(w) = log(w)/log(nmax)
# P(w) = w**-1/log(nmax)
# w(C) = n**C
# Mean: (nmax-1)/log(nmax)
nmean = (nmax - 1) / np.log(nmax)
ncut = int(self.nsamp * config.get("dummy_cut") / nmean)
cut_ranges = np.zeros([self.ndet, ncut, 2], int)
w = (nmax**np.random.uniform(0, 1, size=[self.ndet,
ncut])).astype(int)
np.clip(w, 1, nmax)
cut_ranges[:, :, 0] = np.random.uniform(0,
self.nsamp,
size=[self.ndet,
ncut]).astype(int)
cut_ranges[:, :, 0] = np.sort(cut_ranges[:, :, 0], 1)
cut_ranges[:, :, 1] = cut_ranges[:, :, 0] + w
np.clip(cut_ranges[:, :, 1], 0, self.nsamp)
cut_dummy = sampcut.from_list(cut_ranges, self.nsamp)
print(np.mean(w), nmean, nmax, ncut)
print("cut fraction before", float(self.cut.sum()) / self.cut.size)
self.cut *= cut_dummy
print("cut fraction after", float(self.cut.sum()) / self.cut.size)
filedb.init()
ids = filedb.scans[args.sel]
ntod= len(ids)
cuts = np.zeros([ntod,ndet],dtype=np.uint8)
stats = None
if args.full_stats: stats = np.zeros([ntod,ndet,4])
for si in range(comm.rank, ntod, comm.size):
try:
id = ids[si]
entry = filedb.data[id]
ofile = "%s/%s.txt" % (args.odir, id)
try:
d = actdata.read(entry, fields=["gain","tconst","cut","tod","boresight","hwp"])
d = actdata.calibrate(d, exclude=["tod_fourier","autocut"])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("empty tod")
except (IOError, OSError, errors.DataMissing) as e:
print "Skipped (%s)" % (str(e))
continue
print "Read %s" % id
# Filter the HWP signal
print "no hwp filter"
#d.tod = todfilter.filter_poly_jon(d.tod, d.boresight[1], hwp=d.hwp)
ft = fft.rfft(d.tod)
ps = np.abs(ft)**2/(d.tod.shape[1]*srate)
inds = bins*ps.shape[1]/fmax
bfreqs= np.mean(bins,1)
#tod_moo = fft.irfft(ft, normalize=True)
#ind = np.where(d.dets==640)[0]
# Set up logging
utils.mkdir(root + "log")
logfile = root + "log/log%03d.txt" % comm_world.rank
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, file=logfile, rank=comm_world.rank, shared=False)
# Run through all tods to determine the scanning patterns
L.info("Detecting scanning patterns")
boxes = np.zeros([len(ids), 2, 2])
for ind in range(comm_world.rank, len(ids), comm_world.size):
id = ids[ind]
entry = filedb.data[id]
try:
d = actdata.read(entry, ["boresight", "tconst", "cut", "cut_noiseest"])
d = actdata.calibrate(d, exclude=["autocut"])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("no data")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (ids[ind], str(e)))
continue
# Reorder from az,el to el,az
boxes[ind] = [
np.min(d.boresight[2:0:-1], 1),
np.max(d.boresight[2:0:-1], 1)
]
L.info("%5d: %s" % (ind, id))
boxes = utils.allreduce(boxes, comm_world)
# Prune null boxes
usable = np.all(boxes != 0, (1, 2))
moo = ids[usable]
cow = boxes[usable]
cuts = np.zeros([ntod, ndet], dtype=np.uint8)
stats = None
if args.full_stats: stats = np.zeros([ntod, ndet, 4])
for si in range(comm.rank, ntod, comm.size):
try:
id = ids[si]
entry = filedb.data[id]
ofile = "%s/%s.txt" % (args.odir, id)
try:
d = actdata.read(
entry,
fields=["gain", "tconst", "cut", "tod", "boresight", "hwp"])
d = actdata.calibrate(d, exclude=["tod_fourier", "autocut"])
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("empty tod")
except (IOError, errors.DataMissing) as e:
print "Skipped (%s)" % (e.message)
continue
print "Read %s" % id
# Filter the HWP signal
print "no hwp filter"
#d.tod = todfilter.filter_poly_jon(d.tod, d.boresight[1], hwp=d.hwp)
ft = fft.rfft(d.tod)
ps = np.abs(ft)**2 / (d.tod.shape[1] * srate)
inds = bins * ps.shape[1] / fmax
bfreqs = np.mean(bins, 1)
#tod_moo = fft.irfft(ft, normalize=True)
#ind = np.where(d.dets==640)[0]
# Process each scan independently
myinds = np.arange(len(filelist))[myid::nproc]
for ind in myinds:
ofile = args.odir + "/%s.hdf" % filelist[ind]
if args.c and os.path.isfile(ofile):
L.info("Already done %s" % filelist[ind])
continue
L.info("Processing %s" % filelist[ind])
try:
d = scan.read_scan(filelist[ind])
except IOError:
try:
d = actscan.ACTScan(db[filelist[ind]])
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("all samples cut")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
continue
try:
L.debug("Reading samples")
tod = d.get_samples().astype(dtype)
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
continue
# Measure noise
L.debug("Noise")
ivar = 1 / np.array([
np.median(onlyfinite(get_desloped_var(blockify(t, 20)))) for t in tod
])
