def scan_iterator(filelist,
inds,
reader,
db=None,
dets=None,
quiet=False,
downsample=1,
hwp_resample=False):
"""Given a set of ids/files and a set of indices into that list. Try
to read each of these scans. Returns a list of successfully read scans
and a list of their indices."""
for ind in inds:
try:
if isinstance(filelist[ind], list): raise IOError
d = enscan.read_scan(filelist[ind])
actdata.read(filedb.data[filelist[ind]])
except IOError:
try:
if isinstance(filelist[ind], list):
entry = [db[id] for id in filelist[ind]]
else:
entry = db[filelist[ind]]
d = reader(entry)
if d.ndet == 0 or d.nsamp == 0:
raise errors.DataMissing("Tod contains no valid data")
except errors.DataMissing as e:
if not quiet:
L.debug("Skipped %s (%s)" %
(str(filelist[ind]), e.message))
continue
if dets:
if dets.startswith("@"):
uids = [int(w) for w in open(dets[1:], "r")]
_, det_inds = utils.common_inds([uids, d.dets])
d = d[det_inds]
else:
d = eval("d[%s]" % dets)
hwp_active = np.any(d.hwp_phase[0] != 0)
if hwp_resample and hwp_active:
mapping = enscan.build_hwp_sample_mapping(d.hwp)
d = d.resample(mapping)
d = d[:, ::downsample]
if not quiet: L.debug("Read %s" % str(filelist[ind]))
yield ind, d
m = np.mean(blocks,1)
sd = np.einsum("at,t->a",blocks,s)
A = sd/ss
return np.mean((blocks-m[:,None]-A[:,None]*s[None,:])**2,1)
def onlyfinite(a): return a[np.isfinite(a)]
# Process each scan independently
myinds = np.arange(len(filelist))[myid::nproc]
for ind in myinds:
ofile = args.odir + "/%s.hdf" % filelist[ind]
if args.c and os.path.isfile(ofile):
L.info("Already done %s" % filelist[ind])
continue
L.info("Processing %s" % filelist[ind])
try:
d = scan.read_scan(filelist[ind])
except (IOError, OSError):
try:
d = actscan.ACTScan(db[filelist[ind]])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("all samples cut")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], str(e)))
continue
try:
L.debug("Reading samples")
tod = d.get_samples().astype(dtype)
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], str(e)))
continue
# Measure noise
m = np.mean(blocks,1)
sd = np.einsum("at,t->a",blocks,s)
A = sd/ss
return np.mean((blocks-m[:,None]-A[:,None]*s[None,:])**2,1)
def onlyfinite(a): return a[np.isfinite(a)]
# Process each scan independently
myinds = np.arange(len(filelist))[myid::nproc]
for ind in myinds:
ofile = args.odir + "/%s.hdf" % filelist[ind]
if args.c and os.path.isfile(ofile):
L.info("Already done %s" % filelist[ind])
continue
L.info("Processing %s" % filelist[ind])
try:
d = scan.read_scan(filelist[ind])
except IOError:
try:
d = actscan.ACTScan(db[filelist[ind]])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("all samples cut")
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
continue
try:
L.debug("Reading samples")
tod = d.get_samples().astype(dtype)
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
continue
# Measure noise