from enact import filedb, data
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("id")
parser.add_argument("area")
parser.add_argument("--di", type=int, default=0, help="Index into array of accepted detectors to use.")
args = parser.parse_args()
dtype = np.float64
eqsys = config.get("map_eqsys")
area = enmap.read_map(args.area).astype(dtype)
area = enmap.zeros((3,)+area.shape[-2:], area.wcs, dtype)
entry = filedb.data[args.id]
# First get the raw samples
d = data.read(entry, subdets=[args.di])
raw_tod = d.tod[0,d.sample_offset:d.cutafter].copy()
raw_bore = d.boresight[:,d.sample_offset:d.cutafter].T
# Then some calibrated samples
d = data.calibrate(d)
cal_tod = d.tod[0]
cal_bore = d.boresight.T
# Apply fourier-truncation to raw data
raw_tod = raw_tod[:cal_tod.shape[0]]
raw_bore = raw_bore[:cal_bore.shape[0]]
# And a proper ACTScan
scan = data.ACTScan(entry, subdets=[args.di])
# Detector pointing
det_ipoint = scan.boresight + scan.offsets[0]
toks = args.imap.split(":")
imap_sys, fname = ":".join(toks[:-1]), toks[-1]
imap = bunch.Bunch(sys=imap_sys or None, map=enmap.read_map(fname))
for i in myinds:
id = ids[i]
entry = db[id]
ofile = "%s/%s.hdf" % (args.odir, id)
if os.path.isfile(ofile) and args.resume: continue
t = []
t.append(time.time())
try:
fields = ["gain", "tconst", "cut", "tod", "boresight", "noise_cut"]
if args.spikecut: fields.append("spikes")
if args.imap: fields += ["polangle", "point_offsets", "site"]
d = data.read(entry, fields)
t.append(time.time())
d = data.calibrate(d)
t.append(time.time())
if args.imap:
# Make a full scan object, so we can perform pointing projection
# operations
d.noise = None
scan = data.ACTScan(entry, d=d)
imap.map = imap.map.astype(d.tod.dtype, copy=False)
pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys)
# Subtract input map from tod inplace
pmap.forward(d.tod, imap.map, tmul=1, mmul=-1)
utils.deslope(d.tod, w=8, inplace=True)
ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5
t.append(time.time())
nobj = len(objs) dstep = 10 sstep = 100 filedb.init() db = filedb.data ids = filedb.scans[args.query] info = filedb.select(ids) hprint = False for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] entry = db[id] # Get a few representative samples try: d = data.read(entry, fields=["gain","cut","point_offsets","boresight","site"]) d = data.calibrate(d) except (zipfile.BadZipfile, errors.DataMissing) as e: print "#%s error: %s" % (id,e.message) #print "%s %8.3f %7.3f %8.3f %7.3f %s" % (id, np.nan, np.nan, np.nan, np.nan, "nodata") continue hour = info[ind].fields.hour tags = sorted(list(set(info[ind].tags)-set([id]))) # Get input pointing bore = d.boresight[:,::sstep] offs = d.point_offset.T[:,::dstep] ipoint = np.zeros(bore.shape + offs.shape[1:]) ipoint[0] = utils.ctime2mjd(bore[0,:,None]) ipoint[1:]= bore[1:,:,None]+offs[:,None,:]
# Use power at 10 Hz to compute weights
fref = [9.5, 10.5]
# Create a global (all tods + all dets) 1-D noise spectrum. Result
# will probably have to be highpass filtered, but that can be done elsewhere
myps = np.zeros(args.N)
mynspec = 0
filedb.init()
ids = filedb.scans[args.sel]
for si in range(comm.rank, len(ids), comm.size):
id = ids[si]
entry = filedb.data[id]
print "reading %s" % id
try:
d = data.read(entry)
d = data.calibrate(d, nofft=True)
except (IOError, errors.DataMissing) as e:
print "skipping (%s)" % e.message
continue
n = d.tod.shape[1]
ft = fft.rfft(d.tod)
del d.tod
ps = np.abs(ft)**2 / (n * srate)
inds = np.linspace(0, args.N, ps.shape[1], endpoint=False).astype(int)
for det, ps_det in zip(d.dets, ps):
ps_bin = np.bincount(inds, ps_det, minlength=args.N) / np.bincount(
inds, minlength=args.N)
weight = 1.0
if args.weight:
filelist = [
line.split()[0] for filelist in args.filelists
for line in open(filelist, "r") if line[0] != "#"
]
myinds = range(len(filelist))[myid::nproc]
n = len(filelist)
for i in myinds:
id = filelist[i]
entry = db[id]
ofile = "%s/%s.hdf" % (args.odir, id)
if os.path.isfile(ofile) and args.resume: continue
t = []
t.append(time.time())
try:
d = data.read(entry, ["gain", "tconst", "cut", "tod", "boresight"])
t.append(time.time())
d = data.calibrate(d)
t.append(time.time())
except errors.DataMissing as e:
print "%3d/%d %25s skip (%s)" % (i + 1, n, id, e.message)
continue
except zipfile.BadZipfile:
print "%d/%d %25s bad zip" % (i + 1, n, id)
continue
ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5
t.append(time.time())
if model == "old":
noise = nmat_measure.detvecs_old(ft, d.srate, d.dets)
elif model == "jon":
di = np.where(d.dets == 20)[0]
parser.add_argument("area")
parser.add_argument("--di",
type=int,
default=0,
help="Index into array of accepted detectors to use.")
args = parser.parse_args()
dtype = np.float64
eqsys = config.get("map_eqsys")
area = enmap.read_map(args.area).astype(dtype)
area = enmap.zeros((3, ) + area.shape[-2:], area.wcs, dtype)
entry = filedb.data[args.id]
# First get the raw samples
d = data.read(entry, subdets=[args.di])
raw_tod = d.tod[0, d.sample_offset:d.cutafter].copy()
raw_bore = d.boresight[:, d.sample_offset:d.cutafter].T
# Then some calibrated samples
d = data.calibrate(d)
cal_tod = d.tod[0]
cal_bore = d.boresight.T
# Apply fourier-truncation to raw data
raw_tod = raw_tod[:cal_tod.shape[0]]
raw_bore = raw_bore[:cal_bore.shape[0]]
# And a proper ACTScan
scan = data.ACTScan(entry, subdets=[args.di])
# Detector pointing
det_ipoint = scan.boresight + scan.offsets[0]
dstep = 10
sstep = 100
filedb.init()
db = filedb.data
ids = filedb.scans[args.query]
info = filedb.select(ids)
hprint = False
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
entry = db[id]
# Get a few representative samples
try:
d = data.read(
entry,
fields=["gain", "cut", "point_offsets", "boresight", "site"])
d = data.calibrate(d)
except (zipfile.BadZipfile, errors.DataMissing) as e:
print "#%s error: %s" % (id, e.message)
#print "%s %8.3f %7.3f %8.3f %7.3f %s" % (id, np.nan, np.nan, np.nan, np.nan, "nodata")
continue
hour = info[ind].fields.hour
tags = sorted(list(set(info[ind].tags) - set([id])))
# Get input pointing
bore = d.boresight[:, ::sstep]
offs = d.point_offset.T[:, ::dstep]
ipoint = np.zeros(bore.shape + offs.shape[1:])
ipoint[0] = utils.ctime2mjd(bore[0, :, None])
if args.imap:
toks = args.imap.split(":")
imap_sys, fname = ":".join(toks[:-1]), toks[-1]
imap = bunch.Bunch(sys=imap_sys or None, map=enmap.read_map(fname))
for i in myinds:
id = ids[i]
entry = db[id]
ofile = "%s/%s.hdf" % (args.odir, id)
if os.path.isfile(ofile) and args.resume: continue
t=[]; t.append(time.time())
try:
fields = ["gain","tconst","cut","tod","boresight", "noise_cut"]
if args.spikecut: fields.append("spikes")
if args.imap: fields += ["polangle","point_offsets","site"]
d = data.read(entry, fields) ; t.append(time.time())
d = data.calibrate(d) ; t.append(time.time())
if args.imap:
# Make a full scan object, so we can perform pointing projection
# operations
d.noise = None
scan = data.ACTScan(entry, d=d)
imap.map = imap.map.astype(d.tod.dtype, copy=False)
pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys)
# Subtract input map from tod inplace
pmap.forward(d.tod, imap.map, tmul=1, mmul=-1)
utils.deslope(d.tod, w=8, inplace=True)
ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5 ; t.append(time.time())
spikes = d.spikes[:2].T if args.spikecut else None
if model == "old":
noise = nmat_measure.detvecs_old(ft, d.srate, d.dets)
if comm.rank == 0: hfile = h5py.File(args.odir + "/cum%04d.hdf" % si, "w") for eq in [tod_eq,acc_eq,det_eq]: tot = eq.reduce() if comm.rank == 0: hfile[eq.name + "/rhs"] = tot.rhs hfile[eq.name + "/div"] = tot.div hfile[eq.name + "/sig"] = tot.solve() if comm.rank == 0: hfile.close() for si in range(comm.rank, len(ids)/comm.size*comm.size, comm.size): entry = filedb.data[ids[si]] print "Reading %s" % entry.id try: d = data.read(entry, fields=["gain","polangle","tconst","boresight","cut","tod"], absdets=absdets) d = data.calibrate(d) except (IOError,errors.DataMissing) as e: print "Skipping [%s]" % e.message output_cum(si) continue print "Computing pol tod" ndet, nsamp = d.tod.shape print ndet, nsamp # Estimate white noise level (rough) rms = white_est(d.tod[:,:50000]) weight = (1-d.cut.to_mask())/rms[:,None]**2 if args.filter: d.tod = highpass(d.tod, args.filter)