if os.path.isfile(ofile) and args.resume: continue
t = []
t.append(time.time())
try:
fields = ["gain", "tconst", "cut", "tod", "boresight", "noise_cut"]
if args.spikecut: fields.append("spikes")
if args.imap: fields += ["polangle", "point_offsets", "site"]
d = data.read(entry, fields)
t.append(time.time())
d = data.calibrate(d)
t.append(time.time())
if args.imap:
# Make a full scan object, so we can perform pointing projection
# operations
d.noise = None
scan = data.ACTScan(entry, d=d)
imap.map = imap.map.astype(d.tod.dtype, copy=False)
pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys)
# Subtract input map from tod inplace
pmap.forward(d.tod, imap.map, tmul=1, mmul=-1)
utils.deslope(d.tod, w=8, inplace=True)
ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5
t.append(time.time())
spikes = d.spikes[:2].T if args.spikecut else None
if model == "old":
noise = nmat_measure.detvecs_old(ft, d.srate, d.dets)
elif model == "jon":
noise = nmat_measure.detvecs_jon(ft,
d.srate,
d.dets,
shared,
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, file=logfile, rank=myid)
# And benchmarking
utils.mkdir(root + "bench")
benchfile = root + "bench/bench%03d.txt" % myid
# Read in all our scans
L.info("Reading scans")
tmpinds = np.arange(len(filelist))[myid::nproc]
myscans, myinds = [], []
for ind in tmpinds:
try:
d = scan.read_scan(filelist[ind])
except IOError:
try:
d = data.ACTScan(db[filelist[ind]])
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
continue
d = d[:, ::config.get("downsample")]
if args.ndet > 0: d = d[:args.ndet]
myscans.append(d)
myinds.append(ind)
L.debug("Read %s" % filelist[ind])
nread = comm.allreduce(len(myscans))
L.info("Found %d tods" % nread)
if nread == 0:
L.info("Giving up")
sys.exit(1)
sd = np.einsum("at,t->a",blocks,s)
A = sd/ss
return np.mean((blocks-m[:,None]-A[:,None]*s[None,:])**2,1)
def onlyfinite(a): return a[np.isfinite(a)]
# Process each scan independently
myinds = np.arange(len(filelist))[myid::nproc]
for ind in myinds:
ofile = args.odir + "/%s.hdf" % filelist[ind]
if args.c and os.path.isfile(ofile): continue
L.info("Processing %s" % filelist[ind])
try:
d = scan.read_scan(filelist[ind])
except IOError:
try:
d = data.ACTScan(db[filelist[ind]])
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
continue
try:
# Set up pmat for this scan
L.debug("Pmats")
Psrc = pmat.PmatPtsrc(d, params)
Pcut = pmat.PmatCut(d)
L.debug("Reading samples")
tod = d.get_samples().astype(dtype)
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
continue
entry = filedb.data[args.id] # First get the raw samples d = data.read(entry, subdets=[args.di]) raw_tod = d.tod[0, d.sample_offset:d.cutafter].copy() raw_bore = d.boresight[:, d.sample_offset:d.cutafter].T # Then some calibrated samples d = data.calibrate(d) cal_tod = d.tod[0] cal_bore = d.boresight.T # Apply fourier-truncation to raw data raw_tod = raw_tod[:cal_tod.shape[0]] raw_bore = raw_bore[:cal_bore.shape[0]] # And a proper ACTScan scan = data.ACTScan(entry, subdets=[args.di]) # Detector pointing det_ipoint = scan.boresight + scan.offsets[0] # Manual transformation trf = pmat.pos2pix(scan, None, eqsys) det_opoint_exact = trf(det_ipoint.T).T psi_rot_exact = np.arctan2(det_opoint_exact[:, 3], det_opoint_exact[:, 2]) psi_det = np.arctan2(scan.comps[:, 2], scan.comps[:, 1]) psi_exact = (psi_rot_exact + psi_det) / 2 % np.pi ppix = pmat.PmatMap(scan, area) # Exact to pixels det_opix_exact = ppix.transform(det_ipoint.T).T
def get_scans(area,
signal,
bore,
dets,
noise,
seed=0,
real=None,
noise_override=None):
scans = []
# Get real scan information if necessary
L.debug("real")
if real:
real_scans = []
filedb.init()
db = filedb.data
ids = fileb.scans[real].ids
for id in ids:
try:
real_scans.append(data.ACTScan(db[id]))
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (id, e.message))
# Dets
L.debug("dets")
sim_dets = []
toks = dets.split(":")
if toks[0] == "scattered":
ngroup, nper, rad = int(toks[1]), int(toks[2]), float(toks[3])
sim_dets = [
scansim.dets_scattered(ngroup, nper, rad=rad * np.pi / 180 / 60)
]
margin = rad * np.pi / 180 / 60
elif toks[0] == "real":
ndet = int(toks[1])
dslice = slice(0, ndet) if ndet > 0 else slice(None)
sim_dets = [
bunch.Bunch(comps=s.comps[dslice], offsets=s.offsets[dslice])
for s in real_scans
]
margin = np.max([np.sum(s.offsets**2, 1)**0.5 for s in sim_dets])
else:
raise ValueError
# Boresight. Determines our number of scans
L.debug("bore")
sim_bore = []
toks = bore.split(":")
if toks[0] == "grid":
nscan, density, short = int(toks[1]), float(toks[2]), float(toks[3])
for i in range(nscan):
tbox = shorten(area.box(), i % 2, short)
sim_bore.append(
scansim.scan_grid(tbox,
density * np.pi / 180 / 60,
dir=i,
margin=margin))
elif toks[0] == "real":
sim_bore = [
bunch.Bunch(boresight=s.boresight,
sys=s.sys,
site=s.site,
mjd0=s.mjd0) for s in real_scans
]
else:
raise ValueError
nsim = len(sim_bore)
# Make one det info per scan
sim_dets = sim_dets * (nsim / len(sim_dets)) + sim_dets[:nsim %
len(sim_dets)]
# Noise
L.debug("noise")
sim_nmat = []
toks = noise.split(":")
nonoise = False
if toks[0] == "1/f":
sigma, alpha, fknee = [float(v) for v in toks[1:4]]
nonoise = sigma < 0
for i in range(nsim):
sim_nmat.append(
scansim.oneoverf_noise(sim_dets[i].comps.shape[0],
sim_bore[i].boresight.shape[0],
sigma=np.abs(sigma),
alpha=alpha,
fknee=fknee))
elif toks[0] == "detcorr":
sigma, alpha, fknee = [float(v) for v in toks[1:4]]
nonoise = sigma < 0
for i in range(nsim):
sim_nmat.append(
scansim.oneoverf_detcorr_noise(sim_dets[i].comps.shape[0],
sim_bore[i].boresight.shape[0],
sigma=np.abs(sigma),
alpha=alpha,
fknee=fknee))
elif toks[0] == "real":
scale = 1.0 if len(toks) < 2 else float(toks[1])
for i, s in enumerate(real_scans):
ndet = len(sim_dets[i].offsets)
nmat = s.noise[:ndet] * scale**-2
sim_nmat.append(nmat)
else:
raise ValueError
noise_scale = not nonoise if noise_override is None else noise_override
sim_nmat = sim_nmat * (nsim / len(sim_nmat)) + sim_nmat[:nsim %
len(sim_nmat)]
# Signal
L.debug("signal")
toks = signal.split(":")
if toks[0] == "ptsrc":
# This one always operates in the same coordinates as
nsrc, amp, fwhm = int(toks[1]), float(toks[2]), float(toks[3])
np.random.seed(seed)
sim_srcs = scansim.rand_srcs(area.box(),
nsrc,
amp,
abs(fwhm) * np.pi / 180 / 60,
rand_fwhm=fwhm < 0)
for i in range(nsim):
scans.append(
scansim.SimSrcs(sim_bore[i],
sim_dets[i],
sim_srcs,
sim_nmat[i],
seed=seed + i,
noise_scale=noise_scale))
elif toks[0] == "vsrc":
# Create a single variable source
ra, dec, fwhm = float(toks[1]) * np.pi / 180, float(
toks[2]) * np.pi / 180, float(toks[3]) * np.pi / 180 / 60
amps = [float(t) for t in toks[4].split(",")]
for i in range(nsim):
sim_srcs = bunch.Bunch(pos=np.array([[dec, ra]]),
amps=np.array([[amps[i], 0, 0, 0]]),
beam=np.array([fwhm / (8 * np.log(2)**0.5)
]))
scans.append(
scansim.SimSrcs(sim_bore[i],
sim_dets[i],
sim_srcs,
sim_nmat[i],
seed=seed + i,
noise_scale=noise_scale,
nsigma=20))
elif toks[0] == "cmb":
np.random.seed(seed)
ps = powspec.read_spectrum(toks[1])
sim_map = enmap.rand_map(area.shape, area.wcs, ps)
for i in range(nsim):
scans.append(
scansim.SimMap(sim_bore[i],
sim_dets[i],
sim_map,
sim_nmat[i],
seed=seed + i,
noise_scale=noise_scale))
else:
raise ValueError
return scans
def get_scans(area, signal, bore, dets, noise, seed=0, real=None):
scans = []
# Get real scan information if necessary
L.debug("real")
if real:
real_scans = []
toks = real.split(":")
filelist, fslice = toks[0], ":".join(toks[1:])
filelist = [
line.split()[0] for line in open(filelist, "r") if line[0] != "#"
]
filelist = eval("filelist" + fslice)
db = filedb.ACTdb(config.get("filedb"))
for id in filelist:
try:
real_scans.append(data.ACTScan(db[id]))
except errors.DataMissing as e:
L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
# Dets
L.debug("dets")
sim_dets = []
toks = dets.split(":")
if toks[0] == "scattered":
ngroup, nper, rad = int(toks[1]), int(toks[2]), float(toks[3])
sim_dets = [
scansim.dets_scattered(ngroup, nper, rad=rad * np.pi / 180 / 60)
]
margin = rad * np.pi / 180 / 60
elif toks[0] == "real":
ndet = int(toks[1])
dslice = slice(0, ndet) if ndet > 0 else slice(None)
sim_dets = [
bunch.Bunch(comps=s.comps[dslice], offsets=s.offsets[dslice])
for s in real_scans
]
margin = np.max([np.sum(s.offsets**2, 1)**0.5 for s in sim_dets])
else:
raise ValueError
# Boresight. Determines our number of scans
L.debug("bore")
sim_bore = []
toks = bore.split(":")
if toks[0] == "grid":
nscan, density, short = int(toks[1]), float(toks[2]), float(toks[3])
for i in range(nscan):
tbox = shorten(area.box(), i % 2, short)
sim_bore.append(
scansim.scan_grid(tbox,
density * np.pi / 180 / 60,
dir=i,
margin=margin))
elif toks[0] == "real":
sim_bore = [
bunch.Bunch(boresight=s.boresight,
sys=s.sys,
site=s.site,
mjd0=s.mjd0) for s in real_scans
]
else:
raise ValueError
nsim = len(sim_bore)
# Make one det info per scan
sim_dets = sim_dets * (nsim / len(sim_dets)) + sim_dets[:nsim %
len(sim_dets)]
# Noise
L.debug("noise")
sim_nmat = []
toks = noise.split(":")
noiseless = False
if toks[0] == "1/f":
sigma, alpha, fknee = [float(v) for v in toks[1:4]]
noiseless = sigma < 0
for i in range(nsim):
sim_nmat.append(
scansim.oneoverf_noise(sim_dets[i].comps.shape[0],
sim_bore[i].boresight.shape[0],
sigma=np.abs(sigma),
alpha=alpha,
fknee=fknee))
elif toks[0] == "detcorr":
sigma, alpha, fknee = [float(v) for v in toks[1:4]]
noiseless = sigma < 0
for i in range(nsim):
sim_nmat.append(
scansim.oneoverf_detcorr_noise(sim_dets[i].comps.shape[0],
sim_bore[i].boresight.shape[0],
sigma=np.abs(sigma),
alpha=alpha,
fknee=fknee))
elif toks[0] == "real":
scale = 1.0 if len(toks) < 2 else float(toks[1])
for i, s in enumerate(real_scans):
ndet = len(sim_dets[i].offsets)
nmat = s.noise[:ndet] * scale**-2
sim_nmat.append(nmat)
else:
raise ValueError
sim_nmat = sim_nmat * (nsim / len(sim_nmat)) + sim_nmat[:nsim %
len(sim_nmat)]
# Signal
L.debug("signal")
toks = signal.split(":")
if toks[0] == "ptsrc":
# This one always operates in the same coordinates as
nsrc, amp, fwhm = int(toks[1]), float(toks[2]), float(toks[3])
sim_srcs = scansim.rand_srcs(area.box(),
nsrc,
amp,
abs(fwhm) * np.pi / 180 / 60,
rand_fwhm=fwhm < 0)
for i in range(nsim):
scans.append(
scansim.SimSrcs(sim_bore[i],
sim_dets[i],
sim_srcs,
sim_nmat[i],
seed=seed + i,
nonoise=noiseless))
elif toks[0] == "cmb":
np.random.seed(seed)
ps = powspec.read_spectrum(toks[1])
sim_map = enmap.rand_map(area.shape, area.wcs, ps)
for i in range(nsim):
scans.append(
scansim.SimMap(sim_bore[i],
sim_dets[i],
sim_map,
sim_nmat[i],
seed=seed + i,
nonoise=noiseless))
else:
raise ValueError
return scans