for id in filelist:
f.write("%s\n" % id)
shutil.copyfile(filedb.cjoin(["root","dataset","filedb"]), args.odir + "/filedb.txt")
try: shutil.copyfile(filedb.cjoin(["root","dataset","todinfo"]), args.odir + "/todinfo.txt")
except (IOError, OSError): pass
# Set up logging
utils.mkdir(args.odir + "/log")
logfile = args.odir + "/log/log%03d.txt" % myid
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, file=logfile, rank=myid, shared=False)
# And benchmarking
utils.mkdir(args.odir + "/bench")
benchfile = args.odir + "/bench/bench%03d.txt" % myid
# Read our point source list
params = pointsrcs.src2param(pointsrcs.read(args.srcs))
ntot = len(params)
# Eliminate insignificant sources
params = params[np.abs(params[:,2])>args.minamp]
if comm.rank == 0:
L.info("Got %d sources, keeping %d > %d uK" % (ntot,len(params),args.minamp))
pointsrcs.write(args.odir + "/srcs.txt", pointsrcs.param2src(params))
# Our noise model is slightly different from the main noise model,
# since we assume it is white and independent between detectors,
# which is not strictly true. To minimize error, we measure the
# noise level using a method which is as close as possible to
# how we will use it later
def blockify(tod, w): return tod[:tod.size/w*w].reshape(-1,w)
def get_desloped_var(blocks):
def autocut(d, turnaround=None, ground=None, sun=None, moon=None, max_frac=None, pickup=None):
"""Apply automatic cuts to calibrated data."""
if not config.get("autocut"): return d
ndet, nsamp = d.ndet, d.nsamp
if not ndet or not nsamp: return d
# Insert a cut into d if necessary
if "cut" not in d:
d += dataset.DataField("cut", sampcut.empty(ndet,nsamp))
# insert an autocut datafield, to keep track of how much data each
# automatic cut cost us
d += dataset.DataField("autocut", [])
def addcut(label, dcut, targets="c"):
# det ndet part here allows for broadcasting of cuts from 1-det to full-det
dn = dcut.sum()*d.ndet/dcut.ndet if dcut is not None else 0
if dn == 0: d.autocut.append([label,0,0])
else:
n0, dn = d.cut.sum(), dcut.sum()
dn = dn*d.cut.ndet/dcut.ndet
if "c" in targets: d.cut *= dcut
if "n" in targets: d.cut_noiseest *= dcut
if "b" in targets: d.cut_basic *= dcut
d.autocut.append([ label, dn, d.cut.sum() - n0 ]) # name, mycut, myeffect
if config.get("cut_tconst") and "tau" in d:
addcut("tconst", cuts.tconst_cut(nsamp, d.tau, config.get("cut_tconst")))
if config.get("cut_stationary") and "boresight" in d:
addcut("stationary", cuts.stationary_cut(d.boresight[1]))
if config.get("cut_tod_ends") and "srate" in d:
addcut("tod_ends", cuts.tod_end_cut(nsamp, d.srate))
if config.get("cut_turnaround", turnaround) and "boresight" in d:
addcut("turnaround",cuts.turnaround_cut(d.boresight[1]))
if config.get("cut_ground", ground) and "boresight" in d and "point_offset" in d:
addcut("ground", cuts.ground_cut(d.boresight, d.point_offset))
if config.get("cut_sun", sun) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("avoidance",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Sun", config.get("cut_sun_dist")*np.pi/180))
if config.get("cut_moon", moon) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("moon",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Moon", config.get("cut_moon_dist")*np.pi/180))
if config.get("cut_pickup", pickup) and "boresight" in d and "pickup_cut" in d:
addcut("pickup",cuts.pickup_cut(d.boresight[1], d.dets, d.pickup_cut))
if config.get("cut_mostly_cut"):
addcut("mostly_cut", cuts.cut_mostly_cut_detectors(d.cut_quality))
if config.get("cut_obj"):
objs = utils.split_outside(config.get("cut_obj"),",")
for obj in objs:
toks = obj.split(":")
objname = toks[0]
if objname.startswith("["):
objname = [float(w)*utils.degree for w in objname[1:-1].split(",")]
dist = 0.2*utils.degree
if len(toks) > 1: dist = float(toks[1])*utils.degree
# Hack: only cut for noise estimation purposes if dist is negative
targets = "cnb" if dist > 0 else "n"
addcut(obj, cuts.avoidance_cut(d.boresight, d.point_offset, d.site, objname, dist), targets=targets)
if config.get("cut_srcs"):
cpar = [tok.split(":") for tok in config.get("cut_srcs").split(",")]
names, lims = [], []
for par in cpar:
if par[0] in ["map","nmat"]:
names.append(par[0])
lims.append(float(par[1]))
if any(lims):
params = pointsrcs.src2param(d.pointsrcs)
params[:,5:7] = 1
params[:,7] = 0
# Only bother with sources that are actually strong enough
maxlim = max(lims+[0])
params = params[params[:,2]>maxlim]
cutlist = cuts.point_source_cut(d, params, lims)
for name, c in zip(names, cutlist):
if name == "map": addcut("point_srcs_m", c, "c")
elif name == "nmat": addcut("point_srcs_n", c, "n")
if config.get("cut_extra_srcs"):
srclist = np.loadtxt(config.get("cut_extra_srcs"), usecols=(0,1), ndmin=2)
srclim = float(config.get("cut_extra_lim"))
params = np.zeros([len(srclist),8])
params[:,:2] = srclist[:,1::-1]*utils.degree
params[:,2] = 1
params[:,5:7] = 1
c = cuts.point_source_cut(d, params, srclim)
addcut("point_srcs", c, "nbc")
# What fraction is cut?
cut_fraction = float(d.cut.sum())/d.cut.size
# Get rid of completely cut detectors
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(d.dets[keep])
ndet, nsamp = d.ndet, d.nsamp
def cut_all_if(label, condition):
if condition: dcut = sampcut.full(d.ndet, nsamp)
else: dcut = None
addcut(label, dcut)
cut_all_if("max_frac", config.get("cut_max_frac", max_frac) < cut_fraction)
if "srate" in d:
cut_all_if("tod_mindur", config.get("cut_tod_mindur") > nsamp/d.srate/60)
cut_all_if("tod_mindet", config.get("cut_tod_mindet") > ndet)
# Get rid of completely cut detectors again
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(dets=d.dets[keep])
return d
for id in filelist:
f.write("%s\n" % id)
shutil.copyfile(filedb.cjoin(["root","dataset","filedb"]), args.odir + "/filedb.txt")
try: shutil.copyfile(filedb.cjoin(["root","dataset","todinfo"]), args.odir + "/todinfo.txt")
except IOError: pass
# Set up logging
utils.mkdir(args.odir + "/log")
logfile = args.odir + "/log/log%03d.txt" % myid
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, file=logfile, rank=myid, shared=False)
# And benchmarking
utils.mkdir(args.odir + "/bench")
benchfile = args.odir + "/bench/bench%03d.txt" % myid
# Read our point source list
params = pointsrcs.src2param(pointsrcs.read(args.srcs))
ntot = len(params)
# Eliminate insignificant sources
params = params[np.abs(params[:,2])>args.minamp]
if comm.rank == 0:
L.info("Got %d sources, keeping %d > %d uK" % (ntot,len(params),args.minamp))
pointsrcs.write(args.odir + "/srcs.txt", pointsrcs.param2src(params))
# Our noise model is slightly different from the main noise model,
# since we assume it is white and independent between detectors,
# which is not strictly true. To minimize error, we measure the
# noise level using a method which is as close as possible to
# how we will use it later
def blockify(tod, w): return tod[:tod.size/w*w].reshape(-1,w)
def get_desloped_var(blocks):
if not pertod:
filter = mapmaking.FilterBuddyDmap(myscans,
mysubs,
m,
sys=sys,
mul=-mul,
tmul=tmul)
else:
raise NotImplementedError(
"FIXME: Implement per tod buddy subtraction with dmaps"
)
elif param["name"] == "src":
if param["value"] == 0: continue
if "params" not in param: params = myscans[0].pointsrcs
else: params = pointsrcs.read(param["params"])
params = pointsrcs.src2param(params)
params = params.astype(np.float64)
print "FIXME: how to handle per-source beams? Forcing to relative for now"
params[:, 5:7] = 1
params[:, 7] = 0
filter = mapmaking.FilterAddSrcs(myscans,
params,
sys=param["sys"],
mul=-float(param["mul"]))
else:
raise ValueError("Unrecognized fitler name '%s'" % param["name"])
# Add to normal filters of post-noise-model filters based on parameters
if "postnoise" in param and int(param["postnoise"]) > 0:
print "postnosie"
filters2.append(filter)
else:
filter = mapmaking.FilterBuddyPertod(m, sys=sys, mul=-mul, tmul=tmul, nstep=nstep, prec=prec)
else:
# Warning: This only works if a dmap has already been initialized etc.
m = dmap.read_map(fname, bbox=mybbox, tshape=tshape, comm=comm).astype(dtype)
if not pertod:
filter = mapmaking.FilterBuddyDmap(myscans, mysubs, m, sys=sys, mul=-mul, tmul=tmul)
else:
raise NotImplementedError("FIXME: Implement per tod buddy subtraction with dmaps")
elif param["name"] == "src":
if param["value"] == 0: continue
if "params" not in param: srcs = myscans[0].pointsrcs
else: srcs = pointsrcs.read(param["params"])
# Restrict to chosen amplitude
if "snr" in srcs.dtype.names:
srcs = srcs[srcs.snr >= float(param["snr"])]
srcparam = pointsrcs.src2param(srcs)
srcparam = srcparam.astype(np.float64)
filter = mapmaking.FilterAddSrcs(myscans, srcparam, sys=param["sys"], mul=-float(param["mul"]))
src_filters.append(filter)
elif param["name"] == "beamsym":
if param["value"] == 0: continue
ibeam = float(param["ibeam"])*utils.arcmin*utils.fwhm
obeam = float(param["obeam"])*utils.arcmin*utils.fwhm
filter = mapmaking.FilterBroadenBeamHor(ibeam, obeam)
else:
raise ValueError("Unrecognized fitler name '%s'" % param["name"])
# Add to normal filters of post-noise-model filters based on parameters
if "postnoise" in param and int(param["postnoise"]) > 0:
print "postnosie", param["name"]
filters2.append(filter)
else:
myinds, mysubs, mybbox = scanutils.distribute_scans2(
inds, costs, comm, boxes)
L.info("Rereading shuffled scans")
del myscans # scans do take up some space, even without the tod being read in
myinds, myscans = scanutils.read_scans(
chunk_ids,
myinds,
actscan.ACTScan,
filedb.data,
downsample=config.get("downsample"))
if args.srcsub:
#### 2. Prepare our point source database and the corresponding cuts
src_override = pointsrcs.read(args.srcs) if args.srcs else None
for scan in myscans:
scan.srcparam = pointsrcs.src2param(
src_override if src_override is not None else scan.
pointsrcs)
scan.srcparam, nmerged = planet9.merge_nearby(scan.srcparam)
planet9.cut_srcs_rad(scan, scan.srcparam[nmerged > 1])
hour = (scan.mjd0 % 1) % 24
isday = hour > 11 or hour < 23
if isday:
planet9.cut_bright_srcs_daytime(
scan,
scan.srcparam,
alim_include=args.srclim_day,
errbox=dayerr)
else:
planet9.cut_bright_srcs(scan,
scan.srcparam,
alim_include=args.srclim)
filter = mapmaking.FilterBuddyPertod(m, sys=sys, mul=-mul, tmul=tmul, nstep=nstep, prec=prec)
else:
# Warning: This only works if a dmap has already been initialized etc.
m = dmap.read_map(fname, bbox=mybbox, tshape=tshape, comm=comm).astype(dtype)
if not pertod:
filter = mapmaking.FilterBuddyDmap(myscans, mysubs, m, sys=sys, mul=-mul, tmul=tmul)
else:
raise NotImplementedError("FIXME: Implement per tod buddy subtraction with dmaps")
elif param["name"] == "src":
if param["value"] == 0: continue
if "params" not in param: srcs = myscans[0].pointsrcs
else: srcs = pointsrcs.read(param["params"])
# Restrict to chosen amplitude
if "snr" in srcs.dtype.names:
srcs = srcs[srcs.snr >= float(param["snr"])]
srcparam = pointsrcs.src2param(srcs)
srcparam = srcparam.astype(np.float64)
filter = mapmaking.FilterAddSrcs(myscans, srcparam, sys=param["sys"], mul=-float(param["mul"]))
src_filters.append(filter)
elif param["name"] == "beamsym":
if param["value"] == 0: continue
ibeam = float(param["ibeam"])*utils.arcmin*utils.fwhm
obeam = float(param["obeam"])*utils.arcmin*utils.fwhm
filter = mapmaking.FilterBroadenBeamHor(ibeam, obeam)
else:
raise ValueError("Unrecognized fitler name '%s'" % param["name"])
# Add to normal filters of post-noise-model filters based on parameters
if "postnoise" in param and int(param["postnoise"]) > 0:
print("postnosie", param["name"])
filters2.append(filter)
else: