def get_datasets(config, sel=None):
# Set up boolean arrays for querys
all_tags = set()
for dataset in config.datasets:
all_tags |= dataset.tags
flags = {
flag: np.array([flag in dataset.tags for dataset in config.datasets],
bool)
for flag in all_tags
}
# Extract the relevant datasets
datasets = config.datasets
if sel is not None:
sel = "&".join(["(" + w + ")" for w in utils.split_outside(sel, ",")])
selinds = np.where(eval(sel, flags))[0]
datasets = [datasets[i] for i in selinds]
# Make all paths relative to us instead of the config file
cdir = os.path.dirname(config.path)
# In tiled input format, all input maps have the same geometry
for dataset in datasets:
for split in dataset.splits:
split.map = os.path.join(cdir, split.map)
split.div = os.path.join(cdir, split.div)
# Read the geometry from all the datasets. Also make paths relative to us instead
# of the config file
for dataset in datasets:
shape, wcs = read_geometry(dataset.splits[0].map)
dataset.shape = shape
dataset.wcs = wcs
dataset.beam = setup_beam(dataset.beam_params)
return datasets
def parse_desc(desc, default={}):
res = default.copy()
# Parse normally now that the : are out of the way
for tok in utils.split_outside(desc, ",", "[({", "])}"):
subtoks = tok.split("=")
if len(subtoks) == 1:
res["value"] = subtoks[0]
else:
key, val = subtoks
res[key] = val
return res
def parse_desc(desc, default={}):
res = default.copy()
# Parse normally now that the : are out of the way
for tok in utils.split_outside(desc, ",", "[({", "])}"):
subtoks = tok.split("=")
if len(subtoks) == 1:
res["value"] = subtoks[0]
else:
key, val = subtoks
res[key] = val
return res
def query(self, query=None, apply_default_query=True):
"""Query the database. The query takes the form
tag,tag,tag,...:sort[slice], where all tags must be satisfied for an id to
be returned. More general syntax is also available. For example,
(a+b>c)|foo&bar,cow. This follows standard python and numpy syntax,
except that , is treated as a lower-priority version of &."""
# Make a copy of self.data so we can't modify it without changing ourself
data = self.data.copy()
# First split off any sorting field or slice
if query is None: query = ""
toks = utils.split_outside(query, ":")
query, rest = toks[0], ":".join(toks[1:])
# Hack: Support id fields as tags, even if they contain
# illegal characters..
t1 = time.time()
for id in data["id"]:
if id not in query: continue
query = re.sub(r"""(?<!['"])\b%s\b""" % id, "(id=='%s')" % id,
query)
# Split into ,-separated fields. Fields starting with a "+"
# are taken to be tag markers, and are simply propagated to the
# resulting ids.
toks = utils.split_outside(query, ",")
fields, subid = [], []
override_ids = None
for tok in toks:
if len(tok) == 0: continue
if tok.startswith("+"):
# Tags starting with + will be interpreted as a subid specification
subid.append(tok[1:])
elif tok.startswith("/"):
# Tags starting with / will be interpreted as special query flags
if tok == "/all": apply_default_query = False
else: raise ValueError("Unknown query flag '%s'" % tok)
else:
# Normal field. Perform a few convenience transformations first.
if tok.startswith("@@"):
# Hack. *Force* the given ids to be returned, even if they aren't in the database.
override_ids = load_ids(tok[2:])
continue
elif tok.startswith("@"):
# Restrict dataset to those in the given file
tok = "file_contains('%s',id)" % tok[1:]
elif tok.startswith("~@"):
tok = "~file_contains('%s',id)" % tok[2:]
fields.append(tok)
if override_ids is not None:
# Append subids to our ids, and return immediately. All other fields
# and queries are ignored.
subs = np.array(",".join(subid))
subs = np.full(len(override_ids), subs, subs.dtype)
return append_subs(override_ids, subs)
# Apply our default queries here. These are things that we almost always
# want in our queries, and that it's tedious to have to specify manually
# each time. For example, this would be "selected" for act todinfo queries
if apply_default_query:
fields = fields + utils.split_outside(self.default_query, ",")
subid = ",".join(subid)
# Now evaluate our fields one by one. This is done so that
# function fields can inspect the current state at that point
for field in fields:
scope = np.__dict__.copy()
scope.update(data)
for name, functor in self.functors.iteritems():
scope[name] = functor(data)
with utils.nowarn():
hits = eval(field, scope)
# Restrict all fields to the result
data = dslice(data, hits)
# Split the rest into a sorting field and a slice
toks = rest.split("[")
if len(toks) == 1: sort, fsel, dsel = toks[0], "", ""
elif len(toks) == 2: sort, fsel, dsel = toks[0], "", "[" + toks[1]
else:
sort, fsel, dsel = toks[0], "[" + toks[1], "[" + "[".join(toks[2:])
if self.sort and not sort: sort = self.sort
if sort:
# Evaluate sorting field
field = data[sort]
field = eval("field" + fsel)
data = dslice(data, np.argsort(field))
# Finally apply the data slice
inds = np.arange(len(data["id"]))
inds = eval("inds" + dsel)
data = dslice(data, inds)
# Build our subid extensions and append them to ids
subs = np.array([merge_subid(subid, sub) for sub in data["subids"]])
ids = append_subs(data["id"], subs)
return ids
def autocut(d, turnaround=None, ground=None, sun=None, moon=None, max_frac=None, pickup=None):
"""Apply automatic cuts to calibrated data."""
if not config.get("autocut"): return d
ndet, nsamp = d.ndet, d.nsamp
if not ndet or not nsamp: return d
# Insert a cut into d if necessary
if "cut" not in d:
d += dataset.DataField("cut", sampcut.empty(ndet,nsamp))
# insert an autocut datafield, to keep track of how much data each
# automatic cut cost us
d += dataset.DataField("autocut", [])
def addcut(label, dcut, targets="c"):
# det ndet part here allows for broadcasting of cuts from 1-det to full-det
dn = dcut.sum()*d.ndet/dcut.ndet if dcut is not None else 0
if dn == 0: d.autocut.append([label,0,0])
else:
n0, dn = d.cut.sum(), dcut.sum()
dn = dn*d.cut.ndet/dcut.ndet
if "c" in targets: d.cut *= dcut
if "n" in targets: d.cut_noiseest *= dcut
if "b" in targets: d.cut_basic *= dcut
d.autocut.append([ label, dn, d.cut.sum() - n0 ]) # name, mycut, myeffect
if config.get("cut_tconst") and "tau" in d:
addcut("tconst", cuts.tconst_cut(nsamp, d.tau, config.get("cut_tconst")))
if config.get("cut_stationary") and "boresight" in d:
addcut("stationary", cuts.stationary_cut(d.boresight[1]))
if config.get("cut_tod_ends") and "srate" in d:
addcut("tod_ends", cuts.tod_end_cut(nsamp, d.srate))
if config.get("cut_turnaround", turnaround) and "boresight" in d:
addcut("turnaround",cuts.turnaround_cut(d.boresight[1]))
if config.get("cut_ground", ground) and "boresight" in d and "point_offset" in d:
addcut("ground", cuts.ground_cut(d.boresight, d.point_offset))
if config.get("cut_sun", sun) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("avoidance",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Sun", config.get("cut_sun_dist")*np.pi/180))
if config.get("cut_moon", moon) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("moon",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Moon", config.get("cut_moon_dist")*np.pi/180))
if config.get("cut_pickup", pickup) and "boresight" in d and "pickup_cut" in d:
addcut("pickup",cuts.pickup_cut(d.boresight[1], d.dets, d.pickup_cut))
if config.get("cut_mostly_cut"):
addcut("mostly_cut", cuts.cut_mostly_cut_detectors(d.cut_quality))
if config.get("cut_obj"):
objs = utils.split_outside(config.get("cut_obj"),",")
for obj in objs:
toks = obj.split(":")
objname = toks[0]
if objname.startswith("["):
objname = [float(w)*utils.degree for w in objname[1:-1].split(",")]
dist = 0.2*utils.degree
if len(toks) > 1: dist = float(toks[1])*utils.degree
# Hack: only cut for noise estimation purposes if dist is negative
targets = "cnb" if dist > 0 else "n"
addcut(obj, cuts.avoidance_cut(d.boresight, d.point_offset, d.site, objname, dist), targets=targets)
if config.get("cut_srcs"):
cpar = [tok.split(":") for tok in config.get("cut_srcs").split(",")]
names, lims = [], []
for par in cpar:
if par[0] in ["map","nmat"]:
names.append(par[0])
lims.append(float(par[1]))
if any(lims):
params = pointsrcs.src2param(d.pointsrcs)
params[:,5:7] = 1
params[:,7] = 0
# Only bother with sources that are actually strong enough
maxlim = max(lims+[0])
params = params[params[:,2]>maxlim]
cutlist = cuts.point_source_cut(d, params, lims)
for name, c in zip(names, cutlist):
if name == "map": addcut("point_srcs_m", c, "c")
elif name == "nmat": addcut("point_srcs_n", c, "n")
if config.get("cut_extra_srcs"):
srclist = np.loadtxt(config.get("cut_extra_srcs"), usecols=(0,1), ndmin=2)
srclim = float(config.get("cut_extra_lim"))
params = np.zeros([len(srclist),8])
params[:,:2] = srclist[:,1::-1]*utils.degree
params[:,2] = 1
params[:,5:7] = 1
c = cuts.point_source_cut(d, params, srclim)
addcut("point_srcs", c, "nbc")
# What fraction is cut?
cut_fraction = float(d.cut.sum())/d.cut.size
# Get rid of completely cut detectors
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(d.dets[keep])
ndet, nsamp = d.ndet, d.nsamp
def cut_all_if(label, condition):
if condition: dcut = sampcut.full(d.ndet, nsamp)
else: dcut = None
addcut(label, dcut)
cut_all_if("max_frac", config.get("cut_max_frac", max_frac) < cut_fraction)
if "srate" in d:
cut_all_if("tod_mindur", config.get("cut_tod_mindur") > nsamp/d.srate/60)
cut_all_if("tod_mindet", config.get("cut_tod_mindet") > ndet)
# Get rid of completely cut detectors again
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(dets=d.dets[keep])
return d
parser.add_argument("-a", "--apod-val", type=float, default=2e-1)
parser.add_argument("-A", "--apod-alpha", type=float, default=5)
parser.add_argument("-E", "--apod-edge", type=float, default=120)
parser.add_argument("--kxrad", type=float, default=20)
parser.add_argument("--kx-ymax-scale", type=float, default=1)
parser.add_argument("--highpass", type=float, default=200)
parser.add_argument("--cg-tol", type=float, default=1e-6)
parser.add_argument("--max-ps", type=float, default=0)
parser.add_argument("-F", "--filter", action="store_true")
parser.add_argument("-c", "--cont", action="store_true")
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args()
enmap.extent_model.append("intermediate")
sel = "&".join(["(" + w + ")" for w in utils.split_outside(args.sel, ",")])
dtype = np.float32
comm = mpi.COMM_WORLD
utils.mkdir(args.odir)
# Set up configuration
config = imp.load_source("config", args.config)
# Set up boolean arrays for querys
all_tags = set()
for dataset in config.datasets:
all_tags |= dataset.tags
flags = {
flag: np.array([flag in dataset.tags for dataset in config.datasets], bool)
for flag in all_tags
}