# grad 10.0 0.25
# bilin 0.1 1.00 0.208 0.176
# bilin 1.0 0.01 0.444 0.387
#
# So for the same grid size, bilinear is about 50% slower than gradient,
# but is 100 times more accurate.
# bilinear is memory-limited from grid-sizes of about 0.05 or so.
# gradient is memory-limited from about 0.02
# So for reasonable accuracies, gradient is so memory-limited that
# the quite significant difference in flops is drowned in memory
# overhead.
import numpy as np, argparse, os, time, sys
from enlib import pmat, config, utils, interpol, coordinates, bench, enmap, bunch
config.default("map_bits", 32, "Bits to use for maps")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("--t", type=float, default=56935, help="mjd")
parser.add_argument("--wt", type=float, default=15, help="minutes")
parser.add_argument("--az", type=float, default=90, help="degrees")
parser.add_argument("--waz", type=float, default=80, help="degrees")
parser.add_argument("--el", type=float, default=50, help="degrees")
parser.add_argument("--wel", type=float, default=0, help="degrees")
parser.add_argument("--res", type=float, default=0.5, help="arcmin")
parser.add_argument("--dir",
type=str,
default="1",
help="1 (forward) or -1 (backward), or list")
parser.add_argument("--nsamp", type=int, default=250000)
parser.add_argument("--ndet", type=int, default=1000)
parser.add_argument("--ntime", type=int, default=3)
"syear": lambda id: 2013+np.searchsorted(season_ends, id2ts(id)),
"t5": lambda id: id[:5],
"t": lambda id: id[:id.index(".")],
"date": lambda id: ctime2date(id2ts(id), -9),
"year": lambda id: ctime2date(id2ts(id), -9, "%Y"),
"month": lambda id: ctime2date(id2ts(id), -9, "%m"),
"day": lambda id: ctime2date(id2ts(id), -9, "%d"),
"Udate": lambda id: ctime2date(id2ts(id), 0),
"Uyear": lambda id: ctime2date(id2ts(id), 0, "%Y"),
"Umonth": lambda id: ctime2date(id2ts(id), 0, "%m"),
"Uday": lambda id: ctime2date(id2ts(id), 0, "%d"),
}
# Try to set up default databases. This is optional, and the databases
# will be none if it fails.
config.default("root", ".", "Path to directory where the different metadata sets are")
config.default("dataset", ".", "Path to data set directory relative to data_root")
config.default("filevars", "filevars.py", "File with common definitions for filedbs")
config.default("filedb", "filedb.txt", "File describing the location of the TOD and their metadata. Relative to dataset path.")
config.default("todinfo", "todinfo.hdf","File describing location of the TOD id lists. Relative to dataset path.")
config.default("file_override", "none", "Comma-separated list of field:file, or none to disable")
config.default("patch_dir", "area", "Directory where standard patch geometries are stored.")
config.init()
#class ACTFiles(filedb.FormatDB):
# def __init__(self, file=None, data=None, override=None):
# if file is None and data is None: file = cjoin(["root","dataset","filedb"])
# override = config.get("file_override", override)
# filedb.FormatDB.__init__(self, file=file, data=data, funcs=extractors, override=override)
def setup_filedb():
"""
import numpy as np, time, sys
from enlib import enmap, interpol, utils, coordinates, config, errors, array_ops
from enlib import parallax, bunch, pointsrcs
import pmat_core_32
import pmat_core_64
def get_core(dtype):
if dtype == np.float32:
return pmat_core_32.pmat_core
else:
return pmat_core_64.pmat_core
config.default("pmat_map_order", 0,
"The interpolation order of the map pointing matrix.")
config.default(
"pmat_cut_type", "full",
"The cut sample representation used. 'full' uses one degree of freedom for each cut sample. 'bin:N' uses one degree of freedom for every N samples. 'exp' used one degree of freedom for the first sample, then one for the next two, one for the next 4, and so on, giving high resoultion at the edges of each cut range, and low resolution in the middle."
)
config.default(
"map_sys", "equ",
"The coordinate system of the maps. Can be eg. 'hor', 'equ' or 'gal'.")
config.default(
"pmat_accuracy", 1.0,
"Factor by which to lower accuracy requirement in pointing interpolation. 1.0 corresponds to 1e-3 pixels and 0.1 arc minute in polangle"
)
config.default(
"pmat_interpol_max_size", 100000,
"Maximum mesh size in pointing interpolation. Worst-case time and memory scale at most proportionally with this."
)
# are Nd[nbin,ndet], U[nmode,ndet], S[nbin,nmode],
# and bins[nbin,2].
#
# Noise will be measured in units of uK sqrt(sample)
# (or the square of that for power, which is what
# we measure here), in time domain. We want compatible
# units in frequency domain, such that a flat spectrum
# with amplitude s**2 results in a time series with
# stddev s. The fourier array passed in here must
# already be normalized such that this holds. Compared
# to numpy's ffts, this means dividing the fourier
# array by sqrt(n).
# Our main noise model
config.default("nmat_jon_apod", 0, "Apodization factor to apply for Jon's noise model")
config.default("nmat_jon_downweight", True, "Whether to downweight the lowest frequencies in the noise model.")
config.default("nmat_jon_amp_threshold", "16,16", "low,high threshold (in power) for accepting eigenmodes, relative to median")
config.default("nmat_jon_single_threshold", 0.55, "reject modes that have more than this fraction of its amplitude in a single detector")
config.default("nmat_spike_suppression", 1e-2, "How much to suppress spikes by. This multiplies the uncorrelated noise in those bins")
def detvecs_jon(ft, srate, dets=None, shared=False, cut_bins=None, apodization=None, cut_unit="freq", verbose=False):
"""Build a Detvecs noise matrix based on Jon's noise model.
ft is the *normalized* fourier-transform of a TOD: ft = fft.rfft(d)/nsamp.
srate is the sampling rate, dets is the list of detectors, shared specifies
whether the Detvecs object should use the compressed "shared" layout or not",
and cut_freq_ranges is a [nbin,{freq_from,freq_2}] array of frequencies
to completely cut."""
apodization = config.get("nmat_jon_apod", apodization) or None
downweight = config.get("nmat_jon_downweight")
spike_suppression = config.get("nmat_spike_suppression")
import numpy as np, os
from enlib import config, utils, mpi, enmap, dmap, mapmaking, todfilter, log, scanutils
from enact import filedb, actdata, actscan
config.default("verbosity", 2, "Verbosity")
parser = config.ArgumentParser()
parser.add_argument("imap")
parser.add_argument("idlist")
parser.add_argument("omap")
parser.add_argument("-s", "--sys", type=str, default="cel")
parser.add_argument("--daz", type=float, default=3.0)
parser.add_argument("--nt", type=int, default=10)
parser.add_argument("--dets", type=str, default=0)
parser.add_argument("--ntod", type=int, default=0)
parser.add_argument("-w", "--weighted", type=int, default=1)
parser.add_argument("-D", "--deslope", type=int, default=0)
args = parser.parse_args()
comm = mpi.COMM_WORLD
filedb.init()
ids = [line.split()[0] for line in open(args.idlist, "r")]
if args.ntod: ids = ids[:args.ntod]
is_dmap = os.path.isdir(args.imap)
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, rank=comm.rank)
tshape = (720, 720)
# Read in all our scans
L.info("Reading %d scans" % len(ids))
myinds = np.arange(len(ids))[comm.rank::comm.size]
# 3. May want to encode detector covariance somehow too. Full covmats is
# out of the picture - would be far too big. But could store something
# like average correlation per bin. How would I measure that?
# Calc corr (would be very noisy in such small bins). Mean of
# off-diagonal elements. This mean would be much less noisy.
# Would be bad for correlations of mixed sign, though. For these,
# one could take the square first, and then subtract a noise bias (1).
# Coudl do both. I'm worried that all those covs would be slow.
#
# To avoid needing to keep many gigs in memory, we do this in chunks
# of e.g. 250 tods, making each file just 324 MB large.
import numpy as np, argparse, h5py, os, sys
from enlib import fft, utils, errors, config, mpi, colors, bench
from enact import filedb, actdata
config.default("cut_mostly_cut",False)
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?", default=None)
parser.add_argument("-b", "--nbin", type=int, default=20000)
parser.add_argument("-f", "--fmax", type=float, default=200)
parser.add_argument("-B", "--nbin-det", type=int, default=100)
parser.add_argument("-Z", "--nbin-zoom", type=int, default=100)
parser.add_argument("-F", "--fmax-zoom", type=float, default=10)
parser.add_argument("-C", "--chunk-size", type=int, default=250)
parser.add_argument("--no-autocut", action="store_true")
args = parser.parse_args()
filedb.init()
ids = filedb.scans[args.sel]
# -*- coding: utf-8 -*-
import numpy as np, sys, os, h5py, copy, time
from scipy import optimize
from enlib import fft
from enlib import config, mpi, errors, log, utils, coordinates, pmat, zipper
from enlib import wcs as enwcs, enmap, array_ops
from enlib.cg import CG
from enact import filedb, actdata, actscan
import astropy.io.fits
config.default("verbosity", 1, "Verbosity of output")
config.default("work_az_step", 0.1, "Az resolution for workspace tagging in degrees")
config.default("work_el_step", 0.1, "El resolution for workspace tagging in degrees")
config.default("work_ra_step", 10, "RA resolution for workspace tagging in degrees")
config.default("work_tag_fmt", "%04d_%04d_%03d_%02d", "Format to use for workspace tags")
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
fft.engine = "fftw"
# Fast and incremental mapping program.
# Idea:
# 1. Map in 2 steps: tod -> work and work -> tod
# 2. Work coordinates are pixel-shifted versions of sky coordinates,
# with a different shift for each scanning pattern. Defined in
# 3 steps:
# 1. Shift in RA based on example sweep (fwd and back separately)
# 2. Subdivide pixels in dec to make them approximately equi-spaced
# in azimuth.
# 3. Transpose for memory access efficiency.
# 3. Ignore detector correlations. That way the work-space noise matrix
# is purely horizontal.
# 4. Each tod-detector makes an almost horizontal line in horizontal
# -*- coding: utf-8 -*-
import numpy as np, sys, os, h5py, copy, time
from scipy import optimize
from enlib import fft
from enlib import config, mpi, errors, log, utils, coordinates, pmat, zipper
from enlib import wcs as enwcs, enmap, array_ops
from enlib.cg import CG
from enact import filedb, actdata, actscan
import astropy.io.fits
config.default("verbosity", 1, "Verbosity of output")
config.default("work_az_step", 0.1,
"Az resolution for workspace tagging in degrees")
config.default("work_el_step", 0.1,
"El resolution for workspace tagging in degrees")
config.default("work_ra_step", 10,
"RA resolution for workspace tagging in degrees")
config.default("work_tag_fmt", "%04d_%04d_%03d_%02d",
"Format to use for workspace tags")
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
fft.engine = "fftw"
# Fast and incremental mapping program.
# Idea:
# 1. Map in 2 steps: tod -> work and work -> tod
# 2. Work coordinates are pixel-shifted versions of sky coordinates,
# with a different shift for each scanning pattern. Defined in
# 3 steps:
# 1. Shift in RA based on example sweep (fwd and back separately)
# 2. Subdivide pixels in dec to make them approximately equi-spaced
# in azimuth.
import numpy as np, time, h5py
from scipy import signal
from enlib import config, fft, utils, gapfill, todops, pmat
config.default(
"gfilter_jon_naz", 16,
"The number of azimuth modes to fit/subtract in Jon's polynomial ground filter."
)
config.default(
"gfilter_jon_nt", 10,
"The number of time modes to fit/subtract in Jon's polynomial ground filter."
)
config.default(
"gfilter_jon_nhwp", 0,
"The number of hwp modes to fit/subtract in Jon's polynomial ground filter."
)
config.default(
"gfilter_jon_niter", 3,
"The number of time modes to fit/subtract in Jon's polynomial ground filter."
)
config.default(
"gfilter_jon_phase", False,
"Modify Jon's polynomial ground filter to use phase instead of azimuth.")
def filter_poly_jon(tod,
az,
weights=None,
naz=None,
nt=None,
niter=None,
"syear": lambda id: 2013+np.searchsorted(season_ends, id2ts(id)),
"t5": lambda id: id[:5],
"t": lambda id: id[:id.index(".")],
"date": lambda id: ctime2date(id2ts(id), -9),
"year": lambda id: ctime2date(id2ts(id), -9, "%Y"),
"month": lambda id: ctime2date(id2ts(id), -9, "%m"),
"day": lambda id: ctime2date(id2ts(id), -9, "%d"),
"Udate": lambda id: ctime2date(id2ts(id), 0),
"Uyear": lambda id: ctime2date(id2ts(id), 0, "%Y"),
"Umonth": lambda id: ctime2date(id2ts(id), 0, "%m"),
"Uday": lambda id: ctime2date(id2ts(id), 0, "%d"),
}
# Try to set up default databases. This is optional, and the databases
# will be none if it fails.
config.default("root", ".", "Path to directory where the different metadata sets are")
config.default("dataset", ".", "Path to data set directory relative to data_root")
config.default("filevars", "filevars.py", "File with common definitions for filedbs")
config.default("filedb", "filedb.txt", "File describing the location of the TOD and their metadata. Relative to dataset path.")
config.default("todinfo", "todinfo.hdf","File describing location of the TOD id lists. Relative to dataset path.")
config.default("file_override", "none", "Comma-separated list of field:file, or none to disable")
config.init()
#class ACTFiles(filedb.FormatDB):
# def __init__(self, file=None, data=None, override=None):
# if file is None and data is None: file = cjoin(["root","dataset","filedb"])
# override = config.get("file_override", override)
# filedb.FormatDB.__init__(self, file=file, data=data, funcs=extractors, override=override)
def setup_filedb():
"""Create a default filedb based on the root, dataset and filedb config
mode = sys.argv[1]
mjd0 = 57174
# Handle each mode. These are practically separate programs, but I keep them in one
# command to reduce clutter.
if mode == "map":
# Map mode. Process the actual time-ordered data, producing rhs.fits, div.fits and info.fits
# for each time-chunk.
import numpy as np, os, time
from enlib import utils
with utils.nowarn():
import h5py
from enlib import planet9, enmap, dmap, config, mpi, scanutils, sampcut, pmat, mapmaking
from enlib import log, pointsrcs, gapfill, ephemeris
from enact import filedb, actdata, actscan, cuts as actcuts
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default("map_sys", "cel", "Coordinate system for the maps")
config.default("verbosity", 1, "Verbosity")
parser = config.ArgumentParser()
parser.add_argument("map", help="dummy")
parser.add_argument("sel")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?", default=None)
parser.add_argument("--dt", type=float, default=3)
parser.add_argument("-T", "--Tref", type=float, default=40)
parser.add_argument("--fref", type=float, default=150)
parser.add_argument("--srcs", type=str, default=None)
parser.add_argument("--srclim", type=float, default=200)
parser.add_argument("-S", "--corr-spacing", type=float, default=2)
# Scans through the indicated tods and computes the ratio
# between the power at mid and high frequency to determine
# how white the white noise floor it. Cuts detectors that
# aren't white enough. Also cuts detectors that have suspiciously
# low white noise floors.
import numpy as np, argparse, h5py, os, sys, shutil
from enlib import fft, utils, enmap, errors, config, mpi, todfilter
from enact import filedb, actdata, filters
config.default("gfilter_jon_nhwp", 200, "The number of hwp modes to fit/subtract in Jon's polynomial ground filter.")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-f", type=str, default="10:1,100:1")
parser.add_argument("-R", type=str, default="0.5:3")
parser.add_argument("--max-sens", type=float, default=20, help="Reject detectors more than this times more sensitive than the median at any of the indicated frequencies. Set to 0 to disable.")
parser.add_argument("--full-stats", action="store_true")
args = parser.parse_args()
comm = mpi.COMM_WORLD
srate = 400.
fmax = srate/2
ndet = 32*33
utils.mkdir(args.odir)
tmp = [[float(tok) for tok in word.split(":")] for word in args.f.split(",")]
bins = np.array([[t[0]-t[1]/2,t[0]+t[1]/2] for t in tmp])
rate = [float(w) for w in args.R.split(":")]
filedb.init()
# grad 10.0 0.25
# bilin 0.1 1.00 0.208 0.176
# bilin 1.0 0.01 0.444 0.387
#
# So for the same grid size, bilinear is about 50% slower than gradient,
# but is 100 times more accurate.
# bilinear is memory-limited from grid-sizes of about 0.05 or so.
# gradient is memory-limited from about 0.02
# So for reasonable accuracies, gradient is so memory-limited that
# the quite significant difference in flops is drowned in memory
# overhead.
import numpy as np, argparse, os, time, sys
from enlib import pmat, config, utils, interpol, coordinates, bench, enmap, bunch
config.default("map_bits", 32, "Bits to use for maps")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("--t", type=float, default=56935, help="mjd")
parser.add_argument("--wt", type=float, default=15, help="minutes")
parser.add_argument("--az", type=float, default=90, help="degrees")
parser.add_argument("--waz", type=float, default=80, help="degrees")
parser.add_argument("--el", type=float, default=50, help="degrees")
parser.add_argument("--wel", type=float, default=0, help="degrees")
parser.add_argument("--res", type=float, default=0.5, help="arcmin")
parser.add_argument("--dir", type=str, default="1", help="1 (forward) or -1 (backward), or list")
parser.add_argument("--nsamp", type=int, default=250000)
parser.add_argument("--ndet", type=int, default=1000)
parser.add_argument("--ntime", type=int, default=3)
#parser.add_argument("-T", action="store_true")
parser.add_argument("-H", "--hwp", action="store_true")
parser.add_argument("-i", "--interpolator", type=str, default="all")
import numpy as np, argparse, os, sys, pipes, shutil, warnings
from enlib import utils, pmat, config, errors, mpi, bunch
from enlib import log, bench, scan, ptsrc_data, pointsrcs
from enact import actscan, filedb, todinfo
warnings.filterwarnings("ignore")
config.default("filedb", "filedb.txt", "File describing the location of the TOD and their metadata")
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("filelist")
parser.add_argument("srcs")
parser.add_argument("odir")
parser.add_argument("--ncomp", type=int, default=3)
parser.add_argument("--ndet", type=int, default=0)
parser.add_argument("--minamp", type=float, default=100)
parser.add_argument("-c", action="store_true")
parser.add_argument("--oldformat", action="store_true")
args = parser.parse_args()
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
comm = mpi.COMM_WORLD
myid = comm.rank
nproc = comm.size
filedb.init()
db = filedb.data
filelist = todinfo.get_tods(args.filelist, filedb.scans)
import numpy as np, time, copy, argparse, os, sys, pipes, shutil, re
from enlib import utils
with utils.nowarn(): import h5py
from enlib import enmap, pmat, fft, config, array_ops, mapmaking, nmat, errors, mpi
from enlib import log, bench, dmap, coordinates, scan as enscan, scanutils
from enlib import pointsrcs, bunch, planet9, ephemeris, parallax
from enlib.cg import CG
from enlib.source_model import SourceModel
from enact import actscan, nmat_measure, filedb, todinfo
from enact import actdata
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default("hwp_resample", False, "Whether to resample the TOD to make the HWP equispaced")
config.default("map_cg_nmax", 500, "Max number of CG steps to perform in map-making")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
config.default("task_dist", "size", "How to assign scans to each mpi task. Can be 'plain' for comm.rank:n:comm.size-type assignment, 'size' for equal-total-size assignment. The optimal would be 'time', for equal total time for each, but that's not implemented currently.")
config.default("gfilter_jon", False, "Whether to enable Jon's ground filter.")
config.default("map_ptsrc_handling", "subadd", "How to handle point sources in the map. Can be 'none' for no special treatment, 'subadd' to subtract from the TOD and readd in pixel space, and 'sim' to simulate a pointsource-only TOD.")
config.default("map_ptsrc_sys", "cel", "Coordinate system the point source positions are specified in. Default is 'cel'")
config.default("map_format", "fits", "File format to use when writing maps. Can be 'fits', 'fits.gz' or 'hdf'.")
config.default("resume", 0, "Interval at which to write the internal CG information to allow for restarting. If 0, this will never be written. Also controls whether existing information on disk will be used for restarting if avialable. If negative, restart information will be written, but not used.")
# Special source handling
config.default("src_handling", "none", "Special source handling. 'none' to disable, 'inpaint' to inpaint, 'full' to map srcs with a white noise model and 'solve' to solve for the model errors jointly with the map.")
config.default("src_handling_lim", 10000, "Minimum source amplitude to apply special source handling to it")
config.default("src_handling_list", "", "Override source list")
# Default signal parameters
config.default("signal_sky_default", "use=no,type=map,name=sky,sys=cel,prec=bin", "Default parameters for sky map")
config.default("signal_hor_default", "use=no,type=map,name=hor,sys=hor,prec=bin", "Default parameters for ground map")
# Then read the actual tod
_, tod = try_read(files.read_tod, "dark_tod", entry.tod, ids=dark_dets)
samples = [0,tod.shape[-1]]
return dataset.DataSet([
dataset.DataField("dark_dets", dark_dets),
#dataset.DataField("dark_cut", cuts, samples=samples, sample_index=1),
dataset.DataField("dark_tod", tod, samples=samples, sample_index=1)])
def read_buddies(entry):
dets, buddies = try_read(files.read_buddies, "buddies", entry.buddies)
return dataset.DataSet([
dataset.DataField("buddies_raw", data=buddies),
dataset.DataField("buddies_raw_dets", data=build_detname(dets, entry)),
])
config.default("hwp_fallback", "none", "How to handle missing HWP data. 'none' skips the tod (it it is supposed to have hwp data), while 'raw' falls back on the native hwp data.")
def read_hwp(entry):
dummy = dataset.DataSet([
dataset.DataField("hwp", 0),
dataset.DataField("hwp_id", "none"),
dataset.DataField("hwp_source", "none")])
epochs = try_read(files.read_hwp_epochs, "hwp_epochs", entry.hwp_epochs)
t, _, ar = entry.id.split(".")
t = float(t)
if ar not in epochs: return dummy
for epoch in epochs[ar]:
if t >= epoch[0] and t < epoch[1]:
# Ok, the HWP was active during this period. Try to read it. It can be in
# several different formats.
if entry.hwp_format == "tod":
# HWP angles in the tod, in the new, high-quality format
import numpy as np, time, h5py, copy, argparse, os, sys, pipes, shutil, re
from enlib import enmap, utils, pmat, fft, config, array_ops, mapmaking, nmat, errors, mpi
from enlib import log, bench, dmap, coordinates, scan as enscan, rangelist, scanutils
from enlib import pointsrcs, bunch
from enlib.cg import CG
from enlib.source_model import SourceModel
from enact import actscan, nmat_measure, filedb, todinfo
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("srcs")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("--nmax", type=int, default=10)
parser.add_argument("-s", "--src", type=int, default=None, help="Only analyze given source")
parser.add_argument("-c", "--cont", action="store_true")
args = parser.parse_args()
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
comm = mpi.COMM_WORLD
tcomm = mpi.COMM_SELF
nmax = args.nmax
ncomp = 3
isys = "hor"
utils.mkdir(args.odir)
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, rank=comm.rank, shared=False)
# 2. For each scanning pattern, define a phase pixelization [ndet,{+,-},naz]
# 3. Sort tods in each scanning pattern by date. These tods collectively
# make up an [ntod,ndet,nphase] data cube, but this would have a size of
# roughly 1e3*1e3*1e3 = 1e9 pixels, which is a bit too big. So we will
# output individual enmaps instead, as map_{el}_{az0}_{az1}_{pattern}_{id}.fits,
# where id is the TOD id and pattern is the index into the list of patterns.
# 3. For each tod in a scanning pattern, read a partially calibrated TOD
# and project it onto our map.
import numpy as np, os, h5py, sys, pipes, shutil, warnings
from enlib import config, errors, utils, log, bench, enmap, pmat, mapmaking, mpi, todfilter
from enlib.cg import CG
from enact import actdata, actscan, filedb, todinfo
warnings.filterwarnings("ignore")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("noise_model", "uncorr", "Noise model. Defaults to uncorr, since detector correlations have no effect when one is mapping each detector to separate pixels")
parser = config.ArgumentParser(os.environ["HOME"]+"/.enkirc")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?")
parser.add_argument("--tol", type=float, default=10, help="Tolerance in arcmin for separating scanning patterns")
parser.add_argument("--daz", type=float, default=2, help="Pixel size in azimuth, in arcmin")
parser.add_argument("--nrow", type=int, default=33)
parser.add_argument("--ncol", type=int, default=32)
parser.add_argument("--nstep",type=int, default=20)
parser.add_argument("--nsub", type=int, default=2)
parser.add_argument("--i0", type=int, default=None)
from __future__ import division, print_function
import numpy as np, time, os, sys
from scipy import integrate
from enlib import utils
with utils.nowarn():
import h5py
from enlib import mpi, errors, fft, mapmaking, config, pointsrcs
from enlib import pmat, coordinates, enmap, bench, bunch, nmat, sampcut, gapfill, wcsutils, array_ops
from enact import filedb, actdata, actscan, nmat_measure
config.set("downsample", 1, "Amount to downsample tod by")
config.set("gapfill", "linear",
"Gapfiller to use. Can be 'linear' or 'joneig'")
config.default(
"pmat_interpol_pad", 10.0,
"Number of arcminutes to pad the interpolation coordinate system by")
parser = config.ArgumentParser()
parser.add_argument("catalog")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-s", "--srcs", type=str, default=None)
parser.add_argument("-v", "--verbose", action="count", default=0)
parser.add_argument("-q", "--quiet", action="count", default=0)
parser.add_argument("-H", "--highpass", type=float, default=None)
parser.add_argument("--minamp", type=float, default=None)
parser.add_argument("--minsn", type=float, default=1)
parser.add_argument("--sys", type=str, default="cel")
args = parser.parse_args()
from __future__ import division, print_function
import numpy as np, warnings, time, copy, argparse, os, sys, pipes, shutil, re
from enlib import config, coordinates, mapmaking, bench, scanutils, log, sampcut, dmap
from pixell import utils, enmap, pointsrcs, bunch, mpi, fft
from enact import filedb, actdata, actscan, files, todinfo
from scipy import ndimage
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default(
"verbosity", 1,
"Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages."
)
config.default("tod_window", 5.0,
"Number of samples to window the tod by on each end")
config.default(
"eig_limit", 0.1,
"Pixel condition number below which polarization is dropped to make total intensity more stable. Should be a high value for single-tod maps to avoid thin stripes with really high noise"
)
config.default(
"map_sys", "equ",
"The coordinate system of the maps. Can be eg. 'hor', 'equ' or 'gal'.")
config.default("map_dist", False, "Whether to use distributed maps")
parser = config.ArgumentParser()
parser.add_argument("sel", help="TOD selction query")
parser.add_argument("area", help="Geometry to map")
parser.add_argument("odir", help="Output directory")
parser.add_argument("prefix", nargs="?", help="Output file name prefix")
parser.add_argument("--dets", type=str, default=0, help="Detector slice")
args = parser.parse_args()
import numpy as np, time, h5py, copy, argparse, os, sys, pipes, shutil, re
from enlib import enmap, utils, pmat, fft, config, array_ops, mapmaking, nmat, errors, mpi
from enlib import log, bench, dmap, coordinates, scan as enscan, rangelist, scanutils
from enlib import pointsrcs, bunch
from enlib.cg import CG
from enlib.source_model import SourceModel
from enact import actscan, nmat_measure, filedb, todinfo
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
config.default("map_format", "fits", "File format to use when writing maps. Can be 'fits', 'fits.gz' or 'hdf'.")
config.default("tod_window", 5.0, "Number of samples to window the tod by on each end")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("prefix",nargs="?")
parser.add_argument("--ndet", type=int, default=0, help="Max number of detectors")
args = parser.parse_args()
filedb.init()
utils.mkdir(args.odir)
root = args.odir + "/" + (args.prefix + "_" if args.prefix else "")
log_level = log.verbosity2level(config.get("verbosity"))
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
area = enmap.read_map(args.area)
comm = mpi.COMM_WORLD
ids = filedb.scans[args.sel]
L = log.init(level=log_level, rank=comm.rank)
"""This module provides functions for filling gaps in an array based on ranges or masks."""
import numpy as np, utils
from enlib import fft, config, resample
from enlib.utils import repeat_filler
from enlib.rangelist import Rangelist, Multirange, multify
config.default("gapfill", "linear",
"TOD gapfill method. Can be 'copy' or 'linear'")
config.default("gapfill_context", 10,
"Samples of context to use for matching up edges of cuts.")
def gapfill(arr, ranges, inplace=False, overlap=None):
gapfiller = {
"linear": gapfill_linear,
"joneig": gapfill_joneig,
"copy": gapfill_copy,
"cubic": gapfill_cubic
}[config.get("gapfill")]
overlap = config.get("gapfill_context", overlap)
return gapfiller(arr, ranges, inplace=inplace, overlap=overlap)
@multify
def gapfill_linear(arr, ranges, inplace=False, overlap=None):
"""Returns arr with the ranges given by ranges, which can be [:,{from,to}] or
a Rangelist, filled using linear interpolation."""
ranges = Rangelist(ranges, len(arr), copy=False)
overlap = config.get("gapfill_context", overlap)
if not inplace: arr = np.array(arr)
nr = len(ranges.ranges)
# # define areas of significant acceleration
# lim = config.get("cut_turnaround_lim", lim)
# addaz = np.abs(ddaz)
# sigma = np.std(ddaz)
# for i in range(3):
# sigma = np.std(ddaz[addaz < sigma*4])
# mask = addaz > sigma*lim
# # Build the cut, and grow it by the margin
# cut = sampcut.from_mask(mask)
# margin= utils.nint(config.get("cut_turnaround_margin", margin)*srate/2)
# cut = cut.widen(margin)
# return cut
# New, simpler turnaround cuts. Simply cuts a given number of degrees away from the
# extrema.
config.default("cut_turnaround_margin", 0.2,
"Margin for turnaround cut in degrees.")
def turnaround_cut(az, margin=None):
margin = config.get("cut_turnaround_margin", margin) * utils.degree
# Use percentile just in case there's some outliers (for example a scan that's a bit
# higher than the others.
az1 = np.percentile(az, 0.1)
az2 = np.percentile(az, 99.9)
mask = (az < az1) | (az > az2)
cut = sampcut.from_mask(mask)
return cut
# return res
import numpy as np, os
from enlib import config, utils, mpi, enmap, dmap, mapmaking, todfilter, log, scanutils
from enact import filedb, actdata, actscan
config.default("verbosity", 2, "Verbosity")
parser = config.ArgumentParser()
parser.add_argument("imap")
parser.add_argument("idlist")
parser.add_argument("omap")
parser.add_argument("-s", "--sys", type=str, default="cel")
parser.add_argument( "--daz", type=float, default=3.0)
parser.add_argument( "--nt", type=int, default=10)
parser.add_argument( "--dets",type=str, default=0)
parser.add_argument( "--ntod",type=int, default=0)
parser.add_argument("-w", "--weighted", type=int, default=1)
parser.add_argument("-D", "--deslope", type=int, default=0)
args = parser.parse_args()
comm = mpi.COMM_WORLD
filedb.init()
ids = [line.split()[0] for line in open(args.idlist,"r")]
if args.ntod: ids = ids[:args.ntod]
is_dmap = os.path.isdir(args.imap)
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, rank=comm.rank)
tshape= (720,720)
# Read in all our scans
L.info("Reading %d scans" % len(ids))
myinds = np.arange(len(ids))[comm.rank::comm.size]
from __future__ import division, print_function
import numpy as np, time, os, sys
from scipy import integrate
from enlib import utils
with utils.nowarn():
import h5py
from enlib import mpi, errors, fft, mapmaking, config, pointsrcs
from enlib import pmat, coordinates, enmap, bench, bunch, nmat, sampcut, gapfill, wcsutils, array_ops
from enact import filedb, actdata, actscan, nmat_measure
config.set("downsample", 1, "Amount to downsample tod by")
config.set("gapfill", "linear",
"Gapfiller to use. Can be 'linear' or 'joneig'")
config.default(
"pmat_interpol_pad", 10.0,
"Number of arcminutes to pad the interpolation coordinate system by")
config.default(
"pmat_ptsrc_rsigma", 3,
"Max number of standard deviations away from a point source to compute the beam profile. Larger values are slower but more accurate, but may lead to misleading effective times in cases where a large region around a source is cut."
)
parser = config.ArgumentParser()
parser.add_argument("catalog")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-s", "--srcs", type=str, default=None)
parser.add_argument("-v", "--verbose", action="count", default=0)
parser.add_argument("-q", "--quiet", action="count", default=0)
parser.add_argument("-H", "--highpass", type=float, default=None)
parser.add_argument("--minamp", type=float, default=None)
# Use joneig filtering to clean the area around each point source position.
# Treat the remainder as white noise. Make thumbnail maps in horizontal coordinates
# centered on the fiducial source position. Output as fully self-contained hdf files
# that can be analyzed by the fitter program.
import numpy as np, os, time, h5py, warnings
from astropy.io import fits
from enlib import utils, config, mpi, errors, sampcut, gapfill, cg
from enlib import pmat, coordinates, enmap, bench, bunch
from enact import filedb, actdata, actscan, nmat_measure
config.default("pmat_accuracy", 20.0, "Factor by which to lower accuracy requirement in pointing interpolation. 1.0 corresponds to 1e-3 pixels and 0.1 arc minute in polangle")
config.default("pmat_interpol_max_size", 1000000, "Maximum mesh size in pointing interpolation. Worst-case time and memory scale at most proportionally with this.")
config.default("gapfill", "linear", "TOD gapfill method. Can be 'copy', 'linear' or 'cubic'")
config.set("pmat_cut_type", "full", "Controls the degrees of freedom for the cut pmat. Using 'full' is stronly adviced to avoid unreasonable slowdown for src_thumb_build2")
parser = config.ArgumentParser(os.environ["HOME"] + "./enkirc")
parser.add_argument("srclist")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-A", "--minamp", type=float, default=0)
parser.add_argument("-b", "--box", type=str, default="-10:10,-10:10")
parser.add_argument("-p", "--pad", type=float, default=2)
parser.add_argument("-r", "--res", type=float, default=0.1)
parser.add_argument("-s", "--restrict", type=str, default=None)
parser.add_argument("-m", "--minimaps", action="store_true")
parser.add_argument("-c", "--cont", action="store_true")
parser.add_argument("-C", "--cols", type=str, default="0:1:2")
parser.add_argument( "--nref", type=int, default=2)
args = parser.parse_args()
# The joneig approach requires the mask to be as small as possible, especially in the scan
# direction, but the source must be entirely contained inside it. Previous tests have shown
# that x position can vary between -2' to 2'. If we want 3 sigma margin, then we need to add
import numpy as np, time, h5py
from enlib import config, fft, utils, gapfill, todops, pmat, rangelist
config.default("gfilter_jon_naz", 8, "The number of azimuth modes to fit/subtract in Jon's polynomial ground filter.")
config.default("gfilter_jon_nt", 10, "The number of time modes to fit/subtract in Jon's polynomial ground filter.")
config.default("gfilter_jon_nhwp", 0, "The number of hwp modes to fit/subtract in Jon's polynomial ground filter.")
config.default("gfilter_jon_niter", 3, "The number of time modes to fit/subtract in Jon's polynomial ground filter.")
def filter_poly_jon(tod, az, weights=None, naz=None, nt=None, niter=None, cuts=None, hwp=None, nhwp=None, deslope=True, inplace=True):
"""Fix naz Legendre polynomials in az and nt other polynomials
in t jointly. Then subtract the best fit from the data.
The subtraction is inplace, so tod is modified. If naz or nt are
negative, they are fit for, but not subtracted.
NOTE: This function may leave tod nonperiodic.
"""
#moomoo = tod[:8].copy()
naz = config.get("gfilter_jon_naz", naz)
nt = config.get("gfilter_jon_nt", nt)
nhwp= config.get("gfilter_jon_nhwp", nhwp)
niter = config.get("gfilter_jon_niter", niter)
if not inplace: tod = tod.copy()
do_gapfill = cuts is not None
#print "Mos", naz, nt, nhwp
#print hwp
# No point in iterating if we aren't gapfilling
if not do_gapfill: niter = 1
if hwp is None or np.all(hwp==0): nhwp = 0
naz, asign = np.abs(naz), np.sign(naz)
nt, tsign = np.abs(nt), np.sign(nt)
nhwp,hsign = np.abs(nhwp),np.sign(nhwp)
d = tod.reshape(-1,tod.shape[-1])
# 2. For each scanning pattern, define a phase pixelization [ndet,{+,-},naz]
# 3. Sort tods in each scanning pattern by date. These tods collectively
# make up an [ntod,ndet,nphase] data cube, but this would have a size of
# roughly 1e3*1e3*1e3 = 1e9 pixels, which is a bit too big. So we will
# output individual enmaps instead, as map_{el}_{az0}_{az1}_{pattern}_{id}.fits,
# where id is the TOD id and pattern is the index into the list of patterns.
# 3. For each tod in a scanning pattern, read a partially calibrated TOD
# and project it onto our map.
import numpy as np, os, h5py, sys, pipes, shutil, warnings
from enlib import config, errors, utils, log, bench, enmap, pmat, mapmaking, mpi, todfilter
from enlib.cg import CG
from enact import actdata, actscan, filedb, todinfo
warnings.filterwarnings("ignore")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default(
"verbosity", 1,
"Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages."
)
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default(
"noise_model", "uncorr",
"Noise model. Defaults to uncorr, since detector correlations have no effect when one is mapping each detector to separate pixels"
)
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?")
parser.add_argument(
# are Nd[nbin,ndet], U[nmode,ndet], S[nbin,nmode],
# and bins[nbin,2].
#
# Noise will be measured in units of uK sqrt(sample)
# (or the square of that for power, which is what
# we measure here), in time domain. We want compatible
# units in frequency domain, such that a flat spectrum
# with amplitude s**2 results in a time series with
# stddev s. The fourier array passed in here must
# already be normalized such that this holds. Compared
# to numpy's ffts, this means dividing the fourier
# array by sqrt(n).
# Our main noise model
config.default("nmat_jon_apod", 0, "Apodization factor to apply for Jon's noise model")
config.default("nmat_jon_downweight", True, "Whether to downweight the lowest frequencies in the noise model.")
config.default("nmat_jon_amp_threshold", "16,16", "low,high threshold (in power) for accepting eigenmodes, relative to median")
config.default("nmat_jon_single_threshold", 0.55, "reject modes that have more than this fraction of its amplitude in a single detector")
config.default("nmat_spike_suppression", 1e-2, "How much to suppress spikes by. This multiplies the uncorrelated noise in those bins")
def detvecs_jon(ft, srate, dets=None, shared=False, cut_bins=None, apodization=None, cut_unit="freq", verbose=False):
"""Build a Detvecs noise matrix based on Jon's noise model.
ft is the *normalized* fourier-transform of a TOD: ft = fft.rfft(d)/nsamp.
srate is the sampling rate, dets is the list of detectors, shared specifies
whether the Detvecs object should use the compressed "shared" layout or not",
and cut_freq_ranges is a [nbin,{freq_from,freq_2}] array of frequencies
to completely cut."""
apodization = config.get("nmat_jon_apod", apodization) or None
downweight = config.get("nmat_jon_downweight")
spike_suppression = config.get("nmat_spike_suppression")
import numpy as np, time, h5py, copy, argparse, os, mpi4py.MPI, sys, pipes, shutil
from enlib import enmap, utils, pmat, fft, config, array_ops, map_equation, nmat, errors
from enlib import log, bench, scan
from enlib.cg import CG
from enact import data, nmat_measure, filedb, todinfo
config.default("filedb", "filedb.txt",
"File describing the location of the TOD and their metadata")
config.default("todinfo", "todinfo.txt",
"File describing location of the TOD id lists")
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default("map_precon", "bin", "Preconditioner to use for map-making")
config.default("map_cg_nmax", 1000,
"Max number of CG steps to perform in map-making")
config.default(
"verbosity", 1,
"Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages."
)
config.default(
"task_dist", "size",
"How to assign scans to each mpi task. Can be 'plain' for myid:n:nproc-type assignment, 'size' for equal-total-size assignment. The optimal would be 'time', for equal total time for each, but that's not implemented currently."
)
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("filelist")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?")
parser.add_argument("-d", "--dump", type=int, default=10)
parser.add_argument("--ncomp", type=int, default=3)
import numpy as np, argparse, enlib.scan, os
from enlib import enmap, utils, config, scansim, log, powspec, fft, bunch
from enact import actscan, filedb, nmat_measure
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
parser = config.ArgumentParser()
parser.add_argument("odir")
parser.add_argument("--area", type=str)
parser.add_argument("--bore", type=str, default="grid:2:0.2:0.8")
parser.add_argument("--dets", type=str, default="scattered:3:3:2.0")
parser.add_argument("--signal",type=str, default="ptsrc:100:1e3:-3")
parser.add_argument("--noise", type=str, default="1/f:20:2:0.5")
parser.add_argument("--seed", type=int, default=1)
parser.add_argument("--measure", type=float, default=None)
parser.add_argument("--real", type=str, default=None)
args = parser.parse_args()
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level)
utils.mkdir(args.odir)
if args.area:
area = enmap.read_map(args.area)
if area.ndim == 2: area = area[None]
else:
shape, wcs = enmap.geometry(pos=np.array([[-1,-1],[1,1]])*np.pi/180, shape=(600,600), pre=(3,), proj="car", ref=[0,0])
area = enmap.zeros(shape, wcs)
def get_scans(area, signal, bore, dets, noise, seed=0, real=None, noise_override=None):
scans = []
import numpy as np, argparse, os, sys, pipes, shutil, warnings
from enlib import utils, pmat, config, errors, mpi, bunch
from enlib import log, bench, scan, ptsrc_data, pointsrcs
from enact import actscan, filedb, todinfo
warnings.filterwarnings("ignore")
config.default("filedb", "filedb.txt", "File describing the location of the TOD and their metadata")
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("filelist")
parser.add_argument("srcs")
parser.add_argument("odir")
parser.add_argument("--ncomp", type=int, default=3)
parser.add_argument("--ndet", type=int, default=0)
parser.add_argument("--minamp", type=float, default=100)
parser.add_argument("-c", action="store_true")
parser.add_argument("--oldformat", action="store_true")
args = parser.parse_args()
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
comm = mpi.COMM_WORLD
myid = comm.rank
nproc = comm.size
filedb.init()
db = filedb.data
filelist = todinfo.get_tods(args.filelist, filedb.scans)
import numpy as np, time, h5py, copy, argparse, os, sys, pipes, shutil, re
from enlib import enmap, utils, pmat, fft, config, array_ops, mapmaking, nmat, errors, mpi
from enlib import log, bench, dmap, coordinates, scan as enscan, rangelist, scanutils
from enlib import pointsrcs, bunch
from enlib.cg import CG
from enact import actscan, nmat_measure, filedb, todinfo
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default(
"verbosity", 1,
"Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages."
)
config.default(
"map_format", "fits",
"File format to use when writing maps. Can be 'fits', 'fits.gz' or 'hdf'.")
config.default("tod_window", 5.0,
"Number of samples to window the tod by on each end")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?")
parser.add_argument("--ndet",
type=int,
default=0,
help="Max number of detectors")
args = parser.parse_args()
filedb.init()
self.pix, self.phase = self.data[scan].get_pix_phase()
def free(self):
del self.pix, self.phase
def forward(self, scan, tod, work):
if scan not in self.data: return
self.data[scan].forward(tod, work, self.pix, self.phase)
def backward(self, scan, tod, work):
if scan not in self.data: return
self.data[scan].backward(tod, work, self.pix, self.phase)
config.default(
"dmap_format", "merged",
"How to store dmaps on disk. 'merged': combine into a single fits file before writing. This is memory intensive. 'tiles': Write the tiles that make up the dmap directly to disk."
)
class SignalDmap(Signal):
def __init__(self,
scans,
subinds,
area,
cuts=None,
name="main",
ofmt="{name}",
output=True,
ext="fits",
pmat_order=None,
sys=None,
parser.add_argument("odir")
parser.add_argument("--rcol", type=int, default=6)
parser.add_argument("--dcol", type=int, default=7)
parser.add_argument("--acol", type=int, default=12)
parser.add_argument("-A", "--minamp", type=float, default=0)
parser.add_argument("-f", "--fknee-mul", type=float, default=1.5)
parser.add_argument("-a", "--alpha", type=float, default=5)
parser.add_argument("-R", "--radius", type=float, default=10)
parser.add_argument("-r", "--res", type=float, default=0.1)
parser.add_argument("-s", "--restrict", type=str, default=None)
parser.add_argument("-m", "--minimaps", action="store_true")
parser.add_argument("-c", "--cont", action="store_true")
args = parser.parse_args()
config.default(
"pmat_accuracy", 10.0,
"Factor by which to lower accuracy requirement in pointing interpolation. 1.0 corresponds to 1e-3 pixels and 0.1 arc minute in polangle"
)
filedb.init()
ids = filedb.scans[args.sel]
comm = mpi.COMM_WORLD
R = args.radius * utils.arcmin
res = args.res * utils.arcmin
dtype = np.float32
bsize_fknee = 100
bsize_ivar = 400
utils.mkdir(args.odir)
rfreqs = [12, 80]
drfreqs = [1, 1]
import numpy as np, argparse, mpi4py.MPI, os, h5py
from enlib import utils, config, ptsrc_data, log
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("filelist")
parser.add_argument("srcs")
parser.add_argument("odir")
parser.add_argument("-R", "--radius", type=float, default=5.0)
parser.add_argument("-r", "--resolution", type=float, default=0.25)
args = parser.parse_args()
comm = mpi4py.MPI.COMM_WORLD
myid = comm.rank
nproc = comm.size
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, rank=myid)
# Allow filelist to take the format filename:[slice]
toks = args.filelist.split(":")
filelist, fslice = toks[0], ":".join(toks[1:])
filelist = [line.split()[0] for line in open(filelist,"r") if line[0] != "#"]
filelist = eval("filelist"+fslice)
utils.mkdir(args.odir)
srcs = np.loadtxt(args.srcs).T
# create minimaps around each source
nsrc = srcs.shape[1]
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("--rcol", type=int, default=6)
parser.add_argument("--dcol", type=int, default=7)
parser.add_argument("--acol", type=int, default=12)
parser.add_argument("-A", "--minamp", type=float, default=0)
parser.add_argument("-f", "--fknee-mul", type=float, default=1.5)
parser.add_argument("-a", "--alpha", type=float, default=5)
parser.add_argument("-R", "--radius", type=float, default=10)
parser.add_argument("-r", "--res", type=float, default=0.1)
parser.add_argument("-s", "--restrict", type=str, default=None)
parser.add_argument("-m", "--minimaps", action="store_true")
parser.add_argument("-c", "--cont", action="store_true")
args = parser.parse_args()
config.default("pmat_accuracy", 10.0, "Factor by which to lower accuracy requirement in pointing interpolation. 1.0 corresponds to 1e-3 pixels and 0.1 arc minute in polangle")
filedb.init()
ids = filedb.scans[args.sel]
comm = mpi.COMM_WORLD
R = args.radius*utils.arcmin
res = args.res*utils.arcmin
dtype= np.float32
bsize_fknee = 100
bsize_ivar = 400
utils.mkdir(args.odir)
rfreqs = [12, 80]
drfreqs = [1, 1]
def find_scan_vel(scan, ipos, aspeed, dt=0.1):
# 4. Read off the focalplane correlation function at that position.
#
# No matter which approach I chosse, the simplest thing to do for the
# build-up program is to store
# 1. Offsets for all relevant detectors.
# 2. Covariance for all those detectors relative to center.
# this will just a constant vector for common mode subtraction.
import numpy as np, os, h5py
from enlib import config, utils, mapmaking, scanutils, mpi, log, pmat, enmap, bench, fft
from enact import filedb, actscan, actdata, nmat_measure
# We want a detector-uncorrelated noise model because we will
# assume that when combining the maps later. This lets us choose
# smaller noise bins because the number of DOF is smaller.
config.default("nmat_uncorr_type", "lin")
config.default("nmat_uncorr_nbin", 4000)
config.default("nmat_uncorr_nmin", 1)
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?")
parser.add_argument("-b", "--bsize", type=float, default=1)
parser.add_argument("-v", "--verbosity", type=int, default=1)
parser.add_argument("-d", "--dets", type=str, default=None)
args = parser.parse_args()
comm = mpi.COMM_WORLD
level = log.verbosity2level(args.verbosity)
# which would be 8-42 minutes. And that's using 16 cores! That's too slow. So this one is
# useful for comparing with a faster methods for a few reference tods, but not in general.
# Currently N and P take similar time. Can optimize P more with some effort, but N is dominated
# by ffts, and can't improve much.
from __future__ import division, print_function
import numpy as np, time, astropy.io.fits, os, sys
from scipy import optimize, integrate
from enlib import utils
with utils.nowarn(): import h5py
from enlib import mpi, errors, fft, mapmaking, config, jointmap, pointsrcs
from enlib import pmat, coordinates, enmap, bench, bunch, nmat, sampcut, gapfill, wcsutils, array_ops
from enact import filedb, actdata, actscan, nmat_measure
config.set("downsample", 1, "Amount to downsample tod by")
config.set("gapfill", "linear", "Gapfiller to use. Can be 'linear' or 'joneig'")
config.default("pmat_interpol_pad", 10.0, "Number of arcminutes to pad the interpolation coordinate system by")
config.default("pmat_interpol_max_size", 4000000, "Maximum mesh size in pointing interpolation. Worst-case time and memory scale at most proportionally with this.")
parser = config.ArgumentParser(os.environ["HOME"] + "./enkirc")
parser.add_argument("mode", help="Mode to use. Can be srcs or planet. This sets up useful defaults for other arguments")
parser.add_argument("srcdb_or_planet")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-R", "--radius", type=float, default=12)
parser.add_argument("-r", "--res", type=float, default=0.1)
parser.add_argument("-m", "--method", type=str, default="fixamp")
parser.add_argument("-M", "--minimaps", action="store_true")
parser.add_argument("-c", "--cont", action="store_true")
parser.add_argument("-s", "--srcs", type=str, default=None)
parser.add_argument("-A", "--minamp", type=float, default=None)
parser.add_argument("-v", "--verbose", action="count", default=0)
import numpy as np, time
from enact import nmat_measure, actdata
from enlib import utils, scan, nmat, resample, config, errors, bench, sampcut
config.default("cut_noise_whiteness", False, "Whether to apply the noise_cut or not")
config.default("cut_spikes", True, "Whether to apply the spike cut or not")
config.default("tod_sys", "hor", "Coordinate system the TOD is in. 'hor': Ideal horizontal coordinates. 'tele': non-ideal telescope coordinates.")
config.default("downsample_method", "fft", "Method to use when downsampling the TOD")
config.default("noise_model", "jon", "Which noise model to use. Can be 'file' or 'jon'")
config.default("tod_skip_deconv", False, "Whether to skip the time constant and butterworth deconvolution in actscan")
config.default("dummy_cut", 0.0, "Fraction of dummy cuts to inject *after* gapfilling and noise estimation")
config.default("dummy_cut_len", 1000, "Dummy cuts will be exponentially distributed up to this length")
class ACTScan(scan.Scan):
def __init__(self, entry, subdets=None, d=None, verbose=False, dark=False):
self.fields = ["gain","mce_filter","tags","polangle","tconst","hwp","cut","point_offsets","boresight","site","tod_shape","array_info","beam","pointsrcs", "buddies"]
if dark: self.fields += ["dark"]
if config.get("noise_model") == "file":
self.fields += ["noise"]
else:
if config.get("cut_noise_whiteness"):
self.fields += ["noise_cut"]
if config.get("cut_spikes"):
self.fields += ["spikes"]
if d is None:
d = actdata.read(entry, self.fields, verbose=verbose)
d = actdata.calibrate(d, verbose=verbose)
if subdets is not None:
d.restrict(dets=d.dets[subdets])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("No data in scan")
ndet = d.ndet
# Necessary components for Scan interface
import numpy as np, time, h5py, copy, argparse, os, sys, pipes, shutil, re
from enlib import enmap, utils, pmat, fft, config, array_ops, mapmaking, nmat, errors, mpi
from enlib import log, bench, dmap, coordinates, scan as enscan, rangelist, scanutils
from enlib import pointsrcs, bunch
from enlib.cg import CG
from enact import actscan, nmat_measure, filedb, todinfo
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default(
"verbosity", 1,
"Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages."
)
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("srcs")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("--nmax", type=int, default=10)
parser.add_argument("-s",
"--src",
type=int,
default=None,
help="Only analyze given source")
parser.add_argument("-c", "--cont", action="store_true")
args = parser.parse_args()
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
comm = mpi.COMM_WORLD
tcomm = mpi.COMM_SELF
nmax = args.nmax
# 3. May want to encode detector covariance somehow too. Full covmats is
# out of the picture - would be far too big. But could store something
# like average correlation per bin. How would I measure that?
# Calc corr (would be very noisy in such small bins). Mean of
# off-diagonal elements. This mean would be much less noisy.
# Would be bad for correlations of mixed sign, though. For these,
# one could take the square first, and then subtract a noise bias (1).
# Coudl do both. I'm worried that all those covs would be slow.
#
# To avoid needing to keep many gigs in memory, we do this in chunks
# of e.g. 250 tods, making each file just 324 MB large.
import numpy as np, argparse, h5py, os, sys
from enlib import fft, utils, errors, config, mpi, colors, bench
from enact import filedb, actdata
config.default("cut_mostly_cut", False)
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?", default=None)
parser.add_argument("-b", "--nbin", type=int, default=20000)
parser.add_argument("-f", "--fmax", type=float, default=200)
parser.add_argument("-B", "--nbin-det", type=int, default=100)
parser.add_argument("-Z", "--nbin-zoom", type=int, default=100)
parser.add_argument("-F", "--fmax-zoom", type=float, default=10)
parser.add_argument("-C", "--chunk-size", type=int, default=250)
parser.add_argument("--tconst", action="store_true")
parser.add_argument("--no-autocut", action="store_true")
args = parser.parse_args()
filedb.init()
# # define areas of significant acceleration
# lim = config.get("cut_turnaround_lim", lim)
# addaz = np.abs(ddaz)
# sigma = np.std(ddaz)
# for i in range(3):
# sigma = np.std(ddaz[addaz < sigma*4])
# mask = addaz > sigma*lim
# # Build the cut, and grow it by the margin
# cut = sampcut.from_mask(mask)
# margin= utils.nint(config.get("cut_turnaround_margin", margin)*srate/2)
# cut = cut.widen(margin)
# return cut
# New, simpler turnaround cuts. Simply cuts a given number of degrees away from the
# extrema.
config.default("cut_turnaround_margin", 0.2, "Margin for turnaround cut in degrees.")
def turnaround_cut(az, margin=None):
margin = config.get("cut_turnaround_margin", margin)*utils.degree
# Use percentile just in case there's some outliers (for example a scan that's a bit
# higher than the others.
az1 = np.percentile(az, 0.1)
az2 = np.percentile(az, 99.9)
mask = (az<az1)|(az>az2)
cut = sampcut.from_mask(mask)
return cut
# return res
config.default("cut_ground_az", "57:62,-62:-57,73:75", "Az ranges to consider for ground cut")
config.default("cut_ground_el", "0:38", "El ranges to consider for ground cut")
def ground_cut(bore, det_offs, az_ranges=None, el_ranges=None):
mapping.oimap[None],
order=1)
res.hwp_phase = np.ascontiguousarray(
utils.interpol(res.hwp_phase.T, mapping.oimap[None], order=1).T)
try:
res.dark_tod = utils.interpol(res.dark_tod, mapping.oimap[None])
res.dark_cut = resample_cut(res.dark_cut, mapping)
except AttributeError as e:
pass
res.cut = resample_cut(res.cut, mapping)
res.cut_noiseest = resample_cut(res.cut_noiseest, mapping)
res.noise = resample_noise(res.noise, mapping)
return res
config.default("downsample_method", "fft",
"Method to use when downsampling the TOD")
class H5Scan(Scan):
def __init__(self, fname):
self.fname = fname
with h5py.File(fname, "r") as hfile:
for k in ["boresight", "offsets", "comps", "sys", "mjd0", "dets"]:
setattr(self, k, hfile[k].value)
n = self.boresight.shape[0]
neach = hfile["cut/neach"].value
flat = hfile["cut/flat"].value
self.cut = sampcut.Sampcut(flat, utils.cumsum(neach,
endpoint=True), n)
self.cut_noiseest = self.cut.copy()
self.noise = nmat.read_nmat(hfile, "noise")
import numpy as np, time, h5py, copy, argparse, os, sys, pipes, shutil, re
from enlib import enmap, utils, pmat, fft, config, array_ops, mapmaking, nmat, errors, mpi
from enlib import log, bench, dmap, coordinates, scan as enscan, scanutils
from enlib import pointsrcs, bunch
from enlib.cg import CG
from enlib.source_model import SourceModel
from enact import actscan, nmat_measure, filedb, todinfo
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default("map_cg_nmax", 500,
"Max number of CG steps to perform in map-making")
config.default(
"verbosity", 1,
"Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages."
)
config.default(
"task_dist", "size",
"How to assign scans to each mpi task. Can be 'plain' for comm.rank:n:comm.size-type assignment, 'size' for equal-total-size assignment. The optimal would be 'time', for equal total time for each, but that's not implemented currently."
)
config.default("gfilter_jon", False, "Whether to enable Jon's ground filter.")
config.default(
"map_ptsrc_handling", "subadd",
"How to handle point sources in the map. Can be 'none' for no special treatment, 'subadd' to subtract from the TOD and readd in pixel space, and 'sim' to simulate a pointsource-only TOD."
)
config.default(
"map_ptsrc_sys", "cel",
"Coordinate system the point source positions are specified in. Default is 'cel'"
)
config.default(
"map_format", "fits",
import numpy as np, os, sys, zipfile
from enlib import config, utils, coordinates, targets, mpi
from enact import filedb, files
config.default("filedb", "filedb.txt",
"File describing the location of the TOD and their metadata")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("query")
parser.add_argument("-r", "--focalplane-radius", type=float, default=1.0)
parser.add_argument("-A", "--all-dists", action="store_true")
args = parser.parse_args()
comm = mpi.COMM_WORLD
filedb.init()
db = filedb.data
filelist = filedb.scans[args.query]
hprint = False
for ind in range(comm.rank, len(filelist), comm.size):
id = filelist[ind]
entry = db.query(id, multi=False)
# Get a few representative samples
site = files.read_site(entry.site)
try:
bore = files.read_boresight(entry.tod)[0]
except zipfile.BadZipfile:
print "%s %9.3f %9.3f %9.3f %9.3f %s" % (id, np.nan, np.nan, np.nan,
np.nan, "badzip")
continue
if bore.shape[0] < 3 or bore.shape[1] < 1:
import numpy as np, sys, os
from enlib import enmap, config, log, pmat, mpi, utils, scan as enscan, errors
from enact import actscan, filedb, todinfo
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default(
"verbosity", 1,
"Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages."
)
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("prefix", nargs="?")
args = parser.parse_args()
filedb.init()
ids = filedb.scans[args.sel]
comm = mpi.COMM_WORLD
dtype = np.float64
area = enmap.read_map(args.area).astype(dtype)
utils.mkdir(args.odir)
root = args.odir + "/" + (args.prefix + "_" if args.prefix else "")
# Set up logging
utils.mkdir(root + "log")
logfile = root + "log/log%03d.txt" % comm.rank
log_level = log.verbosity2level(config.get("verbosity"))
# Scans through the indicated tods and computes the ratio
# between the power at mid and high frequency to determine
# how white the white noise floor it. Cuts detectors that
# aren't white enough. Also cuts detectors that have suspiciously
# low white noise floors.
import numpy as np, argparse, h5py, os, sys, shutil
from enlib import fft, utils, enmap, errors, config, mpi, todfilter
from enact import filedb, actdata, filters
config.default(
"gfilter_jon_nhwp", 200,
"The number of hwp modes to fit/subtract in Jon's polynomial ground filter."
)
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-f", type=str, default="10:1,100:1")
parser.add_argument("-R", type=str, default="0.5:3")
parser.add_argument(
"--max-sens",
type=float,
default=20,
help=
"Reject detectors more than this times more sensitive than the median at any of the indicated frequencies. Set to 0 to disable."
)
parser.add_argument("--full-stats", action="store_true")
args = parser.parse_args()
comm = mpi.COMM_WORLD
srate = 400.
# which would be 8-42 minutes. And that's using 16 cores! That's too slow. So this one is
# useful for comparing with a faster methods for a few reference tods, but not in general.
# Currently N and P take similar time. Can optimize P more with some effort, but N is dominated
# by ffts, and can't improve much.
from __future__ import division, print_function
import numpy as np, time, astropy.io.fits, os, sys
from scipy import optimize
from enlib import utils
with utils.nowarn(): import h5py
from enlib import mpi, errors, fft, mapmaking, config, jointmap, pointsrcs
from enlib import pmat, coordinates, enmap, bench, bunch, nmat, sampcut, gapfill, wcsutils
from enact import filedb, actdata, actscan, nmat_measure
config.set("downsample", 1, "Amount to downsample tod by")
config.set("gapfill", "linear", "Gapfiller to use. Can be 'linear' or 'joneig'")
config.default("pmat_interpol_pad", 10.0, "Number of arcminutes to pad the interpolation coordinate system by")
config.default("pmat_interpol_max_size", 4000000, "Maximum mesh size in pointing interpolation. Worst-case time and memory scale at most proportionally with this.")
parser = config.ArgumentParser(os.environ["HOME"] + "./enkirc")
parser.add_argument("mode", help="Mode to use. Can be srcs or planet. This sets up useful defaults for other arguments")
parser.add_argument("srcdb_or_planet")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-R", "--radius", type=float, default=12)
parser.add_argument("-r", "--res", type=float, default=0.1)
parser.add_argument("-m", "--method", type=str, default="fixamp")
parser.add_argument("-M", "--minimaps", action="store_true")
parser.add_argument("-c", "--cont", action="store_true")
parser.add_argument("-s", "--srcs", type=str, default=None)
parser.add_argument("-A", "--minamp", type=float, default=None)
parser.add_argument("-v", "--verbose", action="count", default=0)
