def load_focalplane(args, comm):
"""Load focalplane information
"""
timer = Timer()
gain = None
fp = None
if comm.world_rank == 0:
if args.focalplane is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["fmin"] = 1.0e-5
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["polangle_deg"] = 0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.focalplane, "rb") as p:
fp = pickle.load(p)
if args.gain is not None:
gain = {}
with fits.open(args.gain) as f:
gain["TIME"] = np.array(f["TIME"].data["TIME"])
for i_det, det_name in f["DETECTORS"].data["DETECTORS"]:
gain[det_name] = np.array(f["GAINS"].data[i_det, :])
if comm.comm_world is not None:
if args.gain is not None:
gain = comm.comm_world.bcast(gain, root=0)
fp = comm.comm_world.bcast(fp, root=0)
timer.stop()
if comm.world_rank == 0:
timer.report("Create focalplane ({} dets)".format(len(fp.keys())))
if args.debug:
if comm.world_rank == 0:
outfile = os.path.join(args.outdir, "focalplane.png")
set_backend()
dquats = {x: fp[x]["quat"] for x in fp.keys()}
dfwhm = {x: fp[x]["fwhm"] for x in fp.keys()}
plot_focalplane(dquats, 10.0, 10.0, outfile, fwhm=dfwhm)
# For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in fp.keys():
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.sample_rate * net * net)
return fp, gain, detweights
def plot(
self,
width: float = 4.,
height: float = 4.,
outfile: Optional[Path] = None,
facecolor: Optional[Dict[str, Union[str, Tuple[float, float,
float]]]] = None,
polcolor: Optional[Dict[str, Union[str, Tuple[float, float,
float]]]] = None,
):
df = self.dataframe
if polcolor is None:
data = self.data
kinds = set(name[-1] for name in data)
n_kinds = len(kinds)
if n_kinds == 2:
logger.info(
f'Found detector names ending in 2 characters, treat them as unqiue polarizations: {kinds}'
)
colors = dict(zip(kinds, sns.color_palette("husl", 2)))
polcolor = {name: colors[name[-1]] for name in data}
else:
logger.warn(
'Cannot guess the polarization of detectors, drawing with no polarization color. Define polcolor explicitly to fix this.'
)
return plot_focalplane(
df.quat,
width,
height,
outfile,
fwhm=df.fwhm_arcmin,
facecolor=facecolor,
polcolor=polcolor,
labels=DictValueEqualsKey(),
)
def main():
parser = argparse.ArgumentParser(
description="Simulate fake hexagonal focalplane.",
fromfile_prefix_chars='@')
parser.add_argument("--minpix",
required=False,
type=int,
default=100,
help="minimum number of pixels to use")
parser.add_argument("--out",
required=False,
default="fp_fake",
help="Root name of output pickle file")
parser.add_argument("--fwhm",
required=False,
type=float,
default=5.0,
help="beam FWHM in arcmin")
parser.add_argument("--fwhm_sigma",
required=False,
type=float,
default=0,
help="Relative beam FWHM distribution width")
parser.add_argument("--fov",
required=False,
type=float,
default=5.0,
help="Field of View in degrees")
parser.add_argument("--psd_fknee",
required=False,
type=float,
default=0.05,
help="Detector noise model f_knee in Hz")
parser.add_argument("--psd_NET",
required=False,
type=float,
default=60.0e-6,
help="Detector noise model NET in K*sqrt(sec)")
parser.add_argument("--psd_alpha",
required=False,
type=float,
default=1.0,
help="Detector noise model slope")
parser.add_argument("--psd_fmin",
required=False,
type=float,
default=1.0e-5,
help="Detector noise model f_min in Hz")
parser.add_argument("--bandcenter_ghz",
required=False,
type=float,
help="Band center frequency [GHz]")
parser.add_argument("--bandcenter_sigma",
required=False,
type=float,
default=0,
help="Relative band center distribution width")
parser.add_argument("--bandwidth_ghz",
required=False,
type=float,
help="Bandwidth [GHz]")
parser.add_argument("--bandwidth_sigma",
required=False,
type=float,
default=0,
help="Relative bandwidth distribution width")
parser.add_argument("--random_seed",
required=False,
type=np.int,
default=123456,
help="Random number generator seed for randomized "
"detector parameters")
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
# Guard against being called with multiple processes
if MPI.COMM_WORLD.rank == 0:
# Make one big hexagon layout at the center of the focalplane.
# Compute the number of pixels that is at least the number requested.
test = args.minpix - 1
nrings = 0
while (test - 6 * nrings) > 0:
test -= 6 * nrings
nrings += 1
npix = 1
for r in range(1, nrings + 1):
npix += 6 * r
print("using {} pixels ({} detectors)".format(npix, npix * 2))
# Translate the field-of-view into distance between flag sides
angwidth = args.fov * np.cos(30 * degree)
Apol = tt.hex_pol_angles_qu(npix, offset=0.0)
Bpol = tt.hex_pol_angles_qu(npix, offset=90.0)
Adets = tt.hex_layout(npix, angwidth, "fake_", "A", Apol)
Bdets = tt.hex_layout(npix, angwidth, "fake_", "B", Bpol)
dets = Adets.copy()
dets.update(Bdets)
np.random.seed(args.random_seed)
for indx, d in enumerate(sorted(dets.keys())):
dets[d]["fknee"] = args.psd_fknee
dets[d]["fmin"] = args.psd_fmin
dets[d]["alpha"] = args.psd_alpha
dets[d]["NET"] = args.psd_NET
# This is in degrees, but the input is in arcmin.
dets[d]["fwhm_deg"] = (args.fwhm / 60.0) \
* (1 + np.random.randn()*args.fwhm_sigma)
# This is a fixed value, in arcmin.
dets[d]["fwhm"] = args.fwhm
if args.bandcenter_ghz:
dets[d]["bandcenter_ghz"] \
= args.bandcenter_ghz * (1+np.random.randn()*args.bandcenter_sigma)
if args.bandwidth_ghz:
dets[d]["bandwidth_ghz"] \
= args.bandwidth_ghz * (1+np.random.randn()*args.bandwidth_sigma)
dets[d]["index"] = indx
outfile = "{}_{}".format(args.out, npix)
qdets = {x: y["quat"] for x, y in dets.items()}
beams = {x: (60.0 * y["fwhm_deg"]) for x, y in dets.items()}
tt.plot_focalplane(qdets,
args.fov,
args.fov,
"{}.png".format(outfile),
fwhm=beams)
with open("{}.pkl".format(outfile), "wb") as p:
pickle.dump(dets, p)
return
import sys
import pickle
import toast.tod as tt
from toast.vis import set_backend
focalplane_filename = sys.argv[1]
with open(focalplane_filename, "rb") as p:
fp = pickle.load(p)
outfile = focalplane_filename.replace("pkl", "png")
set_backend()
fp_onlyquat = { det: each_fp["quat"] for det, each_fp in fp.items() }
tt.plot_focalplane(fp_onlyquat, 10.0, 10.0, outfile)
def load_fp(args, comm):
start = MPI.Wtime()
autotimer = timing.auto_timer()
fp = None
# Load focalplane information
nullquat = np.array([0, 0, 0, 1], dtype=np.float64)
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake['quat'] = nullquat
fake['fwhm'] = 30.0
fake['fknee'] = 0.0
fake['fmin'] = 1e-9
fake['alpha'] = 1.0
fake['NET'] = 1.0
fake['color'] = 'r'
fp = {}
# Second detector at 22.5 degree polarization angle
fp['bore1'] = fake
fake2 = {}
zrot = qa.rotation(ZAXIS, 22.5 * degree)
fake2['quat'] = qa.mult(fake['quat'], zrot)
fake2['fwhm'] = 30.0
fake2['fknee'] = 0.0
fake2['fmin'] = 1e-9
fake2['alpha'] = 1.0
fake2['NET'] = 1.0
fake2['color'] = 'r'
fp['bore2'] = fake2
# Third detector at 45 degree polarization angle
fake3 = {}
zrot = qa.rotation(ZAXIS, 45 * degree)
fake3['quat'] = qa.mult(fake['quat'], zrot)
fake3['fwhm'] = 30.0
fake3['fknee'] = 0.0
fake3['fmin'] = 1e-9
fake3['alpha'] = 1.0
fake3['NET'] = 1.0
fake3['color'] = 'r'
fp['bore3'] = fake3
# Fourth detector at 67.5 degree polarization angle
fake4 = {}
zrot = qa.rotation(ZAXIS, 67.5 * degree)
fake4['quat'] = qa.mult(fake['quat'], zrot)
fake4['fwhm'] = 30.0
fake4['fknee'] = 0.0
fake4['fmin'] = 1e-9
fake4['alpha'] = 1.0
fake4['NET'] = 1.0
fake4['color'] = 'r'
fp['bore4'] = fake4
else:
with open(args.fp, 'rb') as p:
fp = pickle.load(p)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print('Create focalplane: {:.2f} seconds'.format(stop - start),
flush=args.flush)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = '{}/focalplane.png'.format(args.outdir)
tt.plot_focalplane(fp, 6, 6, outfile)
detectors = sorted(fp.keys())
detweights = {}
for d in detectors:
net = fp[d]['NET']
detweights[d] = 1.0 / (args.samplerate * net * net)
return fp, detweights
def load_fp(args, comm):
start = MPI.Wtime()
autotimer = timing.auto_timer()
fp = None
# Load focalplane information
nullquat = np.array([0,0,0,1], dtype=np.float64)
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake['quat'] = nullquat
fake['fwhm'] = 30.0
fake['fknee'] = 0.0
fake['fmin'] = 1e-9
fake['alpha'] = 1.0
fake['NET'] = 1.0
fake['color'] = 'r'
fp = {}
# Second detector at 22.5 degree polarization angle
fp['bore1'] = fake
fake2 = {}
zrot = qa.rotation(ZAXIS, 22.5*degree)
fake2['quat'] = qa.mult(fake['quat'], zrot)
fake2['fwhm'] = 30.0
fake2['fknee'] = 0.0
fake2['fmin'] = 1e-9
fake2['alpha'] = 1.0
fake2['NET'] = 1.0
fake2['color'] = 'r'
fp['bore2'] = fake2
# Third detector at 45 degree polarization angle
fake3 = {}
zrot = qa.rotation(ZAXIS, 45*degree)
fake3['quat'] = qa.mult(fake['quat'], zrot)
fake3['fwhm'] = 30.0
fake3['fknee'] = 0.0
fake3['fmin'] = 1e-9
fake3['alpha'] = 1.0
fake3['NET'] = 1.0
fake3['color'] = 'r'
fp['bore3'] = fake3
# Fourth detector at 67.5 degree polarization angle
fake4 = {}
zrot = qa.rotation(ZAXIS, 67.5*degree)
fake4['quat'] = qa.mult(fake['quat'], zrot)
fake4['fwhm'] = 30.0
fake4['fknee'] = 0.0
fake4['fmin'] = 1e-9
fake4['alpha'] = 1.0
fake4['NET'] = 1.0
fake4['color'] = 'r'
fp['bore4'] = fake4
else:
with open(args.fp, 'rb') as p:
fp = pickle.load(p)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print('Create focalplane: {:.2f} seconds'.format(stop-start),
flush=args.flush)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = '{}/focalplane.png'.format(args.outdir)
tt.plot_focalplane(fp, 6, 6, outfile)
detectors = sorted(fp.keys())
detweights = {}
for d in detectors:
net = fp[d]['NET']
detweights[d] = 1.0 / (args.samplerate * net * net)
return fp, detweights
def load_focalplane(args, comm, schedule):
focalplane = None
# Load focalplane information
if comm.comm_world is None or comm.comm_world.rank == 0:
if args.focalplane is None:
detector_data = {}
ZAXIS = np.array([0, 0, 1.0])
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0, 0, 0, 1.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["fmin"] = 1e-9
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
detector_data["bore1"] = fake
# Second detector at 22.5 degree polarization angle
fake2 = {}
zrot = qa.rotation(ZAXIS, np.radians(22.5))
fake2["quat"] = qa.mult(fake["quat"], zrot)
fake2["fwhm"] = 30.0
fake2["fknee"] = 0.0
fake2["fmin"] = 1e-9
fake2["alpha"] = 1.0
fake2["NET"] = 1.0
fake2["color"] = "r"
detector_data["bore2"] = fake2
# Third detector at 45 degree polarization angle
fake3 = {}
zrot = qa.rotation(ZAXIS, np.radians(45))
fake3["quat"] = qa.mult(fake["quat"], zrot)
fake3["fwhm"] = 30.0
fake3["fknee"] = 0.0
fake3["fmin"] = 1e-9
fake3["alpha"] = 1.0
fake3["NET"] = 1.0
fake3["color"] = "r"
detector_data["bore3"] = fake3
# Fourth detector at 67.5 degree polarization angle
fake4 = {}
zrot = qa.rotation(ZAXIS, np.radians(67.5))
fake4["quat"] = qa.mult(fake["quat"], zrot)
fake4["fwhm"] = 30.0
fake4["fknee"] = 0.0
fake4["fmin"] = 1e-9
fake4["alpha"] = 1.0
fake4["NET"] = 1.0
fake4["color"] = "r"
detector_data["bore4"] = fake4
focalplane = Focalplane(detector_data=detector_data,
sample_rate=args.sample_rate)
else:
focalplane = Focalplane(fname_pickle=args.focalplane,
sample_rate=args.sample_rate)
if comm.comm_world is not None:
focalplane = comm.comm_world.bcast(focalplane, root=0)
if args.debug:
if comm.comm_world is None or comm.comm_world.rank == 0:
outfile = "{}/focalplane.png".format(args.outdir)
plot_focalplane(focalplane, 6, 6, outfile)
schedule.telescope.focalplane = focalplane
detweights = focalplane.detweights
return detweights
def main():
# We are going to group our processes in a single group. This is fine
# if we have fewer processes than detectors. Otherwise we should group
# them in a reasonable size that is smaller than the number of detectors
# and which divides evenly into the total number of processes.
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=MPI.COMM_WORLD.size)
# Make a fake focalplane. Plot it just for fun (don't waste time on this
# for very large runs though).
fp = fake_focalplane()
if comm.comm_world.rank == 0:
outfile = "custom_example_focalplane.png"
set_backend()
tt.plot_focalplane(fp, 6.0, 6.0, outfile)
# Read in 2 boresight files
borefiles = [
"../data/custom_example_boresight_1.txt",
"../data/custom_example_boresight_2.txt"
]
# Set up the distributed data
rate = 100.0
data = create_observations(comm, rate, fp, borefiles)
# Configure the healpix pixelization we will use for map-making and
# also the "submap" resolution, which sets granularity of the locally
# stored pieces of the sky.
map_nside = 512
map_npix = 12 * map_nside**2
sub_nside = 4
sub_npix = 12 * sub_nside**2
# Compute a pointing matrix with healpix pixels and weights.
pointing = tt.OpPointingHpix(nside=map_nside, nest=True, mode="IQU",
pixels="pixels", weights="weights")
pointing.exec(data)
# Compute the locally hit submaps
local_submaps = pixel_dist(data, sub_npix)
# Sources of simulated data: scan from a symmetric beam convolved sky
# and then add some simulated noise.
signalmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3,
dtype=np.float64, submap=sub_npix, local=local_submaps)
signalmap.read_healpix_fits("../data/custom_example_sky.fits")
scanmap = tt.OpSimScan(distmap=signalmap, pixels='pixels',
weights='weights', out="sim")
scanmap.exec(data)
nse = tt.OpSimNoise(out="sim", realization=0)
nse.exec(data)
# Accumulate the hits and inverse diagonal pixel covariance, as well as the
# noise weighted map. Here we simply use inverse noise weighting.
detweights = {}
for d in fp.keys():
net = fp[d]["NET"]
detweights[d] = 1.0 / (rate * net * net)
invnpp = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=6,
dtype=np.float64, submap=sub_npix, local=local_submaps)
hits = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=1,
dtype=np.int64, submap=sub_npix, local=local_submaps)
zmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3,
dtype=np.float64, submap=sub_npix, local=local_submaps)
invnpp.data.fill(0.0)
hits.data.fill(0)
zmap.data.fill(0.0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits, zmap=zmap, name="sim")
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
zmap.allreduce()
# Write these products out
hits.write_healpix_fits("custom_example_hits.fits")
invnpp.write_healpix_fits("custom_example_invnpp.fits")
zmap.write_healpix_fits("custom_example_zmap.fits")
# Invert the covariance and write
tm.covariance_invert(invnpp, 1.0e-3)
invnpp.write_healpix_fits("custom_example_npp.fits")
# Apply covariance to make the binned map
tm.covariance_apply(invnpp, zmap)
zmap.write_healpix_fits("custom_example_binned.fits")
MPI.Finalize()
return
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser(
description="Read existing data and make a simple map.",
fromfile_prefix_chars="@",
)
parser.add_argument(
"--groupsize",
required=False,
type=int,
default=0,
help="size of processor groups used to distribute "
"observations",
)
parser.add_argument(
"--hwprpm",
required=False,
type=float,
default=0.0,
help="The rate (in RPM) of the HWP rotation",
)
parser.add_argument(
"--samplerate",
required=False,
default=100.0,
type=np.float,
help="Detector sample rate (Hz)",
)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument("--nside",
required=False,
type=int,
default=64,
help="Healpix NSIDE")
parser.add_argument(
"--subnside",
required=False,
type=int,
default=8,
help="Distributed pixel sub-map NSIDE",
)
parser.add_argument("--coord",
required=False,
default="E",
help="Sky coordinate system [C,E,G]")
parser.add_argument(
"--baseline",
required=False,
type=float,
default=60.0,
help="Destriping baseline length (seconds)",
)
parser.add_argument(
"--noisefilter",
required=False,
default=False,
action="store_true",
help="Destripe with the noise filter enabled",
)
parser.add_argument(
"--madam",
required=False,
default=False,
action="store_true",
help="If specified, use libmadam for map-making",
)
parser.add_argument("--madampar",
required=False,
default=None,
help="Madam parameter file")
parser.add_argument(
"--polyorder",
required=False,
type=int,
help="Polynomial order for the polyfilter",
)
parser.add_argument(
"--wbin_ground",
required=False,
type=float,
help="Ground template bin width [degrees]",
)
parser.add_argument(
"--flush",
required=False,
default=False,
action="store_true",
help="Flush every print statement.",
)
parser.add_argument("--tidas",
required=False,
default=None,
help="Input TIDAS volume")
parser.add_argument("--tidas_detgroup",
required=False,
default=None,
help="TIDAS detector group")
parser.add_argument("--spt3g",
required=False,
default=None,
help="Input SPT3G data directory")
parser.add_argument(
"--spt3g_prefix",
required=False,
default=None,
help="SPT3G data frame file prefix",
)
parser.add_argument(
"--common_flag_mask",
required=False,
default=0,
type=np.uint8,
help="Common flag mask",
)
parser.add_argument(
"--debug",
required=False,
default=False,
action="store_true",
help="Write data distribution info and focalplane plot",
)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
# args = parser.parse_args(sys.argv)
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if (args.tidas is not None) and (args.spt3g is not None):
raise RuntimeError("Cannot read two datasets!")
if (args.tidas is None) and (args.spt3g is None):
raise RuntimeError("No dataset specified!")
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot load")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot load")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# Pixelization
nside = args.nside
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > nside:
subnside = nside
subnpix = 12 * subnside * subnside
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print(
"WARNING: process groupsize does not evenly divide into "
"total number of processes",
flush=True,
)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# Create output directory
mtime = MPI.Wtime()
if comm.comm_world.rank == 0:
if not os.path.isdir(args.outdir):
os.makedirs(args.outdir)
mtime = elapsed(comm.comm_world, mtime, "Creating output directory")
# The distributed timestream data
data = None
if args.tidas is not None:
if args.tidas_detgroup is None:
raise RuntimeError("you must specify the detector group")
data = tds.load_tidas(
comm,
comm.group_size,
args.tidas,
"r",
args.tidas_detgroup,
tds.TODTidas,
group_dets=args.tidas_detgroup,
distintervals="chunks",
)
if args.spt3g is not None:
if args.spt3g_prefix is None:
raise RuntimeError("you must specify the frame file prefix")
data = s3g.load_spt3g(
comm,
comm.group_size,
args.spt3g,
args.spt3g_prefix,
s3g.obsweight_spt3g,
s3g.TOD3G,
)
mtime = elapsed(comm.comm_world, mtime, "Distribute data")
# In debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
mtime = elapsed(comm.comm_world, mtime,
"Dumping debug data distribution")
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
# Just plot the dets from the first TOD
temptod = data.obs[0]["tod"]
# FIXME: change this once we store det info in the metadata.
dfwhm = {x: 10.0 for x in temptod.detectors}
tt.plot_focalplane(temptod.detoffset(),
10.0,
10.0,
outfile,
fwhm=dfwhm)
comm.comm_world.barrier()
mtime = elapsed(comm.comm_world, mtime, "Plotting debug focalplane")
# Compute pointing matrix
pointing = tt.OpPointingHpix(nside=args.nside,
nest=True,
mode="IQU",
hwprpm=args.hwprpm)
pointing.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Expand pointing")
# Mapmaking.
# FIXME: We potentially have a different noise model for every
# observation. We need to have both spt3g and tidas format Noise
# classes which read the information from disk. Then the mapmaking
# operators need to get these noise weights from each observation.
detweights = {d: 1.0 for d in data.obs[0]["tod"].detectors}
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# Filter data if desired
if args.polyorder:
polyfilter = tt.OpPolyFilter(
order=args.polyorder, common_flag_mask=args.common_flag_mask)
polyfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering")
if args.wbin_ground:
groundfilter = tt.OpGroundFilter(
wbin=args.wbin_ground, common_flag_mask=args.common_flag_mask)
groundfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime,
"Ground template filtering")
# Compute pixel space distribution
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
if localpix is None:
raise RuntimeError(
"Process {} has no hit pixels. Perhaps there are fewer "
"detectors than processes in the group?".format(
comm.comm_world.rank))
localsm = np.unique(np.floor_divide(localpix, subnpix))
mtime = elapsed(comm.comm_world, mtime, "Compute local submaps")
# construct distributed maps to store the covariance,
# noise weighted map, and hits
mtime = MPI.Wtime()
invnpp = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=6,
dtype=np.float64,
submap=subnpix,
local=localsm,
)
hits = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=1,
dtype=np.int64,
submap=subnpix,
local=localsm,
)
zmap = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=3,
dtype=np.float64,
submap=subnpix,
local=localsm,
)
# compute the hits and covariance.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(
detweights=detweights,
invnpp=invnpp,
hits=hits,
common_flag_mask=args.common_flag_mask,
)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1")
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1")
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1")
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing N_pp")
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap,
detweights=detweights,
common_flag_mask=args.common_flag_mask)
build_zmap.exec(data)
zmap.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map")
tm.covariance_apply(invnpp, zmap)
mtime = elapsed(comm.comm_world, mtime, "Computing binned map")
zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits"))
mtime = elapsed(comm.comm_world, mtime, "Writing binned map")
else:
# Set up MADAM map making.
pars = {}
pars["temperature_only"] = "F"
pars["force_pol"] = "T"
pars["kfirst"] = "T"
pars["concatenate_messages"] = "T"
pars["write_map"] = "T"
pars["write_binmap"] = "T"
pars["write_matrix"] = "T"
pars["write_wcov"] = "T"
pars["write_hits"] = "T"
pars["nside_cross"] = nside // 2
pars["nside_submap"] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars["base_first"] = args.baseline
pars["nside_map"] = nside
if args.noisefilter:
pars["kfilter"] = "T"
else:
pars["kfilter"] = "F"
pars["fsample"] = args.samplerate
madam = tm.OpMadam(params=pars,
detweights=detweights,
common_flag_mask=args.common_flag_mask)
madam.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking")
comm.comm_world.barrier()
stop = MPI.Wtime()
dur = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(dur), flush=True)
return
dets.update(Bdets)
np.random.seed(args.random_seed)
for indx, d in enumerate(sorted(dets.keys())):
dets[d]["fknee"] = args.psd_fknee
dets[d]["fmin"] = args.psd_fmin
dets[d]["alpha"] = args.psd_alpha
dets[d]["NET"] = args.psd_NET
dets[d]["fwhm_deg"] = fwhm \
* (1 + np.random.randn()*args.fwhm_sigma)
dets[d]["fwhm"] = dets[d]["fwhm_deg"] # Support legacy code
if args.bandcenter_ghz:
dets[d]["bandcenter_ghz"] \
= args.bandcenter_ghz * (1+np.random.randn()*args.bandcenter_sigma)
if args.bandwidth_ghz:
dets[d]["bandwidth_ghz"] \
= args.bandwidth_ghz * (1+np.random.randn()*args.bandwidth_sigma)
dets[d]["index"] = indx
outfile = "{}_{}".format(args.out, npix)
tt.plot_focalplane(dets, args.fov, args.fov, "{}.png".format(outfile))
with open("{}.pkl".format(outfile), "wb") as p:
pickle.dump(dets, p)
tman = timing.timing_manager()
tman.report()
tman.clear()
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description="Simulate satellite "
"boresight pointing and make a noise map.", fromfile_prefix_chars="@" )
parser.add_argument( "--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute observations" )
parser.add_argument( "--samplerate", required=False, type=float,
default=40.0, help="Detector sample rate (Hz)" )
parser.add_argument( "--starttime", required=False, type=float,
default=0.0, help="The overall start time of the simulation" )
parser.add_argument( "--spinperiod", required=False, type=float,
default=10.0, help="The period (in minutes) of the rotation about the "
"spin axis" )
parser.add_argument( "--spinangle", required=False, type=float,
default=30.0, help="The opening angle (in degrees) of the boresight "
"from the spin axis" )
parser.add_argument( "--precperiod", required=False, type=float,
default=50.0, help="The period (in minutes) of the rotation about the "
"precession axis" )
parser.add_argument( "--precangle", required=False, type=float,
default=65.0, help="The opening angle (in degrees) of the spin axis "
"from the precession axis" )
parser.add_argument( "--hwprpm", required=False, type=float,
default=0.0, help="The rate (in RPM) of the HWP rotation" )
parser.add_argument( "--hwpstep", required=False, default=None,
help="For stepped HWP, the angle in degrees of each step" )
parser.add_argument( "--hwpsteptime", required=False, type=float,
default=0.0, help="For stepped HWP, the the time in seconds between "
"steps" )
parser.add_argument( "--obs", required=False, type=float, default=1.0,
help="Number of hours in one science observation" )
parser.add_argument( "--gap", required=False, type=float, default=0.0,
help="Cooler cycle time in hours between science obs" )
parser.add_argument( "--numobs", required=False, type=int, default=1,
help="Number of complete observations" )
parser.add_argument( "--outdir", required=False, default="out",
help="Output directory" )
parser.add_argument( "--debug", required=False, default=False,
action="store_true", help="Write diagnostics" )
parser.add_argument( "--nside", required=False, type=int, default=64,
help="Healpix NSIDE" )
parser.add_argument( "--subnside", required=False, type=int, default=4,
help="Distributed pixel sub-map NSIDE" )
parser.add_argument( "--baseline", required=False, type=float,
default=60.0, help="Destriping baseline length (seconds)" )
parser.add_argument( "--noisefilter", required=False, default=False,
action="store_true", help="Destripe with the noise filter enabled" )
parser.add_argument( "--madam", required=False, default=False,
action="store_true", help="If specified, use libmadam for map-making" )
parser.add_argument( "--madampar", required=False, default=None,
help="Madam parameter file" )
parser.add_argument('--flush',
required=False, default=False, action='store_true',
help='Flush every print statement.')
parser.add_argument( "--MC_start", required=False, type=int, default=0,
help="First Monte Carlo noise realization" )
parser.add_argument( "--MC_count", required=False, type=int, default=1,
help="Number of Monte Carlo noise realizations" )
parser.add_argument( "--fp", required=False, default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
"a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" "
"(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" "
"(float). For optional plotting, the key \"color\" can specify a "
"valid matplotlib color string." )
parser.add_argument('--tidas',
required=False, default=None,
help='Output TIDAS export path')
parser.add_argument('--input_map', required=False,
help='Input map for signal')
parser.add_argument('--input_pysm_model', required=False,
help='Comma separated models for on-the-fly PySM '
'simulation, e.g. s3,d6,f1,a2"')
parser.add_argument('--apply_beam', required=False, action='store_true',
help='Apply beam convolution to input map with gaussian '
'beam parameters defined in focalplane')
parser.add_argument('--input_dipole', required=False,
help='Simulate dipole, possible values are '
'total, orbital, solar')
parser.add_argument('--input_dipole_solar_speed_kms', required=False,
help='Solar system speed [km/s]', type=float,
default=369.0)
parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False,
help='Solar system speed galactic latitude [degrees]',
type=float, default=48.26)
parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False,
help='Solar system speed galactic longitude[degrees]',
type=float, default=263.99)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# get options
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > args.nside:
subnside = args.nside
subnpix = 12 * subnside * subnside
start = MPI.Wtime()
fp = None
# Load focalplane information
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.fp, "rb") as p:
fp = pickle.load(p)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Create focalplane: {:.2f} seconds".format(stop-start),
flush=True)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
tt.plot_focalplane(fp, 10.0, 10.0, outfile)
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
if groupsize > len(fp.keys()):
if comm.comm_world.rank == 0:
print("process group is too large for the number of detectors",
flush=True)
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(fp.keys())
detquats = {}
detindx = None
if "index" in fp[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = fp[d]["quat"]
if detindx is not None:
detindx[d] = fp[d]["index"]
# Distribute the observations uniformly
groupdist = toast.distribute_uniform(args.numobs, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = tt.regular_intervals(args.numobs, args.starttime, 0,
args.samplerate, 3600*args.obs, 3600*args.gap)
# Create the noise model used for all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]["fmin"]
fknee[d] = fp[d]["fknee"]
alpha[d] = fp[d]["alpha"]
NET[d] = fp[d]["NET"]
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
mem_counter = tt.OpMemoryCounter()
# The distributed timestream data
data = toast.Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = tt.TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
firsttime=obsrange[ob].start,
rate=args.samplerate,
spinperiod=args.spinperiod,
spinangle=args.spinangle,
precperiod=args.precperiod,
precangle=args.precangle,
detindx=detindx,
detranks=comm.group_size
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Read parameters, compute data distribution: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = tt.slew_precession_axis(nsim=tod.local_samples[1],
firstsamp=obsoffset, samplerate=args.samplerate,
degday=degday)
tod.set_prec_axis(qprec=precquat)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Construct boresight pointing: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# make a Healpix pointing matrix.
pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime)
pointing.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Pointing generation took {:.3f} s".format(elapsed), flush=True)
start = stop
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = "signal"
if args.input_pysm_model:
simulate_sky_signal(args, comm, data, mem_counter,
[fp], subnpix, localsm, signalname=signalname)
if args.input_dipole:
print("Simulating dipole")
op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole,
solar_speed=args.input_dipole_solar_speed_kms,
solar_gal_lat=args.input_dipole_solar_gal_lat_deg,
solar_gal_lon=args.input_dipole_solar_gal_lon_deg,
out=signalname,
keep_quats=True,
keep_vel=False,
subtract=False,
coord="G",
freq=0, # we could use frequency for quadrupole correction
flag_mask=255, common_flag_mask=255)
op_sim_dipole.exec(data)
# Mapmaking. For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in detectors:
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.samplerate * net * net)
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing N_pp took {:.3f} s".format(elapsed),
flush=True)
start = stop
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Dumping debug data distribution took "
"{:.3f} s".format(elapsed), flush=True)
start = stop
mcstart = start
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# create output directory for this realization
outpath = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(outpath):
os.makedirs(outpath)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Creating output dir {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# clear all signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
if mc == firstmc:
# For the first realization, optionally export the
# timestream data to a TIDAS volume.
if args.tidas is not None:
from toast.tod.tidas import OpTidasExport
tidas_path = os.path.abspath(args.tidas)
export = OpTidasExport(tidas_path, name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Noise simulation {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal",
detweights=detweights)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Building noise weighted map {:04d} took {:.3f} s".format(
mc, elapsed), flush=True)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Computing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.write_healpix_fits(os.path.join(outpath, "binned.fits"))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Writing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
elapsed = stop - mcstart
if comm.comm_world.rank == 0:
print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed),
flush=True)
start = stop
else:
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = cross
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = args.nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# clear all total signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Noise simulation took {:.3f} s".format(elapsed),
flush=True)
start = stop
# create output directory for this realization
pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(pars["path_output"]):
os.makedirs(pars["path_output"])
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open(os.path.join(pars["path_output"],
"distdata.txt"), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
madam = tm.OpMadam(params=pars, detweights=detweights,
name="tot_signal")
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Mapmaking took {:.3f} s".format(elapsed), flush=True)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description="Simulate satellite "
"boresight pointing and make a noise map.", fromfile_prefix_chars="@" )
parser.add_argument( "--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute observations" )
parser.add_argument( "--samplerate", required=False, type=float,
default=40.0, help="Detector sample rate (Hz)" )
parser.add_argument( "--starttime", required=False, type=float,
default=0.0, help="The overall start time of the simulation" )
parser.add_argument( "--spinperiod", required=False, type=float,
default=10.0, help="The period (in minutes) of the rotation about the "
"spin axis" )
parser.add_argument( "--spinangle", required=False, type=float,
default=30.0, help="The opening angle (in degrees) of the boresight "
"from the spin axis" )
parser.add_argument( "--precperiod", required=False, type=float,
default=50.0, help="The period (in minutes) of the rotation about the "
"precession axis" )
parser.add_argument( "--precangle", required=False, type=float,
default=65.0, help="The opening angle (in degrees) of the spin axis "
"from the precession axis" )
parser.add_argument( "--hwprpm", required=False, type=float,
default=0.0, help="The rate (in RPM) of the HWP rotation" )
parser.add_argument( "--hwpstep", required=False, default=None,
help="For stepped HWP, the angle in degrees of each step" )
parser.add_argument( "--hwpsteptime", required=False, type=float,
default=0.0, help="For stepped HWP, the the time in seconds between "
"steps" )
parser.add_argument( "--obs", required=False, type=float, default=1.0,
help="Number of hours in one science observation" )
parser.add_argument( "--gap", required=False, type=float, default=0.0,
help="Cooler cycle time in hours between science obs" )
parser.add_argument( "--numobs", required=False, type=int, default=1,
help="Number of complete observations" )
parser.add_argument( "--outdir", required=False, default="out",
help="Output directory" )
parser.add_argument( "--debug", required=False, default=False,
action="store_true", help="Write diagnostics" )
parser.add_argument( "--nside", required=False, type=int, default=64,
help="Healpix NSIDE" )
parser.add_argument( "--subnside", required=False, type=int, default=4,
help="Distributed pixel sub-map NSIDE" )
parser.add_argument('--coord', required=False, default='E',
help='Sky coordinate system [C,E,G]')
parser.add_argument( "--baseline", required=False, type=float,
default=60.0, help="Destriping baseline length (seconds)" )
parser.add_argument( "--noisefilter", required=False, default=False,
action="store_true", help="Destripe with the noise filter enabled" )
parser.add_argument( "--madam", required=False, default=False,
action="store_true", help="If specified, use libmadam for map-making" )
parser.add_argument( "--madampar", required=False, default=None,
help="Madam parameter file" )
parser.add_argument('--flush',
required=False, default=False, action='store_true',
help='Flush every print statement.')
parser.add_argument( "--MC_start", required=False, type=int, default=0,
help="First Monte Carlo noise realization" )
parser.add_argument( "--MC_count", required=False, type=int, default=1,
help="Number of Monte Carlo noise realizations" )
parser.add_argument( "--fp", required=False, default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
"a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" "
"(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" "
"(float). For optional plotting, the key \"color\" can specify a "
"valid matplotlib color string." )
parser.add_argument( "--gain", required=False, default=None,
help= "Calibrate the input timelines with a set of gains from a"
"FITS file containing 3 extensions:"
"HDU named DETECTORS : table with list of detector names in a column named DETECTORS"
"HDU named TIME: table with common timestamps column named TIME"
"HDU named GAINS: 2D image of floats with one row per detector and one column per value.")
parser.add_argument('--tidas',
required=False, default=None,
help='Output TIDAS export path')
parser.add_argument('--spt3g',
required=False, default=None,
help='Output SPT3G export path')
parser.add_argument('--input_map', required=False,
help='Input map for signal')
parser.add_argument('--input_pysm_model', required=False,
help='Comma separated models for on-the-fly PySM '
'simulation, e.g. s3,d6,f1,a2"')
parser.add_argument('--input_pysm_precomputed_cmb_K_CMB', required=False,
help='Precomputed CMB map for PySM in K_CMB'
'it overrides any model defined in input_pysm_model"')
parser.add_argument('--apply_beam', required=False, action='store_true',
help='Apply beam convolution to input map with gaussian '
'beam parameters defined in focalplane')
parser.add_argument('--input_dipole', required=False,
help='Simulate dipole, possible values are '
'total, orbital, solar')
parser.add_argument('--input_dipole_solar_speed_kms', required=False,
help='Solar system speed [km/s]', type=float,
default=369.0)
parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False,
help='Solar system speed galactic latitude [degrees]',
type=float, default=48.26)
parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False,
help='Solar system speed galactic longitude[degrees]',
type=float, default=263.99)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot export")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# get options
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > args.nside:
subnside = args.nside
subnpix = 12 * subnside * subnside
start = MPI.Wtime()
fp = None
gain = None
# Load focalplane information
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.fp, "rb") as p:
fp = pickle.load(p)
if args.gain is not None:
gain = {}
with fits.open(args.gain) as f:
gain["TIME"] = np.array(f["TIME"].data["TIME"])
for i_det, det_name in f["DETECTORS"].data["DETECTORS"]:
gain[det_name] = np.array(f["GAINS"].data[i_det, :])
if args.gain is not None:
gain = comm.comm_world.bcast(gain, root=0)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Create focalplane ({} dets): {:.2f} seconds"\
.format(len(fp.keys()), stop-start), flush=True)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
dquats = { x : fp[x]["quat"] for x in fp.keys() }
dfwhm = { x : fp[x]["fwhm"] for x in fp.keys() }
tt.plot_focalplane(dquats, 10.0, 10.0, outfile, fwhm=dfwhm)
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
if groupsize > len(fp.keys()):
if comm.comm_world.rank == 0:
print("process group is too large for the number of detectors",
flush=True)
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(fp.keys())
detquats = {}
detindx = None
if "index" in fp[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = fp[d]["quat"]
if detindx is not None:
detindx[d] = fp[d]["index"]
# Distribute the observations uniformly
groupdist = toast.distribute_uniform(args.numobs, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = tt.regular_intervals(args.numobs, args.starttime, 0,
args.samplerate, 3600*args.obs, 3600*args.gap)
# Create the noise model used for all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]["fmin"]
fknee[d] = fp[d]["fknee"]
alpha[d] = fp[d]["alpha"]
NET[d] = fp[d]["NET"]
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
mem_counter = tt.OpMemoryCounter()
# The distributed timestream data
data = toast.Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = tt.TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
coord=args.coord,
firstsamp=obsrange[ob].first,
firsttime=obsrange[ob].start,
rate=args.samplerate,
spinperiod=args.spinperiod,
spinangle=args.spinangle,
precperiod=args.precperiod,
precangle=args.precangle,
detindx=detindx,
detranks=comm.group_size
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Read parameters, compute data distribution: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = np.empty(4*tod.local_samples[1],
dtype=np.float64).reshape((-1,4))
tt.slew_precession_axis(precquat,
firstsamp=(obsoffset + tod.local_samples[0]),
samplerate=args.samplerate, degday=degday)
tod.set_prec_axis(qprec=precquat)
del precquat
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Construct boresight pointing: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# make a Healpix pointing matrix.
pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime)
pointing.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Pointing generation took {:.3f} s".format(elapsed), flush=True)
start = stop
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = "signal"
has_signal = False
if args.input_pysm_model:
has_signal = True
simulate_sky_signal(args, comm, data, mem_counter,
[fp], subnpix, localsm, signalname=signalname)
if args.input_dipole:
print("Simulating dipole")
has_signal = True
op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole,
solar_speed=args.input_dipole_solar_speed_kms,
solar_gal_lat=args.input_dipole_solar_gal_lat_deg,
solar_gal_lon=args.input_dipole_solar_gal_lon_deg,
out=signalname,
keep_quats=False,
keep_vel=False,
subtract=False,
coord=args.coord,
freq=0, # we could use frequency for quadrupole correction
flag_mask=255, common_flag_mask=255)
op_sim_dipole.exec(data)
del op_sim_dipole
# Mapmaking. For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in detectors:
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.samplerate * net * net)
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing N_pp took {:.3f} s".format(elapsed),
flush=True)
start = stop
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Dumping debug data distribution took "
"{:.3f} s".format(elapsed), flush=True)
start = stop
mcstart = start
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# create output directory for this realization
outpath = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(outpath):
os.makedirs(outpath)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Creating output dir {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# clear all signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Noise simulation {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# add sky signal
if has_signal:
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Add sky signal {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
if gain is not None:
op_apply_gain = tt.OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
if mc == firstmc:
# For the first realization, optionally export the
# timestream data. If we had observation intervals defined,
# we could pass "use_interval=True" to the export operators,
# which would ensure breaks in the exported data at
# acceptable places.
if args.tidas is not None:
tidas_path = os.path.abspath(args.tidas)
export = OpTidasExport(tidas_path, TODTidas, backend="hdf5",
use_todchunks=True,
create_opts={"group_dets":"sim"},
ctor_opts={"group_dets":"sim"},
cache_name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Tidas export took {:.3f} s"\
.format(elapsed), flush=True)
start = stop
if args.spt3g is not None:
spt3g_path = os.path.abspath(args.spt3g)
export = Op3GExport(spt3g_path, TOD3G, use_todchunks=True,
export_opts={"prefix" : "sim"},
cache_name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" SPT3G export took {:.3f} s"\
.format(elapsed), flush=True)
start = stop
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal",
detweights=detweights)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Building noise weighted map {:04d} took {:.3f} s".format(
mc, elapsed), flush=True)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Computing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.write_healpix_fits(os.path.join(outpath, "binned.fits"))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Writing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
elapsed = stop - mcstart
if comm.comm_world.rank == 0:
print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed),
flush=True)
start = stop
else:
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = cross
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = args.nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# clear all total signal data from the cache, so that we can generate
# new noise timestreams.
for obs in data.obs:
tod = obs['tod']
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
if has_signal:
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
if gain is not None:
op_apply_gain = tt.OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Noise simulation took {:.3f} s".format(elapsed),
flush=True)
start = stop
# create output directory for this realization
pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(pars["path_output"]):
os.makedirs(pars["path_output"])
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open(os.path.join(pars["path_output"],
"distdata.txt"), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
madam = tm.OpMadam(params=pars, detweights=detweights,
name="tot_signal")
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Mapmaking took {:.3f} s".format(elapsed), flush=True)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser(
description="Read existing data and make a simple map.",
fromfile_prefix_chars="@")
parser.add_argument("--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute "\
"observations")
parser.add_argument("--hwprpm", required=False, type=float,
default=0.0,
help="The rate (in RPM) of the HWP rotation")
parser.add_argument('--samplerate',
required=False, default=100.0, type=np.float,
help='Detector sample rate (Hz)')
parser.add_argument("--outdir", required=False, default="out",
help="Output directory")
parser.add_argument("--nside", required=False, type=int, default=64,
help="Healpix NSIDE")
parser.add_argument("--subnside", required=False, type=int, default=8,
help="Distributed pixel sub-map NSIDE")
parser.add_argument("--coord", required=False, default="E",
help="Sky coordinate system [C,E,G]")
parser.add_argument("--baseline", required=False, type=float,
default=60.0,
help="Destriping baseline length (seconds)")
parser.add_argument("--noisefilter", required=False, default=False,
action="store_true",
help="Destripe with the noise filter enabled")
parser.add_argument("--madam", required=False, default=False,
action="store_true",
help="If specified, use libmadam for map-making")
parser.add_argument("--madampar", required=False, default=None,
help="Madam parameter file")
parser.add_argument("--polyorder",
required=False, type=int,
help="Polynomial order for the polyfilter")
parser.add_argument("--wbin_ground",
required=False, type=float,
help="Ground template bin width [degrees]")
parser.add_argument("--flush", required=False, default=False,
action="store_true",
help="Flush every print statement.")
parser.add_argument("--tidas", required=False, default=None,
help="Input TIDAS volume")
parser.add_argument("--tidas_detgroup", required=False, default=None,
help="TIDAS detector group")
parser.add_argument("--spt3g", required=False, default=None,
help="Input SPT3G data directory")
parser.add_argument("--spt3g_prefix", required=False, default=None,
help="SPT3G data frame file prefix")
parser.add_argument("--common_flag_mask",
required=False, default=0, type=np.uint8,
help="Common flag mask")
parser.add_argument("--debug", required=False, default=False,
action="store_true",
help="Write data distribution info and focalplane plot")
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
#args = parser.parse_args(sys.argv)
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if (args.tidas is not None) and (args.spt3g is not None):
raise RuntimeError("Cannot read two datasets!")
if (args.tidas is None) and (args.spt3g is None):
raise RuntimeError("No dataset specified!")
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot load")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot load")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# Pixelization
nside = args.nside
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > nside:
subnside = nside
subnpix = 12 * subnside * subnside
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# Create output directory
mtime = MPI.Wtime()
if comm.comm_world.rank == 0:
if not os.path.isdir(args.outdir):
os.makedirs(args.outdir)
mtime = elapsed(comm.comm_world, mtime, "Creating output directory")
# The distributed timestream data
data = None
if args.tidas is not None:
if args.tidas_detgroup is None:
raise RuntimeError("you must specify the detector group")
data = tds.load_tidas(comm, comm.group_size, args.tidas,
"r", args.tidas_detgroup, tds.TODTidas,
group_dets=args.tidas_detgroup,
distintervals="chunks")
if args.spt3g is not None:
if args.spt3g_prefix is None:
raise RuntimeError("you must specify the frame file prefix")
data = s3g.load_spt3g(comm, comm.group_size, args.spt3g,
args.spt3g_prefix, s3g.obsweight_spt3g,
s3g.TOD3G)
mtime = elapsed(comm.comm_world, mtime, "Distribute data")
# In debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
mtime = elapsed(comm.comm_world, mtime,
"Dumping debug data distribution")
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
# Just plot the dets from the first TOD
temptod = data.obs[0]["tod"]
# FIXME: change this once we store det info in the metadata.
dfwhm = { x : 10.0 for x in temptod.detectors }
tt.plot_focalplane(temptod.detoffset(), 10.0, 10.0, outfile, fwhm=dfwhm)
comm.comm_world.barrier()
mtime = elapsed(comm.comm_world, mtime,
"Plotting debug focalplane")
# Compute pointing matrix
pointing = tt.OpPointingHpix(
nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm)
pointing.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Expand pointing")
# Mapmaking.
# FIXME: We potentially have a different noise model for every
# observation. We need to have both spt3g and tidas format Noise
# classes which read the information from disk. Then the mapmaking
# operators need to get these noise weights from each observation.
detweights = { d : 1.0 for d in data.obs[0]["tod"].detectors }
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# Filter data if desired
if args.polyorder:
polyfilter = tt.OpPolyFilter(
order=args.polyorder,
common_flag_mask=args.common_flag_mask)
polyfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering")
if args.wbin_ground:
groundfilter = tt.OpGroundFilter(
wbin=args.wbin_ground,
common_flag_mask=args.common_flag_mask)
groundfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Ground template filtering")
# Compute pixel space distribution
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
if localpix is None:
raise RuntimeError(
"Process {} has no hit pixels. Perhaps there are fewer "
"detectors than processes in the group?".format(
comm.comm_world.rank))
localsm = np.unique(np.floor_divide(localpix, subnpix))
mtime = elapsed(comm.comm_world, mtime, "Compute local submaps")
# construct distributed maps to store the covariance,
# noise weighted map, and hits
mtime = MPI.Wtime()
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits, common_flag_mask=args.common_flag_mask)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1")
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1")
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1")
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing N_pp")
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, detweights=detweights,
common_flag_mask=args.common_flag_mask)
build_zmap.exec(data)
zmap.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map")
tm.covariance_apply(invnpp, zmap)
mtime = elapsed(comm.comm_world, mtime, "Computing binned map")
zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits"))
mtime = elapsed(comm.comm_world, mtime, "Writing binned map")
else:
# Set up MADAM map making.
pars = {}
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = nside // 2
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
madam = tm.OpMadam(params=pars, detweights=detweights,
common_flag_mask=args.common_flag_mask)
madam.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking")
comm.comm_world.barrier()
stop = MPI.Wtime()
dur = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(dur), flush=True)
return
def main():
parser = argparse.ArgumentParser(
description="Simulate fake hexagonal focalplane.",
fromfile_prefix_chars='@')
parser.add_argument( "--minpix", required=False, type=int, default=100,
help="minimum number of pixels to use" )
parser.add_argument( "--out", required=False, default="fp_fake",
help="Root name of output pickle file" )
parser.add_argument( "--fwhm", required=False, type=float, default=5.0,
help="beam FWHM in arcmin" )
parser.add_argument( "--fwhm_sigma", required=False, type=float, default=0,
help="Relative beam FWHM distribution width" )
parser.add_argument( "--fov", required=False, type=float, default=5.0,
help="Field of View in degrees" )
parser.add_argument( "--psd_fknee", required=False, type=float,
default=0.05, help="Detector noise model f_knee in Hz" )
parser.add_argument( "--psd_NET", required=False, type=float,
default=60.0e-6, help="Detector noise model NET in K*sqrt(sec)" )
parser.add_argument( "--psd_alpha", required=False, type=float,
default=1.0, help="Detector noise model slope" )
parser.add_argument( "--psd_fmin", required=False, type=float,
default=1.0e-5, help="Detector noise model f_min in Hz" )
parser.add_argument( "--bandcenter_ghz", required=False, type=float,
help="Band center frequency [GHz]" )
parser.add_argument( "--bandcenter_sigma", required=False, type=float,
default=0, help="Relative band center distribution width" )
parser.add_argument( "--bandwidth_ghz", required=False, type=float,
help="Bandwidth [GHz]" )
parser.add_argument( "--bandwidth_sigma", required=False, type=float,
default=0, help="Relative bandwidth distribution width" )
parser.add_argument( "--random_seed", required=False, type=np.int,
default=123456, help="Random number generator seed for randomized "
"detector parameters" )
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
# Guard against being called with multiple processes
if MPI.COMM_WORLD.rank == 0:
# Make one big hexagon layout at the center of the focalplane.
# Compute the number of pixels that is at least the number requested.
test = args.minpix - 1
nrings = 0
while (test - 6 * nrings) > 0:
test -= 6 * nrings
nrings += 1
npix = 1
for r in range(1, nrings+1):
npix += 6 * r
print("using {} pixels ({} detectors)".format(npix, npix*2))
# Translate the field-of-view into distance between flag sides
angwidth = args.fov * np.cos(30 * degree)
Apol = tt.hex_pol_angles_qu(npix, offset=0.0)
Bpol = tt.hex_pol_angles_qu(npix, offset=90.0)
Adets = tt.hex_layout(npix, angwidth, "fake_", "A", Apol)
Bdets = tt.hex_layout(npix, angwidth, "fake_", "B", Bpol)
dets = Adets.copy()
dets.update(Bdets)
np.random.seed(args.random_seed)
for indx, d in enumerate(sorted(dets.keys())):
dets[d]["fknee"] = args.psd_fknee
dets[d]["fmin"] = args.psd_fmin
dets[d]["alpha"] = args.psd_alpha
dets[d]["NET"] = args.psd_NET
# This is in degrees, but the input is in arcmin.
dets[d]["fwhm_deg"] = (args.fwhm / 60.0) \
* (1 + np.random.randn()*args.fwhm_sigma)
# This is a fixed value, in arcmin.
dets[d]["fwhm"] = args.fwhm
if args.bandcenter_ghz:
dets[d]["bandcenter_ghz"] \
= args.bandcenter_ghz * (1+np.random.randn()*args.bandcenter_sigma)
if args.bandwidth_ghz:
dets[d]["bandwidth_ghz"] \
= args.bandwidth_ghz * (1+np.random.randn()*args.bandwidth_sigma)
dets[d]["index"] = indx
outfile = "{}_{}".format(args.out, npix)
qdets = { x : y["quat"] for x, y in dets.items() }
beams = { x : (60.0*y["fwhm_deg"]) for x, y in dets.items() }
tt.plot_focalplane(qdets, args.fov, args.fov, "{}.png".format(outfile),
fwhm=beams)
with open("{}.pkl".format(outfile), "wb") as p:
pickle.dump(dets, p)
return
