def main():
env = Environment.get()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Test the TOAST runtime environment.",
fromfile_prefix_chars="@")
parser.add_argument(
"--groupsize",
required=False,
type=int,
default=0,
help="size of processor groups used to distribute observations",
)
try:
args = parser.parse_args()
except SystemExit:
return
mpiworld, procs, rank = get_world()
if rank == 0:
print(env)
log.info(
"Numba threading layer set to '{}'".format(numba_threading_layer))
if mpiworld is None:
log.info("Running serially with one process")
else:
if rank == 0:
log.info("Running with {} processes".format(procs))
groupsize = args.groupsize
if groupsize <= 0:
groupsize = procs
if rank == 0:
log.info("Using group size of {} processes".format(groupsize))
comm = Comm(world=mpiworld, groupsize=groupsize)
log.info(
"Process {}: world rank {}, group {} of {}, group rank {}".format(
rank, comm.world_rank, comm.group + 1, comm.ngroups,
comm.group_rank))
return
if comm.comm_world is not None:
comm.comm_world.barrier()
tmr.stop()
tmr.clear()
tmr.start()
alltimers = gather_timers(comm=comm.comm_world)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
tmr.stop()
tmr.report("Gather and dump timing info")
timer0.report_clear("toast_satellite_sim.py")
return
if __name__ == "__main__":
try:
main()
except Exception:
# We have an unhandled exception on at least one process. Print a stack
# trace for this process and then abort so that all processes terminate.
mpiworld, procs, rank = get_world()
if procs == 1:
raise
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
lines = ["Proc {}: {}".format(rank, x) for x in lines]
print("".join(lines), flush=True)
if mpiworld is not None:
mpiworld.Abort(6)
def main():
env = Environment.get()
env.enable_function_timers()
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_benchmark (total)")
mpiworld, procs, rank = get_world()
if rank == 0:
log.info("TOAST version = {}".format(env.version()))
log.info("Using a maximum of {} threads per process".format(env.max_threads()))
if mpiworld is None:
log.info("Running serially with one process at {}".format(str(datetime.now())))
else:
if rank == 0:
log.info(
"Running with {} processes at {}".format(procs, str(datetime.now()))
)
cases = {
"tiny": 5000000, # O(1) GB RAM
"xsmall": 50000000, # O(10) GB RAM
"small": 500000000, # O(100) GB RAM
"medium": 5000000000, # O(1) TB RAM
"large": 50000000000, # O(10) TB RAM
"xlarge": 500000000000, # O(100) TB RAM
"heroic": 5000000000000, # O(1000) TB RAM
}
args, comm, n_nodes, n_detector, case, group_seconds, n_group = job_config(
mpiworld, cases
)
# Note: The number of "days" here will just be an approximation of the desired
# data volume since we are doing a realistic schedule for a real observing site.
n_days = int(2.0 * (group_seconds * n_group) / (24 * 3600))
if n_days == 0:
n_days = 1
if rank == 0:
log.info(
"Using {} detectors for approximately {} days".format(n_detector, n_days)
)
# Create the schedule file and input maps on one process
if rank == 0:
create_schedules(args, group_seconds, n_days)
create_input_maps(args)
if mpiworld is not None:
mpiworld.barrier()
if args.dry_run is not None:
if rank == 0:
log.info("Exit from dry run")
# We are done!
sys.exit(0)
gt.start("toast_benchmark (science work)")
# Load and broadcast the schedule file
schedules = pipeline_tools.load_schedule(args, comm)
# Load the weather and append to schedules
pipeline_tools.load_weather(args, comm, schedules)
# Simulate the focalplane
detweights = create_focalplanes(args, comm, schedules, n_detector)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data, total_samples = create_observations(args, comm, schedules)
# handle = None
# if comm.world_rank == 0:
# handle = open(os.path.join(args.outdir, "distdata.txt"), "w")
# data.info(handle)
# if comm.world_rank == 0:
# handle.close()
# if comm.comm_world is not None:
# comm.comm_world.barrier()
# Split the communicator for day and season mapmaking
time_comms = pipeline_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
pipeline_tools.expand_pointing(args, comm, data)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
pipeline_tools.get_elevation_noise(args, comm, data)
# Purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if (args.tidas is None) and (args.spt3g is None):
for ob in data.obs:
tod = ob["tod"]
tod.free_radec_quats()
# Prepare auxiliary information for distributed map objects
signalname = pipeline_tools.scan_sky_signal(args, comm, data, "signal")
# Set up objects to take copies of the TOD at appropriate times
totalname, totalname_freq = setup_sigcopy(args)
# Loop over Monte Carlos
firstmc = args.MC_start
nsimu = args.MC_count
freqs = [float(freq) for freq in args.freq.split(",")]
nfreq = len(freqs)
for mc in range(firstmc, firstmc + nsimu):
pipeline_tools.simulate_atmosphere(args, comm, data, mc, totalname)
# Loop over frequencies with identical focal planes and identical
# atmospheric noise.
for ifreq, freq in enumerate(freqs):
if comm.world_rank == 0:
log.info(
"Processing frequency {}GHz {} / {}, MC = {}".format(
freq, ifreq + 1, nfreq, mc
)
)
# Make a copy of the atmosphere so we can scramble the gains and apply
# frequency-dependent scaling.
pipeline_tools.copy_signal(args, comm, data, totalname, totalname_freq)
pipeline_tools.scale_atmosphere_by_frequency(
args, comm, data, freq=freq, mc=mc, cache_name=totalname_freq
)
pipeline_tools.update_atmospheric_noise_weights(args, comm, data, freq, mc)
# Add previously simulated sky signal to the atmospheric noise.
pipeline_tools.add_signal(
args, comm, data, totalname_freq, signalname, purge=(nsimu == 1)
)
mcoffset = ifreq * 1000000
pipeline_tools.simulate_noise(
args, comm, data, mc + mcoffset, totalname_freq
)
pipeline_tools.scramble_gains(
args, comm, data, mc + mcoffset, totalname_freq
)
outpath = setup_output(args, comm, mc + mcoffset, freq)
# Bin and destripe maps
pipeline_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname_freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
pipeline_tools.apply_polyfilter(args, comm, data, totalname_freq)
pipeline_tools.apply_groundfilter(args, comm, data, totalname_freq)
# Bin filtered maps
pipeline_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname_freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=False,
extra_prefix="filtered",
bin_only=True,
)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
runtime = gt.seconds("toast_benchmark (science work)")
prefactor = 1.0e-3
kilo_samples = 1.0e-3 * total_samples
sample_factor = 1.2
det_factor = 2.0
metric = (
prefactor
* n_detector ** det_factor
* kilo_samples ** sample_factor
/ (n_nodes * runtime)
)
if rank == 0:
msg = "Science Metric: {:0.1e} * ({:d}**{:0.2f}) * ({:0.3e}**{:0.3f}) / ({:0.1f} * {}) = {:0.2f}".format(
prefactor,
n_detector,
det_factor,
kilo_samples,
sample_factor,
runtime,
n_nodes,
metric,
)
log.info("")
log.info(msg)
log.info("")
with open(os.path.join(args.outdir, "log"), "a") as f:
f.write(msg)
f.write("\n\n")
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
with open(os.path.join(args.outdir, "log"), "a") as f:
f.write("Copy of Global Timers:\n")
with open("{}.csv".format(out), "r") as t:
f.write(t.read())
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
log = Logger.get()
parser = argparse.ArgumentParser(
description="Read a toast covariance matrix and invert it.")
parser.add_argument("--input",
required=True,
default=None,
help="The input covariance FITS file")
parser.add_argument(
"--output",
required=False,
default=None,
help="The output inverse covariance FITS file.",
)
parser.add_argument(
"--rcond",
required=False,
default=None,
help="Optionally write the inverse condition number map to this file.",
)
parser.add_argument(
"--single",
required=False,
default=False,
action="store_true",
help="Write the output in single precision.",
)
parser.add_argument(
"--threshold",
required=False,
default=1e-3,
type=np.float,
help="Reciprocal condition number threshold",
)
try:
args = parser.parse_args()
except SystemExit:
return
# get options
infile = args.input
outfile = None
if args.output is not None:
outfile = args.output
else:
inmat = re.match(r"(.*)\.fits", infile)
if inmat is None:
log.error("input file should have .fits extension")
return
inroot = inmat.group(1)
outfile = "{}_inv.fits".format(inroot)
# Get the default communicator
mpiworld, procs, rank = get_world()
# We need to read the header to get the size of the matrix.
# This would be a trivial function call in astropy.fits or
# fitsio, but we don't want to bring in a whole new dependency
# just for that. Instead, we open the file with healpy in memmap
# mode so that nothing is actually read except the header.
nside = 0
ncovnz = 0
if rank == 0:
fake, head = hp.read_map(infile, h=True, memmap=True)
for key, val in head:
if key == "NSIDE":
nside = int(val)
if key == "TFIELDS":
ncovnz = int(val)
if mpiworld is not None:
nside = mpiworld.bcast(nside, root=0)
ncovnz = mpiworld.bcast(ncovnz, root=0)
nnz = int(((np.sqrt(8.0 * ncovnz) - 1.0) / 2.0) + 0.5)
npix = 12 * nside**2
subnside = int(nside / 16)
if subnside == 0:
subnside = 1
subnpix = 12 * subnside**2
nsubmap = int(npix / subnpix)
# divide the submaps as evenly as possible among processes
dist = distribute_uniform(nsubmap, procs)
local = np.arange(dist[rank][0], dist[rank][0] + dist[rank][1])
if rank == 0:
if os.path.isfile(outfile):
os.remove(outfile)
if mpiworld is not None:
mpiworld.barrier()
# create the covariance and inverse condition number map
cov = None
invcov = None
rcond = None
cov = DistPixels(
comm=mpiworld,
dtype=np.float64,
size=npix,
nnz=ncovnz,
submap=subnpix,
local=local,
)
if args.single:
invcov = DistPixels(
comm=mpiworld,
dtype=np.float32,
size=npix,
nnz=ncovnz,
submap=subnpix,
local=local,
)
else:
invcov = cov
if args.rcond is not None:
rcond = DistPixels(
comm=mpiworld,
dtype=np.float64,
size=npix,
nnz=nnz,
submap=subnpix,
local=local,
)
# read the covariance
if rank == 0:
log.info("Reading covariance from {}".format(infile))
cov.read_healpix_fits(infile)
# every process computes its local piece
if rank == 0:
log.info("Inverting covariance")
covariance_invert(cov, args.threshold, rcond=rcond)
if args.single:
invcov.data[:] = cov.data.astype(np.float32)
# write the inverted covariance
if rank == 0:
log.info("Writing inverted covariance to {}".format(outfile))
invcov.write_healpix_fits(outfile)
# write the condition number
if args.rcond is not None:
if rank == 0:
log.info("Writing condition number map")
rcond.write_healpix_fits(args.rcond)
return
def setUp(self):
fixture_name = os.path.splitext(os.path.basename(__file__))[0]
if not toast_available:
print("toast cannot be imported- skipping unit tests", flush=True)
return
self.comm, self.procs, self.rank = get_world()
self.outdir = create_outdir(fixture_name, comm=self.comm)
toastcomm = toast.Comm()
self.data = toast.Data(toastcomm)
# Focalplane
hwfull = get_example()
dets = sim_telescope_detectors(hwfull, "SAT4")
hwfull.data["detectors"] = dets
hw = hwfull.select(match={
"wafer_slot": "w42",
"band": "f030",
"pixel": "00[01]"
})
print(hw.data["detectors"], flush=True)
detquats = {k: v["quat"] for k, v in hw.data["detectors"].items()}
# Samples per observation
self.totsamp = 10000
# Pixelization
nside = 512
self.sim_nside = nside
self.map_nside = nside
# Scan properties
self.site_lon = '-67:47:10'
self.site_lat = '-22:57:30'
self.site_alt = 5200.
self.coord = 'C'
self.azmin = 45
self.azmax = 55
self.el = 60
self.scanrate = 1.0
self.scan_accel = 0.1
self.CES_start = None
# Noise properties
self.rate = 100.0
self.NET = 5.0
self.epsilon = 0.0
self.fmin = 1.0e-5
self.alpha = 1.0
self.fknee = 0.05
tod = TODGround(self.data.comm.comm_group,
detquats,
self.totsamp,
detranks=self.data.comm.group_size,
firsttime=0.0,
rate=self.rate,
site_lon=self.site_lon,
site_lat=self.site_lat,
site_alt=self.site_alt,
azmin=self.azmin,
azmax=self.azmax,
el=self.el,
coord=self.coord,
scanrate=self.scanrate,
scan_accel=self.scan_accel,
CES_start=self.CES_start)
# Analytic noise model
detnames = list(detquats.keys())
drate = {x: self.rate for x in detnames}
dfmin = {x: self.fmin for x in detnames}
dfknee = {x: self.fknee for x in detnames}
dalpha = {x: self.alpha for x in detnames}
dnet = {x: self.NET for x in detnames}
nse = AnalyticNoise(rate=drate,
fmin=dfmin,
detectors=detnames,
fknee=dfknee,
alpha=dalpha,
NET=dnet)
# Single observation
obs = dict()
obs["tod"] = tod
obs["noise"] = nse
obs["id"] = 12345
obs["intervals"] = tod.subscans
obs["site"] = "SimonsObs"
obs["telescope"] = "SAT4"
obs["site_id"] = 1
obs["telescope_id"] = 4
obs["fpradius"] = 5.0
obs["start_time"] = 0
obs["altitude"] = self.site_alt
obs["name"] = "test"
# Add the observation to the dataset
self.data.obs.append(obs)
return
def main():
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate fake hexagonal focalplane.",
fromfile_prefix_chars="@")
parser.add_argument(
"--minpix",
required=False,
type=int,
default=100,
help="minimum number of pixels to use",
)
parser.add_argument(
"--out",
required=False,
default="fp_fake",
help="Root name of output pickle file",
)
parser.add_argument("--fwhm",
required=False,
type=float,
default=5.0,
help="beam FWHM in arcmin")
parser.add_argument(
"--fwhm_sigma",
required=False,
type=float,
default=0,
help="Relative beam FWHM distribution width",
)
parser.add_argument(
"--fov",
required=False,
type=float,
default=5.0,
help="Field of View in degrees",
)
parser.add_argument(
"--psd_fknee",
required=False,
type=float,
default=0.05,
help="Detector noise model f_knee in Hz",
)
parser.add_argument(
"--psd_NET",
required=False,
type=float,
default=60.0e-6,
help="Detector noise model NET in K*sqrt(sec)",
)
parser.add_argument(
"--psd_alpha",
required=False,
type=float,
default=1.0,
help="Detector noise model slope",
)
parser.add_argument(
"--psd_fmin",
required=False,
type=float,
default=1.0e-5,
help="Detector noise model f_min in Hz",
)
parser.add_argument(
"--bandcenter_ghz",
required=False,
type=float,
help="Band center frequency [GHz]",
)
parser.add_argument(
"--bandcenter_sigma",
required=False,
type=float,
default=0,
help="Relative band center distribution width",
)
parser.add_argument("--bandwidth_ghz",
required=False,
type=float,
help="Bandwidth [GHz]")
parser.add_argument(
"--bandwidth_sigma",
required=False,
type=float,
default=0,
help="Relative bandwidth distribution width",
)
parser.add_argument(
"--random_seed",
required=False,
type=np.int,
default=123456,
help="Random number generator seed for randomized "
"detector parameters",
)
try:
args = parser.parse_args()
except SystemExit:
return
# Get the default communicator
mpiworld, procs, rank = get_world()
# Guard against being called with multiple processes
if rank == 0:
# Make one big hexagon layout at the center of the focalplane.
# Compute the number of pixels that is at least the number requested.
test = args.minpix - 1
nrings = 0
while (test - 6 * nrings) > 0:
test -= 6 * nrings
nrings += 1
npix = 1
for r in range(1, nrings + 1):
npix += 6 * r
log.info("using {} pixels ({} detectors)".format(npix, npix * 2))
# Translate the field-of-view into distance between flag sides
angwidth = args.fov * np.cos(30 * degree)
Apol = hex_pol_angles_qu(npix, offset=0.0)
Bpol = hex_pol_angles_qu(npix, offset=90.0)
Adets = hex_layout(npix, angwidth, "fake_", "A", Apol)
Bdets = hex_layout(npix, angwidth, "fake_", "B", Bpol)
dets = Adets.copy()
dets.update(Bdets)
np.random.seed(args.random_seed)
for indx, d in enumerate(sorted(dets.keys())):
dets[d]["fknee"] = args.psd_fknee
dets[d]["fmin"] = args.psd_fmin
dets[d]["alpha"] = args.psd_alpha
dets[d]["NET"] = args.psd_NET
# This is in degrees, but the input is in arcmin.
dets[d]["fwhm_deg"] = (args.fwhm / 60.0) * (
1 + np.random.randn() * args.fwhm_sigma)
# This is a fixed value, in arcmin.
dets[d]["fwhm"] = args.fwhm
if args.bandcenter_ghz:
dets[d]["bandcenter_ghz"] = args.bandcenter_ghz * (
1 + np.random.randn() * args.bandcenter_sigma)
if args.bandwidth_ghz:
dets[d]["bandwidth_ghz"] = args.bandwidth_ghz * (
1 + np.random.randn() * args.bandwidth_sigma)
dets[d]["index"] = indx
outfile = "{}_{}".format(args.out, npix)
qdets = {x: y["quat"] for x, y in dets.items()}
beams = {x: (60.0 * y["fwhm_deg"]) for x, y in dets.items()}
plot_focalplane(qdets,
args.fov,
args.fov,
"{}.png".format(outfile),
fwhm=beams)
with open("{}.pkl".format(outfile), "wb") as p:
pickle.dump(dets, p)
return
import h5py
import numpy as np
import toast
from numba import jit
from toast.mpi import get_world
from toast.op import Operator
from toast.utils import Logger
if TYPE_CHECKING:
from typing import Optional, List
COMM: Optional[toast.mpi.Comm]
PROCS: int
RANK: int
COMM, PROCS, RANK = get_world()
LOGGER = Logger.get()
IS_SERIAL = PROCS == 1
H5_CREATE_KW = {
'compression': 'gzip',
# shuffle minimize the output size
'shuffle': True,
# checksum for data integrity
'fletcher32': True,
# turn off track_times so that identical output gives the same md5sum
'track_times': False
}
@jit(nopython=True, nogil=True, cache=False)
def setUp(self):
fixture_name = os.path.splitext(os.path.basename(__file__))[0]
if not toast_available:
print(
"toast cannot be imported ({})- skipping unit test".format(
toast_import_error),
flush=True,
)
return
self.comm, self.procs, self.rank = get_world()
self.outdir = create_outdir(fixture_name, comm=self.comm)
toastcomm = toast.Comm(world=self.comm)
self.data = toast.Data(toastcomm)
# Focalplane
hwfull = get_example()
dets = sim_telescope_detectors(hwfull, "SAT4")
hwfull.data["detectors"] = dets
hw = hwfull.select(match={
"wafer_slot": "w42",
"band": "SAT_f030",
"pixel": "00[01]"
})
# print(hw.data["detectors"], flush=True)
detquats = {k: v["quat"] for k, v in hw.data["detectors"].items()}
# Samples per observation
self.totsamp = 10000
# Scan properties
self.site_lon = '-67:47:10'
self.site_lat = '-22:57:30'
self.site_alt = 5200.
self.coord = 'C'
self.azmin = 45
self.azmax = 55
self.el = 60
self.scanrate = 1.0
self.scan_accel = 0.1
self.CES_start = None
# Noise properties
self.rate = 100.0
self.NET = 1e-3 # 1 mK NET
self.epsilon = 0.0
self.fmin = 1.0e-5
self.alpha = 1.0
self.fknee = 0.05
for ob in range(3):
ftime = (self.totsamp / self.rate) * ob + 1564015655.88
tod = TODGround(self.data.comm.comm_group,
detquats,
self.totsamp,
detranks=self.data.comm.group_size,
firsttime=ftime,
rate=self.rate,
site_lon=self.site_lon,
site_lat=self.site_lat,
site_alt=self.site_alt,
azmin=self.azmin,
azmax=self.azmax,
el=self.el,
coord=self.coord,
scanrate=self.scanrate,
scan_accel=self.scan_accel,
CES_start=self.CES_start)
# Analytic noise model
detnames = list(detquats.keys())
drate = {x: self.rate for x in detnames}
dfmin = {x: self.fmin for x in detnames}
dfknee = {x: self.fknee for x in detnames}
dalpha = {x: self.alpha for x in detnames}
dnet = {x: self.NET for x in detnames}
nse = AnalyticNoise(rate=drate,
fmin=dfmin,
detectors=detnames,
fknee=dfknee,
alpha=dalpha,
NET=dnet)
# Single observation
obs = dict()
obs["tod"] = tod
obs["noise"] = nse
obs["id"] = 12345
obs["intervals"] = tod.subscans
obs["site"] = "SimonsObs"
obs["telescope"] = "SAT4"
obs["site_id"] = 1
obs["telescope_id"] = 4
obs["fpradius"] = 5.0
obs["start_time"] = ftime
obs["altitude"] = self.site_alt
obs["name"] = "test_{:02}".format(ob)
# Add a focalplane dictionary with just the detector index
focalplane = {}
for idet, det in enumerate(detnames):
focalplane[det] = {"index": idet}
obs["focalplane"] = focalplane
# Add the observation to the dataset
self.data.obs.append(obs)
nse = toast.tod.OpSimNoise(out="signal", realization=0)
nse.exec(self.data)
return
