def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_ground_sim (total)")
mpiworld, procs, rank, comm = get_comm()
args, comm = parse_arguments(comm)
# Initialize madam parameters
madampars = setup_madam(args)
# Load and broadcast the schedule file
schedule = load_schedule(args, comm)[0]
# load or simulate the focalplane
detweights = load_focalplane(args, comm, schedule)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data = create_observations(args, comm, schedule)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
expand_pointing(args, comm, data)
# Prepare auxiliary information for distributed map objects
localpix, localsm, subnpix = get_submaps(args, comm, data)
# Scan input map
signalname = scan_sky_signal(args, comm, data, localsm, subnpix, "signal")
# Simulate noise
if signalname is None:
signalname = "signal"
mc = 0
simulate_noise(args, comm, data, mc, signalname)
# Set up objects to take copies of the TOD at appropriate times
signalname_madam, sigcopy_madam, sigclear = setup_sigcopy(
args, comm, signalname)
npp, zmap = init_binner(args,
comm,
data,
detweights,
subnpix=subnpix,
localsm=localsm)
output_tidas(args, comm, data, signalname)
outpath = setup_output(args, comm)
# Make a copy of the signal for Madam
copy_signal_madam(args, comm, data, sigcopy_madam)
# Bin unprocessed signal for reference
apply_binner(args, comm, data, npp, zmap, detweights, outpath, signalname)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
apply_polyfilter(args, comm, data, signalname)
apply_groundfilter(args, comm, data, signalname)
# Bin the filtered signal
apply_binner(
args,
comm,
data,
npp,
zmap,
detweights,
outpath,
signalname,
prefix="filtered",
)
data.obs[0]["tod"].cache.report()
clear_signal(args, comm, data, sigclear)
data.obs[0]["tod"].cache.report()
# Now run Madam on the unprocessed copy of the signal
if args.use_madam:
apply_madam(args, comm, data, madampars, outpath, detweights,
signalname_madam)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.report_clear("Gather and dump timing info")
return
def create_observations(args, comm, focalplane, groupsize):
timer = Timer()
timer.start()
log = Logger.get()
if groupsize > len(focalplane.keys()):
if comm.world_rank == 0:
log.error("process group is too large for the number of detectors")
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(focalplane.keys())
detquats = {}
detindx = None
if "index" in focalplane[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = focalplane[d]["quat"]
if detindx is not None:
detindx[d] = focalplane[d]["index"]
# Distribute the observations uniformly
groupdist = distribute_uniform(args.obs_num, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = regular_intervals(
args.obs_num,
args.start_time,
0,
args.sample_rate,
3600 * args.obs_time_h,
3600 * args.gap_h,
)
noise = pipeline_tools.get_analytic_noise(args, comm, focalplane)
# The distributed timestream data
data = Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
coord=args.coord,
firstsamp=obsrange[ob].first,
firsttime=obsrange[ob].start,
rate=args.sample_rate,
spinperiod=args.spin_period_min,
spinangle=args.spin_angle_deg,
precperiod=args.prec_period_min,
precangle=args.prec_angle_deg,
detindx=detindx,
detranks=comm.group_size,
hwprpm=args.hwp_rpm,
hwpstep=args.hwp_step_deg,
hwpsteptime=args.hwp_step_time_s,
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
obs["focalplane"] = pipeline_tools.Focalplane(focalplane)
data.obs.append(obs)
if comm.world_rank == 0:
timer.report_clear("Read parameters, compute data distribution")
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = np.empty(4 * tod.local_samples[1],
dtype=np.float64).reshape((-1, 4))
slew_precession_axis(
precquat,
firstsamp=(obsoffset + tod.local_samples[0]),
samplerate=args.sample_rate,
degday=degday,
)
tod.set_prec_axis(qprec=precquat)
del precquat
if comm.world_rank == 0:
timer.report_clear("Construct boresight pointing")
return data
def _prepare(self):
""" Examine the data object.
"""
log = Logger.get()
timer = Timer()
timer.start()
nsamp = self._count_samples()
# Determine the detectors and the pointing matrix non-zeros
# from the first observation. Mappraiser will expect these to remain
# unchanged across observations.
tod = self._data.obs[0]["tod"]
if self._dets is None:
dets = tod.local_dets
else:
dets = [det for det in tod.local_dets if det in self._dets]
ndet = len(dets)
# We get the number of Non-zero pointing weights per pixel, from the
# shape of the data from the first detector
nnzname = "{}_{}".format(self._weights, dets[0])
nnz_full = tod.cache.reference(nnzname).shape[1]
if nnz_full != 3:
raise RuntimeError("OpMappraiser: Don't know how to make a map "
"with nnz={}".format(nnz_full))
nnz = 3
nnz_stride = 1
else:
nnz = nnz_full
nnz_stride = 1
if "nside" not in self._params:
raise RuntimeError(
'OpMappraiser: "nside" must be set in the parameter dictionary'
)
nside = int(self._params["nside"])
# Inspect the valid intervals across all observations to
# determine the number of samples per detector
# N.B: Above comment is from OpMadam, for now
# it only gives frequency binning of the PSDs
psdfreqs = self._get_period_ranges(dets)
self._comm.Barrier()
if self._rank == 0 and self._verbose:
timer.report_clear("Collect dataset dimensions")
return (
dets,
nsamp,
ndet,
nnz,
nnz_full,
nnz_stride,
psdfreqs,
nside,
)
def create_observations(args, comm):
timer = Timer()
timer.start()
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# create the TOD for this observation
if args.noisefile != 'RIMO':
do_eff_cache = True
else:
do_eff_cache = False
tods = []
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
tods.append(
tp.Exchange(
comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
pntflagmask=args.pntflagmask,
do_eff_cache=do_eff_cache,
coord=args.coord,
))
rimo = tods[0].rimo
# Make output directory
if comm.world_rank == 0:
os.makedirs(args.out, exist_ok=True)
if args.noisefile != 'RIMO':
# We split MPI_COMM_WORLD into single process groups, each of
# which is assigned one or more observations (rings)
comm = Comm(groupsize=1)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = Data(comm)
for iobs, tod in enumerate(tods):
if args.noisefile != 'RIMO':
# Use a toast helper method to optimally distribute rings between
# processes.
dist = distribute_discrete(tod.ringsizes, comm.world_size)
my_first_ring, my_n_ring = dist[comm.world_rank]
for my_ring in range(my_first_ring, my_first_ring + my_n_ring):
ringtod = tp.Exchange.from_tod(tod,
my_ring,
comm.comm_group,
noisefile=args.noisefile)
ob = {}
ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring)
ob['id'] = ringtod.globalfirst_ring
ob['tod'] = ringtod
ob['intervals'] = ringtod.valid_intervals
ob['baselines'] = None
ob['noise'] = ringtod.noise
data.obs.append(ob)
else:
ob = {}
ob['name'] = 'observation{:04}'.format(iobs)
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
data.obs.append(ob)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Metadata queries")
return data
def _synthesize_map(self, fn_cmb, there):
""" Synthesize the stored alm expansion into a map
and place the map in node-shared memory.
"""
timer = Timer()
timer.start()
if not there:
# Use libsharp to perform the synthesis across the communicator
if self._quickpolbeam is None:
beam = hp.gauss_beam(fwhm=self._fwhm,
lmax=self._lmax,
pol=True)
beam = beam[:, 0:3].copy()
else:
beam = np.array(hp.read_cl(self._quickpolbeam))
if beam.ndim == 1:
beam = np.vstack([beam, beam, beam])
beam = beam[:, :self._lmax + 1].T.copy()
almT = self._alm[0].reshape(1, 1, -1)
self._alminfo.almxfl(almT, np.ascontiguousarray(beam[:, 0:1]))
my_outmap = synthesis(self._grid,
self._alminfo,
almT,
spin=0,
comm=self._comm)[0]
my_outmap = [my_outmap]
if self._pol:
almP = self._alm[1:3].reshape(1, 2, -1)
self._alminfo.almxfl(almP, np.ascontiguousarray(beam[:,
(1, 2)]))
my_outmap.append(
synthesis(self._grid,
self._alminfo,
almP,
spin=2,
comm=self._comm)[0])
# Discard the a_lm
del self._alm
my_outmap = np.vstack(my_outmap)
my_pixels = self._dist_rings.local_pixels
my_maptemp = np.zeros([self._nnz, self._npix], dtype=np.float)
maptemp = np.zeros([self._nnz, self._npix], dtype=np.float)
my_maptemp[:, my_pixels] = my_outmap
self._comm.Reduce(my_maptemp, maptemp)
del my_maptemp
maptemp = hp.reorder(maptemp, r2n=True)
timer.stop()
if self._global_rank == 0:
timer.report("synthesize CMB map")
# Save the CMB map
os.makedirs(CMBCACHE, exist_ok=True)
header = [("fwhm", np.degrees(self._fwhm),
"gaussian smoothing (deg)")]
hp.write_map(fn_cmb,
maptemp,
extra_header=header,
overwrite=True,
nest=True)
print("CMB map saved in {}".format(fn_cmb), flush=True)
else:
if self._global_rank == 0:
print("Loading cached CMB map from {}".format(fn_cmb),
flush=True)
if self._rank == 0:
maptemp = hp.read_map(fn_cmb,
None,
nest=True,
verbose=False,
dtype=np.float32)
if not self._pol:
maptemp = maptemp[0]
else:
maptemp = None
self.mapsampler = MapSampler(
None,
pol=self._pol,
comm=self._comm,
preloaded_map=maptemp,
nest=True,
plug_holes=False,
use_shmem=True,
)
del maptemp
return
def create_observations(args, comm, schedules):
""" Create and distribute TOAST observations for every CES in
schedules.
Args:
schedules (iterable) : a list of Schedule objects.
"""
log = Logger.get()
timer = Timer()
timer.start()
data = Data(comm)
# Loop over the schedules, distributing each schedule evenly across
# the process groups. For now, we'll assume that each schedule has
# the same number of operational days and the number of process groups
# matches the number of operational days. Relaxing these constraints
# will cause the season break to occur on different process groups
# for different schedules and prevent splitting the communicator.
for schedule in schedules:
telescope = schedule.telescope
all_ces = schedule.ceslist
nces = len(all_ces)
breaks = pipeline_tools.get_breaks(comm, all_ces, args)
groupdist = distribute_uniform(nces, comm.ngroups, breaks=breaks)
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ices in range(group_firstobs, group_firstobs + group_numobs):
obs = create_observation(args, comm, telescope, all_ces[ices])
data.obs.append(obs)
if comm.comm_world is None or comm.comm_group.rank == 0:
log.info("Group # {:4} has {} observations.".format(comm.group, len(data.obs)))
if len(data.obs) == 0:
raise RuntimeError(
"Too many tasks. Every MPI task must "
"be assigned to at least one observation."
)
if comm.comm_world is not None:
comm.comm_world.barrier()
timer.stop()
if comm.world_rank == 0:
timer.report("Simulated scans")
# Split the data object for each telescope for separate mapmaking.
# We could also split by site.
if len(schedules) > 1:
telescope_data = data.split("telescope_name")
if len(telescope_data) == 1:
# Only one telescope available
telescope_data = []
else:
telescope_data = []
telescope_data.insert(0, ("all", data))
return data, telescope_data
def parse_arguments(comm):
timer = Timer()
timer.start()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"atmosphere and make maps for some number of noise Monte Carlos.",
fromfile_prefix_chars="@",
)
pipeline_tools.add_dist_args(parser)
pipeline_tools.add_debug_args(parser)
pipeline_tools.add_todground_args(parser)
pipeline_tools.add_pointing_args(parser)
pipeline_tools.add_polyfilter_args(parser)
pipeline_tools.add_groundfilter_args(parser)
pipeline_tools.add_atmosphere_args(parser)
pipeline_tools.add_noise_args(parser)
pipeline_tools.add_gainscrambler_args(parser)
pipeline_tools.add_madam_args(parser)
pipeline_tools.add_mapmaker_args(parser)
pipeline_tools.add_sky_map_args(parser)
pipeline_tools.add_pysm_args(parser)
pipeline_tools.add_sss_args(parser)
pipeline_tools.add_tidas_args(parser)
pipeline_tools.add_spt3g_args(parser)
pipeline_tools.add_mc_args(parser)
parser.add_argument(
"--outdir", required=False, default="out", help="Output directory"
)
parser.add_argument(
"--madam",
required=False,
action="store_true",
help="Use libmadam for map-making",
dest="use_madam",
)
parser.add_argument(
"--no-madam",
required=False,
action="store_false",
help="Do not use libmadam for map-making [default]",
dest="use_madam",
)
parser.set_defaults(use_madam=False)
parser.add_argument(
"--focalplane",
required=False,
default=None,
help="Pickle file containing a dictionary of detector "
"properties. The keys of this dict are the detector "
"names, and each value is also a dictionary with keys "
'"quat" (4 element ndarray), "fwhm" (float, arcmin), '
'"fknee" (float, Hz), "alpha" (float), and '
'"NET" (float).',
)
parser.add_argument(
"--freq",
required=True,
help="Comma-separated list of frequencies with identical focal planes."
" They override the bandpasses in the focalplane for the purpose of"
" scaling the atmospheric signal but not for simulating the sky signal.",
)
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if args.tidas is not None:
if not tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
if args.spt3g is not None:
if not spt3g_available:
raise RuntimeError("SPT3G not found- cannot export")
if len(args.freq.split(",")) != 1:
# Multi frequency run. We don't support multiple copies of
# scanned signal.
if args.input_map:
raise RuntimeError(
"Multiple frequencies are not supported when scanning from a map"
)
if args.simulate_atmosphere and args.weather is None:
raise RuntimeError("Cannot simulate atmosphere without a TOAST weather file")
if comm.world_rank == 0:
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.world_rank == 0:
os.makedirs(args.outdir, exist_ok=True)
timer.stop()
if comm.world_rank == 0:
timer.report("Parsed parameters")
return args, comm
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_so_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = toast_tools.get_comm()
memreport("at the beginning of the pipeline", comm.comm_world)
args, comm = parse_arguments(comm)
# Load and broadcast the schedule file
schedules = toast_tools.load_schedule(args, comm)
# Load the weather and append to schedules
toast_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = so_tools.load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = so_tools.create_observations(args, comm, schedules)
memreport("after creating observations", comm.comm_world)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
so_tools.get_elevation_noise(args, comm, data)
totalname = "total"
# Split the communicator for day and season mapmaking
time_comms = toast_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
toast_tools.expand_pointing(args, comm, data)
memreport("after pointing", comm.comm_world)
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
madam = None
for mc in range(firstmc, firstmc + nmc):
timer_mc = Timer()
timer_mc.start()
outpath = setup_output(args, comm, mc)
outprefix = args.map_prefix + "_filtered"
if args.madam:
outmap = os.path.join(outpath, outprefix + "_bmap.fits")
else:
outmap = os.path.join(outpath, outprefix + "_binned.fits")
if os.path.isfile(outmap):
if comm.world_rank == 0:
log.info("{} exists, skipping".format(outmap))
continue
if comm.world_rank == 0:
log.info("Processing MC = {} into {}".format(mc, outmap))
# Ensure there is no stale signal in the cache
toast.tod.OpCacheClear(totalname).exec(data)
if args.pysm_model:
if schedules is not None:
focalplanes = [
s.telescope.focalplane.detector_data for s in schedules
]
else:
focalplanes = [telescope.focalplane.detector_data]
so_tools.simulate_sky_signal(args,
comm,
data,
focalplanes,
totalname,
mc=mc)
else:
toast_tools.scan_sky_signal(args, comm, data, totalname, mc=mc)
memreport("after PySM", comm.comm_world)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
toast_tools.apply_polyfilter(args, comm, data, totalname)
toast_tools.apply_groundfilter(args, comm, data, totalname)
memreport("after filter", comm.comm_world)
# Bin maps
timer_map = Timer()
timer_map.start()
if args.madam:
if madam is None:
madampars = {}
madampars["temperature_only"] = False
for name in [
"kfirst", "write_map", "write_matrix",
"write_wcov", "write_hits"
]:
madampars[name] = False
madampars["write_binmap"] = True
madampars["concatenate_messages"] = True
madampars["allreduce"] = True
madampars["nside_submap"] = args.nside_submap
madampars["reassign_submaps"] = True
madampars["pixlim_map"] = 1e-2
madampars["pixmode_map"] = 2
# Instead of fixed detector weights, we'll want to use scaled noise
# PSD:s that include the atmospheric noise
madampars["radiometers"] = True
madampars["noise_weights_from_psd"] = True
madampars["nside_map"] = args.nside
madampars["fsample"] = args.sample_rate
madampars["path_output"] = outpath
madampars["file_root"] = outprefix
if args.madam_concatenate_messages:
# Collective communication is fast but requires memory
madampars["concatenate_messages"] = True
if args.madam_allreduce:
# Every process will allocate a copy of every observed submap.
madampars["allreduce"] = True
else:
# Every process will allocate complete send and receive buffers
madampars["allreduce"] = False
else:
# Slow but memory-efficient point-to-point communication. Allocate
# only enough memory to communicate with one process at a time.
madampars["concatenate_messages"] = False
madampars["allreduce"] = False
madam = toast.todmap.OpMadam(
params=madampars,
detweights=detweights,
name=totalname,
common_flag_mask=args.common_flag_mask,
purge_tod=True,
mcmode=False,
conserve_memory=args.madam_conserve_memory,
)
else:
madam.params["path_output"] = outpath
madam.exec(data)
del madam
madam = None
else:
mapmaker = toast.todmap.OpMapMaker(
nside=args.nside,
nnz=3,
name=totalname,
outdir=outpath,
outprefix=outprefix + "_",
write_hits=False,
zip_maps=False,
write_wcov_inv=False,
write_wcov=False,
write_binned=True,
write_destriped=False,
write_rcond=False,
rcond_limit=1e-3,
baseline_length=None,
common_flag_mask=args.common_flag_mask,
)
mapmaker.exec(data)
if comm.world_rank == 0:
timer_map.report_clear("Bin map")
memreport("after filter & bin", comm.comm_world)
if comm.world_rank == 0:
timer_mc.report_clear(
"Monte Carlo iteration # {:05}".format(mc))
if comm.comm_world is not None:
comm.comm_world.barrier()
memreport("at the end of the pipeline", comm.comm_world)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.report_clear("Gather and dump timing info")
if comm.world_rank == 0:
timer0.report_clear("toast_so_tf.py pipeline")
return
def parse_arguments(comm):
timer = Timer()
timer.start()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"atmosphere and make maps for some number of noise Monte Carlos.",
fromfile_prefix_chars="@",
)
toast_tools.add_dist_args(parser)
toast_tools.add_todground_args(parser)
toast_tools.add_pointing_args(parser)
toast_tools.add_polyfilter_args(parser)
toast_tools.add_groundfilter_args(parser)
toast_tools.add_noise_args(parser)
toast_tools.add_sky_map_args(parser)
toast_tools.add_mc_args(parser)
so_tools.add_hw_args(parser)
so_tools.add_so_noise_args(parser)
so_tools.add_pysm_args(parser)
so_tools.add_export_args(parser)
toast_tools.add_debug_args(parser)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument("--map-prefix",
required=False,
default="toast",
help="Output map prefix")
parser.add_argument(
"--madam",
required=False,
action="store_true",
help="Use libmadam to bin the signal",
dest="madam",
)
parser.add_argument(
"--no-madam",
required=False,
action="store_false",
help="Do not use libMadam to bin the signal",
dest="madam",
)
parser.set_defaults(madam=False)
parser.add_argument(
"--madam-conserve-memory",
required=False,
action="store_true",
help="Stage the Madam buffer packing",
dest="madam_conserve_memory",
)
parser.add_argument(
"--no-madam-conserve-memory",
required=False,
action="store_false",
help="Do not stage the Madam buffer packing",
dest="madam_conserve_memory",
)
parser.set_defaults(madam_conserve_memory=True)
parser.add_argument(
"--madam-allreduce",
required=False,
action="store_true",
help="Use the allreduce communication pattern in Madam",
dest="madam_allreduce",
)
parser.add_argument(
"--no-madam-allreduce",
required=False,
action="store_false",
help="Do not use the allreduce communication pattern in Madam",
dest="madam_allreduce",
)
parser.set_defaults(madam_allreduce=False)
parser.add_argument(
"--madam-concatenate-messages",
required=False,
action="store_true",
help="Use the alltoallv commucation pattern in Madam",
dest="madam_concatenate_messages",
)
parser.add_argument(
"--no-madam-concatenate-messages",
required=False,
action="store_false",
help="Use the point-to-point communication pattern in Madam",
dest="madam_concatenate_messages",
)
parser.set_defaults(madam_concatenate_messages=True)
try:
args = parser.parse_args()
except SystemExit as e:
sys.exit()
if len(args.bands.split(",")) != 1:
# Multi frequency run. We don't support multiple copies of
# scanned signal.
if args.input_map:
raise RuntimeError(
"Multiple frequencies are not supported when scanning from a map"
)
if comm.world_rank == 0:
log.info("\n")
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
log.info("\n")
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.world_rank == 0:
if not os.path.isdir(args.outdir):
try:
os.makedirs(args.outdir)
except FileExistsError:
pass
timer.report_clear("Parse arguments")
return args, comm
def simulate_sky_signal(args,
comm,
data,
focalplanes,
subnpix,
localsm,
signalname=None):
""" Use PySM to simulate smoothed sky signal.
"""
log = Logger.get()
timer = Timer()
timer.start()
# Convolve a signal TOD from PySM
if comm.world_rank == 0:
log.info("Simulating sky signal with PySM")
map_dist = (None if comm is None else pysm.MapDistribution(
nside=args.nside, mpi_comm=comm.comm_rank))
pysm_component_objects = []
pysm_model = []
for model_tag in args.pysm_model.split(","):
if not model_tag.startswith("SO"):
pysm_model.append(model_tag)
else:
if so_pysm_models is None:
raise RuntimeError(
"{} requires so_pysm_models".format(model_tag))
if model_tag == "SO_x1_cib":
pysm_component_objects.append(
so_pysm_models.WebSkyCIB(
websky_version="0.3",
interpolation_kind="linear",
nside=args.nside,
map_dist=map_dist,
))
elif model_tag == "SO_x1_ksz":
pysm_component_objects.append(
so_pysm_models.WebSkySZ(
version="0.3",
nside=args.nside,
map_dist=map_dist,
sz_type="kinetic",
))
elif model_tag == "SO_x1_tsz":
pysm_component_objects.append(
so_pysm_models.WebSkySZ(
version="0.3",
nside=args.nside,
map_dist=map_dist,
sz_type="thermal",
))
elif model_tag.startswith("SO_x1_cmb"):
lensed = "unlensed" not in model_tag
include_solar_dipole = "solar" in model_tag
pysm_component_objects.append(
so_pysm_models.WebSkyCMBMap(
websky_version="0.3",
lensed=lensed,
include_solar_dipole=include_solar_dipole,
seed=1,
nside=args.nside,
map_dist=map_dist,
))
else:
pysm_component_objects.append(
so_pysm_models.get_so_models(model_tag,
args.nside,
map_dist=map_dist))
if signalname is None:
signalname = "pysmsignal"
op_sim_pysm = OpSimPySM(
comm=comm.comm_rank,
out=signalname,
pysm_model=pysm_model,
pysm_component_objects=pysm_component_objects,
focalplanes=focalplanes,
nside=args.nside,
subnpix=subnpix,
localsm=localsm,
apply_beam=args.pysm_apply_beam,
coord="G", # setting G doesn't perform any rotation
map_dist=map_dist,
)
assert args.coord in "CQ", "Input SO models are always in Equatorial coordinates"
op_sim_pysm.exec(data)
if comm.comm_world is not None:
comm.comm_world.barrier()
timer.stop()
if comm.world_rank == 0:
timer.report("PySM")
return signalname
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Simple dipole pipeline',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory for '
'pointing')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Which OBT flag bits to raise for HCM maneuvers')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use')
parser.add_argument('--obtfirst',
required=False,
default=None,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
help='Last OBT to use')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
# Dipole parameters
dipogroup = parser.add_mutually_exclusive_group()
dipogroup.add_argument('--dipole',
dest='dipole',
required=False,
default=False,
action='store_true',
help='Simulate dipole')
dipogroup.add_argument('--solsys-dipole',
required=False,
default=False,
action='store_true',
help='Simulate solar system dipole')
dipogroup.add_argument('--orbital-dipole',
required=False,
default=False,
action='store_true',
help='Simulate orbital dipole')
dipo_parameters_group = parser.add_argument_group('dipole_parameters')
dipo_parameters_group.add_argument(
'--solsys_speed',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_speed"],
help='Solar system speed wrt. CMB rest frame in km/s. Default is '
'Planck 2015 best fit value')
dipo_parameters_group.add_argument(
'--solsys-glon',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_glon"],
help='Solar system velocity direction longitude in degrees')
dipo_parameters_group.add_argument(
'--solsys-glat',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_glat"],
help='Solar system velocity direction latitude in degrees')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.world_rank == 0:
print('All parameters:')
print(args, flush=True)
timer = Timer()
timer.start()
do_dipole = args.dipole or args.solsys_dipole or args.orbital_dipole
if not do_dipole:
raise RuntimeError(
"You have to set dipole, solsys-dipole or orbital-dipole")
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# create the TOD for this observation
tods = []
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
# create the TOD for this observation
tods.append(
tp.Exchange(comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
pntflagmask=args.pntflagmask,
do_eff_cache=False))
# Make output directory
if not os.path.isdir(args.out) and comm.world_rank == 0:
os.makedirs(args.out)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
for iobs, tod in enumerate(tods):
ob = {}
ob['name'] = 'observation{:04}'.format(iobs)
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
ob['noise_simu'] = tod.noise
data.obs.append(ob)
rimo = tods[0].rimo
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Metadata queries")
# Always read the signal and flags, even if the signal is later
# overwritten. There is no overhead for the signal because it is
# interlaced with the flags.
tod_name = 'signal'
timestamps_name = 'timestamps'
flags_name = 'flags'
common_flags_name = 'common_flags'
reader = tp.OpInputPlanck(signal_name=tod_name,
flags_name=flags_name,
timestamps_name=timestamps_name,
commonflags_name=common_flags_name)
if comm.world_rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Read")
"""
# Clear the signal
eraser = tp.OpCacheMath(in1=tod_name, in2=0, multiply=True,
out=tod_name)
if comm.world_rank == 0:
print('Erasing TOD', flush=True)
eraser.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Erase")
"""
# make a planck Healpix pointing matrix
mode = 'IQU'
nside = 512
pointing = tp.OpPointingPlanck(nside=nside,
mode=mode,
RIMO=rimo,
margin=0,
apply_flags=True,
keep_vel=do_dipole,
keep_pos=False,
keep_phase=False,
keep_quats=do_dipole)
pointing.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Pointing Matrix")
flags_name = 'flags'
common_flags_name = 'common_flags'
# Simulate the dipole
if args.dipole:
dipomode = 'total'
elif args.solsys_dipole:
dipomode = 'solsys'
else:
dipomode = 'orbital'
dipo = tp.OpDipolePlanck(args.freq,
solsys_speed=args.solsys_speed,
solsys_glon=args.solsys_glon,
solsys_glat=args.solsys_glat,
mode=dipomode,
output='dipole',
keep_quats=False,
npipe_mode=True)
dipo.exec(data)
dipo = tp.OpDipolePlanck(args.freq,
solsys_speed=args.solsys_speed,
solsys_glon=args.solsys_glon,
solsys_glat=args.solsys_glat,
mode=dipomode,
output='dipole4pi',
keep_quats=False,
npipe_mode=False,
lfi_mode=False)
dipo.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Dipole")
# Write out the values in ASCII
for iobs, obs in enumerate(data.obs):
tod = obs["tod"]
times = tod.local_times()
velocity = tod.local_velocity()
for det in tod.local_dets:
quat = tod.local_pointing(det)
angles = np.vstack(qarray.to_angles(quat)).T
signal = tod.local_signal(det)
dipole = tod.local_signal(det, "dipole")
dipole4pi = tod.local_signal(det, "dipole4pi")
fname_out = os.path.join(
args.out,
"{}_dipole.{}.{}.{}.txt".format(dipomode, comm.world_rank,
iobs, det))
with open(fname_out, "w") as fout:
for t, ang, vel, sig, dipo, dipo4pi in zip(
times, angles, velocity, signal, dipole, dipole4pi):
fout.write(
(10 * " {}" + "\n").format(t, *ang, *vel, sig, dipo,
dipo4pi))
print("{} : Wrote {}".format(comm.world_rank, fname_out))
if comm.world_rank == 0:
timer.report_clear("Write dipole")
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def _stage_data(
self,
nsamp,
ndet,
nnz,
nnz_full,
nnz_stride,
psdfreqs,
detectors,
nside,
):
""" create Mappraiser-compatible buffers
Collect the TOD into Mappraiser buffers. Process pixel weights
Separate from the rest to reduce the memory high water mark
When the user has set purge=True
Moving data between toast and Mappraiser buffers has an overhead.
We perform the operation in a staggered fashion to have the
overhead only once per node.
"""
log = Logger.get()
nodecomm = self._comm.Split_type(MPI.COMM_TYPE_SHARED, self._rank)
# Check if the user has elected to stagger staging the data on each
# node to avoid exhausting memory
if self._conserve_memory:
if self._conserve_memory == 1:
nread = nodecomm.size
else:
nread = min(self._conserve_memory, nodecomm.size)
else:
nread = 1
self._comm.Barrier()
timer_tot = Timer()
timer_tot.start()
# Stage time (Tpltz blocks in Mappraiser), it is never purged
# so the staging is never stepped
timer = Timer()
# THIS STEP IS SKIPPED: we do not have timestamps, nor do we build Toeplitz blocks
# from TOAST psds which comprise detector noise only - a psd fit is done when staging noise -
#timer.start()
#invtt_list = self._stage_time(detectors, nsamp, psdfreqs)
#self._mappraiser_invtt = np.array([np.array(invtt_i, dtype= mappraiser.INVTT_TYPE) for invtt_i in invtt_list])
#del invtt_list
#self._mappraiser_invtt = np.concatenate(self._mappraiser_invtt)
#if self._verbose:
# nodecomm.Barrier()
# if self._rank == 0:
# timer.report_clear("Stage time")
#memreport("after staging time", self._comm) # DEBUG
#count_caches(
# self._data, self._comm, nodecomm, self._cache, "after staging time"
#) # DEBUG
# Stage signal. If signal is not being purged, staging is not stepped
timer.start()
signal_dtype, local_blocks_sizes = self._stage_signal(
detectors, nsamp, ndet, nodecomm, nread)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage signal")
memreport("after staging signal", self._comm) # DEBUG
count_caches(self._data, self._comm, nodecomm, self._cache,
"after staging signal") # DEBUG
# Stage noise. If noise is not being purged, staging is not stepped
timer.start()
invtt_list, noise_dtype = self._stage_noise(detectors, nsamp, ndet,
nodecomm, nread)
self._mappraiser_invtt = np.array([
np.array(invtt_i, dtype=mappraiser.INVTT_TYPE)
for invtt_i in invtt_list
])
del invtt_list
self._mappraiser_invtt = np.concatenate(self._mappraiser_invtt)
if self._params["uniform_w"] == 1:
self._mappraiser_invtt = np.ones_like(self._mappraiser_invtt)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage noise")
memreport("after staging noise", self._comm) # DEBUG
count_caches(self._data, self._comm, nodecomm, self._cache,
"after staging noise") # DEBUG
# Stage pixels
timer_step = Timer()
timer_step.start()
for iread in range(nread):
nodecomm.Barrier()
timer.start()
if nodecomm.rank % nread == iread:
pixels_dtype = self._stage_pixels(detectors, nsamp, ndet, nnz,
nside)
if self._verbose and nread > 1:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage pixels {} / {}".format(
iread + 1, nread))
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer_step.report_clear("Stage pixels")
memreport("after staging pixels", self._comm) # DEBUG
count_caches(self._data, self._comm, nodecomm, self._cache,
"after staging pixels") # DEBUG
# Stage pixel weights
timer_step.start()
weight_dtype = self._stage_pixweights(
detectors,
nsamp,
ndet,
nnz,
nnz_full,
nnz_stride,
nodecomm,
nread,
)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer_step.report_clear("Stage pixel weights")
memreport("after staging pixel weights", self._comm) # DEBUG
count_caches(self._data, self._comm, nodecomm, self._cache,
"after staging pixel weights") # DEBUG
del nodecomm
if self._rank == 0 and self._verbose:
timer_tot.report_clear("Stage all data")
# detweights is either a dictionary of weights specified at
# construction time, or else we use uniform weighting.
# N.B: This is essentially useless in current implementation
detw = {}
if self._detw is None:
for idet, det in enumerate(detectors):
detw[det] = 1.0
else:
detw = self._detw
detweights = np.zeros(ndet, dtype=np.float64)
for idet, det in enumerate(detectors):
detweights[idet] = detw[det]
# Get global array of data sizes of the full communicator
data_size_proc = np.array(self._comm.allgather(
len(self._mappraiser_signal)),
dtype=np.int32)
# Get number of local observations
nobsloc = len(self._data.obs)
return data_size_proc, nobsloc, local_blocks_sizes, signal_dtype, noise_dtype, pixels_dtype, weight_dtype
def _stage_pixweights(
self,
detectors,
nsamp,
ndet,
nnz,
nnz_full,
nnz_stride,
nodecomm,
nread,
):
"""Now collect the pixel weights
"""
log = Logger.get()
timer = Timer()
# Determine if we can purge the pixel weights and avoid keeping two
# copies of the weights in memory
purge = self._purge_weights or (nnz == nnz_full)
if not purge:
nread = 1
nodecomm = MPI.COMM_SELF
for iread in range(nread):
nodecomm.Barrier()
timer.start()
if nodecomm.rank % nread == iread:
self._mappraiser_pixweights = self._cache.create(
"pixweights", mappraiser.WEIGHT_TYPE,
(nsamp * ndet * nnz, ))
self._mappraiser_pixweights[:] = 0
global_offset = 0
for iobs, obs in enumerate(self._data.obs):
tod = obs["tod"]
for idet, det in enumerate(detectors):
# get the pixels and weights for the valid intervals
# from the cache
weightsname = "{}_{}".format(self._weights, det)
weights = tod.cache.reference(weightsname)
weight_dtype = weights.dtype
offset = global_offset
nn = len(weights)
dwslice = slice(
(idet * nsamp + offset) * nnz,
(idet * nsamp + offset + nn) * nnz,
)
self._mappraiser_pixweights[dwslice] = weights.flatten(
)[::nnz_stride]
offset += nn
del weights
# Purge the weights but restore them from the Mappraiser
# buffers when purge_weights=False.
if purge:
for idet, det in enumerate(detectors):
weightsname = "{}_{}".format(self._weights, det)
tod.cache.clear(pattern=weightsname)
global_offset = offset
if self._verbose and nread > 1:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage pixel weights {} / {}".format(
iread + 1, nread))
return weight_dtype
def _stage_noise(self, detectors, nsamp, ndet, nodecomm, nread):
""" Stage noise timestream (detector noise + atmosphere)
"""
log = Logger.get()
timer = Timer()
# Determine if we can purge the signal and avoid keeping two
# copies in memory
purge = self._noise_name is not None and self._purge_tod
if not purge:
nread = 1
nodecomm = MPI.COMM_SELF
for iread in range(nread):
nodecomm.Barrier()
timer.start()
if nodecomm.rank % nread == iread:
self._mappraiser_noise = self._cache.create(
"noise", mappraiser.SIGNAL_TYPE, (nsamp * ndet, ))
if self._noise_name == None:
self._mappraiser_noise = np.zeros_like(
self._mappraiser_noise)
invtt_list = []
for i in range(len(self._data.obs) * len(detectors)):
invtt_list.append(
np.ones(1)) #Must be used with lambda = 1
return invtt_list, self._mappraiser_noise.dtype
self._mappraiser_noise[:] = np.nan
global_offset = 0
invtt_list = []
# fknee_list = []
# fmin_list = []
# alpha_list = []
# logsigma2_list = []
for iobs, obs in enumerate(self._data.obs):
tod = obs["tod"]
for idet, det in enumerate(detectors):
# Get the signal.
noise = tod.local_signal(det, self._noise_name)
# if self._rank ==0 and idet == 0:
# print("|noise| = {}".format(np.sum(noise**2)))
noise_dtype = noise.dtype
offset = global_offset
nn = len(noise)
invtt = self._noise2invtt(
noise, nn, idet
) #, logsigma2, alpha, fknee, fmin = self._noise2invtt(noise, nn, idet)
invtt_list.append(invtt)
# logsigma2_list.append(logsigma2)
# alpha_list.append(alpha)
# fknee_list.append(fknee)
# fmin_list.append(fmin)
dslice = slice(idet * nsamp + offset,
idet * nsamp + offset + nn)
self._mappraiser_noise[dslice] = noise
offset += nn
del noise
# Purge only after all detectors are staged in case some are aliased
# cache.clear() will not fail if the object was already
# deleted as an alias
if purge:
for det in detectors:
cachename = "{}_{}".format(self._noise_name, det)
tod.cache.clear(cachename)
global_offset = offset
if self._verbose and nread > 1:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage noise {} / {}".format(
iread + 1, nread))
# sendcounts = np.array(self._comm.gather(len(fknee_list), 0))
#
# Fknee_list = None
# Fmin_list = None
# Alpha_list = None
# Logsigma2_list = None
#
# if self._rank ==0:
# Fknee_list = np.empty(sum(sendcounts))
# Fmin_list = np.empty(sum(sendcounts))
# Alpha_list = np.empty(sum(sendcounts))
# Logsigma2_list = np.empty(sum(sendcounts))
#
# self._comm.Gatherv(np.array(fknee_list),(Fknee_list,sendcounts),0)
# self._comm.Gatherv(np.array(fmin_list),(Fmin_list, sendcounts),0)
# self._comm.Gatherv(np.array(alpha_list),(Alpha_list, sendcounts),0)
# self._comm.Gatherv(np.array(logsigma2_list),(Logsigma2_list, sendcounts),0)
# if self._rank ==0:
# np.save("fknee.npy",Fknee_list)
# np.save("fmin.npy", Fmin_list)
# np.save("logsigma2.npy",Logsigma2_list)
# np.save("alpha.npy",Alpha_list)
return invtt_list, noise_dtype
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
memreport("at beginning of main", mpiworld)
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--nside',
required=False,
type=np.int,
default=512,
help='Map resolution')
parser.add_argument('--nside_cross',
required=False,
type=np.int,
default=512,
help='Destriping resolution')
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help='Write data distribution info to file')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_in_diode0',
required=False,
default=None,
help='Input Exchange Format File directory, '
'LFI diode 0')
parser.add_argument('--effdir_in_diode1',
required=False,
default=None,
help='Input Exchange Format File directory, '
'LFI diode 1')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory '
'for pointing')
parser.add_argument('--effdir_out',
required=False,
help='Output directory for destriped TOD')
parser.add_argument('--effdir_out_diode0',
required=False,
help='Output directory for destriped TOD, LFI diode 0')
parser.add_argument('--effdir_out_diode1',
required=False,
help='Output directory for destriped TOD, LFI diode 1')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Pointing flag mask')
parser.add_argument('--bad_intervals',
required=False,
help='Path to bad interval file.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
type=np.int,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
type=np.int,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use (can be a list)')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use (can be a list)')
parser.add_argument('--obtfirst',
required=False,
default=None,
type=np.float,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
type=np.float,
help='Last OBT to use')
parser.add_argument('--madampar',
required=False,
default=None,
help='Madam parameter file')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--madam_prefix', required=False, help='map prefix')
parser.add_argument('--split_mask',
required=False,
default=None,
help='Intensity mask, non-zero pixels are not split.')
parser.add_argument('--save_leakage_matrices',
dest='save_leakage_matrices',
default=False,
action='store_true',
help='Compile and write out the leakage projection '
'matrices.')
# noise parameters
parser.add_argument('--noisefile',
required=False,
default='RIMO',
help='Path to noise PSD files for noise filter. '
'Tag DETECTOR will be replaced with detector name.')
parser.add_argument('--static_noise',
dest='static_noise',
required=False,
default=False,
action='store_true',
help='Assume constant noise PSD')
parser.add_argument('--filterfile',
required=False,
help='Extra filter file.')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.comm_world.rank == 0:
print('All parameters:')
print(args, flush=True)
timer = Timer()
timer.start()
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# create the TOD for this observation
if args.noisefile != 'RIMO' and not args.static_noise:
do_eff_cache = True
else:
do_eff_cache = False
tods = []
if args.static_noise:
noisefile = args.noisefile
else:
noisefile = 'RIMO'
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
tods.append(
tp.Exchange(comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_in_diode0=args.effdir_in_diode0,
effdir_in_diode1=args.effdir_in_diode1,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
pntflagmask=args.pntflagmask,
do_eff_cache=do_eff_cache,
noisefile=noisefile))
rimo = tods[0].rimo
# Make output directory
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.makedirs(args.out)
# Read in madam parameter file
# Allow more than one entry, gather into a list
repeated_keys = ['detset', 'detset_nopol', 'survey']
pars = {}
if comm.comm_world.rank == 0:
pars['kfirst'] = False
pars['temperature_only'] = True
pars['base_first'] = 60.0
pars['nside_map'] = args.nside
pars['nside_cross'] = min(args.nside, args.nside_cross)
pars['nside_submap'] = 16
pars['write_map'] = False
pars['write_binmap'] = True
pars['write_matrix'] = False
pars['write_wcov'] = False
pars['write_hits'] = True
pars['kfilter'] = False
pars['info'] = 3
pars['pixlim_map'] = 1e-3
pars['pixlim_cross'] = 1e-3
if args.madampar:
pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*')
comment = re.compile(r'^#.*')
with open(args.madampar, 'r') as f:
for line in f:
if not comment.match(line):
result = pat.match(line)
if result:
key, value = result.group(1), result.group(2)
if key in repeated_keys:
if key not in pars:
pars[key] = []
pars[key].append(value)
else:
pars[key] = value
# Command line parameters override the ones in the madam parameter file
if 'file_root' not in pars:
pars['file_root'] = 'madam'
if args.madam_prefix is not None:
pars['file_root'] = args.madam_prefix
sfreq = '{:03}'.format(args.freq)
if sfreq not in pars['file_root']:
pars['file_root'] += '_' + sfreq
try:
fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq]
except Exception:
fsample = 180.3737
pars['fsample'] = fsample
pars['path_output'] = args.out
if args.save_leakage_matrices:
pars['write_leakmatrix'] = True
pars = comm.comm_world.bcast(pars, root=0)
if args.noisefile != 'RIMO':
# We split MPI_COMM_WORLD into single process groups, each of
# which is assigned one or more observations (rings)
comm = toast.Comm(groupsize=1)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
for iobs, tod in enumerate(tods):
if args.noisefile != 'RIMO' and not args.static_noise:
# Use a toast helper method to optimally distribute rings between
# processes.
dist = toast.distribute_discrete(tod.ringsizes, comm.world_size)
my_first_ring, my_n_ring = dist[comm.comm_world.rank]
for my_ring in range(my_first_ring, my_first_ring + my_n_ring):
ringtod = tp.Exchange.from_tod(tod,
my_ring,
comm.comm_group,
noisefile=args.noisefile)
ob = {}
ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring)
ob['id'] = ringtod.globalfirst_ring
ob['tod'] = ringtod
ob['intervals'] = ringtod.valid_intervals
ob['baselines'] = None
ob['noise'] = ringtod.noise
data.obs.append(ob)
else:
ob = {}
ob['name'] = 'observation{:04}'.format(iobs)
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
data.obs.append(ob)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Metadata queries")
if args.effdir_out is not None or (args.effdir_out_diode0 is not None
and args.effdir_out_diode1 is not None):
do_output = True
else:
do_output = False
# Read in the signal
timer.clear()
timer.start()
reader = tp.OpInputPlanck(signal_name='signal', flags_name='flags')
if comm.comm_world.rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Read")
tod_name = 'signal'
flags_name = 'flags'
# Optionally filter the signal
apply_filter(args, data)
# Optionally flag bad intervals
if args.bad_intervals is not None:
timer = Timer()
timer.start()
flagger = tp.OpBadIntervals(path=args.bad_intervals)
flagger.exec(data)
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Apply {}".format(args.bad_intervals))
# make a planck Healpix pointing matrix
timer.clear()
timer.start()
mode = 'IQU'
if pars['temperature_only'] == 'T':
mode = 'I'
nside = int(pars['nside_map'])
pointing = tp.OpPointingPlanck(nside=nside,
mode=mode,
RIMO=rimo,
margin=0,
apply_flags=(not do_output),
keep_vel=False,
keep_pos=False,
keep_phase=False,
keep_quats=False)
pointing.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Pointing Matrix, mode = {}".format(mode))
for obs in data.obs:
obs['tod'].purge_eff_cache()
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
if d[-1] in '01' and d[-2] != '-':
det = to_radiometer(d)
else:
det = d
net = tod.rimo[det].net
fsample = tod.rimo[det].fsample
detweights[d] = 1.0 / (fsample * net * net)
if do_output:
name_out = 'madam_tod'
else:
name_out = None
timer.clear()
timer.start()
try:
madam = toast.todmap.OpMadam(name=tod_name,
flag_name=flags_name,
apply_flags=do_output,
params=pars,
detweights=detweights,
purge=True,
name_out=name_out,
translate_timestamps=False)
except Exception as e:
raise Exception('{:4} : ERROR: failed to initialize Madam: {}'.format(
comm.comm_world.rank, e))
madam.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Madam")
if do_output:
timer = Timer()
timer.start()
writer = tp.OpOutputPlanck(signal_name='madam_tod',
flags_name=None,
commonflags_name=None,
effdir_out=args.effdir_out,
effdir_out_diode0=args.effdir_out_diode0,
effdir_out_diode1=args.effdir_out_diode1)
writer.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Madam output")
memreport("at end of main", mpiworld)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def get_hardware(args, comm, verbose=False):
""" Get the hardware configuration, either from file or by simulating.
Then trim it down to the bands that were selected.
"""
log = Logger.get()
timer = Timer()
timer.start()
telescope = get_telescope(args, comm, verbose=verbose)
if comm.world_rank == 0:
if args.hardware:
log.info("Loading hardware configuration from {}...".format(
args.hardware))
if args.hardware.endswith(".pkl"):
with open(args.hardware, "rb") as fin:
hw = pickle.load(fin)
else:
hw = hardware.Hardware(args.hardware)
timer.report_clear("Load {}".format(args.hardware))
else:
log.info("Simulating default hardware configuration")
hw = hardware.get_example()
timer.report_clear("Get example hardware")
hw.data["detectors"] = hardware.sim_telescope_detectors(
hw, telescope.name)
timer.report_clear("Get telescope detectors")
# Construct a running index for all detectors across all
# telescopes for independent noise realizations
det_index = {}
for idet, det in enumerate(sorted(hw.data["detectors"])):
det_index[det] = idet
match = {"band": args.bands.replace(",", "|")}
if args.tubes is None:
tubes = None
else:
tubes = args.tubes.split(",")
# If one provides both telescopes and tubes, the tubes matching *either*
# will be concatenated
# hw = hw.select(telescopes=[telescope.name], tubes=tubes, match=match)
hw = hw.select(tubes=tubes, match=match)
ndetector = len(hw.data["detectors"])
if ndetector == 0:
raise RuntimeError("No detectors match query: telescope={}, "
"tubes={}, match={}".format(
telescope.name, tubes, match))
if args.thinfp:
# Only accept a fraction of the detectors for
# testing and development
thin_index = {}
for idet, det in enumerate(sorted(hw.data["detectors"])):
thin_index[det] = idet
delete_detectors = []
for det_name in hw.data["detectors"].keys():
if (thin_index[det_name] // 2) % args.thinfp != 0:
delete_detectors.append(det_name)
for det_name in delete_detectors:
del hw.data["detectors"][det_name]
if args.radiusfp_deg:
# Only accept detectors inside the given radius
radius = np.radians(args.radiusfp_deg)
# Comparing to the cosine of the radius avoids repeated
# arccos evaluations
cosradius = np.cos(radius)
inside = radius > 0
delete_detectors = []
for det_name, det_data in hw.data["detectors"].items():
det_quat = det_data["quat"]
vec = qa.rotate(det_quat, ZAXIS)
rcos = np.dot(vec, ZAXIS)
if (rcos < cosradius and inside) or (rcos > cosradius
and not inside):
delete_detectors.append(det_name)
for det_name in delete_detectors:
del hw.data["detectors"][det_name]
ndetector = len(hw.data["detectors"])
log.info(
"Telescope = {} tubes = {} bands = {}, thinfp = {} radiusfp = {} "
"matches {} detectors".format(
telescope.name,
args.tubes,
args.bands,
args.thinfp,
args.radiusfp_deg,
ndetector,
))
timer.report_clear("Trim detectors")
else:
hw = None
det_index = None
if comm.comm_world is not None:
hw = comm.comm_world.bcast(hw)
det_index = comm.comm_world.bcast(det_index)
return hw, telescope, det_index
def get_hardware(args, comm, verbose=False):
""" Get the hardware configuration, either from file or by simulating.
Then trim it down to the bands that were selected.
"""
log = Logger.get()
telescope = get_telescope(args, comm, verbose=verbose)
timer = Timer()
if comm.world_rank == 0:
timer.start()
if args.hardware:
log.info("Loading hardware configuration from {}..."
"".format(args.hardware))
hw = Hardware(args.hardware)
timer.report_clear("Load hardware map")
else:
log.info("Simulating default hardware configuration")
hw = get_example()
hw.data["detectors"] = sim_telescope_detectors(hw, telescope.name)
timer.report_clear("Simulate hardware map")
# Construct a running index for all detectors across all
# telescopes for independent noise realizations
det_index = {}
for idet, det in enumerate(sorted(hw.data["detectors"])):
det_index[det] = idet
match = {"band": args.bands.replace(",", "|")}
tube_slots = None
if args.wafer_slots is not None:
match["wafer_slot"] = args.wafer_slots.split(",")
elif args.tube_slots is not None:
tube_slots = args.tube_slots.split(",")
# If one provides both telescopes and tube_slots, the tube_slots matching *either*
# will be concatenated
#hw = hw.select(telescopes=[telescope.name], tube_slots=tube_slots, match=match)
hw = hw.select(tube_slots=tube_slots, match=match)
if args.thinfp:
# Only accept a fraction of the detectors for
# testing and development
delete_detectors = []
for det_name in hw.data["detectors"].keys():
if (det_index[det_name] // 4) % args.thinfp != 0:
delete_detectors.append(det_name)
for det_name in delete_detectors:
del hw.data["detectors"][det_name]
ndetector = len(hw.data["detectors"])
if ndetector == 0:
raise RuntimeError("No detectors match query: telescope={}, "
"tube_slots={}, match={}".format(
telescope.name, tube_slots, match))
log.info(
f"Telescope = {telescope.name} tube_slots = {args.tube_slots}, "
f"wafer_slots = {args.wafer_slots}, bands = {args.bands}, "
f"thinfp = {args.thinfp} matches {ndetector} detectors")
timer.report_clear("Select detectors")
else:
hw = None
det_index = None
if comm.comm_world is not None:
hw = comm.comm_world.bcast(hw)
det_index = comm.comm_world.bcast(det_index)
if comm.world_rank == 0:
timer.report_clear("Broadcast hardware map")
return hw, telescope, det_index
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
if comm.comm_world.rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Planck Ringset making',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--read_eff',
dest='read_eff',
default=False,
action='store_true',
help='Read and co-add the signal from effdir')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory for '
'pointing')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use')
parser.add_argument('--obtfirst',
required=False,
default=None,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
help='Last OBT to use')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
dipogroup = parser.add_mutually_exclusive_group()
dipogroup.add_argument('--dipole',
dest='dipole',
required=False,
default=False,
action='store_true',
help='Simulate dipole')
dipogroup.add_argument('--solsys_dipole',
dest='solsys_dipole',
required=False,
default=False,
action='store_true',
help='Simulate solar system dipole')
dipogroup.add_argument('--orbital_dipole',
dest='orbital_dipole',
required=False,
default=False,
action='store_true',
help='Simulate orbital dipole')
# dipole parameters
dipo_parameters_group = parser.add_argument_group('dipole_parameters')
dipo_parameters_group.add_argument(
'--solsys_speed',
dest='solsys_speed',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_speed"],
help='Solar system speed wrt. CMB rest frame in km/s. '
'Default is Planck 2015 best fit value')
dipo_parameters_group.add_argument(
'--solsys_glon',
dest='solsys_glon',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_glon"],
help='Solar system velocity direction longitude in degrees')
dipo_parameters_group.add_argument(
'--solsys_glat',
dest='solsys_glat',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_glat"],
help='Solar system velocity direction latitude in degrees')
# libconviqt parameters
parser.add_argument('--lmax',
required=False,
default=1024,
type=np.int,
help='Simulation lmax')
parser.add_argument('--fwhm',
required=False,
default=0.0,
type=np.float,
help='Sky fwhm [arcmin] to deconvolve')
parser.add_argument('--beammmax',
required=False,
default=None,
type=np.int,
help='Beam mmax')
parser.add_argument('--order',
required=False,
default=11,
type=np.int,
help='Iteration order')
parser.add_argument('--pxx',
required=False,
default=False,
action='store_true',
help='Beams are in Pxx frame, not Dxx')
parser.add_argument('--skyfile',
required=False,
default=None,
help='Path to sky alm files. Tag DETECTOR will be '
'replaced with detector name.')
parser.add_argument('--beamfile',
required=False,
default=None,
help='Path to beam alm files. Tag DETECTOR will be '
'replaced with detector name.')
parser.add_argument('--nopol',
dest='nopol',
default=False,
action='store_true',
help='Sky and beam should be treated unpolarized')
# noise simulation parameters
parser.add_argument('--add_noise',
dest='add_noise',
default=False,
action='store_true',
help='Simulate noise')
parser.add_argument('--noisefile',
required=False,
default='RIMO',
help='Path to noise PSD files for noise filter. '
'Tag DETECTOR will be replaced with detector name.')
parser.add_argument('--noisefile_simu',
required=False,
default='RIMO',
help='Path to noise PSD files for noise simulation. '
'Tag DETECTOR will be replaced with detector name.')
parser.add_argument('--mc',
required=False,
default=0,
type=np.int,
help='Noise realization')
# ringset parameters
parser.add_argument('--nside_ring',
required=False,
default=128,
type=np.int,
help='Ringset resolution')
parser.add_argument('--ring_root',
required=False,
default='ringset',
help='Root filename for ringsets (setting to empty '
'disables ringset output).')
args = parser.parse_args()
if comm.world_rank == 0:
print('All parameters:')
print(args, flush=True)
timer = Timer()
timer.start()
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# Make output directory
if comm.world_rank == 0:
os.makedirs(args.out, exist_ok=True)
do_dipole = args.dipole or args.solsys_dipole or args.orbital_dipole
do_convolve = args.skyfile is not None and args.beamfile is not None
do_noise = args.add_noise
if not do_noise and args.noisefile_simu != 'RIMO':
raise RuntimeError('Did you mean to simulate noise? add_noise = {} '
'but noisefile_simu = {}'.format(
args.add_noise, args.noisefile_simu))
if comm.world_rank == 0:
print('read_eff = {}'.format(args.read_eff))
print('do_dipole = {}'.format(do_dipole))
print('solsys_speed = {}'.format(args.solsys_speed))
print('solsys_glon = {}'.format(args.solsys_glon))
print('solsys_glat = {}'.format(args.solsys_glat))
print('do_convolve = {}'.format(do_convolve))
print('do_noise = {}'.format(do_noise), flush=True)
# create the TOD for the whole data span (loads ring database and caches
# directory contents)
if do_noise and args.noisefile_simu != 'RIMO':
do_eff_cache = True
else:
do_eff_cache = False
tods = []
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
tods.append(
tp.Exchange(comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
do_eff_cache=do_eff_cache))
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Metadata queries")
if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO':
# We split MPI_COMM_WORLD into single process groups, each of
# which is assigned one or more observations (rings)
comm = Comm(groupsize=1)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = Data(comm)
for tod in tods:
if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO':
# Use a toast helper method to optimally distribute rings between
# processes.
dist = distribute_discrete(tod.ringsizes, procs)
my_first_ring, my_n_ring = dist[comm.world_rank]
for my_ring in range(my_first_ring, my_first_ring + my_n_ring):
ringtod = tp.Exchange.from_tod(
tod,
my_ring,
comm.comm_group,
noisefile=args.noisefile,
noisefile_simu=args.noisefile_simu)
ob = {}
ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring)
ob['id'] = ringtod.globalfirst_ring
ob['tod'] = ringtod
ob['intervals'] = ringtod.valid_intervals
ob['baselines'] = None
ob['noise'] = ringtod.noise
ob['noise_simu'] = ringtod.noise_simu
data.obs.append(ob)
else:
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
ob = {}
ob['name'] = 'mission'
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
data.obs.append(ob)
rimo = tods[0].rimo
fsample = rimo[detectors[0]].fsample
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Create observations")
# make a planck Healpix pointing matrix
mode = 'IQU'
pointing = tp.OpPointingPlanck(nside=args.nside_ring,
mode=mode,
RIMO=rimo,
margin=0,
apply_flags=False,
keep_vel=do_dipole,
keep_pos=False,
keep_phase=False,
keep_quats=(do_dipole or do_convolve))
pointing.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Pointing Matrix")
# Always read the signal because we always need the flags
reader = tp.OpInputPlanck(signal_name='tod')
if comm.world_rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
comm.comm_world.barrier()
if comm.world_rank == 0:
timer.report_clear("Read")
tod_name = 'tod'
# Clear the signal if we don't need it
if not args.read_eff:
eraser = tp.OpCacheMath(in1='tod', in2=0, multiply=True, out='tod')
if comm.comm_world.rank == 0:
print('Erasing TOD', flush=True)
eraser.exec(data)
comm.comm_world.barrier()
if comm.world_rank == 0:
timer.report_clear("Erase")
if do_convolve:
# simulate the TOD by convolving the sky with the beams
detectordata = []
for det in tod.detectors:
skyfile = args.skyfile.replace('DETECTOR', det)
beamfile = args.beamfile.replace('DETECTOR', det)
epsilon = rimo[det].epsilon
# Getting the right polarization angle can be a sensitive matter.
# Dxx beams are always defined without psi_uv or psi_pol rotation
# but some Pxx beams may require psi_pol to be removed and psi_uv
# left in.
if args.pxx:
# Beam is in the polarization basis.
# No extra rotations are needed
psipol = np.radians(rimo[det].psi_pol)
else:
# Beam is in the detector basis. Convolver needs to remove
# the last rotation into the polarization sensitive frame.
psipol = np.radians(rimo[det].psi_uv + rimo[det].psi_pol)
detectordata.append((det, skyfile, beamfile, epsilon, psipol))
# always construct conviqt with dxx=True and modify the psipol
# to produce the desired rotation.
conviqt = tt.OpSimConviqt(args.lmax,
args.beammmax,
detectordata,
pol=(not args.nopol),
fwhm=args.fwhm,
order=args.order,
calibrate=True,
dxx=True,
out='tod',
quat_name='quats',
apply_flags=False)
conviqt.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Convolution")
tod_name = 'tod'
if do_dipole:
# Simulate the dipole
if args.dipole:
dipomode = 'total'
elif args.solsys_dipole:
dipomode = 'solsys'
else:
dipomode = 'orbital'
dipo = tp.OpDipolePlanck(args.freq,
solsys_speed=args.solsys_speed,
solsys_glon=args.solsys_glon,
solsys_glat=args.solsys_glat,
mode=dipomode,
output='tod',
add_to_existing=args.read_eff)
dipo.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Dipole")
tod_name = 'tod'
if do_noise:
nse = tt.OpSimNoise(out='tod',
realization=args.mc,
component=0,
noise='noise_simu',
rate=fsample)
nse.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Noise simulation")
tod_name = 'tod'
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
net = tod.rimo[d].net
fsample = tod.rimo[d].fsample
detweights[d] = 1.0 / (fsample * net * net)
# Make rings
ringmaker = tp.OpRingMaker(args.nside_ring,
args.nside_ring,
signal=tod_name,
fileroot=args.ring_root,
out=args.out,
detmask=args.flagmask,
commonmask=args.obtmask)
ringmaker.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Ringmaking")
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_ground_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = pipeline_tools.get_comm()
args, comm = parse_arguments(comm)
if args.use_madam:
# Initialize madam parameters
madampars = pipeline_tools.setup_madam(args)
# Load and broadcast the schedule file
schedules = pipeline_tools.load_schedule(args, comm)
# Load the weather and append to schedules
pipeline_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = create_observations(args, comm, schedules)
# Split the communicator for day and season mapmaking
time_comms = pipeline_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
pipeline_tools.expand_pointing(args, comm, data)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
pipeline_tools.get_elevation_noise(args, comm, data)
# Purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if (args.tidas is None) and (args.spt3g is None):
for ob in data.obs:
tod = ob["tod"]
tod.free_radec_quats()
# Prepare auxiliary information for distributed map objects
if args.pysm_model:
focalplanes = [s.telescope.focalplane.detector_data for s in schedules]
signalname = pipeline_tools.simulate_sky_signal(
args, comm, data, focalplanes, "signal"
)
else:
signalname = pipeline_tools.scan_sky_signal(args, comm, data, "signal")
# Set up objects to take copies of the TOD at appropriate times
totalname, totalname_freq = setup_sigcopy(args)
# Loop over Monte Carlos
firstmc = args.MC_start
nsimu = args.MC_count
freqs = [float(freq) for freq in args.freq.split(",")]
nfreq = len(freqs)
for mc in range(firstmc, firstmc + nsimu):
pipeline_tools.simulate_atmosphere(args, comm, data, mc, totalname)
# Loop over frequencies with identical focal planes and identical
# atmospheric noise.
for ifreq, freq in enumerate(freqs):
if comm.world_rank == 0:
log.info(
"Processing frequency {}GHz {} / {}, MC = {}".format(
freq, ifreq + 1, nfreq, mc
)
)
# Make a copy of the atmosphere so we can scramble the gains and apply
# frequency-dependent scaling.
pipeline_tools.copy_signal(args, comm, data, totalname, totalname_freq)
pipeline_tools.scale_atmosphere_by_frequency(
args, comm, data, freq=freq, mc=mc, cache_name=totalname_freq
)
pipeline_tools.update_atmospheric_noise_weights(args, comm, data, freq, mc)
# Add previously simulated sky signal to the atmospheric noise.
pipeline_tools.add_signal(
args, comm, data, totalname_freq, signalname, purge=(nsimu == 1)
)
mcoffset = ifreq * 1000000
pipeline_tools.simulate_noise(
args, comm, data, mc + mcoffset, totalname_freq
)
pipeline_tools.simulate_sss(args, comm, data, mc + mcoffset, totalname_freq)
pipeline_tools.scramble_gains(
args, comm, data, mc + mcoffset, totalname_freq
)
if (mc == firstmc) and (ifreq == 0):
# For the first realization and frequency, optionally
# export the timestream data.
pipeline_tools.output_tidas(args, comm, data, totalname)
pipeline_tools.output_spt3g(args, comm, data, totalname)
outpath = setup_output(args, comm, mc + mcoffset, freq)
# Bin and destripe maps
if args.use_madam:
pipeline_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname_freq,
freq=freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
else:
pipeline_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname_freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
pipeline_tools.apply_polyfilter(args, comm, data, totalname_freq)
pipeline_tools.apply_groundfilter(args, comm, data, totalname_freq)
# Bin filtered maps
if args.use_madam:
pipeline_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname_freq,
freq=freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=False,
extra_prefix="filtered",
bin_only=True,
)
else:
pipeline_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname_freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=False,
extra_prefix="filtered",
bin_only=True,
)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
timer0.report_clear("toast_ground_sim.py")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_s4_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = toast_tools.get_comm()
memreport("at the beginning of the pipeline", comm.comm_world)
args, comm = parse_arguments(comm)
# Initialize madam parameters
madampars = toast_tools.setup_madam(args)
# Load and broadcast the schedule file
schedules = toast_tools.load_schedule(args, comm)
# Load the weather and append to schedules
toast_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = s4_tools.load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = s4_tools.create_observations(args, comm, schedules)
memreport("after creating observations", comm.comm_world)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
s4_tools.get_elevation_noise(args, comm, data)
totalname = "total"
# Split the communicator for day and season mapmaking
time_comms = toast_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
toast_tools.expand_pointing(args, comm, data)
# Only purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if args.tidas is None:
for ob in data.obs:
tod = ob["tod"]
try:
tod.free_radec_quats()
except AttributeError:
# These TOD objects do not have RA/Dec quaternions
pass
memreport("after pointing", comm.comm_world)
# Prepare auxiliary information for distributed map objects
memreport("after submaps", comm.comm_world)
# Set up objects to take copies of the TOD at appropriate times
if args.pysm_model:
if schedules is not None:
focalplanes = [
s.telescope.focalplane.detector_data for s in schedules
]
else:
focalplanes = [telescope.focalplane.detector_data]
signalname = s4_tools.simulate_sky_signal(args, comm, data,
focalplanes)
else:
signalname = toast_tools.scan_sky_signal(args, comm, data)
memreport("after PySM", comm.comm_world)
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc + nmc):
if comm.world_rank == 0:
log.info("Processing MC = {}".format(mc))
# Uncomment to run with new TOAST
#toast_tools.draw_weather(args, comm, data, mc)
outpath = setup_output(args, comm, mc)
if outputs_exist(args, comm, outpath):
if comm.world_rank == 0:
log.info("Outputs already exist, skipping.")
continue
toast.tod.OpCacheClear(totalname).exec(data)
toast_tools.simulate_atmosphere(args, comm, data, mc, totalname)
s4_tools.scale_atmosphere_by_bandpass(args, comm, data, totalname, mc)
memreport("after atmosphere", comm.comm_world)
# update_atmospheric_noise_weights(args, comm, data, freq, mc)
toast_tools.add_signal(args,
comm,
data,
totalname,
signalname,
purge=(mc == firstmc + nmc - 1))
memreport("after adding sky", comm.comm_world)
toast_tools.simulate_noise(args, comm, data, mc, totalname)
memreport("after simulating noise", comm.comm_world)
toast_tools.simulate_sss(args, comm, data, mc, totalname)
memreport("after simulating SSS", comm.comm_world)
toast_tools.scramble_gains(args, comm, data, mc, totalname)
if mc == firstmc:
# For the first realization and frequency, optionally
# export the timestream data.
toast_tools.output_tidas(args, comm, data, totalname)
memreport("after export", comm.comm_world)
if args.no_maps:
continue
# Bin and destripe maps
pairdiff(data, args, comm, totalname, mc == firstmc)
if not args.skip_madam:
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
memreport("after madam", comm.comm_world)
if (args.filterbin_ground_order is not None
or args.filterbin_poly_order is not None):
toast_tools.apply_filterbin(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if (args.apply_polyfilter or args.apply_polyfilter2D
or args.apply_common_mode_filter or args.apply_groundfilter):
# Filter signal
toast_tools.apply_common_mode_filter(args, comm, data, totalname)
toast_tools.apply_polyfilter2D(args, comm, data, totalname)
toast_tools.apply_polyfilter(args, comm, data, totalname)
toast_tools.apply_groundfilter(args, comm, data, totalname)
memreport("after filter", comm.comm_world)
# Bin maps
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(args.skip_madam and mc == firstmc),
extra_prefix="filtered",
bin_only=(not args.skip_madam),
)
memreport("after filter & bin", comm.comm_world)
if comm.comm_world is not None:
comm.comm_world.barrier()
memreport("at the end of the pipeline", comm.comm_world)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
timer0.stop()
if comm.world_rank == 0:
timer0.report("toast_s4_sim.py pipeline")
return
def main():
timer0 = Timer()
timer0.start()
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
memreport("At start of pipeline", mpiworld)
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help='Write data distribution info to file')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory '
'for pointing')
parser.add_argument('--coord',
default='G',
help='Coordinate system, "G", "E" or "C"')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Pointing flag mask')
parser.add_argument('--bad_intervals',
required=False,
help='Path to bad interval file.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
type=np.int,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
type=np.int,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use (can be a list)')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use (can be a list)')
parser.add_argument('--obtfirst',
required=False,
default=None,
type=np.float,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
type=np.float,
help='Last OBT to use')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--catalog', required=True, help='Target catalog file')
parser.add_argument('--radius',
required=True,
type=np.float,
help='Search radius about the source [arc min]')
parser.add_argument('--mask',
required=False,
help='Mask defining region of the sky to accept')
parser.add_argument('--bg',
required=False,
help='Background map to subtract')
parser.add_argument('--recalib_bg',
dest='recalib_bg',
default=False,
action='store_true',
help='Recalibrate bg map for each ring.')
parser.add_argument('--full_rings',
dest='full_rings',
default=False,
action='store_true',
help='Extract impacted rings entirely.')
# noise parameters
parser.add_argument('--noisefile',
required=False,
default='RIMO',
help='Path to noise PSD files for noise filter. '
'Tag DETECTOR will be replaced with detector name.')
# Dipole parameters
dipogroup = parser.add_mutually_exclusive_group()
dipogroup.add_argument('--dipole',
dest='dipole',
required=False,
default=False,
action='store_true',
help='Simulate dipole')
dipogroup.add_argument('--solsys_dipole',
dest='solsys_dipole',
required=False,
default=False,
action='store_true',
help='Simulate solar system dipole')
dipogroup.add_argument('--orbital_dipole',
dest='orbital_dipole',
required=False,
default=False,
action='store_true',
help='Simulate orbital dipole')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.world_rank == 0:
print('All parameters:')
print(args, flush=True)
data = create_observations(args, comm)
rimo = data.obs[0]["tod"].rimo
memreport("After create observations", mpiworld)
# Read in the signal
timer = Timer()
timer.start()
reader = tp.OpInputPlanck(signal_name='signal', flags_name='flags')
if comm.world_rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Reading")
tod_name = 'signal'
flags_name = 'flags'
memreport("After read", mpiworld)
# Optionally flag bad intervals
if args.bad_intervals is not None:
flagger = tp.OpBadIntervals(path=args.bad_intervals)
flagger.exec(data)
if comm.world_rank == 0:
timer.report_clear("Applying {}".format(args.bad_intervals))
do_dipole = (args.dipole or args.solsys_dipole or args.orbital_dipole)
# make a planck Healpix pointing matrix
pointing = tp.OpPointingPlanck(nside=1024,
mode='IQU',
RIMO=rimo,
margin=0,
apply_flags=False,
keep_vel=do_dipole,
keep_pos=False,
keep_phase=True,
keep_quats=True)
pointing.exec(data)
memreport("After pointing", mpiworld)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Pointing Matrix")
# Optionally subtract the dipole
if do_dipole:
if args.dipole:
dipomode = 'total'
elif args.solsys_dipole:
dipomode = 'solsys'
else:
dipomode = 'orbital'
dipo = tp.OpDipolePlanck(args.freq,
mode=dipomode,
output='dipole',
keep_quats=True)
dipo.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Dipole")
subtractor = tp.OpCacheMath(in1=tod_name,
in2='dipole',
subtract=True,
out=tod_name)
if comm.comm_world.rank == 0:
print('Subtracting dipole', flush=True)
subtractor.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Dipole subtraction")
memreport("After dipole", mpiworld)
extract = tp.OpExtractPlanck(rimo,
args.catalog,
args.radius,
mpiworld,
common_flag_mask=args.obtmask,
flag_mask=args.flagmask,
maskfile=args.mask,
bg=args.bg,
full_rings=args.full_rings,
recalibrate_bg=args.recalib_bg,
out=args.out)
extract.exec(data)
memreport("After extract", mpiworld)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Extraction")
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.report_clear("Gather and dump timing info")
timer0.report_clear("Full pipeline")
return
def parse_arguments(comm):
timer = Timer()
timer.start()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"atmosphere and make maps for some number of noise Monte Carlos.",
fromfile_prefix_chars="@",
)
toast_tools.add_dist_args(parser)
toast_tools.add_todground_args(parser)
toast_tools.add_pointing_args(parser)
toast_tools.add_polyfilter_args(parser)
toast_tools.add_polyfilter2D_args(parser)
toast_tools.add_common_mode_filter_args(parser)
toast_tools.add_groundfilter_args(parser)
toast_tools.add_atmosphere_args(parser)
toast_tools.add_noise_args(parser)
toast_tools.add_gainscrambler_args(parser)
toast_tools.add_madam_args(parser)
toast_tools.add_filterbin_args(parser)
toast_tools.add_sky_map_args(parser)
toast_tools.add_sss_args(parser)
toast_tools.add_tidas_args(parser)
toast_tools.add_mc_args(parser)
s4_tools.add_hw_args(parser)
s4_tools.add_s4_noise_args(parser)
s4_tools.add_pysm_args(parser)
toast_tools.add_debug_args(parser)
parser.add_argument(
"--no-maps",
required=False,
default=False,
action="store_true",
help="Disable all mapmaking.",
)
parser.add_argument(
"--skip-madam",
required=False,
default=False,
action="store_true",
help="Skip the first Madam call.",
)
parser.add_argument(
"--pairdiff",
required=False,
default=False,
action="store_true",
help="Pair-difference TOD and pointing.",
)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
try:
args = parser.parse_args()
except SystemExit as e:
sys.exit()
if len(args.bands.split(",")) != 1:
# Multi frequency run. We don't support multiple copies of
# scanned signal.
if args.input_map:
raise RuntimeError(
"Multiple frequencies are not supported when scanning from a map"
)
if args.simulate_atmosphere and args.weather is None:
raise RuntimeError(
"Cannot simulate atmosphere without a TOAST weather file")
if comm.world_rank == 0:
log.info("\n")
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
log.info("\n")
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.world_rank == 0:
os.makedirs(args.outdir, exist_ok=True)
timer.report_clear("Parse arguments")
return args, comm
def cache_effdirs(self, effdir_in, effdir_in_diode0, effdir_in_diode1,
effdir_out, effdir_dark, effdir_pntg, effdir_fsl,
extra_effdirs, effdir_flags):
""" Cache the metadata so we don't need to look for files
while reading and writing
"""
if effdir_in is not None and PATTERN_SEPARATOR in effdir_in:
self.effdir_in, self.effdir_in_pattern = effdir_in.split(
PATTERN_SEPARATOR)
else:
self.effdir_in, self.effdir_in_pattern = effdir_in, None
self.effdir_in_diode0 = effdir_in_diode0
self.effdir_in_diode1 = effdir_in_diode1
if effdir_out is not None and PATTERN_SEPARATOR in effdir_out:
self.effdir_out, self.effdir_out_pattern = effdir_out.split(
PATTERN_SEPARATOR)
else:
self.effdir_out, self.effdir_out_pattern = effdir_out, None
self.effdir_out = effdir_out
if effdir_dark is not None:
self.effdir_dark = effdir_dark
else:
self.effdir_dark = self.effdir_in
if effdir_pntg is not None:
self.effdir_pntg = effdir_pntg
else:
self.effdir_pntg = self.effdir_in
self.effdir_fsl = effdir_fsl
self.extra_effdirs = extra_effdirs
if effdir_flags is None:
self.effdir_flags = self.effdir_in
self.effdir_flags_pattern = self.effdir_in_pattern
else:
if PATTERN_SEPARATOR in effdir_flags:
(self.effdir_flags, self.effdir_flags_pattern
) = effdir_flags.split(PATTERN_SEPARATOR)
else:
(self.effdir_flags,
self.effdir_flags_pattern) = effdir_flags, None
if self.rank == 0:
all_effdirs = [
self.effdir_in, self.effdir_out, self.effdir_pntg,
self.effdir_dark, self.effdir_fsl, self.effdir_flags,
self.effdir_in_diode0, self.effdir_in_diode1
]
if self.extra_effdirs is not None:
for effdir in self.extra_effdirs:
all_effdirs.append(effdir)
for effdir in all_effdirs:
if effdir is None:
continue
if effdir in filenames_cache:
continue
print('Building a list of files under {} ...'.format(effdir),
end='',
flush=True)
timer = Timer()
timer.start()
filenames_cache[effdir] = sorted(list_files(effdir))
timer.stop()
timer.report("List files")
if self.comm is None:
self.filenames = filenames_cache
else:
self.filenames = self.comm.bcast(filenames_cache, root=0)
return
def scale_atmosphere_by_bandpass(args,
comm,
data,
totalname,
mc,
verbose=False):
""" Scale atmospheric fluctuations by bandpass.
Assume that cached signal under totalname is pure atmosphere
and scale the absorption coefficient according to the bandpass.
If the focalplane is included in the observation and defines
bandpasses for the detectors, the scaling is computed for each
detector separately.
"""
if not args.simulate_atmosphere:
return
timer = Timer()
log = Logger.get()
if comm.world_rank == 0 and verbose:
log.info("Scaling atmosphere by bandpass")
timer.start()
for obs in data.obs:
tod = obs["tod"]
todcomm = tod.mpicomm
site_id = obs["site_id"]
weather = obs["weather"]
if "focalplane" in obs:
focalplane = obs["focalplane"]
else:
focalplane = None
start_time = obs["start_time"]
weather.set(site_id, mc, start_time)
altitude = obs["altitude"]
air_temperature = weather.air_temperature
surface_pressure = weather.surface_pressure
pwv = weather.pwv
# Use the entire processing group to sample the absorption
# coefficient as a function of frequency
freqmin = 0
freqmax = 1000
nfreq = 10001
freqstep = (freqmax - freqmin) / (nfreq - 1)
if todcomm is None:
nfreq_task = nfreq
my_ifreq_min = 0
my_ifreq_max = nfreq
else:
nfreq_task = int(nfreq // todcomm.size) + 1
my_ifreq_min = nfreq_task * todcomm.rank
my_ifreq_max = min(nfreq, nfreq_task * (todcomm.rank + 1))
my_nfreq = my_ifreq_max - my_ifreq_min
if my_nfreq > 0:
if atm_available_utils:
my_freqs = freqmin + np.arange(my_ifreq_min,
my_ifreq_max) * freqstep
my_absorption = atm_absorption_coefficient_vec(
altitude,
air_temperature,
surface_pressure,
pwv,
my_freqs[0],
my_freqs[-1],
my_nfreq,
)
else:
raise RuntimeError(
"Atmosphere utilities from libaatm are not available")
else:
my_freqs = np.array([])
my_absorption = np.array([])
if todcomm is None:
freqs = my_freqs
absorption = my_absorption
else:
freqs = np.hstack(todcomm.allgather(my_freqs))
absorption = np.hstack(todcomm.allgather(my_absorption))
# loading = atm_atmospheric_loading(altitude, pwv, freq)
for det in tod.local_dets:
# Use detector bandpass from the focalplane
center = focalplane[det]["bandcenter_ghz"]
width = focalplane[det]["bandwidth_ghz"]
nstep = 101
# Interpolate the absorption coefficient to do a top hat
# integral across the bandpass
det_freqs = np.linspace(center - width / 2, center + width / 2,
nstep)
absorption_det = np.mean(np.interp(det_freqs, freqs, absorption))
cachename = "{}_{}".format(totalname, det)
ref = tod.cache.reference(cachename)
ref *= absorption_det
del ref
if comm.comm_world is not None:
comm.comm_world.barrier()
timer.stop()
if comm.world_rank == 0 and verbose:
timer.report("Atmosphere scaling")
return
def export_TOD(args,
comm,
data,
totalname,
schedules,
other=None,
verbose=True):
if args.export is None:
return
log = Logger.get()
timer = Timer()
# Only import spt3g if we are writing out so3g files
from spt3g import core as core3g
from ..export import ToastExport
path = os.path.abspath(args.export)
key = args.export_key
if key is not None:
if key not in ALLOWED_KEYS:
raise RuntimeError(
f"Cannot export data, --export-key='{key}' not in {ALLOWED_KEYS}"
)
prefix = "{}_{}".format(args.bands, key)
det_groups = {}
for obs in data.obs:
for (det_name, det_data) in obs["focalplane"].items():
value = det_data[key]
if value not in det_groups:
det_groups[value] = []
det_groups[value].append(det_name)
else:
prefix = args.bands
det_groups = None
if comm.world_rank == 0 and verbose:
log.info("Exporting data to directory tree at {}".format(path))
timer.start()
export = ToastExport(
path,
prefix=prefix,
use_intervals=True,
cache_name=totalname,
cache_copy=other,
mask_flag_common=TODGround.TURNAROUND,
filesize=2**30,
units=core3g.G3TimestreamUnits.Tcmb,
detgroups=det_groups,
compress=args.compress,
)
export.exec(data)
if comm.comm_world is not None:
comm.comm_world.Barrier()
timer.stop()
if comm.world_rank == 0 and verbose:
timer.report("Wrote simulated data to {}:{}" "".format(path, "total"))
return
def get_analytic_noise(args, comm, focalplane, verbose=True):
""" Create a TOAST noise object.
Create a noise object from the 1/f noise parameters contained in the
focalplane database.
"""
timer = Timer()
timer.start()
detectors = sorted(focalplane.keys())
fmins = {}
fknees = {}
alphas = {}
NETs = {}
rates = {}
indices = {}
for d in detectors:
rates[d] = args.sample_rate
fmins[d] = focalplane[d]["fmin"]
fknees[d] = focalplane[d]["fknee"]
alphas[d] = focalplane[d]["alpha"]
NETs[d] = focalplane[d]["NET"]
indices[d] = focalplane[d]["index"]
if args.common_mode_noise:
# Add an extra "virtual" detector for common mode noise for
# every optics tube
fmin, fknee, alpha, net = np.array(
args.common_mode_noise.split(",")).astype(np.float64)
hw = hardware.get_example()
for itube, tube in enumerate(sorted(hw.data["tubes"].keys())):
d = "common_mode_{}".format(tube)
detectors.append(d)
rates[d] = args.sample_rate
fmins[d] = fmin
fknees[d] = fknee
alphas[d] = alpha
NETs[d] = net
indices[d] = 100000 + itube
noise = AnalyticNoise(
rate=rates,
fmin=fmins,
detectors=detectors,
fknee=fknees,
alpha=alphas,
NET=NETs,
indices=indices,
)
if args.common_mode_noise:
# Update the mixing matrix in the noise operator
mixmatrix = {}
keys = set()
for det in focalplane.keys():
tube = focalplane[det]["tube"]
common = "common_mode_{}".format(tube)
mixmatrix[det] = {det: 1, common: 1}
keys.add(det)
keys.add(common)
# There should probably be an accessor method to update the
# mixmatrix in the TOAST Noise object.
if noise._mixmatrix is not None:
raise RuntimeError("Did not expect non-empty mixing matrix")
noise._mixmatrix = mixmatrix
noise._keys = list(sorted(keys))
timer.stop()
if comm.world_rank == 0 and verbose:
timer.report("Creating noise model")
return noise
def main():
env = Environment.get()
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_satellite_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = pipeline_tools.get_comm()
args, comm, groupsize = parse_arguments(comm, procs)
# Parse options
tmr = Timer()
tmr.start()
if comm.world_rank == 0:
os.makedirs(args.outdir, exist_ok=True)
focalplane, gain, detweights = load_focalplane(args, comm)
if comm.world_rank == 0:
tmr.report_clear("Load focalplane")
data = create_observations(args, comm, focalplane, groupsize)
if comm.world_rank == 0:
tmr.report_clear("Create observations")
pipeline_tools.expand_pointing(args, comm, data)
if comm.world_rank == 0:
tmr.report_clear("Expand pointing")
signalname = None
skyname = pipeline_tools.simulate_sky_signal(args, comm, data,
[focalplane], "signal")
if skyname is not None:
signalname = skyname
if comm.world_rank == 0:
tmr.report_clear("Simulate sky signal")
skyname = pipeline_tools.apply_conviqt(args, comm, data, "signal")
if skyname is not None:
signalname = skyname
if comm.world_rank == 0:
tmr.report_clear("Apply beam convolution")
diponame = pipeline_tools.simulate_dipole(args, comm, data, "signal")
if diponame is not None:
signalname = diponame
if comm.world_rank == 0:
tmr.report_clear("Simulate dipole")
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.world_rank == 0:
handle = open(os.path.join(args.outdir, "distdata.txt"), "w")
data.info(handle)
if comm.world_rank == 0:
handle.close()
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear("Dumping data distribution")
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.world_rank == 0:
handle = open(os.path.join(args.outdir, "distdata.txt"), "w")
data.info(handle)
if comm.world_rank == 0:
handle.close()
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear("Dumping data distribution")
# Mapmaking.
if args.use_madam:
# Initialize madam parameters
madampars = pipeline_tools.setup_madam(args)
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear("Initialize madam map-making")
# Loop over Monte Carlos
firstmc = args.MC_start
nmc = args.MC_count
for mc in range(firstmc, firstmc + nmc):
mctmr = Timer()
mctmr.start()
# create output directory for this realization
outpath = os.path.join(args.outdir, "mc_{:03d}".format(mc))
pipeline_tools.simulate_noise(args,
comm,
data,
mc,
"tot_signal",
overwrite=True)
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear(" Simulate noise {:04d}".format(mc))
# add sky signal
pipeline_tools.add_signal(args, comm, data, "tot_signal", signalname)
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear(" Add sky signal {:04d}".format(mc))
if gain is not None:
op_apply_gain = OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear(" Apply gains {:04d}".format(mc))
if mc == firstmc:
# For the first realization, optionally export the
# timestream data. If we had observation intervals defined,
# we could pass "use_interval=True" to the export operators,
# which would ensure breaks in the exported data at
# acceptable places.
pipeline_tools.output_tidas(args, comm, data, "tot_signal")
pipeline_tools.output_spt3g(args, comm, data, "tot_signal")
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear(" Write TOD snapshot {:04d}".format(mc))
if args.use_madam:
pipeline_tools.apply_madam(args, comm, data, madampars, outpath,
detweights, "tot_signal")
else:
pipeline_tools.apply_mapmaker(args, comm, data, outpath,
"tot_signal")
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear(" Map-making {:04d}".format(mc))
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
mctmr.report_clear(" Monte Carlo loop {:04d}".format(mc))
gt.stop_all()
if comm.comm_world is not None:
comm.comm_world.barrier()
tmr.stop()
tmr.clear()
tmr.start()
alltimers = gather_timers(comm=comm.comm_world)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
tmr.stop()
tmr.report("Gather and dump timing info")
timer0.report_clear("toast_satellite_sim.py")
return
def _unstage_data(
self,
nsamp,
nnz,
nnz_full,
detectors,
signal_type,
noise_type,
pixels_dtype,
nside,
weight_dtype,
):
""" Clear Mappraiser buffers, [restore pointing into TOAST caches-> not done currently].
"""
log = Logger.get()
# self._mappraiser_timestamps = None
# self._cache.destroy("timestamps")
if self._conserve_memory:
nodecomm = self._comm.Split_type(MPI.COMM_TYPE_SHARED, self._rank)
nread = nodecomm.size
else:
nodecomm = MPI.COMM_SELF
nread = 1
self._comm.Barrier()
timer_tot = Timer()
timer_tot.start()
for iread in range(nread):
timer_step = Timer()
timer_step.start()
timer = Timer()
timer.start()
if nodecomm.rank % nread == iread:
self._unstage_signal(detectors, nsamp, signal_type)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Unstage signal {} / {}".format(
iread + 1, nread))
if nodecomm.rank % nread == iread:
self._unstage_noise(detectors, nsamp, noise_type)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Unstage noise {} / {}".format(
iread + 1, nread))
if nodecomm.rank % nread == iread:
self._unstage_pixels(detectors, nsamp, pixels_dtype, nside)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Unstage pixels {} / {}".format(
iread + 1, nread))
if nodecomm.rank % nread == iread:
self._unstage_pixweights(detectors, nsamp, weight_dtype, nnz,
nnz_full)
nodecomm.Barrier()
if self._verbose and self._rank == 0:
timer.report_clear("Unstage pixel weights {} / {}".format(
iread + 1, nread))
if self._rank == 0 and self._verbose and nread > 1:
timer_step.report_clear("Unstage data {} / {}".format(
iread + 1, nread))
self._comm.Barrier()
if self._rank == 0 and self._verbose:
timer_tot.report_clear("Unstage all data")
del nodecomm
return
