def main():
env = Environment.get()
env.enable_function_timers()
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_benchmark (total)")
mpiworld, procs, rank = get_world()
if rank == 0:
log.info("TOAST version = {}".format(env.version()))
log.info("Using a maximum of {} threads per process".format(env.max_threads()))
if mpiworld is None:
log.info("Running serially with one process at {}".format(str(datetime.now())))
else:
if rank == 0:
log.info(
"Running with {} processes at {}".format(procs, str(datetime.now()))
)
cases = {
"tiny": 5000000, # O(1) GB RAM
"xsmall": 50000000, # O(10) GB RAM
"small": 500000000, # O(100) GB RAM
"medium": 5000000000, # O(1) TB RAM
"large": 50000000000, # O(10) TB RAM
"xlarge": 500000000000, # O(100) TB RAM
"heroic": 5000000000000, # O(1000) TB RAM
}
args, comm, n_nodes, n_detector, case, group_seconds, n_group = job_config(
mpiworld, cases
)
# Note: The number of "days" here will just be an approximation of the desired
# data volume since we are doing a realistic schedule for a real observing site.
n_days = int(2.0 * (group_seconds * n_group) / (24 * 3600))
if n_days == 0:
n_days = 1
if rank == 0:
log.info(
"Using {} detectors for approximately {} days".format(n_detector, n_days)
)
# Create the schedule file and input maps on one process
if rank == 0:
create_schedules(args, group_seconds, n_days)
create_input_maps(args)
if mpiworld is not None:
mpiworld.barrier()
if args.dry_run is not None:
if rank == 0:
log.info("Exit from dry run")
# We are done!
sys.exit(0)
gt.start("toast_benchmark (science work)")
# Load and broadcast the schedule file
schedules = pipeline_tools.load_schedule(args, comm)
# Load the weather and append to schedules
pipeline_tools.load_weather(args, comm, schedules)
# Simulate the focalplane
detweights = create_focalplanes(args, comm, schedules, n_detector)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data, total_samples = create_observations(args, comm, schedules)
# handle = None
# if comm.world_rank == 0:
# handle = open(os.path.join(args.outdir, "distdata.txt"), "w")
# data.info(handle)
# if comm.world_rank == 0:
# handle.close()
# if comm.comm_world is not None:
# comm.comm_world.barrier()
# Split the communicator for day and season mapmaking
time_comms = pipeline_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
pipeline_tools.expand_pointing(args, comm, data)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
pipeline_tools.get_elevation_noise(args, comm, data)
# Purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if (args.tidas is None) and (args.spt3g is None):
for ob in data.obs:
tod = ob["tod"]
tod.free_radec_quats()
# Prepare auxiliary information for distributed map objects
signalname = pipeline_tools.scan_sky_signal(args, comm, data, "signal")
# Set up objects to take copies of the TOD at appropriate times
totalname, totalname_freq = setup_sigcopy(args)
# Loop over Monte Carlos
firstmc = args.MC_start
nsimu = args.MC_count
freqs = [float(freq) for freq in args.freq.split(",")]
nfreq = len(freqs)
for mc in range(firstmc, firstmc + nsimu):
pipeline_tools.simulate_atmosphere(args, comm, data, mc, totalname)
# Loop over frequencies with identical focal planes and identical
# atmospheric noise.
for ifreq, freq in enumerate(freqs):
if comm.world_rank == 0:
log.info(
"Processing frequency {}GHz {} / {}, MC = {}".format(
freq, ifreq + 1, nfreq, mc
)
)
# Make a copy of the atmosphere so we can scramble the gains and apply
# frequency-dependent scaling.
pipeline_tools.copy_signal(args, comm, data, totalname, totalname_freq)
pipeline_tools.scale_atmosphere_by_frequency(
args, comm, data, freq=freq, mc=mc, cache_name=totalname_freq
)
pipeline_tools.update_atmospheric_noise_weights(args, comm, data, freq, mc)
# Add previously simulated sky signal to the atmospheric noise.
pipeline_tools.add_signal(
args, comm, data, totalname_freq, signalname, purge=(nsimu == 1)
)
mcoffset = ifreq * 1000000
pipeline_tools.simulate_noise(
args, comm, data, mc + mcoffset, totalname_freq
)
pipeline_tools.scramble_gains(
args, comm, data, mc + mcoffset, totalname_freq
)
outpath = setup_output(args, comm, mc + mcoffset, freq)
# Bin and destripe maps
pipeline_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname_freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
pipeline_tools.apply_polyfilter(args, comm, data, totalname_freq)
pipeline_tools.apply_groundfilter(args, comm, data, totalname_freq)
# Bin filtered maps
pipeline_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname_freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=False,
extra_prefix="filtered",
bin_only=True,
)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
runtime = gt.seconds("toast_benchmark (science work)")
prefactor = 1.0e-3
kilo_samples = 1.0e-3 * total_samples
sample_factor = 1.2
det_factor = 2.0
metric = (
prefactor
* n_detector ** det_factor
* kilo_samples ** sample_factor
/ (n_nodes * runtime)
)
if rank == 0:
msg = "Science Metric: {:0.1e} * ({:d}**{:0.2f}) * ({:0.3e}**{:0.3f}) / ({:0.1f} * {}) = {:0.2f}".format(
prefactor,
n_detector,
det_factor,
kilo_samples,
sample_factor,
runtime,
n_nodes,
metric,
)
log.info("")
log.info(msg)
log.info("")
with open(os.path.join(args.outdir, "log"), "a") as f:
f.write(msg)
f.write("\n\n")
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
with open(os.path.join(args.outdir, "log"), "a") as f:
f.write("Copy of Global Timers:\n")
with open("{}.csv".format(out), "r") as t:
f.write(t.read())
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_ground_sim (total)")
mpiworld, procs, rank, comm = get_comm()
args, comm = parse_arguments(comm)
# Initialize madam parameters
madampars = setup_madam(args)
# Load and broadcast the schedule file
schedule = load_schedule(args, comm)[0]
# load or simulate the focalplane
detweights = load_focalplane(args, comm, schedule)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data = create_observations(args, comm, schedule)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
expand_pointing(args, comm, data)
# Scan input map
signalname = scan_sky_signal(args, comm, data, "signal")
# Simulate noise
if signalname is None:
signalname = "signal"
mc = 0
simulate_noise(args, comm, data, mc, signalname)
# Set up objects to take copies of the TOD at appropriate times
signalname_madam, sigcopy_madam, sigclear = setup_sigcopy(
args, comm, signalname)
npp, zmap = init_binner(args, comm, data, detweights)
output_tidas(args, comm, data, signalname)
outpath = setup_output(args, comm)
# Make a copy of the signal for Madam
copy_signal_madam(args, comm, data, sigcopy_madam)
# Bin unprocessed signal for reference
apply_binner(args, comm, data, npp, zmap, detweights, outpath, signalname)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
apply_polyfilter(args, comm, data, signalname)
apply_groundfilter(args, comm, data, signalname)
# Bin the filtered signal
apply_binner(
args,
comm,
data,
npp,
zmap,
detweights,
outpath,
signalname,
prefix="filtered",
)
data.obs[0]["tod"].cache.report()
clear_signal(args, comm, data, sigclear)
data.obs[0]["tod"].cache.report()
# Now run Madam on the unprocessed copy of the signal
if args.use_madam:
apply_madam(args, comm, data, madampars, outpath, detweights,
signalname_madam)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.report_clear("Gather and dump timing info")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_so_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = toast_tools.get_comm()
memreport("at the beginning of the pipeline", comm.comm_world)
args, comm = parse_arguments(comm)
if args.use_madam:
# Initialize madam parameters
madampars = toast_tools.setup_madam(args)
else:
madampars = None
if args.import_dir is not None:
schedules = None
data, telescope_data, detweights = so_tools.load_observations(
args, comm)
memreport("after load", comm.comm_world)
totalname = "signal"
else:
# Load and broadcast the schedule file
schedules = toast_tools.load_schedule(args, comm)
# Load the weather and append to schedules
toast_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = so_tools.load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = so_tools.create_observations(
args, comm, schedules)
memreport("after creating observations", comm.comm_world)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
so_tools.get_elevation_noise(args, comm, data)
totalname = "total"
# Split the communicator for day and season mapmaking
time_comms = toast_tools.get_time_communicators(args, comm, data)
# Rotate the LAT focalplane around the boresight based on co-rotator position
so_tools.rotate_focalplane(args, data, comm)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
toast_tools.expand_pointing(args, comm, data)
# Flag Solar system objects
so_tools.apply_flag_sso(args, comm, data)
# Optionally, output h_n maps
so_tools.compute_h_n(args, comm, data)
# Optionally, output crosslinking map
so_tools.compute_crosslinking(args, comm, data, detweights)
# Optionally, output cadence map
so_tools.compute_cadence_map(args, comm, data)
# Only purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if not (args.tidas is None) and (args.export is None):
for ob in data.obs:
tod = ob["tod"]
try:
tod.free_radec_quats()
except AttributeError:
# These TOD objects do not have RA/Dec quaternions
pass
memreport("after pointing", comm.comm_world)
# Set up objects to take copies of the TOD at appropriate times
if args.pysm_model:
if schedules is not None:
focalplanes = [
s.telescope.focalplane.detector_data for s in schedules
]
else:
focalplanes = [telescope.focalplane.detector_data]
signalname = so_tools.simulate_sky_signal(args, comm, data,
focalplanes)
else:
signalname = toast_tools.scan_sky_signal(args, comm, data)
memreport("after PySM", comm.comm_world)
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc + nmc):
if comm.world_rank == 0:
log.info("Processing MC = {}".format(mc))
toast_tools.draw_weather(args, comm, data, mc)
toast_tools.simulate_atmosphere(args, comm, data, mc, totalname)
#so_tools.scale_atmosphere_by_bandpass(args, comm, data, totalname, mc)
toast_tools.scale_atmosphere_by_frequency(
args,
comm,
data,
cache_name=totalname,
mc=mc,
)
memreport("after atmosphere", comm.comm_world)
so_tools.simulate_hwpss(args, comm, data, mc, totalname)
# update_atmospheric_noise_weights(args, comm, data, freq, mc)
toast_tools.add_signal(args,
comm,
data,
totalname,
signalname,
purge=(mc == firstmc + nmc - 1))
memreport("after adding sky", comm.comm_world)
toast_tools.simulate_sss(args, comm, data, mc, totalname)
memreport("after simulating SSS", comm.comm_world)
toast_tools.simulate_noise(args, comm, data, mc, totalname)
memreport("after simulating noise", comm.comm_world)
so_tools.apply_sim_sso(args, comm, data, mc, totalname)
memreport("after simulating SSO", comm.comm_world)
so_tools.convolve_time_constant(args, comm, data, totalname)
memreport("after convolving with time constant", comm.comm_world)
# DEBUG begin
"""
import matplotlib.pyplot as plt
tod = data.obs[0]['tod']
times = tod.local_times()
for det in tod.local_dets:
sig = tod.local_signal(det, totalname)
plt.plot(times, sig, label=det)
plt.legend(loc='best')
fnplot = 'debug_{}.png'.format(args.madam_prefix)
plt.savefig(fnplot)
plt.close()
print('DEBUG plot saved in', fnplot)
return
"""
# DEBUG end
toast_tools.scramble_gains(args, comm, data, mc, totalname)
so_tools.deconvolve_time_constant(args,
comm,
data,
totalname,
realization=mc)
memreport("after deconvolving time constant", comm.comm_world)
if mc == firstmc:
# For the first realization and frequency, optionally
# export the timestream data.
toast_tools.output_tidas(args, comm, data, totalname)
so_tools.export_TOD(args, comm, data, totalname, schedules)
memreport("after export", comm.comm_world)
if args.no_maps:
continue
outpath = setup_output(args, comm, mc)
# Optionally demodulate signal
so_tools.demodulate(args, comm, data, totalname, detweights, madampars)
# Bin and destripe maps
if args.use_madam:
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
else:
toast_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
memreport("after destriper", comm.comm_world)
if (args.filterbin_ground_order is not None
or args.filterbin_poly_order is not None):
toast_tools.apply_filterbin(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
toast_tools.apply_polyfilter(args, comm, data, totalname)
memreport("after polyfilter", comm.comm_world)
# Ground filter
memreport("after demodulation", comm.comm_world)
toast_tools.apply_groundfilter(args, comm, data, totalname)
memreport("after groundfilter", comm.comm_world)
# Bin maps
if args.use_madam:
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=args.demodulate,
extra_prefix="filtered",
bin_only=True,
)
else:
toast_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=False,
extra_prefix="filtered",
bin_only=True,
)
memreport("after filter & bin", comm.comm_world)
if args.demodulate and args.MC_count > 1:
if comm.world_rank == 0:
log.info("WARNING: demodulation and MC iterations are "
"incompatible. Terminating after first MC.")
break
if comm.comm_world is not None:
comm.comm_world.barrier()
memreport("at the end of the pipeline", comm.comm_world)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
timer0.stop()
if comm.world_rank == 0:
timer0.report("toast_so_sim.py pipeline")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_ground_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = get_comm()
args, comm = parse_arguments(comm)
# Initialize madam parameters
madampars = setup_madam(args)
# Load and broadcast the schedule file
schedules = load_schedule(args, comm)
# Load the weather and append to schedules
load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = create_observations(args, comm, schedules)
# Split the communicator for day and season mapmaking
time_comms = get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
expand_pointing(args, comm, data)
# Purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if (args.tidas is None) and (args.spt3g is None):
for ob in data.obs:
tod = ob["tod"]
tod.free_radec_quats()
# Prepare auxiliary information for distributed map objects
_, localsm, subnpix = get_submaps(args, comm, data)
if args.pysm_model:
focalplanes = [s.telescope.focalplane.detector_data for s in schedules]
signalname = simulate_sky_signal(args, comm, data, focalplanes,
subnpix, localsm, "signal")
else:
signalname = scan_sky_signal(args, comm, data, localsm, subnpix,
"signal")
# Set up objects to take copies of the TOD at appropriate times
totalname, totalname_freq = setup_sigcopy(args)
# Loop over Monte Carlos
firstmc = args.MC_start
nsimu = args.MC_count
freqs = [float(freq) for freq in args.freq.split(",")]
nfreq = len(freqs)
for mc in range(firstmc, firstmc + nsimu):
simulate_atmosphere(args, comm, data, mc, totalname)
# Loop over frequencies with identical focal planes and identical
# atmospheric noise.
for ifreq, freq in enumerate(freqs):
if comm.world_rank == 0:
log.info("Processing frequency {}GHz {} / {}, MC = {}".format(
freq, ifreq + 1, nfreq, mc))
# Make a copy of the atmosphere so we can scramble the gains and apply
# frequency-dependent scaling.
copy_signal(args, comm, data, totalname, totalname_freq)
scale_atmosphere_by_frequency(args,
comm,
data,
freq=freq,
mc=mc,
cache_name=totalname_freq)
update_atmospheric_noise_weights(args, comm, data, freq, mc)
# Add previously simulated sky signal to the atmospheric noise.
add_signal(args,
comm,
data,
totalname_freq,
signalname,
purge=(nsimu == 1))
mcoffset = ifreq * 1000000
simulate_noise(args, comm, data, mc + mcoffset, totalname_freq)
simulate_sss(args, comm, data, mc + mcoffset, totalname_freq)
scramble_gains(args, comm, data, mc + mcoffset, totalname_freq)
if (mc == firstmc) and (ifreq == 0):
# For the first realization and frequency, optionally
# export the timestream data.
output_tidas(args, comm, data, totalname)
output_spt3g(args, comm, data, totalname)
outpath = setup_output(args, comm, mc + mcoffset, freq)
# Bin and destripe maps
apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname_freq,
freq=freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
apply_polyfilter(args, comm, data, totalname_freq)
apply_groundfilter(args, comm, data, totalname_freq)
# Bin filtered maps
apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname_freq,
freq=freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=False,
extra_prefix="filtered",
bin_only=True,
)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
timer0.report_clear("toast_ground_sim.py")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_s4_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = toast_tools.get_comm()
memreport("at the beginning of the pipeline", comm.comm_world)
args, comm = parse_arguments(comm)
# Initialize madam parameters
madampars = toast_tools.setup_madam(args)
# Load and broadcast the schedule file
schedules = toast_tools.load_schedule(args, comm)
# Load the weather and append to schedules
toast_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = s4_tools.load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = s4_tools.create_observations(args, comm, schedules)
memreport("after creating observations", comm.comm_world)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
s4_tools.get_elevation_noise(args, comm, data)
totalname = "total"
# Split the communicator for day and season mapmaking
time_comms = toast_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
toast_tools.expand_pointing(args, comm, data)
# Only purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if args.tidas is None:
for ob in data.obs:
tod = ob["tod"]
try:
tod.free_radec_quats()
except AttributeError:
# These TOD objects do not have RA/Dec quaternions
pass
memreport("after pointing", comm.comm_world)
# Prepare auxiliary information for distributed map objects
memreport("after submaps", comm.comm_world)
# Set up objects to take copies of the TOD at appropriate times
if args.pysm_model:
if schedules is not None:
focalplanes = [
s.telescope.focalplane.detector_data for s in schedules
]
else:
focalplanes = [telescope.focalplane.detector_data]
signalname = s4_tools.simulate_sky_signal(args, comm, data,
focalplanes)
else:
signalname = toast_tools.scan_sky_signal(args, comm, data)
memreport("after PySM", comm.comm_world)
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc + nmc):
if comm.world_rank == 0:
log.info("Processing MC = {}".format(mc))
# Uncomment to run with new TOAST
#toast_tools.draw_weather(args, comm, data, mc)
outpath = setup_output(args, comm, mc)
if outputs_exist(args, comm, outpath):
if comm.world_rank == 0:
log.info("Outputs already exist, skipping.")
continue
toast.tod.OpCacheClear(totalname).exec(data)
toast_tools.simulate_atmosphere(args, comm, data, mc, totalname)
s4_tools.scale_atmosphere_by_bandpass(args, comm, data, totalname, mc)
memreport("after atmosphere", comm.comm_world)
# update_atmospheric_noise_weights(args, comm, data, freq, mc)
toast_tools.add_signal(args,
comm,
data,
totalname,
signalname,
purge=(mc == firstmc + nmc - 1))
memreport("after adding sky", comm.comm_world)
toast_tools.simulate_noise(args, comm, data, mc, totalname)
memreport("after simulating noise", comm.comm_world)
toast_tools.simulate_sss(args, comm, data, mc, totalname)
memreport("after simulating SSS", comm.comm_world)
toast_tools.scramble_gains(args, comm, data, mc, totalname)
if mc == firstmc:
# For the first realization and frequency, optionally
# export the timestream data.
toast_tools.output_tidas(args, comm, data, totalname)
memreport("after export", comm.comm_world)
if args.no_maps:
continue
# Bin and destripe maps
pairdiff(data, args, comm, totalname, mc == firstmc)
if not args.skip_madam:
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
memreport("after madam", comm.comm_world)
if (args.filterbin_ground_order is not None
or args.filterbin_poly_order is not None):
toast_tools.apply_filterbin(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if (args.apply_polyfilter or args.apply_polyfilter2D
or args.apply_common_mode_filter or args.apply_groundfilter):
# Filter signal
toast_tools.apply_common_mode_filter(args, comm, data, totalname)
toast_tools.apply_polyfilter2D(args, comm, data, totalname)
toast_tools.apply_polyfilter(args, comm, data, totalname)
toast_tools.apply_groundfilter(args, comm, data, totalname)
memreport("after filter", comm.comm_world)
# Bin maps
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(args.skip_madam and mc == firstmc),
extra_prefix="filtered",
bin_only=(not args.skip_madam),
)
memreport("after filter & bin", comm.comm_world)
if comm.comm_world is not None:
comm.comm_world.barrier()
memreport("at the end of the pipeline", comm.comm_world)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
timer0.stop()
if comm.world_rank == 0:
timer0.report("toast_s4_sim.py pipeline")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_so_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = toast_tools.get_comm()
memreport("at the beginning of the pipeline", comm.comm_world)
args, comm = parse_arguments(comm)
# Load and broadcast the schedule file
schedules = toast_tools.load_schedule(args, comm)
# Load the weather and append to schedules
toast_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = so_tools.load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = so_tools.create_observations(args, comm, schedules)
memreport("after creating observations", comm.comm_world)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
so_tools.get_elevation_noise(args, comm, data)
totalname = "total"
# Split the communicator for day and season mapmaking
time_comms = toast_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
toast_tools.expand_pointing(args, comm, data)
memreport("after pointing", comm.comm_world)
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
madam = None
for mc in range(firstmc, firstmc + nmc):
timer_mc = Timer()
timer_mc.start()
outpath = setup_output(args, comm, mc)
outprefix = args.map_prefix + "_filtered"
if args.madam:
outmap = os.path.join(outpath, outprefix + "_bmap.fits")
else:
outmap = os.path.join(outpath, outprefix + "_binned.fits")
if os.path.isfile(outmap):
if comm.world_rank == 0:
log.info("{} exists, skipping".format(outmap))
continue
if comm.world_rank == 0:
log.info("Processing MC = {} into {}".format(mc, outmap))
# Ensure there is no stale signal in the cache
toast.tod.OpCacheClear(totalname).exec(data)
if args.pysm_model:
if schedules is not None:
focalplanes = [
s.telescope.focalplane.detector_data for s in schedules
]
else:
focalplanes = [telescope.focalplane.detector_data]
so_tools.simulate_sky_signal(args,
comm,
data,
focalplanes,
totalname,
mc=mc)
else:
toast_tools.scan_sky_signal(args, comm, data, totalname, mc=mc)
memreport("after PySM", comm.comm_world)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
toast_tools.apply_polyfilter(args, comm, data, totalname)
toast_tools.apply_groundfilter(args, comm, data, totalname)
memreport("after filter", comm.comm_world)
# Bin maps
timer_map = Timer()
timer_map.start()
if args.madam:
if madam is None:
madampars = {}
madampars["temperature_only"] = False
for name in [
"kfirst", "write_map", "write_matrix",
"write_wcov", "write_hits"
]:
madampars[name] = False
madampars["write_binmap"] = True
madampars["concatenate_messages"] = True
madampars["allreduce"] = True
madampars["nside_submap"] = args.nside_submap
madampars["reassign_submaps"] = True
madampars["pixlim_map"] = 1e-2
madampars["pixmode_map"] = 2
# Instead of fixed detector weights, we'll want to use scaled noise
# PSD:s that include the atmospheric noise
madampars["radiometers"] = True
madampars["noise_weights_from_psd"] = True
madampars["nside_map"] = args.nside
madampars["fsample"] = args.sample_rate
madampars["path_output"] = outpath
madampars["file_root"] = outprefix
if args.madam_concatenate_messages:
# Collective communication is fast but requires memory
madampars["concatenate_messages"] = True
if args.madam_allreduce:
# Every process will allocate a copy of every observed submap.
madampars["allreduce"] = True
else:
# Every process will allocate complete send and receive buffers
madampars["allreduce"] = False
else:
# Slow but memory-efficient point-to-point communication. Allocate
# only enough memory to communicate with one process at a time.
madampars["concatenate_messages"] = False
madampars["allreduce"] = False
madam = toast.todmap.OpMadam(
params=madampars,
detweights=detweights,
name=totalname,
common_flag_mask=args.common_flag_mask,
purge_tod=True,
mcmode=False,
conserve_memory=args.madam_conserve_memory,
)
else:
madam.params["path_output"] = outpath
madam.exec(data)
del madam
madam = None
else:
mapmaker = toast.todmap.OpMapMaker(
nside=args.nside,
nnz=3,
name=totalname,
outdir=outpath,
outprefix=outprefix + "_",
write_hits=False,
zip_maps=False,
write_wcov_inv=False,
write_wcov=False,
write_binned=True,
write_destriped=False,
write_rcond=False,
rcond_limit=1e-3,
baseline_length=None,
common_flag_mask=args.common_flag_mask,
)
mapmaker.exec(data)
if comm.world_rank == 0:
timer_map.report_clear("Bin map")
memreport("after filter & bin", comm.comm_world)
if comm.world_rank == 0:
timer_mc.report_clear(
"Monte Carlo iteration # {:05}".format(mc))
if comm.comm_world is not None:
comm.comm_world.barrier()
memreport("at the end of the pipeline", comm.comm_world)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.report_clear("Gather and dump timing info")
if comm.world_rank == 0:
timer0.report_clear("toast_so_tf.py pipeline")
return