def _MLmap(self, data_size_proc, nb_blocks_loc, local_blocks_sizes, nnz):
""" Compute the ML map
"""
if self._verbose:
memreport("just before calling libmappraiser.MLmap", self._comm)
# Compute the Maximum Likelihood map
# os.environ["OMP_NUM_THREADS"] = "1"
mappraiser.MLmap(
self._comm,
self._params,
data_size_proc,
nb_blocks_loc,
local_blocks_sizes,
nnz,
self._mappraiser_pixels,
self._mappraiser_pixweights,
self._mappraiser_signal,
self._mappraiser_noise,
self._params["Lambda"],
self._mappraiser_invtt,
)
# os.environ["OMP_NUM_THREADS"] = "4"
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
print(
"Running with {} processes at {}".format(
procs, str(datetime.datetime.now())
)
)
parser = argparse.ArgumentParser(
description="Simple on-the-fly signal convolution + MADAM Mapmaking",
fromfile_prefix_chars="@",
)
parser.add_argument("--lmax", required=True, type=np.int, help="Simulation lmax")
parser.add_argument(
"--fwhm", required=True, type=np.float, help="Sky fwhm [arcmin] to deconvolve"
)
parser.add_argument("--beammmax", required=True, type=np.int, help="Beam mmax")
parser.add_argument("--order", default=11, type=np.int, help="Iteration order")
parser.add_argument(
"--pxx",
required=False,
default=False,
action="store_true",
help="Beams are in Pxx frame, not Dxx",
)
parser.add_argument(
"--normalize",
required=False,
default=False,
action="store_true",
help="Normalize the beams",
)
parser.add_argument(
"--skyfile",
required=True,
help="Path to sky alm files. Tag DETECTOR will be "
"replaced with detector name.",
)
parser.add_argument(
"--remove_monopole",
required=False,
default=False,
action="store_true",
help="Remove the sky monopole before convolution",
)
parser.add_argument(
"--remove_dipole",
required=False,
default=False,
action="store_true",
help="Remove the sky dipole before convolution",
)
parser.add_argument(
"--beamfile",
required=True,
help="Path to beam alm files. Tag DETECTOR will be "
"replaced with detector name.",
)
parser.add_argument("--rimo", required=True, help="RIMO file")
parser.add_argument("--freq", required=True, type=np.int, help="Frequency")
parser.add_argument(
"--dets", required=False, default=None, help="Detector list (comma separated)"
)
parser.add_argument(
"--effdir", required=True, help="Input Exchange Format File directory"
)
parser.add_argument(
"--effdir_pntg",
required=False,
help="Input Exchange Format File directory " "for pointing",
)
parser.add_argument(
"--effdir_out", required=False, help="Output directory for convolved TOD"
)
parser.add_argument(
"--obtmask", required=False, default=1, type=np.int, help="OBT flag mask"
)
parser.add_argument(
"--flagmask", required=False, default=1, type=np.int, help="Quality flag mask"
)
parser.add_argument("--ringdb", required=True, help="Ring DB file")
parser.add_argument(
"--odfirst", required=False, default=None, type=np.int, help="First OD to use"
)
parser.add_argument(
"--odlast", required=False, default=None, type=np.int, help="Last OD to use"
)
parser.add_argument(
"--ringfirst",
required=False,
default=None,
type=np.int,
help="First ring to use",
)
parser.add_argument(
"--ringlast", required=False, default=None, type=np.int, help="Last ring to use"
)
parser.add_argument(
"--obtfirst",
required=False,
default=None,
type=np.float,
help="First OBT to use",
)
parser.add_argument(
"--obtlast", required=False, default=None, type=np.float, help="Last OBT to use"
)
parser.add_argument("--madam_prefix", required=False, help="map prefix")
parser.add_argument(
"--madampar", required=False, default=None, help="Madam parameter file"
)
parser.add_argument(
"--obtmask_madam", required=False, type=np.int, help="OBT flag mask for Madam"
)
parser.add_argument(
"--flagmask_madam",
required=False,
type=np.int,
help="Quality flag mask for Madam",
)
parser.add_argument(
"--skip_madam",
required=False,
default=False,
action="store_true",
help="Do not run Madam on the convolved timelines",
)
parser.add_argument("--out", required=False, default=".", help="Output directory")
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
timer = Timer()
timer.start()
odrange = None
if args.odfirst is not None and args.odlast is not None:
odrange = (args.odfirst, args.odlast)
ringrange = None
if args.ringfirst is not None and args.ringlast is not None:
ringrange = (args.ringfirst, args.ringlast)
obtrange = None
if args.obtfirst is not None and args.obtlast is not None:
obtrange = (args.obtfirst, args.obtlast)
detectors = None
if args.dets is not None:
detectors = re.split(",", args.dets)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
# Ensure output directory exists
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.makedirs(args.out)
# Read in madam parameter file
# Allow more than one entry, gather into a list
repeated_keys = ["detset", "detset_nopol", "survey"]
pars = {}
if comm.comm_world.rank == 0:
pars["kfirst"] = False
pars["temperature_only"] = True
pars["base_first"] = 60.0
pars["nside_map"] = 512
pars["nside_cross"] = 512
pars["nside_submap"] = 16
pars["write_map"] = False
pars["write_binmap"] = True
pars["write_matrix"] = False
pars["write_wcov"] = False
pars["write_hits"] = True
pars["kfilter"] = False
pars["info"] = 3
if args.madampar:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if not comment.match(line):
result = pat.match(line)
if result:
key, value = result.group(1), result.group(2)
if key in repeated_keys:
if key not in pars:
pars[key] = []
pars[key].append(value)
else:
pars[key] = value
# Command line parameters override the ones in the madam parameter file
if "file_root" not in pars:
pars["file_root"] = "madam"
if args.madam_prefix is not None:
pars["file_root"] = args.madam_prefix
sfreq = "{:03}".format(args.freq)
if sfreq not in pars["file_root"]:
pars["file_root"] += "_" + sfreq
try:
fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq]
except Exception:
fsample = 180.3737
pars["fsample"] = fsample
pars["path_output"] = args.out
print("All parameters:")
print(args, flush=True)
pars = comm.comm_world.bcast(pars, root=0)
memreport("after parameters", MPI.COMM_WORLD)
# madam only supports a single observation. Normally
# we would have multiple observations with some subset
# assigned to each process group.
# create the TOD for this observation
tod = tp.Exchange(
comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
do_eff_cache=False,
)
# normally we would get the intervals from somewhere else, but since
# the Exchange TOD already had to get that information, we can
# get it from there.
ob = {}
ob["name"] = "mission"
ob["id"] = 0
ob["tod"] = tod
ob["intervals"] = tod.valid_intervals
ob["baselines"] = None
ob["noise"] = tod.noise
# Add the bare minimum focal plane information for the conviqt operator
focalplane = {}
for det in tod.detectors:
if args.pxx:
# Beam is in the polarization basis.
# No extra rotations are needed
psipol = tod.rimo[det].psi_pol
else:
# Beam is in the detector basis. Convolver needs to remove
# the last rotation into the polarization sensitive frame.
psipol = tod.rimo[det].psi_uv + tod.rimo[det].psi_pol
focalplane[det] = {
"pol_leakage" : tod.rimo[det].epsilon,
"pol_angle_deg" : psipol,
}
ob["focalplane"] = focalplane
data.obs.append(ob)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Metadata queries")
loader = tp.OpInputPlanck(
commonflags_name="common_flags", flags_name="flags", margin=0
)
loader.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Data read and cache")
tod.cache.report()
memreport("after loading", mpiworld)
# make a planck Healpix pointing matrix
mode = "IQU"
if pars["temperature_only"] == "T":
mode = "I"
nside = int(pars["nside_map"])
pointing = tp.OpPointingPlanck(
nside=nside,
mode=mode,
RIMO=tod.RIMO,
margin=0,
apply_flags=False,
keep_vel=False,
keep_pos=False,
keep_phase=False,
keep_quats=True,
)
pointing.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Pointing Matrix took, mode = {}".format(mode))
memreport("after pointing", mpiworld)
# simulate the TOD by convolving the sky with the beams
if comm.comm_world.rank == 0:
print("Convolving TOD", flush=True)
for pattern in args.beamfile.split(","):
skyfiles = {}
beamfiles = {}
for det in tod.detectors:
freq = "{:03}".format(tp.utilities.det2freq(det))
if "LFI" in det:
psmdet = "{}_{}".format(freq, det[3:])
if det.endswith("M"):
arm = "y"
else:
arm = "x"
graspdet = "{}_{}_{}".format(freq[1:], det[3:5], arm)
else:
psmdet = det.replace("-", "_")
graspdet = det
skyfile = (
args.skyfile.replace("FREQ", freq)
.replace("PSMDETECTOR", psmdet)
.replace("DETECTOR", det)
)
skyfiles[det] = skyfile
beamfile = pattern.replace("GRASPDETECTOR", graspdet).replace(
"DETECTOR", det
)
beamfiles[det] = beamfile
if comm.comm_world.rank == 0:
print("Convolving {} with {}".format(skyfile, beamfile), flush=True)
conviqt = OpSimConviqt(
comm.comm_world,
skyfiles,
beamfiles,
lmax=args.lmax,
beammmax=args.beammmax,
pol=True,
fwhm=args.fwhm,
order=args.order,
calibrate=True,
dxx=True,
out="conviqt_tod",
apply_flags=False,
remove_monopole=args.remove_monopole,
remove_dipole=args.remove_dipole,
verbosity=1,
normalize_beam=args.normalize,
)
conviqt.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Convolution")
memreport("after conviqt", mpiworld)
if args.effdir_out is not None:
if comm.comm_world.rank == 0:
print("Writing TOD", flush=True)
tod.set_effdir_out(args.effdir_out, None)
writer = tp.OpOutputPlanck(
signal_name="conviqt_tod",
flags_name="flags",
commonflags_name="common_flags",
)
writer.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Conviqt output")
memreport("after writing", mpiworld)
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
net = tod.rimo[d].net
fsample = tod.rimo[d].fsample
detweights[d] = 1.0 / (fsample * net * net)
if not args.skip_madam:
if comm.comm_world.rank == 0:
print("Calling Madam", flush=True)
try:
if args.obtmask_madam is None:
obtmask = args.obtmask
else:
obtmask = args.obtmask_madam
if args.flagmask_madam is None:
flagmask = args.flagmask
else:
flagmask = args.flagmask_madam
madam = OpMadam(
params=pars,
detweights=detweights,
name="conviqt_tod",
flag_name="flags",
purge=True,
name_out="madam_tod",
common_flag_mask=obtmask,
flag_mask=flagmask,
)
except Exception as e:
raise Exception(
"{:4} : ERROR: failed to initialize Madam: {}".format(
comm.comm_world.rank, e
)
)
madam.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Madam took {:.3f} s")
memreport("after madam", mpiworld)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_so_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = toast_tools.get_comm()
memreport("at the beginning of the pipeline", comm.comm_world)
args, comm = parse_arguments(comm)
if args.use_madam:
# Initialize madam parameters
madampars = toast_tools.setup_madam(args)
else:
madampars = None
if args.import_dir is not None:
schedules = None
data, telescope_data, detweights = so_tools.load_observations(
args, comm)
memreport("after load", comm.comm_world)
totalname = "signal"
else:
# Load and broadcast the schedule file
schedules = toast_tools.load_schedule(args, comm)
# Load the weather and append to schedules
toast_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = so_tools.load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = so_tools.create_observations(
args, comm, schedules)
memreport("after creating observations", comm.comm_world)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
so_tools.get_elevation_noise(args, comm, data)
totalname = "total"
# Split the communicator for day and season mapmaking
time_comms = toast_tools.get_time_communicators(args, comm, data)
# Rotate the LAT focalplane around the boresight based on co-rotator position
so_tools.rotate_focalplane(args, data, comm)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
toast_tools.expand_pointing(args, comm, data)
# Flag Solar system objects
so_tools.apply_flag_sso(args, comm, data)
# Optionally, output h_n maps
so_tools.compute_h_n(args, comm, data)
# Optionally, output crosslinking map
so_tools.compute_crosslinking(args, comm, data, detweights)
# Optionally, output cadence map
so_tools.compute_cadence_map(args, comm, data)
# Only purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if not (args.tidas is None) and (args.export is None):
for ob in data.obs:
tod = ob["tod"]
try:
tod.free_radec_quats()
except AttributeError:
# These TOD objects do not have RA/Dec quaternions
pass
memreport("after pointing", comm.comm_world)
# Set up objects to take copies of the TOD at appropriate times
if args.pysm_model:
if schedules is not None:
focalplanes = [
s.telescope.focalplane.detector_data for s in schedules
]
else:
focalplanes = [telescope.focalplane.detector_data]
signalname = so_tools.simulate_sky_signal(args, comm, data,
focalplanes)
else:
signalname = toast_tools.scan_sky_signal(args, comm, data)
memreport("after PySM", comm.comm_world)
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc + nmc):
if comm.world_rank == 0:
log.info("Processing MC = {}".format(mc))
toast_tools.draw_weather(args, comm, data, mc)
toast_tools.simulate_atmosphere(args, comm, data, mc, totalname)
#so_tools.scale_atmosphere_by_bandpass(args, comm, data, totalname, mc)
toast_tools.scale_atmosphere_by_frequency(
args,
comm,
data,
cache_name=totalname,
mc=mc,
)
memreport("after atmosphere", comm.comm_world)
so_tools.simulate_hwpss(args, comm, data, mc, totalname)
# update_atmospheric_noise_weights(args, comm, data, freq, mc)
toast_tools.add_signal(args,
comm,
data,
totalname,
signalname,
purge=(mc == firstmc + nmc - 1))
memreport("after adding sky", comm.comm_world)
toast_tools.simulate_sss(args, comm, data, mc, totalname)
memreport("after simulating SSS", comm.comm_world)
toast_tools.simulate_noise(args, comm, data, mc, totalname)
memreport("after simulating noise", comm.comm_world)
so_tools.apply_sim_sso(args, comm, data, mc, totalname)
memreport("after simulating SSO", comm.comm_world)
so_tools.convolve_time_constant(args, comm, data, totalname)
memreport("after convolving with time constant", comm.comm_world)
# DEBUG begin
"""
import matplotlib.pyplot as plt
tod = data.obs[0]['tod']
times = tod.local_times()
for det in tod.local_dets:
sig = tod.local_signal(det, totalname)
plt.plot(times, sig, label=det)
plt.legend(loc='best')
fnplot = 'debug_{}.png'.format(args.madam_prefix)
plt.savefig(fnplot)
plt.close()
print('DEBUG plot saved in', fnplot)
return
"""
# DEBUG end
toast_tools.scramble_gains(args, comm, data, mc, totalname)
so_tools.deconvolve_time_constant(args,
comm,
data,
totalname,
realization=mc)
memreport("after deconvolving time constant", comm.comm_world)
if mc == firstmc:
# For the first realization and frequency, optionally
# export the timestream data.
toast_tools.output_tidas(args, comm, data, totalname)
so_tools.export_TOD(args, comm, data, totalname, schedules)
memreport("after export", comm.comm_world)
if args.no_maps:
continue
outpath = setup_output(args, comm, mc)
# Optionally demodulate signal
so_tools.demodulate(args, comm, data, totalname, detweights, madampars)
# Bin and destripe maps
if args.use_madam:
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
else:
toast_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
memreport("after destriper", comm.comm_world)
if (args.filterbin_ground_order is not None
or args.filterbin_poly_order is not None):
toast_tools.apply_filterbin(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
toast_tools.apply_polyfilter(args, comm, data, totalname)
memreport("after polyfilter", comm.comm_world)
# Ground filter
memreport("after demodulation", comm.comm_world)
toast_tools.apply_groundfilter(args, comm, data, totalname)
memreport("after groundfilter", comm.comm_world)
# Bin maps
if args.use_madam:
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=args.demodulate,
extra_prefix="filtered",
bin_only=True,
)
else:
toast_tools.apply_mapmaker(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=False,
extra_prefix="filtered",
bin_only=True,
)
memreport("after filter & bin", comm.comm_world)
if args.demodulate and args.MC_count > 1:
if comm.world_rank == 0:
log.info("WARNING: demodulation and MC iterations are "
"incompatible. Terminating after first MC.")
break
if comm.comm_world is not None:
comm.comm_world.barrier()
memreport("at the end of the pipeline", comm.comm_world)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
timer0.stop()
if comm.world_rank == 0:
timer0.report("toast_so_sim.py pipeline")
return
def main():
timer0 = Timer()
timer0.start()
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
memreport("At start of pipeline", mpiworld)
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help='Write data distribution info to file')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory '
'for pointing')
parser.add_argument('--coord',
default='G',
help='Coordinate system, "G", "E" or "C"')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Pointing flag mask')
parser.add_argument('--bad_intervals',
required=False,
help='Path to bad interval file.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
type=np.int,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
type=np.int,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use (can be a list)')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use (can be a list)')
parser.add_argument('--obtfirst',
required=False,
default=None,
type=np.float,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
type=np.float,
help='Last OBT to use')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--catalog', required=True, help='Target catalog file')
parser.add_argument('--radius',
required=True,
type=np.float,
help='Search radius about the source [arc min]')
parser.add_argument('--mask',
required=False,
help='Mask defining region of the sky to accept')
parser.add_argument('--bg',
required=False,
help='Background map to subtract')
parser.add_argument('--recalib_bg',
dest='recalib_bg',
default=False,
action='store_true',
help='Recalibrate bg map for each ring.')
parser.add_argument('--full_rings',
dest='full_rings',
default=False,
action='store_true',
help='Extract impacted rings entirely.')
# noise parameters
parser.add_argument('--noisefile',
required=False,
default='RIMO',
help='Path to noise PSD files for noise filter. '
'Tag DETECTOR will be replaced with detector name.')
# Dipole parameters
dipogroup = parser.add_mutually_exclusive_group()
dipogroup.add_argument('--dipole',
dest='dipole',
required=False,
default=False,
action='store_true',
help='Simulate dipole')
dipogroup.add_argument('--solsys_dipole',
dest='solsys_dipole',
required=False,
default=False,
action='store_true',
help='Simulate solar system dipole')
dipogroup.add_argument('--orbital_dipole',
dest='orbital_dipole',
required=False,
default=False,
action='store_true',
help='Simulate orbital dipole')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.world_rank == 0:
print('All parameters:')
print(args, flush=True)
data = create_observations(args, comm)
rimo = data.obs[0]["tod"].rimo
memreport("After create observations", mpiworld)
# Read in the signal
timer = Timer()
timer.start()
reader = tp.OpInputPlanck(signal_name='signal', flags_name='flags')
if comm.world_rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Reading")
tod_name = 'signal'
flags_name = 'flags'
memreport("After read", mpiworld)
# Optionally flag bad intervals
if args.bad_intervals is not None:
flagger = tp.OpBadIntervals(path=args.bad_intervals)
flagger.exec(data)
if comm.world_rank == 0:
timer.report_clear("Applying {}".format(args.bad_intervals))
do_dipole = (args.dipole or args.solsys_dipole or args.orbital_dipole)
# make a planck Healpix pointing matrix
pointing = tp.OpPointingPlanck(nside=1024,
mode='IQU',
RIMO=rimo,
margin=0,
apply_flags=False,
keep_vel=do_dipole,
keep_pos=False,
keep_phase=True,
keep_quats=True)
pointing.exec(data)
memreport("After pointing", mpiworld)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Pointing Matrix")
# Optionally subtract the dipole
if do_dipole:
if args.dipole:
dipomode = 'total'
elif args.solsys_dipole:
dipomode = 'solsys'
else:
dipomode = 'orbital'
dipo = tp.OpDipolePlanck(args.freq,
mode=dipomode,
output='dipole',
keep_quats=True)
dipo.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Dipole")
subtractor = tp.OpCacheMath(in1=tod_name,
in2='dipole',
subtract=True,
out=tod_name)
if comm.comm_world.rank == 0:
print('Subtracting dipole', flush=True)
subtractor.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Dipole subtraction")
memreport("After dipole", mpiworld)
extract = tp.OpExtractPlanck(rimo,
args.catalog,
args.radius,
mpiworld,
common_flag_mask=args.obtmask,
flag_mask=args.flagmask,
maskfile=args.mask,
bg=args.bg,
full_rings=args.full_rings,
recalibrate_bg=args.recalib_bg,
out=args.out)
extract.exec(data)
memreport("After extract", mpiworld)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Extraction")
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.report_clear("Gather and dump timing info")
timer0.report_clear("Full pipeline")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
memreport("at beginning of main", mpiworld)
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--nside',
required=False,
type=np.int,
default=512,
help='Map resolution')
parser.add_argument('--nside_cross',
required=False,
type=np.int,
default=512,
help='Destriping resolution')
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help='Write data distribution info to file')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_in_diode0',
required=False,
default=None,
help='Input Exchange Format File directory, '
'LFI diode 0')
parser.add_argument('--effdir_in_diode1',
required=False,
default=None,
help='Input Exchange Format File directory, '
'LFI diode 1')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory '
'for pointing')
parser.add_argument('--effdir_out',
required=False,
help='Output directory for destriped TOD')
parser.add_argument('--effdir_out_diode0',
required=False,
help='Output directory for destriped TOD, LFI diode 0')
parser.add_argument('--effdir_out_diode1',
required=False,
help='Output directory for destriped TOD, LFI diode 1')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Pointing flag mask')
parser.add_argument('--bad_intervals',
required=False,
help='Path to bad interval file.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
type=np.int,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
type=np.int,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use (can be a list)')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use (can be a list)')
parser.add_argument('--obtfirst',
required=False,
default=None,
type=np.float,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
type=np.float,
help='Last OBT to use')
parser.add_argument('--madampar',
required=False,
default=None,
help='Madam parameter file')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--madam_prefix', required=False, help='map prefix')
parser.add_argument('--split_mask',
required=False,
default=None,
help='Intensity mask, non-zero pixels are not split.')
parser.add_argument('--save_leakage_matrices',
dest='save_leakage_matrices',
default=False,
action='store_true',
help='Compile and write out the leakage projection '
'matrices.')
# noise parameters
parser.add_argument('--noisefile',
required=False,
default='RIMO',
help='Path to noise PSD files for noise filter. '
'Tag DETECTOR will be replaced with detector name.')
parser.add_argument('--static_noise',
dest='static_noise',
required=False,
default=False,
action='store_true',
help='Assume constant noise PSD')
parser.add_argument('--filterfile',
required=False,
help='Extra filter file.')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.comm_world.rank == 0:
print('All parameters:')
print(args, flush=True)
timer = Timer()
timer.start()
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# create the TOD for this observation
if args.noisefile != 'RIMO' and not args.static_noise:
do_eff_cache = True
else:
do_eff_cache = False
tods = []
if args.static_noise:
noisefile = args.noisefile
else:
noisefile = 'RIMO'
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
tods.append(
tp.Exchange(comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_in_diode0=args.effdir_in_diode0,
effdir_in_diode1=args.effdir_in_diode1,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
pntflagmask=args.pntflagmask,
do_eff_cache=do_eff_cache,
noisefile=noisefile))
rimo = tods[0].rimo
# Make output directory
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.makedirs(args.out)
# Read in madam parameter file
# Allow more than one entry, gather into a list
repeated_keys = ['detset', 'detset_nopol', 'survey']
pars = {}
if comm.comm_world.rank == 0:
pars['kfirst'] = False
pars['temperature_only'] = True
pars['base_first'] = 60.0
pars['nside_map'] = args.nside
pars['nside_cross'] = min(args.nside, args.nside_cross)
pars['nside_submap'] = 16
pars['write_map'] = False
pars['write_binmap'] = True
pars['write_matrix'] = False
pars['write_wcov'] = False
pars['write_hits'] = True
pars['kfilter'] = False
pars['info'] = 3
pars['pixlim_map'] = 1e-3
pars['pixlim_cross'] = 1e-3
if args.madampar:
pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*')
comment = re.compile(r'^#.*')
with open(args.madampar, 'r') as f:
for line in f:
if not comment.match(line):
result = pat.match(line)
if result:
key, value = result.group(1), result.group(2)
if key in repeated_keys:
if key not in pars:
pars[key] = []
pars[key].append(value)
else:
pars[key] = value
# Command line parameters override the ones in the madam parameter file
if 'file_root' not in pars:
pars['file_root'] = 'madam'
if args.madam_prefix is not None:
pars['file_root'] = args.madam_prefix
sfreq = '{:03}'.format(args.freq)
if sfreq not in pars['file_root']:
pars['file_root'] += '_' + sfreq
try:
fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq]
except Exception:
fsample = 180.3737
pars['fsample'] = fsample
pars['path_output'] = args.out
if args.save_leakage_matrices:
pars['write_leakmatrix'] = True
pars = comm.comm_world.bcast(pars, root=0)
if args.noisefile != 'RIMO':
# We split MPI_COMM_WORLD into single process groups, each of
# which is assigned one or more observations (rings)
comm = toast.Comm(groupsize=1)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
for iobs, tod in enumerate(tods):
if args.noisefile != 'RIMO' and not args.static_noise:
# Use a toast helper method to optimally distribute rings between
# processes.
dist = toast.distribute_discrete(tod.ringsizes, comm.world_size)
my_first_ring, my_n_ring = dist[comm.comm_world.rank]
for my_ring in range(my_first_ring, my_first_ring + my_n_ring):
ringtod = tp.Exchange.from_tod(tod,
my_ring,
comm.comm_group,
noisefile=args.noisefile)
ob = {}
ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring)
ob['id'] = ringtod.globalfirst_ring
ob['tod'] = ringtod
ob['intervals'] = ringtod.valid_intervals
ob['baselines'] = None
ob['noise'] = ringtod.noise
data.obs.append(ob)
else:
ob = {}
ob['name'] = 'observation{:04}'.format(iobs)
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
data.obs.append(ob)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Metadata queries")
if args.effdir_out is not None or (args.effdir_out_diode0 is not None
and args.effdir_out_diode1 is not None):
do_output = True
else:
do_output = False
# Read in the signal
timer.clear()
timer.start()
reader = tp.OpInputPlanck(signal_name='signal', flags_name='flags')
if comm.comm_world.rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Read")
tod_name = 'signal'
flags_name = 'flags'
# Optionally filter the signal
apply_filter(args, data)
# Optionally flag bad intervals
if args.bad_intervals is not None:
timer = Timer()
timer.start()
flagger = tp.OpBadIntervals(path=args.bad_intervals)
flagger.exec(data)
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Apply {}".format(args.bad_intervals))
# make a planck Healpix pointing matrix
timer.clear()
timer.start()
mode = 'IQU'
if pars['temperature_only'] == 'T':
mode = 'I'
nside = int(pars['nside_map'])
pointing = tp.OpPointingPlanck(nside=nside,
mode=mode,
RIMO=rimo,
margin=0,
apply_flags=(not do_output),
keep_vel=False,
keep_pos=False,
keep_phase=False,
keep_quats=False)
pointing.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Pointing Matrix, mode = {}".format(mode))
for obs in data.obs:
obs['tod'].purge_eff_cache()
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
if d[-1] in '01' and d[-2] != '-':
det = to_radiometer(d)
else:
det = d
net = tod.rimo[det].net
fsample = tod.rimo[det].fsample
detweights[d] = 1.0 / (fsample * net * net)
if do_output:
name_out = 'madam_tod'
else:
name_out = None
timer.clear()
timer.start()
try:
madam = toast.todmap.OpMadam(name=tod_name,
flag_name=flags_name,
apply_flags=do_output,
params=pars,
detweights=detweights,
purge=True,
name_out=name_out,
translate_timestamps=False)
except Exception as e:
raise Exception('{:4} : ERROR: failed to initialize Madam: {}'.format(
comm.comm_world.rank, e))
madam.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Madam")
if do_output:
timer = Timer()
timer.start()
writer = tp.OpOutputPlanck(signal_name='madam_tod',
flags_name=None,
commonflags_name=None,
effdir_out=args.effdir_out,
effdir_out_diode0=args.effdir_out_diode0,
effdir_out_diode1=args.effdir_out_diode1)
writer.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Madam output")
memreport("at end of main", mpiworld)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_s4_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = toast_tools.get_comm()
memreport("at the beginning of the pipeline", comm.comm_world)
args, comm = parse_arguments(comm)
# Initialize madam parameters
madampars = toast_tools.setup_madam(args)
# Load and broadcast the schedule file
schedules = toast_tools.load_schedule(args, comm)
# Load the weather and append to schedules
toast_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = s4_tools.load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = s4_tools.create_observations(args, comm, schedules)
memreport("after creating observations", comm.comm_world)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
s4_tools.get_elevation_noise(args, comm, data)
totalname = "total"
# Split the communicator for day and season mapmaking
time_comms = toast_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
toast_tools.expand_pointing(args, comm, data)
# Only purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if args.tidas is None:
for ob in data.obs:
tod = ob["tod"]
try:
tod.free_radec_quats()
except AttributeError:
# These TOD objects do not have RA/Dec quaternions
pass
memreport("after pointing", comm.comm_world)
# Prepare auxiliary information for distributed map objects
memreport("after submaps", comm.comm_world)
# Set up objects to take copies of the TOD at appropriate times
if args.pysm_model:
if schedules is not None:
focalplanes = [
s.telescope.focalplane.detector_data for s in schedules
]
else:
focalplanes = [telescope.focalplane.detector_data]
signalname = s4_tools.simulate_sky_signal(args, comm, data,
focalplanes)
else:
signalname = toast_tools.scan_sky_signal(args, comm, data)
memreport("after PySM", comm.comm_world)
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc + nmc):
if comm.world_rank == 0:
log.info("Processing MC = {}".format(mc))
# Uncomment to run with new TOAST
#toast_tools.draw_weather(args, comm, data, mc)
outpath = setup_output(args, comm, mc)
if outputs_exist(args, comm, outpath):
if comm.world_rank == 0:
log.info("Outputs already exist, skipping.")
continue
toast.tod.OpCacheClear(totalname).exec(data)
toast_tools.simulate_atmosphere(args, comm, data, mc, totalname)
s4_tools.scale_atmosphere_by_bandpass(args, comm, data, totalname, mc)
memreport("after atmosphere", comm.comm_world)
# update_atmospheric_noise_weights(args, comm, data, freq, mc)
toast_tools.add_signal(args,
comm,
data,
totalname,
signalname,
purge=(mc == firstmc + nmc - 1))
memreport("after adding sky", comm.comm_world)
toast_tools.simulate_noise(args, comm, data, mc, totalname)
memreport("after simulating noise", comm.comm_world)
toast_tools.simulate_sss(args, comm, data, mc, totalname)
memreport("after simulating SSS", comm.comm_world)
toast_tools.scramble_gains(args, comm, data, mc, totalname)
if mc == firstmc:
# For the first realization and frequency, optionally
# export the timestream data.
toast_tools.output_tidas(args, comm, data, totalname)
memreport("after export", comm.comm_world)
if args.no_maps:
continue
# Bin and destripe maps
pairdiff(data, args, comm, totalname, mc == firstmc)
if not args.skip_madam:
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
memreport("after madam", comm.comm_world)
if (args.filterbin_ground_order is not None
or args.filterbin_poly_order is not None):
toast_tools.apply_filterbin(
args,
comm,
data,
outpath,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if (args.apply_polyfilter or args.apply_polyfilter2D
or args.apply_common_mode_filter or args.apply_groundfilter):
# Filter signal
toast_tools.apply_common_mode_filter(args, comm, data, totalname)
toast_tools.apply_polyfilter2D(args, comm, data, totalname)
toast_tools.apply_polyfilter(args, comm, data, totalname)
toast_tools.apply_groundfilter(args, comm, data, totalname)
memreport("after filter", comm.comm_world)
# Bin maps
toast_tools.apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(args.skip_madam and mc == firstmc),
extra_prefix="filtered",
bin_only=(not args.skip_madam),
)
memreport("after filter & bin", comm.comm_world)
if comm.comm_world is not None:
comm.comm_world.barrier()
memreport("at the end of the pipeline", comm.comm_world)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
timer0.stop()
if comm.world_rank == 0:
timer0.report("toast_s4_sim.py pipeline")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_so_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = toast_tools.get_comm()
memreport("at the beginning of the pipeline", comm.comm_world)
args, comm = parse_arguments(comm)
# Load and broadcast the schedule file
schedules = toast_tools.load_schedule(args, comm)
# Load the weather and append to schedules
toast_tools.load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = so_tools.load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = so_tools.create_observations(args, comm, schedules)
memreport("after creating observations", comm.comm_world)
# Optionally rewrite the noise PSD:s in each observation to include
# elevation-dependence
so_tools.get_elevation_noise(args, comm, data)
totalname = "total"
# Split the communicator for day and season mapmaking
time_comms = toast_tools.get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
toast_tools.expand_pointing(args, comm, data)
memreport("after pointing", comm.comm_world)
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
madam = None
for mc in range(firstmc, firstmc + nmc):
timer_mc = Timer()
timer_mc.start()
outpath = setup_output(args, comm, mc)
outprefix = args.map_prefix + "_filtered"
if args.madam:
outmap = os.path.join(outpath, outprefix + "_bmap.fits")
else:
outmap = os.path.join(outpath, outprefix + "_binned.fits")
if os.path.isfile(outmap):
if comm.world_rank == 0:
log.info("{} exists, skipping".format(outmap))
continue
if comm.world_rank == 0:
log.info("Processing MC = {} into {}".format(mc, outmap))
# Ensure there is no stale signal in the cache
toast.tod.OpCacheClear(totalname).exec(data)
if args.pysm_model:
if schedules is not None:
focalplanes = [
s.telescope.focalplane.detector_data for s in schedules
]
else:
focalplanes = [telescope.focalplane.detector_data]
so_tools.simulate_sky_signal(args,
comm,
data,
focalplanes,
totalname,
mc=mc)
else:
toast_tools.scan_sky_signal(args, comm, data, totalname, mc=mc)
memreport("after PySM", comm.comm_world)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
toast_tools.apply_polyfilter(args, comm, data, totalname)
toast_tools.apply_groundfilter(args, comm, data, totalname)
memreport("after filter", comm.comm_world)
# Bin maps
timer_map = Timer()
timer_map.start()
if args.madam:
if madam is None:
madampars = {}
madampars["temperature_only"] = False
for name in [
"kfirst", "write_map", "write_matrix",
"write_wcov", "write_hits"
]:
madampars[name] = False
madampars["write_binmap"] = True
madampars["concatenate_messages"] = True
madampars["allreduce"] = True
madampars["nside_submap"] = args.nside_submap
madampars["reassign_submaps"] = True
madampars["pixlim_map"] = 1e-2
madampars["pixmode_map"] = 2
# Instead of fixed detector weights, we'll want to use scaled noise
# PSD:s that include the atmospheric noise
madampars["radiometers"] = True
madampars["noise_weights_from_psd"] = True
madampars["nside_map"] = args.nside
madampars["fsample"] = args.sample_rate
madampars["path_output"] = outpath
madampars["file_root"] = outprefix
if args.madam_concatenate_messages:
# Collective communication is fast but requires memory
madampars["concatenate_messages"] = True
if args.madam_allreduce:
# Every process will allocate a copy of every observed submap.
madampars["allreduce"] = True
else:
# Every process will allocate complete send and receive buffers
madampars["allreduce"] = False
else:
# Slow but memory-efficient point-to-point communication. Allocate
# only enough memory to communicate with one process at a time.
madampars["concatenate_messages"] = False
madampars["allreduce"] = False
madam = toast.todmap.OpMadam(
params=madampars,
detweights=detweights,
name=totalname,
common_flag_mask=args.common_flag_mask,
purge_tod=True,
mcmode=False,
conserve_memory=args.madam_conserve_memory,
)
else:
madam.params["path_output"] = outpath
madam.exec(data)
del madam
madam = None
else:
mapmaker = toast.todmap.OpMapMaker(
nside=args.nside,
nnz=3,
name=totalname,
outdir=outpath,
outprefix=outprefix + "_",
write_hits=False,
zip_maps=False,
write_wcov_inv=False,
write_wcov=False,
write_binned=True,
write_destriped=False,
write_rcond=False,
rcond_limit=1e-3,
baseline_length=None,
common_flag_mask=args.common_flag_mask,
)
mapmaker.exec(data)
if comm.world_rank == 0:
timer_map.report_clear("Bin map")
memreport("after filter & bin", comm.comm_world)
if comm.world_rank == 0:
timer_mc.report_clear(
"Monte Carlo iteration # {:05}".format(mc))
if comm.comm_world is not None:
comm.comm_world.barrier()
memreport("at the end of the pipeline", comm.comm_world)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.report_clear("Gather and dump timing info")
if comm.world_rank == 0:
timer0.report_clear("toast_so_tf.py pipeline")
return
def _stage_data(
self,
nsamp,
ndet,
nnz,
nnz_full,
nnz_stride,
psdfreqs,
detectors,
nside,
):
""" create Mappraiser-compatible buffers
Collect the TOD into Mappraiser buffers. Process pixel weights
Separate from the rest to reduce the memory high water mark
When the user has set purge=True
Moving data between toast and Mappraiser buffers has an overhead.
We perform the operation in a staggered fashion to have the
overhead only once per node.
"""
log = Logger.get()
nodecomm = self._comm.Split_type(MPI.COMM_TYPE_SHARED, self._rank)
# Check if the user has elected to stagger staging the data on each
# node to avoid exhausting memory
if self._conserve_memory:
if self._conserve_memory == 1:
nread = nodecomm.size
else:
nread = min(self._conserve_memory, nodecomm.size)
else:
nread = 1
self._comm.Barrier()
timer_tot = Timer()
timer_tot.start()
# Stage time (Tpltz blocks in Mappraiser), it is never purged
# so the staging is never stepped
timer = Timer()
# THIS STEP IS SKIPPED: we do not have timestamps, nor do we build Toeplitz blocks
# from TOAST psds which comprise detector noise only - a psd fit is done when staging noise -
#timer.start()
#invtt_list = self._stage_time(detectors, nsamp, psdfreqs)
#self._mappraiser_invtt = np.array([np.array(invtt_i, dtype= mappraiser.INVTT_TYPE) for invtt_i in invtt_list])
#del invtt_list
#self._mappraiser_invtt = np.concatenate(self._mappraiser_invtt)
#if self._verbose:
# nodecomm.Barrier()
# if self._rank == 0:
# timer.report_clear("Stage time")
#memreport("after staging time", self._comm) # DEBUG
#count_caches(
# self._data, self._comm, nodecomm, self._cache, "after staging time"
#) # DEBUG
# Stage signal. If signal is not being purged, staging is not stepped
timer.start()
signal_dtype, local_blocks_sizes = self._stage_signal(
detectors, nsamp, ndet, nodecomm, nread)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage signal")
memreport("after staging signal", self._comm) # DEBUG
count_caches(self._data, self._comm, nodecomm, self._cache,
"after staging signal") # DEBUG
# Stage noise. If noise is not being purged, staging is not stepped
timer.start()
invtt_list, noise_dtype = self._stage_noise(detectors, nsamp, ndet,
nodecomm, nread)
self._mappraiser_invtt = np.array([
np.array(invtt_i, dtype=mappraiser.INVTT_TYPE)
for invtt_i in invtt_list
])
del invtt_list
self._mappraiser_invtt = np.concatenate(self._mappraiser_invtt)
if self._params["uniform_w"] == 1:
self._mappraiser_invtt = np.ones_like(self._mappraiser_invtt)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage noise")
memreport("after staging noise", self._comm) # DEBUG
count_caches(self._data, self._comm, nodecomm, self._cache,
"after staging noise") # DEBUG
# Stage pixels
timer_step = Timer()
timer_step.start()
for iread in range(nread):
nodecomm.Barrier()
timer.start()
if nodecomm.rank % nread == iread:
pixels_dtype = self._stage_pixels(detectors, nsamp, ndet, nnz,
nside)
if self._verbose and nread > 1:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage pixels {} / {}".format(
iread + 1, nread))
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer_step.report_clear("Stage pixels")
memreport("after staging pixels", self._comm) # DEBUG
count_caches(self._data, self._comm, nodecomm, self._cache,
"after staging pixels") # DEBUG
# Stage pixel weights
timer_step.start()
weight_dtype = self._stage_pixweights(
detectors,
nsamp,
ndet,
nnz,
nnz_full,
nnz_stride,
nodecomm,
nread,
)
if self._verbose:
nodecomm.Barrier()
if self._rank == 0:
timer_step.report_clear("Stage pixel weights")
memreport("after staging pixel weights", self._comm) # DEBUG
count_caches(self._data, self._comm, nodecomm, self._cache,
"after staging pixel weights") # DEBUG
del nodecomm
if self._rank == 0 and self._verbose:
timer_tot.report_clear("Stage all data")
# detweights is either a dictionary of weights specified at
# construction time, or else we use uniform weighting.
# N.B: This is essentially useless in current implementation
detw = {}
if self._detw is None:
for idet, det in enumerate(detectors):
detw[det] = 1.0
else:
detw = self._detw
detweights = np.zeros(ndet, dtype=np.float64)
for idet, det in enumerate(detectors):
detweights[idet] = detw[det]
# Get global array of data sizes of the full communicator
data_size_proc = np.array(self._comm.allgather(
len(self._mappraiser_signal)),
dtype=np.int32)
# Get number of local observations
nobsloc = len(self._data.obs)
return data_size_proc, nobsloc, local_blocks_sizes, signal_dtype, noise_dtype, pixels_dtype, weight_dtype
