def apply_flag_sso(args, comm, data, verbose=True):
if args.flag_sso is None:
return
if comm.world_rank == 0 and verbose:
print(f"Flagging SSO:s", flush=True)
timer = Timer()
timer.start()
sso_names = []
sso_radii = []
for arg in args.flag_sso:
sso_name, sso_radius = arg.split(",")
sso_radius = np.radians(float(sso_radius) / 60)
sso_names.append(sso_name)
sso_radii.append(sso_radius)
flag_sso = OpFlagSSO(sso_names, sso_radii, flag_mask=args.flag_sso_mask)
flag_sso.exec(data)
if comm.world_rank == 0 and verbose:
timer.report_clear(f"Flag {sso_names}")
return
def load_observations(args, comm):
"""Load existing data and put it in TOAST observations.
"""
# This import is not at the top of the file to avoid
# loading spt3g through so3g unnecessarily
from ...io.toast_load import load_data
log = Logger.get()
if args.import_obs is not None:
import_obs = args.import_obs.split(",")
else:
import_obs = None
hw, telescope, det_index = get_hardware(args, comm, verbose=True)
focalplane = get_focalplane(args, comm, hw, det_index, verbose=True)
detweights = focalplane.detweights
telescope.focalplane = focalplane
if comm.world_rank == 0:
log.info("Loading TOD from {}".format(args.import_dir))
timer = Timer()
timer.start()
data = load_data(
args.import_dir,
obs=import_obs,
comm=comm,
prefix=args.import_prefix,
dets=hw,
detranks=comm.group_size,
)
if comm.world_rank == 0:
timer.report_clear("Load data")
telescope_data = [("all", data)]
site = telescope.site
focalplane = telescope.focalplane
if args.weather is not None:
weather = Weather(args.weather)
else:
weather = None
for obs in data.obs:
#obs["baselines"] = None
obs["noise"] = focalplane.noise
#obs["id"] = int(ces.mjdstart * 10000)
#obs["intervals"] = tod.subscans
obs["site"] = site.name
obs["site_id"] = site.id
obs["telescope"] = telescope.name
obs["telescope_id"] = telescope.id
obs["fpradius"] = focalplane.radius
obs["weather"] = weather
#obs["start_time"] = ces.start_time
obs["altitude"] = site.alt
#obs["season"] = ces.season
#obs["date"] = ces.start_date
#obs["MJD"] = ces.mjdstart
obs["focalplane"] = focalplane.detector_data
#obs["rising"] = ces.rising
#obs["mindist_sun"] = ces.mindist_sun
#obs["mindist_moon"] = ces.mindist_moon
#obs["el_sun"] = ces.el_sun
return data, telescope_data, detweights
def _stage_signal(self, detectors, nsamp, ndet, nodecomm, nread):
""" Stage signal
"""
log = Logger.get()
timer = Timer()
# Determine if we can purge the signal and avoid keeping two
# copies in memory
purge = self._name is not None and self._purge_tod
if not purge:
nread = 1
nodecomm = MPI.COMM_SELF
for iread in range(nread):
nodecomm.Barrier()
timer.start()
if nodecomm.rank % nread == iread:
self._mappraiser_signal = self._cache.create(
"signal", mappraiser.SIGNAL_TYPE, (nsamp * ndet, ))
self._mappraiser_signal[:] = np.nan
global_offset = 0
local_blocks_sizes = []
for iobs, obs in enumerate(self._data.obs):
tod = obs["tod"]
for idet, det in enumerate(detectors):
# Get the signal.
signal = tod.local_signal(det, self._name)
signal_dtype = signal.dtype
offset = global_offset
local_V_size = len(signal)
dslice = slice(idet * nsamp + offset,
idet * nsamp + offset + local_V_size)
self._mappraiser_signal[dslice] = signal
offset += local_V_size
local_blocks_sizes.append(local_V_size)
del signal
# Purge only after all detectors are staged in case some are aliased
# cache.clear() will not fail if the object was already
# deleted as an alias
if purge:
for det in detectors:
cachename = "{}_{}".format(self._name, det)
tod.cache.clear(cachename)
global_offset = offset
local_blocks_sizes = np.array(local_blocks_sizes,
dtype=np.int32)
if self._verbose and nread > 1:
nodecomm.Barrier()
if self._rank == 0:
timer.report_clear("Stage signal {} / {}".format(
iread + 1, nread))
return signal_dtype, local_blocks_sizes
def convolve_time_constant(args, comm, data, name, verbose=True):
if not args.tau_convolve:
return
log = Logger.get()
timer = Timer()
timer.start()
tauop = OpTimeConst(name=name, tau=args.tau_value, inverse=False)
tauop.exec(data)
timer.report_clear("Convolve time constant")
return
def simulate_hwpss(args, comm, data, mc, name):
if not args.simulate_hwpss:
return
log = Logger.get()
timer = Timer()
timer.start()
hwpssop = OpSimHWPSS(name=name, fname_hwpss=args.hwpss_file, mc=mc)
hwpssop.exec(data)
timer.report_clear("Simulate HWPSS")
return
def rotate_focalplane(args, data, comm):
""" The LAT focalplane projected on the sky rotates as the cryostat
(co-rotator) tilts. Usually the tilt is the same as the observing
elevation to maintain constant angle between the mirror and the cryostat.
This method must be called *before* expanding the detector pointing
from boresight.
"""
log = Logger.get()
timer = Timer()
timer.start()
for obs in data.obs:
if obs["telescope"] != "LAT":
continue
tod = obs["tod"]
cache_name = "corotator_angle_deg"
if tod.cache.exists(cache_name):
corotator_angle = tod.cache.reference(cache_name)
else:
# If a vector of co-rotator angles isn't already cached,
# make one now from the observation metadata. This will
# ensure they get recorded in the so3g files.
corotator_angle = obs["corotator_angle_deg"]
offset, nsample = tod.local_samples
tod.cache.put(cache_name, np.zeros(nsample) + corotator_angle)
el = np.degrees(tod.read_boresight_el())
rot = qa.rotation(
ZAXIS, np.radians(corotator_angle + el + LAT_COROTATOR_OFFSET_DEG)
)
quats = tod.read_boresight()
quats[:] = qa.mult(quats, rot)
try:
# If there are horizontal boresight quaternions, they need
# to be rotated as well.
quats = tod.read_boresight(azel=True)
quats[:] = qa.mult(quats, rot)
except Exception as e:
pass
if comm.comm_world is None or comm.comm_world.rank == 0:
timer.report_clear("Rotate focalplane")
return
def get_elevation_noise(args, comm, data, key="noise"):
""" Insert elevation-dependent noise
"""
if args.no_elevation_noise:
return
timer = Timer()
timer.start()
# fsample = args.sample_rate
for obs in data.obs:
tod = obs["tod"]
fp = obs["focalplane"]
noise = obs[key]
for det in tod.local_dets:
if det not in noise.keys:
raise RuntimeError(
'Detector "{}" does not have a PSD in the noise object'.
format(det))
A = fp[det]["A"]
C = fp[det]["C"]
psd = noise.psd(det)
# We only consider a small range of samples for the elevation
n = tod.local_samples[1]
istart = max(0, n // 2 - 1000)
istop = min(n, n // 2 + 1000)
try:
# Some TOD classes provide a shortcut to Az/El
el = tod.read_azel(detector=det,
local_start=istart,
n=istop - istart)[1]
except Exception:
azelquat = tod.read_pntg(detector=det,
azel=True,
local_start=istart,
n=istop - istart)
# Convert Az/El quaternion of the detector back into
# angles for the simulation.
theta = qa.to_position(azelquat)[0]
el = np.pi / 2 - theta
el = np.median(el)
# Scale the analytical noise PSD. Pivot is at el = 50 deg.
psd[:] *= (A / np.sin(el) + C)**2
if comm.world_rank == 0:
timer.report_clear("Elevation noise")
return
def apply_polyfilter(args, comm, data, cache_name=None, verbose=True):
"""Apply the polynomial filter to data under `cache_name`."""
if not args.apply_polyfilter:
return
log = Logger.get()
timer = Timer()
timer.start()
if comm.world_rank == 0 and verbose:
log.info("Polyfiltering signal")
polyfilter = OpPolyFilter(order=args.poly_order,
name=cache_name,
common_flag_mask=args.common_flag_mask)
polyfilter.exec(data)
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0 and verbose:
timer.report_clear("Polynomial filtering")
return
def apply_common_mode_filter(args, comm, data, cache_name=None, verbose=True):
"""Apply the common mode filter to data under `cache_name`."""
if not args.apply_common_mode_filter:
return
log = Logger.get()
timer = Timer()
timer.start()
if comm.world_rank == 0 and verbose:
log.info("Common mode filtering signal")
commonfilter = OpCommonModeFilter(
name=cache_name,
common_flag_mask=args.common_flag_mask,
focalplane_key=args.common_mode_filter_key,
)
commonfilter.exec(data)
if comm.world_rank == 0 and verbose:
timer.report_clear("Common mode filtering")
return
def compute_crosslinking(args, comm, data, detweights=None, verbose=True):
if not args.write_crosslinking:
return
log = Logger.get()
timer = Timer()
timer.start()
crosslinking = OpCrossLinking(
outdir=args.out,
outprefix=args.crosslinking_prefix,
common_flag_mask=args.common_flag_mask,
flag_mask=255,
zip_maps=args.hn_zip,
rcond_limit=1e-3,
detweights=detweights,
)
crosslinking.exec(data)
timer.report_clear("Compute crosslinking map")
return
def compute_cadence_map(args, comm, data, verbose=True):
if not args.write_cadence_map:
return
log = Logger.get()
if comm.world_rank == 0:
log.info("Computing cadence map")
timer = Timer()
timer.start()
cadence = OpCadenceMap(
outdir=args.out,
outprefix=args.cadence_map_prefix,
common_flag_mask=args.common_flag_mask,
flag_mask=255,
)
cadence.exec(data)
if comm.world_rank == 0:
timer.report_clear("Compute cadence map")
return
def compute_h_n(args, comm, data, verbose=True):
if args.hn_max < args.hn_min:
return
log = Logger.get()
timer = Timer()
timer.start()
hnop = OpHn(
outdir=args.hn_outdir,
outprefix=args.hn_prefix,
nmin=args.hn_min,
nmax=args.hn_max,
common_flag_mask=args.common_flag_mask,
flag_mask=255,
zip_maps=args.hn_zip,
)
hnop.exec(data)
timer.report_clear("Compute h_n")
return
def apply_groundfilter(args, comm, data, cache_name=None, verbose=True):
if not args.apply_groundfilter:
return
log = Logger.get()
timer = Timer()
timer.start()
if comm.world_rank == 0 and verbose:
log.info("Ground-filtering signal")
groundfilter = OpGroundFilter(
filter_order=args.ground_order,
name=cache_name,
common_flag_mask=args.common_flag_mask,
)
groundfilter.exec(data)
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0 and verbose:
timer.report_clear("Ground filtering")
return
def demodulate(args,
comm,
data,
name,
detweights=None,
madampars=None,
verbose=True):
if not args.demodulate:
return
log = Logger.get()
timer = Timer()
timer.start()
if detweights is not None:
# Copy the detector weights to demodulated TOD
modulated = [
detname for detname in detweights if "demod" not in detname
]
for detname in modulated:
detweight = detweights[detname]
for demodkey in ["demod0", "demod4r", "demod4i"]:
demod_name = "{}_{}".format(demodkey, detname)
detweights[demod_name] = detweight
del detweights[detname]
if madampars is not None:
# Filtering will affect the high frequency end of the noise PSD
madampars["radiometers"] = False
# Intensity and polarization will be decoupled in the noise matrix
madampars["allow_decoupling"] = True
demod = OpDemod(
name=name,
wkernel=args.demod_wkernel,
fmax=args.demod_fmax,
nskip=args.demod_nskip,
do_2f=args.demod_2f,
)
demod.exec(data)
timer.report_clear("Demodulate")
return
def apply_sim_sso(args, comm, data, mc, totalname, verbose=True):
if args.simulate_sso is None:
return
if args.beam_file is None:
raise RuntimeError("Cannot simulate SSOs without a beam file")
timer = Timer()
timer.start()
for sso_name in args.simulate_sso.split(","):
if comm.world_rank == 0 and verbose:
print("Simulating {}".format(sso_name), flush=True)
sim_sso = OpSimSSO(sso_name, args.beam_file, out=totalname)
sim_sso.exec(data)
if comm.world_rank == 0 and verbose:
timer.report_clear("Simulate {}".format(sso_name))
return
def load_focalplanes(args, comm, schedules, verbose=False):
""" Attach a focalplane to each of the schedules.
Args:
schedules (list) : List of Schedule instances.
Each schedule has two members, telescope
and ceslist, a list of CES objects.
Returns:
detweights (dict) : Inverse variance noise weights for every
detector across all focal planes. In [K_CMB^-2].
They can be used to bin the TOD.
"""
# log = Logger.get()
timer = Timer()
timer.start()
# Load focalplane information
timer1 = Timer()
timer1.start()
hw, telescope, det_index = get_hardware(args, comm, verbose=verbose)
focalplane = get_focalplane(args, comm, hw, det_index, verbose=verbose)
telescope.focalplane = focalplane
if comm.world_rank == 0 and verbose:
timer1.report_clear("Collect focaplane information")
for schedule in schedules:
# Replace the telescope created from reading the observing schedule but
# keep the weather object
weather = schedule.telescope.site.weather
schedule.telescope = telescope
schedule.telescope.site.weather = weather
detweights = telescope.focalplane.detweights
timer.stop()
if (comm.comm_world is None or comm.world_rank == 0) and verbose:
timer.report("Loading focalplane")
return detweights
def deconvolve_time_constant(args,
comm,
data,
name,
realization=0,
verbose=True):
if not args.tau_deconvolve:
return
log = Logger.get()
timer = Timer()
timer.start()
tauop = OpTimeConst(
name=name,
tau=args.tau_value,
inverse=True,
tau_sigma=args.tau_sigma,
realization=realization,
)
tauop.exec(data)
timer.report_clear("De-convolve time constant")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
print(
"Running with {} processes at {}".format(
procs, str(datetime.datetime.now())
)
)
parser = argparse.ArgumentParser(
description="Simple on-the-fly signal convolution + MADAM Mapmaking",
fromfile_prefix_chars="@",
)
parser.add_argument("--lmax", required=True, type=np.int, help="Simulation lmax")
parser.add_argument(
"--fwhm", required=True, type=np.float, help="Sky fwhm [arcmin] to deconvolve"
)
parser.add_argument("--beammmax", required=True, type=np.int, help="Beam mmax")
parser.add_argument("--order", default=11, type=np.int, help="Iteration order")
parser.add_argument(
"--pxx",
required=False,
default=False,
action="store_true",
help="Beams are in Pxx frame, not Dxx",
)
parser.add_argument(
"--normalize",
required=False,
default=False,
action="store_true",
help="Normalize the beams",
)
parser.add_argument(
"--skyfile",
required=True,
help="Path to sky alm files. Tag DETECTOR will be "
"replaced with detector name.",
)
parser.add_argument(
"--remove_monopole",
required=False,
default=False,
action="store_true",
help="Remove the sky monopole before convolution",
)
parser.add_argument(
"--remove_dipole",
required=False,
default=False,
action="store_true",
help="Remove the sky dipole before convolution",
)
parser.add_argument(
"--beamfile",
required=True,
help="Path to beam alm files. Tag DETECTOR will be "
"replaced with detector name.",
)
parser.add_argument("--rimo", required=True, help="RIMO file")
parser.add_argument("--freq", required=True, type=np.int, help="Frequency")
parser.add_argument(
"--dets", required=False, default=None, help="Detector list (comma separated)"
)
parser.add_argument(
"--effdir", required=True, help="Input Exchange Format File directory"
)
parser.add_argument(
"--effdir_pntg",
required=False,
help="Input Exchange Format File directory " "for pointing",
)
parser.add_argument(
"--effdir_out", required=False, help="Output directory for convolved TOD"
)
parser.add_argument(
"--obtmask", required=False, default=1, type=np.int, help="OBT flag mask"
)
parser.add_argument(
"--flagmask", required=False, default=1, type=np.int, help="Quality flag mask"
)
parser.add_argument("--ringdb", required=True, help="Ring DB file")
parser.add_argument(
"--odfirst", required=False, default=None, type=np.int, help="First OD to use"
)
parser.add_argument(
"--odlast", required=False, default=None, type=np.int, help="Last OD to use"
)
parser.add_argument(
"--ringfirst",
required=False,
default=None,
type=np.int,
help="First ring to use",
)
parser.add_argument(
"--ringlast", required=False, default=None, type=np.int, help="Last ring to use"
)
parser.add_argument(
"--obtfirst",
required=False,
default=None,
type=np.float,
help="First OBT to use",
)
parser.add_argument(
"--obtlast", required=False, default=None, type=np.float, help="Last OBT to use"
)
parser.add_argument("--madam_prefix", required=False, help="map prefix")
parser.add_argument(
"--madampar", required=False, default=None, help="Madam parameter file"
)
parser.add_argument(
"--obtmask_madam", required=False, type=np.int, help="OBT flag mask for Madam"
)
parser.add_argument(
"--flagmask_madam",
required=False,
type=np.int,
help="Quality flag mask for Madam",
)
parser.add_argument(
"--skip_madam",
required=False,
default=False,
action="store_true",
help="Do not run Madam on the convolved timelines",
)
parser.add_argument("--out", required=False, default=".", help="Output directory")
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
timer = Timer()
timer.start()
odrange = None
if args.odfirst is not None and args.odlast is not None:
odrange = (args.odfirst, args.odlast)
ringrange = None
if args.ringfirst is not None and args.ringlast is not None:
ringrange = (args.ringfirst, args.ringlast)
obtrange = None
if args.obtfirst is not None and args.obtlast is not None:
obtrange = (args.obtfirst, args.obtlast)
detectors = None
if args.dets is not None:
detectors = re.split(",", args.dets)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
# Ensure output directory exists
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.makedirs(args.out)
# Read in madam parameter file
# Allow more than one entry, gather into a list
repeated_keys = ["detset", "detset_nopol", "survey"]
pars = {}
if comm.comm_world.rank == 0:
pars["kfirst"] = False
pars["temperature_only"] = True
pars["base_first"] = 60.0
pars["nside_map"] = 512
pars["nside_cross"] = 512
pars["nside_submap"] = 16
pars["write_map"] = False
pars["write_binmap"] = True
pars["write_matrix"] = False
pars["write_wcov"] = False
pars["write_hits"] = True
pars["kfilter"] = False
pars["info"] = 3
if args.madampar:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if not comment.match(line):
result = pat.match(line)
if result:
key, value = result.group(1), result.group(2)
if key in repeated_keys:
if key not in pars:
pars[key] = []
pars[key].append(value)
else:
pars[key] = value
# Command line parameters override the ones in the madam parameter file
if "file_root" not in pars:
pars["file_root"] = "madam"
if args.madam_prefix is not None:
pars["file_root"] = args.madam_prefix
sfreq = "{:03}".format(args.freq)
if sfreq not in pars["file_root"]:
pars["file_root"] += "_" + sfreq
try:
fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq]
except Exception:
fsample = 180.3737
pars["fsample"] = fsample
pars["path_output"] = args.out
print("All parameters:")
print(args, flush=True)
pars = comm.comm_world.bcast(pars, root=0)
memreport("after parameters", MPI.COMM_WORLD)
# madam only supports a single observation. Normally
# we would have multiple observations with some subset
# assigned to each process group.
# create the TOD for this observation
tod = tp.Exchange(
comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
do_eff_cache=False,
)
# normally we would get the intervals from somewhere else, but since
# the Exchange TOD already had to get that information, we can
# get it from there.
ob = {}
ob["name"] = "mission"
ob["id"] = 0
ob["tod"] = tod
ob["intervals"] = tod.valid_intervals
ob["baselines"] = None
ob["noise"] = tod.noise
# Add the bare minimum focal plane information for the conviqt operator
focalplane = {}
for det in tod.detectors:
if args.pxx:
# Beam is in the polarization basis.
# No extra rotations are needed
psipol = tod.rimo[det].psi_pol
else:
# Beam is in the detector basis. Convolver needs to remove
# the last rotation into the polarization sensitive frame.
psipol = tod.rimo[det].psi_uv + tod.rimo[det].psi_pol
focalplane[det] = {
"pol_leakage" : tod.rimo[det].epsilon,
"pol_angle_deg" : psipol,
}
ob["focalplane"] = focalplane
data.obs.append(ob)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Metadata queries")
loader = tp.OpInputPlanck(
commonflags_name="common_flags", flags_name="flags", margin=0
)
loader.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Data read and cache")
tod.cache.report()
memreport("after loading", mpiworld)
# make a planck Healpix pointing matrix
mode = "IQU"
if pars["temperature_only"] == "T":
mode = "I"
nside = int(pars["nside_map"])
pointing = tp.OpPointingPlanck(
nside=nside,
mode=mode,
RIMO=tod.RIMO,
margin=0,
apply_flags=False,
keep_vel=False,
keep_pos=False,
keep_phase=False,
keep_quats=True,
)
pointing.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Pointing Matrix took, mode = {}".format(mode))
memreport("after pointing", mpiworld)
# simulate the TOD by convolving the sky with the beams
if comm.comm_world.rank == 0:
print("Convolving TOD", flush=True)
for pattern in args.beamfile.split(","):
skyfiles = {}
beamfiles = {}
for det in tod.detectors:
freq = "{:03}".format(tp.utilities.det2freq(det))
if "LFI" in det:
psmdet = "{}_{}".format(freq, det[3:])
if det.endswith("M"):
arm = "y"
else:
arm = "x"
graspdet = "{}_{}_{}".format(freq[1:], det[3:5], arm)
else:
psmdet = det.replace("-", "_")
graspdet = det
skyfile = (
args.skyfile.replace("FREQ", freq)
.replace("PSMDETECTOR", psmdet)
.replace("DETECTOR", det)
)
skyfiles[det] = skyfile
beamfile = pattern.replace("GRASPDETECTOR", graspdet).replace(
"DETECTOR", det
)
beamfiles[det] = beamfile
if comm.comm_world.rank == 0:
print("Convolving {} with {}".format(skyfile, beamfile), flush=True)
conviqt = OpSimConviqt(
comm.comm_world,
skyfiles,
beamfiles,
lmax=args.lmax,
beammmax=args.beammmax,
pol=True,
fwhm=args.fwhm,
order=args.order,
calibrate=True,
dxx=True,
out="conviqt_tod",
apply_flags=False,
remove_monopole=args.remove_monopole,
remove_dipole=args.remove_dipole,
verbosity=1,
normalize_beam=args.normalize,
)
conviqt.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Convolution")
memreport("after conviqt", mpiworld)
if args.effdir_out is not None:
if comm.comm_world.rank == 0:
print("Writing TOD", flush=True)
tod.set_effdir_out(args.effdir_out, None)
writer = tp.OpOutputPlanck(
signal_name="conviqt_tod",
flags_name="flags",
commonflags_name="common_flags",
)
writer.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Conviqt output")
memreport("after writing", mpiworld)
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
net = tod.rimo[d].net
fsample = tod.rimo[d].fsample
detweights[d] = 1.0 / (fsample * net * net)
if not args.skip_madam:
if comm.comm_world.rank == 0:
print("Calling Madam", flush=True)
try:
if args.obtmask_madam is None:
obtmask = args.obtmask
else:
obtmask = args.obtmask_madam
if args.flagmask_madam is None:
flagmask = args.flagmask
else:
flagmask = args.flagmask_madam
madam = OpMadam(
params=pars,
detweights=detweights,
name="conviqt_tod",
flag_name="flags",
purge=True,
name_out="madam_tod",
common_flag_mask=obtmask,
flag_mask=flagmask,
)
except Exception as e:
raise Exception(
"{:4} : ERROR: failed to initialize Madam: {}".format(
comm.comm_world.rank, e
)
)
madam.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Madam took {:.3f} s")
memreport("after madam", mpiworld)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking',
fromfile_prefix_chars='@')
parser.add_argument('--skip_madam',
dest='skip_madam',
default=False,
action='store_true',
help='D not make maps with Madam.')
parser.add_argument('--skip_noise',
dest='skip_noise',
default=False,
action='store_true',
help='Do not add simulated noise to the TOD.')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help='Write data distribution info to file')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir2',
required=False,
help='Additional input Exchange Format File directory')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory for '
'pointing')
parser.add_argument('--effdir_fsl',
required=False,
help='Input Exchange Format File directory for '
'straylight')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Which OBT flag bits to raise for HCM maneuvers')
parser.add_argument('--bad_intervals',
required=False,
help='Path to bad interval file.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use')
parser.add_argument('--obtfirst',
required=False,
default=None,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
help='Last OBT to use')
parser.add_argument('--read_eff',
dest='read_eff',
default=False,
action='store_true',
help='Read and co-add the signal from effdir')
parser.add_argument('--decalibrate',
required=False,
help='Path to calibration file to decalibrate with. '
'You can use python string formatting, assuming '
'.format(mc)')
parser.add_argument('--calibrate',
required=False,
help='Path to calibration file to calibrate with. '
'You can use python string formatting, assuming '
'.format(mc)')
parser.add_argument('--madampar',
required=False,
default=None,
help='Madam parameter file')
parser.add_argument('--nside',
required=False,
default=None,
type=np.int,
help='Madam resolution')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--madam_prefix', required=False, help='map prefix')
parser.add_argument('--make_rings',
dest='make_rings',
default=False,
action='store_true',
help='Compile ringsets.')
parser.add_argument('--nside_ring',
required=False,
default=128,
type=np.int,
help='Ringset resolution')
parser.add_argument('--ring_root',
required=False,
default='ringset',
help='Root filename for ringsets (setting to empty '
'disables ringset output).')
parser.add_argument('--MC_start',
required=False,
default=0,
type=np.int,
help='First Monte Carlo noise realization')
parser.add_argument('--MC_count',
required=False,
default=1,
type=np.int,
help='Number of Monte Carlo noise realizations')
# noise parameters
parser.add_argument('--noisefile',
required=False,
default='RIMO',
help='Path to noise PSD files for noise filter. '
'Tag DETECTOR will be replaced with detector name.')
parser.add_argument('--noisefile_simu',
required=False,
default='RIMO',
help='Path to noise PSD files for noise simulation. '
'Tag DETECTOR will be replaced with detector name.')
# Dipole parameters
dipogroup = parser.add_mutually_exclusive_group()
dipogroup.add_argument('--dipole',
dest='dipole',
required=False,
default=False,
action='store_true',
help='Simulate dipole')
dipogroup.add_argument('--solsys_dipole',
dest='solsys_dipole',
required=False,
default=False,
action='store_true',
help='Simulate solar system dipole')
dipogroup.add_argument('--orbital_dipole',
dest='orbital_dipole',
required=False,
default=False,
action='store_true',
help='Simulate orbital dipole')
dipo_parameters_group = parser.add_argument_group('dipole_parameters')
dipo_parameters_group.add_argument(
'--solsys_speed',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_speed"],
help='Solar system speed wrt. CMB rest frame in km/s. Default is '
'Planck 2015 best fit value')
dipo_parameters_group.add_argument(
'--solsys_glon',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_glon"],
help='Solar system velocity direction longitude in degrees')
dipo_parameters_group.add_argument(
'--solsys_glat',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_glat"],
help='Solar system velocity direction latitude in degrees')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.world_rank == 0:
print('All parameters:')
print(args, flush=True)
if args.MC_count < 1:
raise RuntimeError('MC_count = {} < 1. Nothing done.'
''.format(args.MC_count))
timer = Timer()
timer.start()
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# create the TOD for this observation
if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO':
do_eff_cache = True
else:
do_eff_cache = False
tods = []
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
# create the TOD for this observation
tods.append(
tp.Exchange(comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
extra_effdirs=[args.effdir2, args.effdir_fsl],
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
pntflagmask=args.pntflagmask,
do_eff_cache=do_eff_cache))
# Make output directory
if not os.path.isdir(args.out) and comm.world_rank == 0:
os.makedirs(args.out)
# Read in madam parameter file
# Allow more than one entry, gather into a list
repeated_keys = ['detset', 'detset_nopol', 'survey']
pars = {}
if comm.world_rank == 0:
pars['kfirst'] = False
pars['temperature_only'] = True
pars['base_first'] = 60.0
pars['nside_submap'] = 16
pars['write_map'] = False
pars['write_binmap'] = True
pars['write_matrix'] = False
pars['write_wcov'] = False
pars['write_hits'] = True
pars['kfilter'] = False
pars['info'] = 3
if args.madampar:
pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*')
comment = re.compile(r'^#.*')
with open(args.madampar, 'r') as f:
for line in f:
if not comment.match(line):
result = pat.match(line)
if result:
key, value = result.group(1), result.group(2)
if key in repeated_keys:
if key not in pars:
pars[key] = []
pars[key].append(value)
else:
pars[key] = value
# Command line parameters override the ones in the madam parameter file
if 'file_root' not in pars:
pars['file_root'] = 'madam'
if args.madam_prefix is not None:
pars['file_root'] = args.madam_prefix
sfreq = '{:03}'.format(args.freq)
if sfreq not in pars['file_root']:
pars['file_root'] += '_' + sfreq
try:
fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq]
except Exception:
fsample = 180.3737
pars['fsample'] = fsample
pars['path_output'] = args.out
pars = comm.comm_world.bcast(pars, root=0)
madam_mcmode = True
if 'nsubchunk' in pars and int(pars['nsubchunk']) > 1:
madam_mcmode = False
if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO':
# We split MPI_COMM_WORLD into single process groups, each of
# which is assigned one or more observations (rings)
comm = toast.Comm(groupsize=1)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
for iobs, tod in enumerate(tods):
if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO':
# Use a toast helper method to optimally distribute rings between
# processes.
dist = distribute_discrete(tod.ringsizes, comm.world_size)
my_first_ring, my_n_ring = dist[comm.world_rank]
for my_ring in range(my_first_ring, my_first_ring + my_n_ring):
ringtod = tp.Exchange.from_tod(
tod,
my_ring,
comm.comm_group,
noisefile=args.noisefile,
noisefile_simu=args.noisefile_simu)
ob = {}
ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring)
ob['id'] = ringtod.globalfirst_ring
ob['tod'] = ringtod
ob['intervals'] = ringtod.valid_intervals
ob['baselines'] = None
ob['noise'] = ringtod.noise
ob['noise_simu'] = ringtod.noise_simu
data.obs.append(ob)
else:
ob = {}
ob['name'] = 'observation{:04}'.format(iobs)
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
ob['noise_simu'] = tod.noise
data.obs.append(ob)
rimo = tods[0].rimo
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Metadata queries")
# Always read the signal and flags, even if the signal is later
# overwritten. There is no overhead for the signal because it is
# interlaced with the flags.
tod_name = 'signal'
timestamps_name = 'timestamps'
flags_name = 'flags'
common_flags_name = 'common_flags'
reader = tp.OpInputPlanck(signal_name=tod_name,
flags_name=flags_name,
timestamps_name=timestamps_name,
commonflags_name=common_flags_name)
if comm.world_rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Read")
# Clear the signal if we don't need it
if not args.read_eff:
eraser = tp.OpCacheMath(in1=tod_name,
in2=0,
multiply=True,
out=tod_name)
if comm.world_rank == 0:
print('Erasing TOD', flush=True)
eraser.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Erase")
# Optionally flag bad intervals
if args.bad_intervals is not None:
flagger = tp.OpBadIntervals(path=args.bad_intervals)
flagger.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Apply {}".format(args.bad_intervals))
# Now read an optional second TOD to add with the first
if args.effdir2 is not None:
# Read the extra TOD and add it to the first one
reader = tp.OpInputPlanck(signal_name='tod2',
flags_name=None,
timestamps_name=None,
commonflags_name=None,
effdir=args.effdir2)
if comm.world_rank == 0:
print('Reading extra TOD from {}'.format(args.effdir2), flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
print("Reading took {:.3f} s".format(elapsed), flush=True)
adder = tp.OpCacheMath(in1=tod_name,
in2='signal2',
add=True,
out=tod_name)
if comm.world_rank == 0:
print('Adding TODs', flush=True)
adder.exec(data)
# Erase the extra cache object
for ob in data.obs:
tod = ob['tod']
tod.cache.clear('signal2_.*')
if args.effdir_fsl is not None:
# Read the straylight signal into the tod cache under
# "fsl_<detector>"
reader = tp.OpInputPlanck(signal_name='fsl',
flags_name=None,
timestamps_name=None,
commonflags_name=None,
effdir=args.effdir_fsl)
if comm.world_rank == 0:
print('Reading straylight signal from {}'.format(args.effdir_fsl),
flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Read FSL")
do_fsl = True
else:
do_fsl = False
# make a planck Healpix pointing matrix
mode = 'IQU'
if pars['temperature_only'] == 'T':
mode = 'I'
if args.nside is None:
if 'nside_map' in pars:
nside = int(pars['nside_map'])
else:
raise RuntimeError(
'Nside must be set either in the Madam parameter file or on '
'the command line')
else:
nside = args.nside
pars['nside_map'] = nside
if 'nside_cross' not in pars or pars['nside_cross'] > pars['nside_map']:
pars['nside_cross'] = pars['nside_map']
do_dipole = args.dipole or args.solsys_dipole or args.orbital_dipole
pointing = tp.OpPointingPlanck(nside=nside,
mode=mode,
RIMO=rimo,
margin=0,
apply_flags=True,
keep_vel=do_dipole,
keep_pos=False,
keep_phase=False,
keep_quats=do_dipole)
pointing.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Pointing Matrix")
flags_name = 'flags'
common_flags_name = 'common_flags'
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
net = tod.rimo[d].net
fsample = tod.rimo[d].fsample
detweights[d] = 1.0 / (fsample * net * net)
if args.debug:
with open("debug_planck_exchange_madam.txt", "w") as f:
data.info(f)
if do_dipole:
# Simulate the dipole
if args.dipole:
dipomode = 'total'
elif args.solsys_dipole:
dipomode = 'solsys'
else:
dipomode = 'orbital'
dipo = tp.OpDipolePlanck(args.freq,
solsys_speed=args.solsys_speed,
solsys_glon=args.solsys_glon,
solsys_glat=args.solsys_glat,
mode=dipomode,
output='dipole',
keep_quats=False)
dipo.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Dipole")
# Loop over Monte Carlos
madam = None
for mc in range(args.MC_start, args.MC_start + args.MC_count):
out = "{}/{:05d}".format(args.out, mc)
if comm.world_rank == 0:
if not os.path.isdir(out):
os.makedirs(out)
# clear all noise data from the cache, so that we can generate
# new noise timestreams.
for ob in data.obs:
ob['tod'].cache.clear("noise_.*")
tod_name = 'signal'
if do_dipole:
adder = tp.OpCacheMath(in1=tod_name,
in2='dipole',
add=True,
out='noise')
adder.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Add dipole".format(mc))
tod_name = 'noise'
# Simulate noise
if not args.skip_noise:
tod_name = 'noise'
nse = toast.tod.OpSimNoise(out=tod_name,
realization=mc,
component=0,
noise='noise_simu',
rate=fsample)
if comm.world_rank == 0:
print('Simulating noise from {}'.format(args.noisefile_simu),
flush=True)
nse.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Noise simulation".format(mc))
# If we didn't add the dipole, we need to add the input
# signal with the noise we just simulated
if args.read_eff and not do_dipole:
adder = tp.OpCacheMath(in1=tod_name,
in2='signal',
add=True,
out=tod_name)
adder.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Add input signal".format(mc))
# Make rings
if args.make_rings:
ringmaker = tp.OpRingMaker(args.nside_ring,
nside,
signal=tod_name,
fileroot=args.ring_root,
out=out,
commonmask=args.obtmask,
detmask=args.flagmask)
ringmaker.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Ringmaking".format(mc))
# Apply calibration errors
if args.decalibrate is not None:
fn = args.decalibrate
try:
fn = fn.format(mc)
except Exception:
pass
if comm.world_rank == 0:
print('Decalibrating with {}'.format(fn), flush=True)
decalibrator = tp.OpCalibPlanck(signal_in=tod_name,
signal_out='noise',
file_gain=fn,
decalibrate=True)
decalibrator.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Decalibrate".format(mc))
tod_name = 'noise'
if args.calibrate is not None:
fn = args.calibrate
try:
fn = fn.format(mc)
except Exception:
pass
if comm.world_rank == 0:
print('Calibrating with {}'.format(fn), flush=True)
calibrator = tp.OpCalibPlanck(signal_in=tod_name,
signal_out='noise',
file_gain=fn)
calibrator.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Calibrate".format(mc))
tod_name = 'noise'
# Subtract the dipole and straylight
if do_dipole:
subtractor = tp.OpCacheMath(in1=tod_name,
in2='dipole',
subtract=True,
out='noise')
subtractor.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Subtract dipole".format(mc))
tod_name = 'noise'
if do_fsl:
subtractor = tp.OpCacheMath(in1=tod_name,
in2='fsl',
subtract=True,
out='noise')
subtractor.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Subtract straylight".format(mc))
tod_name = 'noise'
# Make the map
if not args.skip_madam:
# Make maps
if madam is None:
try:
madam = toast.todmap.OpMadam(params=pars,
detweights=detweights,
purge_tod=True,
name=tod_name,
apply_flags=False,
name_out=None,
noise='noise',
mcmode=madam_mcmode,
translate_timestamps=False)
except Exception as e:
raise Exception(
'{:4} : ERROR: failed to initialize Madam: '
'{}'.format(comm.world_rank, e))
madam.params['path_output'] = out
madam.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Mapmaking".format(mc))
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def apply_mappraiser(
args,
comm,
data,
params,
signalname,
noisename,
time_comms=None,
telescope_data=None,
verbose=True,
):
""" Use libmappraiser to run the ML map-making
Args:
time_comms (iterable) : Series of disjoint communicators that
map, e.g., seasons and days. Each entry is a tuple of
the form (`name`, `communicator`)
telescope_data (iterable) : series of disjoint TOAST data
objects. Each entry is tuple of the form (`name`, `data`).
"""
if comm.comm_world is None:
raise RuntimeError("Mappraiser requires MPI")
log = Logger.get()
total_timer = Timer()
total_timer.start()
if comm.world_rank == 0 and verbose:
log.info("Making maps")
mappraiser = OpMappraiser(
params= params,
purge=True,
name=signalname,
noise_name = noisename,
conserve_memory=args.conserve_memory,
)
if time_comms is None:
time_comms = [("all", comm.comm_world)]
if telescope_data is None:
telescope_data = [("all", data)]
timer = Timer()
for time_name, time_comm in time_comms:
for tele_name, tele_data in telescope_data:
if len(time_name.split("-")) == 3:
# Special rules for daily maps
if args.do_daymaps:
continue
if len(telescope_data) > 1 and tele_name == "all":
# Skip daily maps over multiple telescopes
continue
timer.start()
# N.B: code below is for Madam but may be useful to copy in Mappraiser
# once we start doing multiple maps in one run
# madam.params["file_root"] = "{}_telescope_{}_time_{}".format(
# file_root, tele_name, time_name
# )
# if time_comm == comm.comm_world:
# madam.params["info"] = info
# else:
# # Cannot have verbose output from concurrent mapmaking
# madam.params["info"] = 0
# if (time_comm is None or time_comm.rank == 0) and verbose:
# log.info("Mapping {}".format(madam.params["file_root"]))
mappraiser.exec(tele_data, time_comm)
if time_comm is not None:
time_comm.barrier()
if comm.world_rank == 0 and verbose:
timer.report_clear("Mapping {}_telescope_{}_time_{}".format(
args.outpath,
tele_name,
time_name,
))
if comm.comm_world is not None:
comm.comm_world.barrier()
total_timer.stop()
if comm.world_rank == 0 and verbose:
total_timer.report("Mappraiser total")
return
# normally we would get the intervals from somewhere else, but since
# the Exchange TOD already had to get that information, we can
# get it from there.
ob = {}
ob["name"] = "mission"
ob["id"] = 0
ob["tod"] = tod
ob["intervals"] = tod.valid_intervals
ob["baselines"] = None
ob["noise"] = tod.noise
data.obs.append(ob)
if comm.world_rank == 0:
timer.report_clear("Metadata queries")
ring_starts = [t.first for t in tod.valid_intervals]
ring_times = [t.start for t in tod.valid_intervals]
ring_lens = [(t.last - t.first) for t in tod.valid_intervals]
ring_offset = tod.globalfirst_ring
for interval in tod.valid_intervals:
if interval.last < tod.local_samples[0]:
ring_offset += 1
ring_number = ring_offset - 1
globalfirst = tod.globalfirst
ringtable = ringdb_table_name(args.freq)
def main():
env = Environment.get()
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_satellite_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = get_comm()
args, comm, groupsize = parse_arguments(comm, procs)
# Parse options
tmr = Timer()
tmr.start()
if comm.world_rank == 0:
os.makedirs(args.outdir, exist_ok=True)
focalplane, gain, detweights = load_focalplane(args, comm)
data = create_observations(args, comm, focalplane, groupsize)
expand_pointing(args, comm, data)
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = None
skyname = simulate_sky_signal(args, comm, data, [focalplane], subnpix,
localsm, "signal")
if skyname is not None:
signalname = skyname
diponame = simulate_dipole(args, comm, data, "signal")
if diponame is not None:
signalname = diponame
# Mapmaking.
if not args.use_madam:
if comm.world_rank == 0:
log.info("Not using Madam, will only make a binned map")
npp, zmap = init_binner(args,
comm,
data,
detweights,
subnpix=subnpix,
localsm=localsm)
# Loop over Monte Carlos
firstmc = args.MC_start
nmc = args.MC_count
for mc in range(firstmc, firstmc + nmc):
mctmr = Timer()
mctmr.start()
outpath = os.path.join(args.outdir, "mc_{:03d}".format(mc))
simulate_noise(args, comm, data, mc, "tot_signal", overwrite=True)
# add sky signal
add_signal(args, comm, data, "tot_signal", signalname)
if gain is not None:
timer = Timer()
timer.start()
op_apply_gain = OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
if comm.world_rank == 0:
timer.report_clear(" Apply gains {:04d}".format(mc))
if mc == firstmc:
# For the first realization, optionally export the
# timestream data. If we had observation intervals defined,
# we could pass "use_interval=True" to the export operators,
# which would ensure breaks in the exported data at
# acceptable places.
output_tidas(args, comm, data, "tot_signal")
output_spt3g(args, comm, data, "tot_signal")
apply_binner(args, comm, data, npp, zmap, detweights, outpath,
"tot_signal")
if comm.world_rank == 0:
mctmr.report_clear(" Map-making {:04d}".format(mc))
else:
# Initialize madam parameters
madampars = setup_madam(args)
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.world_rank == 0:
handle = open(os.path.join(args.outdir, "distdata.txt"), "w")
data.info(handle)
if comm.world_rank == 0:
handle.close()
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear("Dumping data distribution")
# Loop over Monte Carlos
firstmc = args.MC_start
nmc = args.MC_count
for mc in range(firstmc, firstmc + nmc):
mctmr = Timer()
mctmr.start()
# create output directory for this realization
outpath = os.path.join(args.outdir, "mc_{:03d}".format(mc))
simulate_noise(args, comm, data, mc, "tot_signal", overwrite=True)
# add sky signal
add_signal(args, comm, data, "tot_signal", signalname)
if gain is not None:
op_apply_gain = OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
tmr.report_clear(" Apply gains {:04d}".format(mc))
apply_madam(args, comm, data, madampars, outpath, detweights,
"tot_signal")
if comm.comm_world is not None:
comm.comm_world.barrier()
if comm.world_rank == 0:
mctmr.report_clear(" Map-making {:04d}".format(mc))
gt.stop_all()
if comm.comm_world is not None:
comm.comm_world.barrier()
tmr.stop()
tmr.clear()
tmr.start()
alltimers = gather_timers(comm=comm.comm_world)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
tmr.stop()
tmr.report("Gather and dump timing info")
timer0.report_clear("toast_satellite_sim.py")
return
def create_observations(args, comm, schedule):
"""Simulate constant elevation scans.
Simulate constant elevation scans at "site" matching entries in
"all_ces". Each operational day is assigned to a different
process group to allow making day maps.
"""
timer = Timer()
log = Logger.get()
data = Data(comm)
telescope = schedule.telescope
site = telescope.site
focalplane = telescope.focalplane
all_ces = schedule.ceslist
nces = len(all_ces)
breaks = get_breaks(comm, all_ces, args)
nbreak = len(breaks)
groupdist = distribute_uniform(nces, comm.ngroups, breaks=breaks)
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
if comm.comm_group is not None:
ndetrank = comm.comm_group.size
else:
ndetrank = 1
for ices in range(group_firstobs, group_firstobs + group_numobs):
ces = all_ces[ices]
totsamples = int((ces.stop_time - ces.start_time) * args.sample_rate)
# create the single TOD for this observation
try:
tod = TODGround(
comm.comm_group,
focalplane.detquats,
totsamples,
detranks=ndetrank,
firsttime=ces.start_time,
rate=args.sample_rate,
site_lon=site.lon,
site_lat=site.lat,
site_alt=site.alt,
azmin=ces.azmin,
azmax=ces.azmax,
el=ces.el,
scanrate=args.scan_rate,
scan_accel=args.scan_accel,
cosecant_modulation=args.scan_cosecant_modulate,
CES_start=None,
CES_stop=None,
sun_angle_min=args.sun_angle_min,
coord=args.coord,
sampsizes=None,
report_timing=args.debug,
)
except RuntimeError as e:
raise RuntimeError("Failed to create the CES scan: {}".format(e))
# Create the (single) observation
ob = {}
ob["name"] = "CES-{}-{}-{}".format(ces.name, ces.scan, ces.subscan)
ob["tod"] = tod
if len(tod.subscans) > 0:
ob["intervals"] = tod.subscans
else:
raise RuntimeError("{} has no valid intervals".format(ob["name"]))
ob["baselines"] = None
ob["noise"] = focalplane.noise
ob["id"] = int(ces.mjdstart * 10000)
data.obs.append(ob)
for ob in data.obs:
tod = ob["tod"]
tod.free_azel_quats()
if comm.comm_world is None or comm.comm_group.rank == 0:
log.info("Group # {:4} has {} observations.".format(
comm.group, len(data.obs)))
if len(data.obs) == 0:
raise RuntimeError("Too many tasks. Every MPI task must "
"be assigned to at least one observation.")
if comm.world_rank == 0:
timer.report_clear("Simulate scans")
return data
def parse_arguments(comm):
timer = Timer()
timer.start()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"atmosphere and make maps for some number of noise Monte Carlos.",
fromfile_prefix_chars="@",
)
toast_tools.add_dist_args(parser)
toast_tools.add_todground_args(parser)
toast_tools.add_pointing_args(parser)
toast_tools.add_polyfilter_args(parser)
toast_tools.add_groundfilter_args(parser)
toast_tools.add_atmosphere_args(parser)
toast_tools.add_noise_args(parser)
toast_tools.add_gainscrambler_args(parser)
toast_tools.add_madam_args(parser)
toast_tools.add_mapmaker_args(parser)
toast_tools.add_filterbin_args(parser)
toast_tools.add_sky_map_args(parser)
toast_tools.add_sss_args(parser)
toast_tools.add_tidas_args(parser)
toast_tools.add_mc_args(parser)
so_tools.add_corotator_args(parser)
so_tools.add_time_constant_args(parser)
so_tools.add_demodulation_args(parser)
so_tools.add_h_n_args(parser)
so_tools.add_crosslinking_args(parser)
so_tools.add_cadence_map_args(parser)
so_tools.add_hw_args(parser)
so_tools.add_so_noise_args(parser)
so_tools.add_pysm_args(parser)
so_tools.add_export_args(parser)
toast_tools.add_debug_args(parser)
so_tools.add_import_args(parser)
so_tools.add_sim_sso_args(parser)
so_tools.add_flag_sso_args(parser)
so_tools.add_sim_hwpss_args(parser)
parser.add_argument(
"--no-maps",
required=False,
default=False,
action="store_true",
help="Disable all mapmaking.",
)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument(
"--madam",
required=False,
action="store_true",
help="Use libmadam for map-making",
dest="use_madam",
)
parser.add_argument(
"--no-madam",
required=False,
action="store_false",
help="Do not use libmadam for map-making [default]",
dest="use_madam",
)
parser.set_defaults(use_madam=True)
try:
args = parser.parse_args()
except SystemExit as e:
sys.exit()
if len(args.bands.split(",")) != 1:
# Multi frequency run. We don't support multiple copies of
# scanned signal.
if args.input_map:
raise RuntimeError(
"Multiple frequencies are not supported when scanning from a map"
)
if args.weather is None:
raise RuntimeError("You must provide a TOAST weather file")
if comm.world_rank == 0:
log.info("\n")
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
log.info("\n")
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.world_rank == 0:
if not os.path.isdir(args.outdir):
try:
os.makedirs(args.outdir)
except FileExistsError:
pass
timer.report_clear("Parse arguments")
return args, comm
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
print('Running with {} processes at {}'
''.format(procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Planck Ringset making',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--nosingle',
dest='nosingle',
required=False,
default=False,
action='store_true',
help='Do not compute single detector PSDs')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory '
'for pointing')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--skymask', required=False, help='Pixel mask file')
parser.add_argument('--skymap', required=False, help='Sky estimate file')
parser.add_argument('--skypol',
dest='skypol',
required=False,
default=False,
action='store_true',
help='Sky estimate is polarized')
parser.add_argument('--no_spin_harmonics',
dest='no_spin_harmonics',
required=False,
default=False,
action='store_true',
help='Do not include PSD bins with spin harmonics')
parser.add_argument('--calibrate',
required=False,
help='Path to calibration file to calibrate with.')
parser.add_argument('--calibrate_signal_estimate',
dest='calibrate_signal_estimate',
required=False,
default=False,
action='store_true',
help='Calibrate '
'the signal estimate using linear regression.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
type=np.int,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
type=np.int,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
type=np.int,
help='First ring to use')
parser.add_argument('--ringlast',
required=False,
default=None,
type=np.int,
help='Last ring to use')
parser.add_argument('--obtfirst',
required=False,
default=None,
type=np.float,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
type=np.float,
help='Last OBT to use')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--nbin_psd',
required=False,
default=1000,
type=np.int,
help='Number of logarithmically '
'distributed spectral bins to write.')
parser.add_argument('--lagmax',
required=False,
default=100000,
type=np.int,
help='Maximum lag to evaluate for the '
'autocovariance function [samples].')
parser.add_argument('--stationary_period',
required=False,
default=86400.,
type=np.float,
help='Length of a stationary interval [seconds].')
# Dipole parameters
dipogroup = parser.add_mutually_exclusive_group()
dipogroup.add_argument('--dipole',
dest='dipole',
required=False,
default=False,
action='store_true',
help='Simulate dipole')
dipogroup.add_argument('--solsys_dipole',
dest='solsys_dipole',
required=False,
default=False,
action='store_true',
help='Simulate solar system dipole')
dipogroup.add_argument('--orbital_dipole',
dest='orbital_dipole',
required=False,
default=False,
action='store_true',
help='Simulate orbital dipole')
# Extra filter
parser.add_argument('--filterfile',
required=False,
help='Extra filter file.')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.comm_world.rank == 0:
print('All parameters:')
print(args, flush=True)
timer = Timer()
timer.start()
odrange = None
if args.odfirst is not None and args.odlast is not None:
odrange = (args.odfirst, args.odlast)
ringrange = None
if args.ringfirst is not None and args.ringlast is not None:
ringrange = (args.ringfirst, args.ringlast)
obtrange = None
if args.obtfirst is not None and args.obtlast is not None:
obtrange = (args.obtfirst, args.obtlast)
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
if args.nosingle and len(detectors) != 2:
raise RuntimeError('You cannot skip the single detectors PSDs '
'without multiple detectors.')
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
# Make output directory
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.mkdir(args.out)
# create the TOD for this observation
tod = tp.Exchange(
comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
do_eff_cache=False,
)
rimo = tod.rimo
ob = {}
ob['name'] = 'mission'
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
data.obs.append(ob)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Metadata queries")
# Read the signal
tod_name = 'signal'
flags_name = 'flags'
reader = tp.OpInputPlanck(signal_name=tod_name, flags_name=flags_name)
if comm.comm_world.rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Reading")
if args.calibrate is not None:
fn = args.calibrate
if comm.comm_world.rank == 0:
print('Calibrating with {}'.format(fn), flush=True)
calibrator = tp.OpCalibPlanck(signal_in=tod_name,
signal_out=tod_name,
file_gain=fn)
calibrator.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Calibrate")
# Optionally subtract the dipole
do_dipole = (args.dipole or args.solsys_dipole or args.orbital_dipole)
if do_dipole:
if args.dipole:
dipomode = 'total'
elif args.solsys_dipole:
dipomode = 'solsys'
else:
dipomode = 'orbital'
dipo = tp.OpDipolePlanck(args.freq,
mode=dipomode,
output='dipole',
keep_quats=True)
dipo.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Dipole")
subtractor = tp.OpCacheMath(in1=tod_name,
in2='dipole',
subtract=True,
out=tod_name)
if comm.comm_world.rank == 0:
print('Subtracting dipole', flush=True)
subtractor.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Dipole subtraction")
# Optionally filter the signal
apply_filter(args, data)
timer.clear()
# Estimate noise
noise_estimator = tp.OpNoiseEstim(
signal=tod_name,
flags=flags_name,
detmask=args.flagmask,
commonmask=args.obtmask,
maskfile=args.skymask,
mapfile=args.skymap,
out=args.out,
rimo=rimo,
pol=args.skypol,
nbin_psd=args.nbin_psd,
lagmax=args.lagmax,
stationary_period=args.stationary_period,
nosingle=args.nosingle,
no_spin_harmonics=args.no_spin_harmonics,
calibrate_signal_estimate=args.calibrate_signal_estimate)
noise_estimator.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Noise estimation")
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def get_analytic_noise(args, comm, focalplane, verbose=True):
""" Create a TOAST noise object.
Create a noise object from the 1/f noise parameters contained in the
focalplane database. Optionally add thermal common modes.
"""
timer = Timer()
timer.start()
detectors = sorted(focalplane.detector_data.keys())
fmins = {}
fknees = {}
alphas = {}
NETs = {}
rates = {}
indices = {}
for d in detectors:
rates[d] = args.sample_rate
fmins[d] = focalplane[d]["fmin"]
fknees[d] = focalplane[d]["fknee"]
alphas[d] = focalplane[d]["alpha"]
NETs[d] = focalplane[d]["NET"]
indices[d] = focalplane[d]["index"]
ncommon = 0
coupling_strength_distributions = []
common_modes = []
if args.common_mode_noise:
# Add an extra "virtual" detector for common mode noise for
# every optics tube
for common_mode in args.common_mode_noise.split(";"):
ncommon += 1
try:
fmin, fknee, alpha, net, center, width = np.array(
common_mode.split(",")).astype(np.float64)
except ValueError:
fmin, fknee, alpha, net = np.array(
common_mode.split(",")).astype(np.float64)
center, width = 1, 0
coupling_strength_distributions.append([center, width])
hw = get_example()
for itube, tube_slot in enumerate(
sorted(hw.data["tube_slots"].keys())):
d = "common_mode_{}_{}".format(ncommon - 1, tube_slot)
detectors.append(d)
common_modes.append(d)
rates[d] = args.sample_rate
fmins[d] = fmin
fknees[d] = fknee
alphas[d] = alpha
NETs[d] = net
indices[d] = ncommon * 100000 + itube
noise = AnalyticNoise(
rate=rates,
fmin=fmins,
detectors=detectors,
fknee=fknees,
alpha=alphas,
NET=NETs,
indices=indices,
)
if args.common_mode_noise:
mixmatrix = {}
keys = set()
if args.common_mode_only:
detweight = 0
else:
detweight = 1
for icommon in range(ncommon):
# Update the mixing matrix in the noise operator
center, width = coupling_strength_distributions[icommon]
np.random.seed(1001 + icommon)
couplings = center + np.random.randn(1000000) * width
for det in focalplane.detector_data.keys():
if det not in mixmatrix:
mixmatrix[det] = {det: detweight}
keys.add(det)
tube_slot = focalplane[det]["tube_slot"]
common = "common_mode_{}_{}".format(icommon, tube_slot)
index = focalplane[det]["index"]
mixmatrix[det][common] = couplings[index]
keys.add(common)
# Add a diagonal entries, even if we wouldn't usually ask for
# the common mode alone.
for common in common_modes:
mixmatrix[common] = {common: 1}
# There should probably be an accessor method to update the
# mixmatrix in the TOAST Noise object.
if noise._mixmatrix is not None:
raise RuntimeError("Did not expect non-empty mixing matrix")
noise._mixmatrix = mixmatrix
noise._keys = list(sorted(keys))
focalplane._noise = noise
if comm.world_rank == 0 and verbose:
timer.report_clear("Creating noise model")
return noise
def create_observations(args, comm, focalplane, groupsize):
timer = Timer()
timer.start()
if groupsize > len(focalplane.keys()):
if comm.world_rank == 0:
log.error("process group is too large for the number of detectors")
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(focalplane.keys())
detquats = {}
detindx = None
if "index" in focalplane[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = focalplane[d]["quat"]
if detindx is not None:
detindx[d] = focalplane[d]["index"]
# Distribute the observations uniformly
groupdist = distribute_uniform(args.obs_num, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = regular_intervals(
args.obs_num,
args.start_time,
0,
args.sample_rate,
3600 * args.obs_time_h,
3600 * args.gap_h,
)
noise = get_analytic_noise(args, comm, focalplane)
# The distributed timestream data
data = Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
coord=args.coord,
firstsamp=obsrange[ob].first,
firsttime=obsrange[ob].start,
rate=args.sample_rate,
spinperiod=args.spin_period_min,
spinangle=args.spin_angle_deg,
precperiod=args.prec_period_min,
precangle=args.prec_angle_deg,
detindx=detindx,
detranks=comm.group_size,
hwprpm=hwprpm,
hwpstep=hwpstep,
hwpsteptime=hwpsteptime,
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
if comm.world_rank == 0:
timer.report_clear("Read parameters, compute data distribution")
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = np.empty(4 * tod.local_samples[1],
dtype=np.float64).reshape((-1, 4))
slew_precession_axis(
precquat,
firstsamp=(obsoffset + tod.local_samples[0]),
samplerate=args.sample_rate,
degday=degday,
)
tod.set_prec_axis(qprec=precquat)
del precquat
if comm.world_rank == 0:
timer.report_clear("Construct boresight pointing")
return data
def parse_arguments(comm):
timer = Timer()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"and map astrophysical signal.",
fromfile_prefix_chars="@",
)
add_dist_args(parser)
add_debug_args(parser)
add_todground_args(parser)
add_pointing_args(parser)
add_polyfilter_args(parser)
add_groundfilter_args(parser)
add_gainscrambler_args(parser)
add_noise_args(parser)
add_sky_map_args(parser)
add_tidas_args(parser)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
add_madam_args(parser)
add_binner_args(parser)
parser.add_argument(
"--madam",
required=False,
action="store_true",
help="Use libmadam for map-making",
dest="use_madam",
)
parser.add_argument(
"--no-madam",
required=False,
action="store_false",
help="Do not use libmadam for map-making [default]",
dest="use_madam",
)
parser.set_defaults(use_madam=False)
parser.add_argument(
"--focalplane",
required=False,
default=None,
help="Pickle file containing a dictionary of detector "
"properties. The keys of this dict are the detector "
"names, and each value is also a dictionary with keys "
'"quat" (4 element ndarray), "fwhm" (float, arcmin), '
'"fknee" (float, Hz), "alpha" (float), and '
'"NET" (float). For optional plotting, the key "color"'
" can specify a valid matplotlib color string.",
)
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if args.tidas is not None:
if not tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
if comm.comm_world is None or comm.world_rank == 0:
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.comm_world is None or comm.comm_world.rank == 0:
os.makedirs(args.outdir, exist_ok=True)
if comm.comm_world is None or comm.world_rank == 0:
timer.report_clear("Parsed parameters")
return args, comm
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_ground_sim (total)")
mpiworld, procs, rank, comm = get_comm()
args, comm = parse_arguments(comm)
# Initialize madam parameters
madampars = setup_madam(args)
# Load and broadcast the schedule file
schedule = load_schedule(args, comm)[0]
# load or simulate the focalplane
detweights = load_focalplane(args, comm, schedule)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data = create_observations(args, comm, schedule)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
expand_pointing(args, comm, data)
# Scan input map
signalname = scan_sky_signal(args, comm, data, "signal")
# Simulate noise
if signalname is None:
signalname = "signal"
mc = 0
simulate_noise(args, comm, data, mc, signalname)
# Set up objects to take copies of the TOD at appropriate times
signalname_madam, sigcopy_madam, sigclear = setup_sigcopy(
args, comm, signalname)
npp, zmap = init_binner(args, comm, data, detweights)
output_tidas(args, comm, data, signalname)
outpath = setup_output(args, comm)
# Make a copy of the signal for Madam
copy_signal_madam(args, comm, data, sigcopy_madam)
# Bin unprocessed signal for reference
apply_binner(args, comm, data, npp, zmap, detweights, outpath, signalname)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
apply_polyfilter(args, comm, data, signalname)
apply_groundfilter(args, comm, data, signalname)
# Bin the filtered signal
apply_binner(
args,
comm,
data,
npp,
zmap,
detweights,
outpath,
signalname,
prefix="filtered",
)
data.obs[0]["tod"].cache.report()
clear_signal(args, comm, data, sigclear)
data.obs[0]["tod"].cache.report()
# Now run Madam on the unprocessed copy of the signal
if args.use_madam:
apply_madam(args, comm, data, madampars, outpath, detweights,
signalname_madam)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.report_clear("Gather and dump timing info")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_ground_sim (total)")
timer0 = Timer()
timer0.start()
mpiworld, procs, rank, comm = get_comm()
args, comm = parse_arguments(comm)
# Initialize madam parameters
madampars = setup_madam(args)
# Load and broadcast the schedule file
schedules = load_schedule(args, comm)
# Load the weather and append to schedules
load_weather(args, comm, schedules)
# load or simulate the focalplane
detweights = load_focalplanes(args, comm, schedules)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data, telescope_data = create_observations(args, comm, schedules)
# Split the communicator for day and season mapmaking
time_comms = get_time_communicators(args, comm, data)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
expand_pointing(args, comm, data)
# Purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if (args.tidas is None) and (args.spt3g is None):
for ob in data.obs:
tod = ob["tod"]
tod.free_radec_quats()
# Prepare auxiliary information for distributed map objects
_, localsm, subnpix = get_submaps(args, comm, data)
if args.pysm_model:
focalplanes = [s.telescope.focalplane.detector_data for s in schedules]
signalname = simulate_sky_signal(args, comm, data, focalplanes,
subnpix, localsm, "signal")
else:
signalname = scan_sky_signal(args, comm, data, localsm, subnpix,
"signal")
# Set up objects to take copies of the TOD at appropriate times
totalname, totalname_freq = setup_sigcopy(args)
# Loop over Monte Carlos
firstmc = args.MC_start
nsimu = args.MC_count
freqs = [float(freq) for freq in args.freq.split(",")]
nfreq = len(freqs)
for mc in range(firstmc, firstmc + nsimu):
simulate_atmosphere(args, comm, data, mc, totalname)
# Loop over frequencies with identical focal planes and identical
# atmospheric noise.
for ifreq, freq in enumerate(freqs):
if comm.world_rank == 0:
log.info("Processing frequency {}GHz {} / {}, MC = {}".format(
freq, ifreq + 1, nfreq, mc))
# Make a copy of the atmosphere so we can scramble the gains and apply
# frequency-dependent scaling.
copy_signal(args, comm, data, totalname, totalname_freq)
scale_atmosphere_by_frequency(args,
comm,
data,
freq=freq,
mc=mc,
cache_name=totalname_freq)
update_atmospheric_noise_weights(args, comm, data, freq, mc)
# Add previously simulated sky signal to the atmospheric noise.
add_signal(args,
comm,
data,
totalname_freq,
signalname,
purge=(nsimu == 1))
mcoffset = ifreq * 1000000
simulate_noise(args, comm, data, mc + mcoffset, totalname_freq)
simulate_sss(args, comm, data, mc + mcoffset, totalname_freq)
scramble_gains(args, comm, data, mc + mcoffset, totalname_freq)
if (mc == firstmc) and (ifreq == 0):
# For the first realization and frequency, optionally
# export the timestream data.
output_tidas(args, comm, data, totalname)
output_spt3g(args, comm, data, totalname)
outpath = setup_output(args, comm, mc + mcoffset, freq)
# Bin and destripe maps
apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname_freq,
freq=freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=(mc == firstmc),
)
if args.apply_polyfilter or args.apply_groundfilter:
# Filter signal
apply_polyfilter(args, comm, data, totalname_freq)
apply_groundfilter(args, comm, data, totalname_freq)
# Bin filtered maps
apply_madam(
args,
comm,
data,
madampars,
outpath,
detweights,
totalname_freq,
freq=freq,
time_comms=time_comms,
telescope_data=telescope_data,
first_call=False,
extra_prefix="filtered",
bin_only=True,
)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.outdir, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
timer0.report_clear("toast_ground_sim.py")
return
