def apply_flag_sso(args, comm, data, verbose=True): if args.flag_sso is None: return if comm.world_rank == 0 and verbose: print(f"Flagging SSO:s", flush=True) timer = Timer() timer.start() sso_names = [] sso_radii = [] for arg in args.flag_sso: sso_name, sso_radius = arg.split(",") sso_radius = np.radians(float(sso_radius) / 60) sso_names.append(sso_name) sso_radii.append(sso_radius) flag_sso = OpFlagSSO(sso_names, sso_radii, flag_mask=args.flag_sso_mask) flag_sso.exec(data) if comm.world_rank == 0 and verbose: timer.report_clear(f"Flag {sso_names}") return
def load_observations(args, comm): """Load existing data and put it in TOAST observations. """ # This import is not at the top of the file to avoid # loading spt3g through so3g unnecessarily from ...io.toast_load import load_data log = Logger.get() if args.import_obs is not None: import_obs = args.import_obs.split(",") else: import_obs = None hw, telescope, det_index = get_hardware(args, comm, verbose=True) focalplane = get_focalplane(args, comm, hw, det_index, verbose=True) detweights = focalplane.detweights telescope.focalplane = focalplane if comm.world_rank == 0: log.info("Loading TOD from {}".format(args.import_dir)) timer = Timer() timer.start() data = load_data( args.import_dir, obs=import_obs, comm=comm, prefix=args.import_prefix, dets=hw, detranks=comm.group_size, ) if comm.world_rank == 0: timer.report_clear("Load data") telescope_data = [("all", data)] site = telescope.site focalplane = telescope.focalplane if args.weather is not None: weather = Weather(args.weather) else: weather = None for obs in data.obs: #obs["baselines"] = None obs["noise"] = focalplane.noise #obs["id"] = int(ces.mjdstart * 10000) #obs["intervals"] = tod.subscans obs["site"] = site.name obs["site_id"] = site.id obs["telescope"] = telescope.name obs["telescope_id"] = telescope.id obs["fpradius"] = focalplane.radius obs["weather"] = weather #obs["start_time"] = ces.start_time obs["altitude"] = site.alt #obs["season"] = ces.season #obs["date"] = ces.start_date #obs["MJD"] = ces.mjdstart obs["focalplane"] = focalplane.detector_data #obs["rising"] = ces.rising #obs["mindist_sun"] = ces.mindist_sun #obs["mindist_moon"] = ces.mindist_moon #obs["el_sun"] = ces.el_sun return data, telescope_data, detweights
def _stage_signal(self, detectors, nsamp, ndet, nodecomm, nread): """ Stage signal """ log = Logger.get() timer = Timer() # Determine if we can purge the signal and avoid keeping two # copies in memory purge = self._name is not None and self._purge_tod if not purge: nread = 1 nodecomm = MPI.COMM_SELF for iread in range(nread): nodecomm.Barrier() timer.start() if nodecomm.rank % nread == iread: self._mappraiser_signal = self._cache.create( "signal", mappraiser.SIGNAL_TYPE, (nsamp * ndet, )) self._mappraiser_signal[:] = np.nan global_offset = 0 local_blocks_sizes = [] for iobs, obs in enumerate(self._data.obs): tod = obs["tod"] for idet, det in enumerate(detectors): # Get the signal. signal = tod.local_signal(det, self._name) signal_dtype = signal.dtype offset = global_offset local_V_size = len(signal) dslice = slice(idet * nsamp + offset, idet * nsamp + offset + local_V_size) self._mappraiser_signal[dslice] = signal offset += local_V_size local_blocks_sizes.append(local_V_size) del signal # Purge only after all detectors are staged in case some are aliased # cache.clear() will not fail if the object was already # deleted as an alias if purge: for det in detectors: cachename = "{}_{}".format(self._name, det) tod.cache.clear(cachename) global_offset = offset local_blocks_sizes = np.array(local_blocks_sizes, dtype=np.int32) if self._verbose and nread > 1: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Stage signal {} / {}".format( iread + 1, nread)) return signal_dtype, local_blocks_sizes
def convolve_time_constant(args, comm, data, name, verbose=True): if not args.tau_convolve: return log = Logger.get() timer = Timer() timer.start() tauop = OpTimeConst(name=name, tau=args.tau_value, inverse=False) tauop.exec(data) timer.report_clear("Convolve time constant") return
def simulate_hwpss(args, comm, data, mc, name): if not args.simulate_hwpss: return log = Logger.get() timer = Timer() timer.start() hwpssop = OpSimHWPSS(name=name, fname_hwpss=args.hwpss_file, mc=mc) hwpssop.exec(data) timer.report_clear("Simulate HWPSS") return
def rotate_focalplane(args, data, comm): """ The LAT focalplane projected on the sky rotates as the cryostat (co-rotator) tilts. Usually the tilt is the same as the observing elevation to maintain constant angle between the mirror and the cryostat. This method must be called *before* expanding the detector pointing from boresight. """ log = Logger.get() timer = Timer() timer.start() for obs in data.obs: if obs["telescope"] != "LAT": continue tod = obs["tod"] cache_name = "corotator_angle_deg" if tod.cache.exists(cache_name): corotator_angle = tod.cache.reference(cache_name) else: # If a vector of co-rotator angles isn't already cached, # make one now from the observation metadata. This will # ensure they get recorded in the so3g files. corotator_angle = obs["corotator_angle_deg"] offset, nsample = tod.local_samples tod.cache.put(cache_name, np.zeros(nsample) + corotator_angle) el = np.degrees(tod.read_boresight_el()) rot = qa.rotation( ZAXIS, np.radians(corotator_angle + el + LAT_COROTATOR_OFFSET_DEG) ) quats = tod.read_boresight() quats[:] = qa.mult(quats, rot) try: # If there are horizontal boresight quaternions, they need # to be rotated as well. quats = tod.read_boresight(azel=True) quats[:] = qa.mult(quats, rot) except Exception as e: pass if comm.comm_world is None or comm.comm_world.rank == 0: timer.report_clear("Rotate focalplane") return
def get_elevation_noise(args, comm, data, key="noise"): """ Insert elevation-dependent noise """ if args.no_elevation_noise: return timer = Timer() timer.start() # fsample = args.sample_rate for obs in data.obs: tod = obs["tod"] fp = obs["focalplane"] noise = obs[key] for det in tod.local_dets: if det not in noise.keys: raise RuntimeError( 'Detector "{}" does not have a PSD in the noise object'. format(det)) A = fp[det]["A"] C = fp[det]["C"] psd = noise.psd(det) # We only consider a small range of samples for the elevation n = tod.local_samples[1] istart = max(0, n // 2 - 1000) istop = min(n, n // 2 + 1000) try: # Some TOD classes provide a shortcut to Az/El el = tod.read_azel(detector=det, local_start=istart, n=istop - istart)[1] except Exception: azelquat = tod.read_pntg(detector=det, azel=True, local_start=istart, n=istop - istart) # Convert Az/El quaternion of the detector back into # angles for the simulation. theta = qa.to_position(azelquat)[0] el = np.pi / 2 - theta el = np.median(el) # Scale the analytical noise PSD. Pivot is at el = 50 deg. psd[:] *= (A / np.sin(el) + C)**2 if comm.world_rank == 0: timer.report_clear("Elevation noise") return
def apply_polyfilter(args, comm, data, cache_name=None, verbose=True): """Apply the polynomial filter to data under `cache_name`.""" if not args.apply_polyfilter: return log = Logger.get() timer = Timer() timer.start() if comm.world_rank == 0 and verbose: log.info("Polyfiltering signal") polyfilter = OpPolyFilter(order=args.poly_order, name=cache_name, common_flag_mask=args.common_flag_mask) polyfilter.exec(data) if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0 and verbose: timer.report_clear("Polynomial filtering") return
def apply_common_mode_filter(args, comm, data, cache_name=None, verbose=True): """Apply the common mode filter to data under `cache_name`.""" if not args.apply_common_mode_filter: return log = Logger.get() timer = Timer() timer.start() if comm.world_rank == 0 and verbose: log.info("Common mode filtering signal") commonfilter = OpCommonModeFilter( name=cache_name, common_flag_mask=args.common_flag_mask, focalplane_key=args.common_mode_filter_key, ) commonfilter.exec(data) if comm.world_rank == 0 and verbose: timer.report_clear("Common mode filtering") return
def compute_crosslinking(args, comm, data, detweights=None, verbose=True): if not args.write_crosslinking: return log = Logger.get() timer = Timer() timer.start() crosslinking = OpCrossLinking( outdir=args.out, outprefix=args.crosslinking_prefix, common_flag_mask=args.common_flag_mask, flag_mask=255, zip_maps=args.hn_zip, rcond_limit=1e-3, detweights=detweights, ) crosslinking.exec(data) timer.report_clear("Compute crosslinking map") return
def compute_cadence_map(args, comm, data, verbose=True): if not args.write_cadence_map: return log = Logger.get() if comm.world_rank == 0: log.info("Computing cadence map") timer = Timer() timer.start() cadence = OpCadenceMap( outdir=args.out, outprefix=args.cadence_map_prefix, common_flag_mask=args.common_flag_mask, flag_mask=255, ) cadence.exec(data) if comm.world_rank == 0: timer.report_clear("Compute cadence map") return
def compute_h_n(args, comm, data, verbose=True): if args.hn_max < args.hn_min: return log = Logger.get() timer = Timer() timer.start() hnop = OpHn( outdir=args.hn_outdir, outprefix=args.hn_prefix, nmin=args.hn_min, nmax=args.hn_max, common_flag_mask=args.common_flag_mask, flag_mask=255, zip_maps=args.hn_zip, ) hnop.exec(data) timer.report_clear("Compute h_n") return
def apply_groundfilter(args, comm, data, cache_name=None, verbose=True): if not args.apply_groundfilter: return log = Logger.get() timer = Timer() timer.start() if comm.world_rank == 0 and verbose: log.info("Ground-filtering signal") groundfilter = OpGroundFilter( filter_order=args.ground_order, name=cache_name, common_flag_mask=args.common_flag_mask, ) groundfilter.exec(data) if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0 and verbose: timer.report_clear("Ground filtering") return
def demodulate(args, comm, data, name, detweights=None, madampars=None, verbose=True): if not args.demodulate: return log = Logger.get() timer = Timer() timer.start() if detweights is not None: # Copy the detector weights to demodulated TOD modulated = [ detname for detname in detweights if "demod" not in detname ] for detname in modulated: detweight = detweights[detname] for demodkey in ["demod0", "demod4r", "demod4i"]: demod_name = "{}_{}".format(demodkey, detname) detweights[demod_name] = detweight del detweights[detname] if madampars is not None: # Filtering will affect the high frequency end of the noise PSD madampars["radiometers"] = False # Intensity and polarization will be decoupled in the noise matrix madampars["allow_decoupling"] = True demod = OpDemod( name=name, wkernel=args.demod_wkernel, fmax=args.demod_fmax, nskip=args.demod_nskip, do_2f=args.demod_2f, ) demod.exec(data) timer.report_clear("Demodulate") return
def apply_sim_sso(args, comm, data, mc, totalname, verbose=True): if args.simulate_sso is None: return if args.beam_file is None: raise RuntimeError("Cannot simulate SSOs without a beam file") timer = Timer() timer.start() for sso_name in args.simulate_sso.split(","): if comm.world_rank == 0 and verbose: print("Simulating {}".format(sso_name), flush=True) sim_sso = OpSimSSO(sso_name, args.beam_file, out=totalname) sim_sso.exec(data) if comm.world_rank == 0 and verbose: timer.report_clear("Simulate {}".format(sso_name)) return
def load_focalplanes(args, comm, schedules, verbose=False): """ Attach a focalplane to each of the schedules. Args: schedules (list) : List of Schedule instances. Each schedule has two members, telescope and ceslist, a list of CES objects. Returns: detweights (dict) : Inverse variance noise weights for every detector across all focal planes. In [K_CMB^-2]. They can be used to bin the TOD. """ # log = Logger.get() timer = Timer() timer.start() # Load focalplane information timer1 = Timer() timer1.start() hw, telescope, det_index = get_hardware(args, comm, verbose=verbose) focalplane = get_focalplane(args, comm, hw, det_index, verbose=verbose) telescope.focalplane = focalplane if comm.world_rank == 0 and verbose: timer1.report_clear("Collect focaplane information") for schedule in schedules: # Replace the telescope created from reading the observing schedule but # keep the weather object weather = schedule.telescope.site.weather schedule.telescope = telescope schedule.telescope.site.weather = weather detweights = telescope.focalplane.detweights timer.stop() if (comm.comm_world is None or comm.world_rank == 0) and verbose: timer.report("Loading focalplane") return detweights
def deconvolve_time_constant(args, comm, data, name, realization=0, verbose=True): if not args.tau_deconvolve: return log = Logger.get() timer = Timer() timer.start() tauop = OpTimeConst( name=name, tau=args.tau_value, inverse=True, tau_sigma=args.tau_sigma, realization=realization, ) tauop.exec(data) timer.report_clear("De-convolve time constant") return
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.comm_world.rank == 0: print( "Running with {} processes at {}".format( procs, str(datetime.datetime.now()) ) ) parser = argparse.ArgumentParser( description="Simple on-the-fly signal convolution + MADAM Mapmaking", fromfile_prefix_chars="@", ) parser.add_argument("--lmax", required=True, type=np.int, help="Simulation lmax") parser.add_argument( "--fwhm", required=True, type=np.float, help="Sky fwhm [arcmin] to deconvolve" ) parser.add_argument("--beammmax", required=True, type=np.int, help="Beam mmax") parser.add_argument("--order", default=11, type=np.int, help="Iteration order") parser.add_argument( "--pxx", required=False, default=False, action="store_true", help="Beams are in Pxx frame, not Dxx", ) parser.add_argument( "--normalize", required=False, default=False, action="store_true", help="Normalize the beams", ) parser.add_argument( "--skyfile", required=True, help="Path to sky alm files. Tag DETECTOR will be " "replaced with detector name.", ) parser.add_argument( "--remove_monopole", required=False, default=False, action="store_true", help="Remove the sky monopole before convolution", ) parser.add_argument( "--remove_dipole", required=False, default=False, action="store_true", help="Remove the sky dipole before convolution", ) parser.add_argument( "--beamfile", required=True, help="Path to beam alm files. Tag DETECTOR will be " "replaced with detector name.", ) parser.add_argument("--rimo", required=True, help="RIMO file") parser.add_argument("--freq", required=True, type=np.int, help="Frequency") parser.add_argument( "--dets", required=False, default=None, help="Detector list (comma separated)" ) parser.add_argument( "--effdir", required=True, help="Input Exchange Format File directory" ) parser.add_argument( "--effdir_pntg", required=False, help="Input Exchange Format File directory " "for pointing", ) parser.add_argument( "--effdir_out", required=False, help="Output directory for convolved TOD" ) parser.add_argument( "--obtmask", required=False, default=1, type=np.int, help="OBT flag mask" ) parser.add_argument( "--flagmask", required=False, default=1, type=np.int, help="Quality flag mask" ) parser.add_argument("--ringdb", required=True, help="Ring DB file") parser.add_argument( "--odfirst", required=False, default=None, type=np.int, help="First OD to use" ) parser.add_argument( "--odlast", required=False, default=None, type=np.int, help="Last OD to use" ) parser.add_argument( "--ringfirst", required=False, default=None, type=np.int, help="First ring to use", ) parser.add_argument( "--ringlast", required=False, default=None, type=np.int, help="Last ring to use" ) parser.add_argument( "--obtfirst", required=False, default=None, type=np.float, help="First OBT to use", ) parser.add_argument( "--obtlast", required=False, default=None, type=np.float, help="Last OBT to use" ) parser.add_argument("--madam_prefix", required=False, help="map prefix") parser.add_argument( "--madampar", required=False, default=None, help="Madam parameter file" ) parser.add_argument( "--obtmask_madam", required=False, type=np.int, help="OBT flag mask for Madam" ) parser.add_argument( "--flagmask_madam", required=False, type=np.int, help="Quality flag mask for Madam", ) parser.add_argument( "--skip_madam", required=False, default=False, action="store_true", help="Do not run Madam on the convolved timelines", ) parser.add_argument("--out", required=False, default=".", help="Output directory") try: args = parser.parse_args() except SystemExit: sys.exit(0) timer = Timer() timer.start() odrange = None if args.odfirst is not None and args.odlast is not None: odrange = (args.odfirst, args.odlast) ringrange = None if args.ringfirst is not None and args.ringlast is not None: ringrange = (args.ringfirst, args.ringlast) obtrange = None if args.obtfirst is not None and args.obtlast is not None: obtrange = (args.obtfirst, args.obtlast) detectors = None if args.dets is not None: detectors = re.split(",", args.dets) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) # Ensure output directory exists if not os.path.isdir(args.out) and comm.comm_world.rank == 0: os.makedirs(args.out) # Read in madam parameter file # Allow more than one entry, gather into a list repeated_keys = ["detset", "detset_nopol", "survey"] pars = {} if comm.comm_world.rank == 0: pars["kfirst"] = False pars["temperature_only"] = True pars["base_first"] = 60.0 pars["nside_map"] = 512 pars["nside_cross"] = 512 pars["nside_submap"] = 16 pars["write_map"] = False pars["write_binmap"] = True pars["write_matrix"] = False pars["write_wcov"] = False pars["write_hits"] = True pars["kfilter"] = False pars["info"] = 3 if args.madampar: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if not comment.match(line): result = pat.match(line) if result: key, value = result.group(1), result.group(2) if key in repeated_keys: if key not in pars: pars[key] = [] pars[key].append(value) else: pars[key] = value # Command line parameters override the ones in the madam parameter file if "file_root" not in pars: pars["file_root"] = "madam" if args.madam_prefix is not None: pars["file_root"] = args.madam_prefix sfreq = "{:03}".format(args.freq) if sfreq not in pars["file_root"]: pars["file_root"] += "_" + sfreq try: fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq] except Exception: fsample = 180.3737 pars["fsample"] = fsample pars["path_output"] = args.out print("All parameters:") print(args, flush=True) pars = comm.comm_world.bcast(pars, root=0) memreport("after parameters", MPI.COMM_WORLD) # madam only supports a single observation. Normally # we would have multiple observations with some subset # assigned to each process group. # create the TOD for this observation tod = tp.Exchange( comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, do_eff_cache=False, ) # normally we would get the intervals from somewhere else, but since # the Exchange TOD already had to get that information, we can # get it from there. ob = {} ob["name"] = "mission" ob["id"] = 0 ob["tod"] = tod ob["intervals"] = tod.valid_intervals ob["baselines"] = None ob["noise"] = tod.noise # Add the bare minimum focal plane information for the conviqt operator focalplane = {} for det in tod.detectors: if args.pxx: # Beam is in the polarization basis. # No extra rotations are needed psipol = tod.rimo[det].psi_pol else: # Beam is in the detector basis. Convolver needs to remove # the last rotation into the polarization sensitive frame. psipol = tod.rimo[det].psi_uv + tod.rimo[det].psi_pol focalplane[det] = { "pol_leakage" : tod.rimo[det].epsilon, "pol_angle_deg" : psipol, } ob["focalplane"] = focalplane data.obs.append(ob) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Metadata queries") loader = tp.OpInputPlanck( commonflags_name="common_flags", flags_name="flags", margin=0 ) loader.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Data read and cache") tod.cache.report() memreport("after loading", mpiworld) # make a planck Healpix pointing matrix mode = "IQU" if pars["temperature_only"] == "T": mode = "I" nside = int(pars["nside_map"]) pointing = tp.OpPointingPlanck( nside=nside, mode=mode, RIMO=tod.RIMO, margin=0, apply_flags=False, keep_vel=False, keep_pos=False, keep_phase=False, keep_quats=True, ) pointing.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Pointing Matrix took, mode = {}".format(mode)) memreport("after pointing", mpiworld) # simulate the TOD by convolving the sky with the beams if comm.comm_world.rank == 0: print("Convolving TOD", flush=True) for pattern in args.beamfile.split(","): skyfiles = {} beamfiles = {} for det in tod.detectors: freq = "{:03}".format(tp.utilities.det2freq(det)) if "LFI" in det: psmdet = "{}_{}".format(freq, det[3:]) if det.endswith("M"): arm = "y" else: arm = "x" graspdet = "{}_{}_{}".format(freq[1:], det[3:5], arm) else: psmdet = det.replace("-", "_") graspdet = det skyfile = ( args.skyfile.replace("FREQ", freq) .replace("PSMDETECTOR", psmdet) .replace("DETECTOR", det) ) skyfiles[det] = skyfile beamfile = pattern.replace("GRASPDETECTOR", graspdet).replace( "DETECTOR", det ) beamfiles[det] = beamfile if comm.comm_world.rank == 0: print("Convolving {} with {}".format(skyfile, beamfile), flush=True) conviqt = OpSimConviqt( comm.comm_world, skyfiles, beamfiles, lmax=args.lmax, beammmax=args.beammmax, pol=True, fwhm=args.fwhm, order=args.order, calibrate=True, dxx=True, out="conviqt_tod", apply_flags=False, remove_monopole=args.remove_monopole, remove_dipole=args.remove_dipole, verbosity=1, normalize_beam=args.normalize, ) conviqt.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Convolution") memreport("after conviqt", mpiworld) if args.effdir_out is not None: if comm.comm_world.rank == 0: print("Writing TOD", flush=True) tod.set_effdir_out(args.effdir_out, None) writer = tp.OpOutputPlanck( signal_name="conviqt_tod", flags_name="flags", commonflags_name="common_flags", ) writer.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Conviqt output") memreport("after writing", mpiworld) # for now, we pass in the noise weights from the RIMO. detweights = {} for d in tod.detectors: net = tod.rimo[d].net fsample = tod.rimo[d].fsample detweights[d] = 1.0 / (fsample * net * net) if not args.skip_madam: if comm.comm_world.rank == 0: print("Calling Madam", flush=True) try: if args.obtmask_madam is None: obtmask = args.obtmask else: obtmask = args.obtmask_madam if args.flagmask_madam is None: flagmask = args.flagmask else: flagmask = args.flagmask_madam madam = OpMadam( params=pars, detweights=detweights, name="conviqt_tod", flag_name="flags", purge=True, name_out="madam_tod", common_flag_mask=obtmask, flag_mask=flagmask, ) except Exception as e: raise Exception( "{:4} : ERROR: failed to initialize Madam: {}".format( comm.comm_world.rank, e ) ) madam.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Madam took {:.3f} s") memreport("after madam", mpiworld) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() if comm.world_rank == 0: print("Running with {} processes at {}".format( procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking', fromfile_prefix_chars='@') parser.add_argument('--skip_madam', dest='skip_madam', default=False, action='store_true', help='D not make maps with Madam.') parser.add_argument('--skip_noise', dest='skip_noise', default=False, action='store_true', help='Do not add simulated noise to the TOD.') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--debug', dest='debug', default=False, action='store_true', help='Write data distribution info to file') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir2', required=False, help='Additional input Exchange Format File directory') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory for ' 'pointing') parser.add_argument('--effdir_fsl', required=False, help='Input Exchange Format File directory for ' 'straylight') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--pntflagmask', required=False, default=0, type=np.int, help='Which OBT flag bits to raise for HCM maneuvers') parser.add_argument('--bad_intervals', required=False, help='Path to bad interval file.') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, help='First ring to use') parser.add_argument('--ringlast', required=False, default=None, help='Last ring to use') parser.add_argument('--obtfirst', required=False, default=None, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, help='Last OBT to use') parser.add_argument('--read_eff', dest='read_eff', default=False, action='store_true', help='Read and co-add the signal from effdir') parser.add_argument('--decalibrate', required=False, help='Path to calibration file to decalibrate with. ' 'You can use python string formatting, assuming ' '.format(mc)') parser.add_argument('--calibrate', required=False, help='Path to calibration file to calibrate with. ' 'You can use python string formatting, assuming ' '.format(mc)') parser.add_argument('--madampar', required=False, default=None, help='Madam parameter file') parser.add_argument('--nside', required=False, default=None, type=np.int, help='Madam resolution') parser.add_argument('--out', required=False, default='.', help='Output directory') parser.add_argument('--madam_prefix', required=False, help='map prefix') parser.add_argument('--make_rings', dest='make_rings', default=False, action='store_true', help='Compile ringsets.') parser.add_argument('--nside_ring', required=False, default=128, type=np.int, help='Ringset resolution') parser.add_argument('--ring_root', required=False, default='ringset', help='Root filename for ringsets (setting to empty ' 'disables ringset output).') parser.add_argument('--MC_start', required=False, default=0, type=np.int, help='First Monte Carlo noise realization') parser.add_argument('--MC_count', required=False, default=1, type=np.int, help='Number of Monte Carlo noise realizations') # noise parameters parser.add_argument('--noisefile', required=False, default='RIMO', help='Path to noise PSD files for noise filter. ' 'Tag DETECTOR will be replaced with detector name.') parser.add_argument('--noisefile_simu', required=False, default='RIMO', help='Path to noise PSD files for noise simulation. ' 'Tag DETECTOR will be replaced with detector name.') # Dipole parameters dipogroup = parser.add_mutually_exclusive_group() dipogroup.add_argument('--dipole', dest='dipole', required=False, default=False, action='store_true', help='Simulate dipole') dipogroup.add_argument('--solsys_dipole', dest='solsys_dipole', required=False, default=False, action='store_true', help='Simulate solar system dipole') dipogroup.add_argument('--orbital_dipole', dest='orbital_dipole', required=False, default=False, action='store_true', help='Simulate orbital dipole') dipo_parameters_group = parser.add_argument_group('dipole_parameters') dipo_parameters_group.add_argument( '--solsys_speed', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_speed"], help='Solar system speed wrt. CMB rest frame in km/s. Default is ' 'Planck 2015 best fit value') dipo_parameters_group.add_argument( '--solsys_glon', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_glon"], help='Solar system velocity direction longitude in degrees') dipo_parameters_group.add_argument( '--solsys_glat', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_glat"], help='Solar system velocity direction latitude in degrees') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.world_rank == 0: print('All parameters:') print(args, flush=True) if args.MC_count < 1: raise RuntimeError('MC_count = {} < 1. Nothing done.' ''.format(args.MC_count)) timer = Timer() timer.start() nrange = 1 odranges = None if args.odfirst is not None and args.odlast is not None: odranges = [] firsts = [int(i) for i in str(args.odfirst).split(',')] lasts = [int(i) for i in str(args.odlast).split(',')] for odfirst, odlast in zip(firsts, lasts): odranges.append((odfirst, odlast)) nrange = len(odranges) ringranges = None if args.ringfirst is not None and args.ringlast is not None: ringranges = [] firsts = [int(i) for i in str(args.ringfirst).split(',')] lasts = [int(i) for i in str(args.ringlast).split(',')] for ringfirst, ringlast in zip(firsts, lasts): ringranges.append((ringfirst, ringlast)) nrange = len(ringranges) obtranges = None if args.obtfirst is not None and args.obtlast is not None: obtranges = [] firsts = [float(i) for i in str(args.obtfirst).split(',')] lasts = [float(i) for i in str(args.obtlast).split(',')] for obtfirst, obtlast in zip(firsts, lasts): obtranges.append((obtfirst, obtlast)) nrange = len(obtranges) if odranges is None: odranges = [None] * nrange if ringranges is None: ringranges = [None] * nrange if obtranges is None: obtranges = [None] * nrange detectors = None if args.dets is not None: detectors = re.split(',', args.dets) # create the TOD for this observation if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO': do_eff_cache = True else: do_eff_cache = False tods = [] for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges): # create the TOD for this observation tods.append( tp.Exchange(comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, extra_effdirs=[args.effdir2, args.effdir_fsl], effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, pntflagmask=args.pntflagmask, do_eff_cache=do_eff_cache)) # Make output directory if not os.path.isdir(args.out) and comm.world_rank == 0: os.makedirs(args.out) # Read in madam parameter file # Allow more than one entry, gather into a list repeated_keys = ['detset', 'detset_nopol', 'survey'] pars = {} if comm.world_rank == 0: pars['kfirst'] = False pars['temperature_only'] = True pars['base_first'] = 60.0 pars['nside_submap'] = 16 pars['write_map'] = False pars['write_binmap'] = True pars['write_matrix'] = False pars['write_wcov'] = False pars['write_hits'] = True pars['kfilter'] = False pars['info'] = 3 if args.madampar: pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*') comment = re.compile(r'^#.*') with open(args.madampar, 'r') as f: for line in f: if not comment.match(line): result = pat.match(line) if result: key, value = result.group(1), result.group(2) if key in repeated_keys: if key not in pars: pars[key] = [] pars[key].append(value) else: pars[key] = value # Command line parameters override the ones in the madam parameter file if 'file_root' not in pars: pars['file_root'] = 'madam' if args.madam_prefix is not None: pars['file_root'] = args.madam_prefix sfreq = '{:03}'.format(args.freq) if sfreq not in pars['file_root']: pars['file_root'] += '_' + sfreq try: fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq] except Exception: fsample = 180.3737 pars['fsample'] = fsample pars['path_output'] = args.out pars = comm.comm_world.bcast(pars, root=0) madam_mcmode = True if 'nsubchunk' in pars and int(pars['nsubchunk']) > 1: madam_mcmode = False if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO': # We split MPI_COMM_WORLD into single process groups, each of # which is assigned one or more observations (rings) comm = toast.Comm(groupsize=1) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) for iobs, tod in enumerate(tods): if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO': # Use a toast helper method to optimally distribute rings between # processes. dist = distribute_discrete(tod.ringsizes, comm.world_size) my_first_ring, my_n_ring = dist[comm.world_rank] for my_ring in range(my_first_ring, my_first_ring + my_n_ring): ringtod = tp.Exchange.from_tod( tod, my_ring, comm.comm_group, noisefile=args.noisefile, noisefile_simu=args.noisefile_simu) ob = {} ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring) ob['id'] = ringtod.globalfirst_ring ob['tod'] = ringtod ob['intervals'] = ringtod.valid_intervals ob['baselines'] = None ob['noise'] = ringtod.noise ob['noise_simu'] = ringtod.noise_simu data.obs.append(ob) else: ob = {} ob['name'] = 'observation{:04}'.format(iobs) ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise ob['noise_simu'] = tod.noise data.obs.append(ob) rimo = tods[0].rimo if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Metadata queries") # Always read the signal and flags, even if the signal is later # overwritten. There is no overhead for the signal because it is # interlaced with the flags. tod_name = 'signal' timestamps_name = 'timestamps' flags_name = 'flags' common_flags_name = 'common_flags' reader = tp.OpInputPlanck(signal_name=tod_name, flags_name=flags_name, timestamps_name=timestamps_name, commonflags_name=common_flags_name) if comm.world_rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Read") # Clear the signal if we don't need it if not args.read_eff: eraser = tp.OpCacheMath(in1=tod_name, in2=0, multiply=True, out=tod_name) if comm.world_rank == 0: print('Erasing TOD', flush=True) eraser.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Erase") # Optionally flag bad intervals if args.bad_intervals is not None: flagger = tp.OpBadIntervals(path=args.bad_intervals) flagger.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Apply {}".format(args.bad_intervals)) # Now read an optional second TOD to add with the first if args.effdir2 is not None: # Read the extra TOD and add it to the first one reader = tp.OpInputPlanck(signal_name='tod2', flags_name=None, timestamps_name=None, commonflags_name=None, effdir=args.effdir2) if comm.world_rank == 0: print('Reading extra TOD from {}'.format(args.effdir2), flush=True) reader.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: print("Reading took {:.3f} s".format(elapsed), flush=True) adder = tp.OpCacheMath(in1=tod_name, in2='signal2', add=True, out=tod_name) if comm.world_rank == 0: print('Adding TODs', flush=True) adder.exec(data) # Erase the extra cache object for ob in data.obs: tod = ob['tod'] tod.cache.clear('signal2_.*') if args.effdir_fsl is not None: # Read the straylight signal into the tod cache under # "fsl_<detector>" reader = tp.OpInputPlanck(signal_name='fsl', flags_name=None, timestamps_name=None, commonflags_name=None, effdir=args.effdir_fsl) if comm.world_rank == 0: print('Reading straylight signal from {}'.format(args.effdir_fsl), flush=True) reader.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Read FSL") do_fsl = True else: do_fsl = False # make a planck Healpix pointing matrix mode = 'IQU' if pars['temperature_only'] == 'T': mode = 'I' if args.nside is None: if 'nside_map' in pars: nside = int(pars['nside_map']) else: raise RuntimeError( 'Nside must be set either in the Madam parameter file or on ' 'the command line') else: nside = args.nside pars['nside_map'] = nside if 'nside_cross' not in pars or pars['nside_cross'] > pars['nside_map']: pars['nside_cross'] = pars['nside_map'] do_dipole = args.dipole or args.solsys_dipole or args.orbital_dipole pointing = tp.OpPointingPlanck(nside=nside, mode=mode, RIMO=rimo, margin=0, apply_flags=True, keep_vel=do_dipole, keep_pos=False, keep_phase=False, keep_quats=do_dipole) pointing.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Pointing Matrix") flags_name = 'flags' common_flags_name = 'common_flags' # for now, we pass in the noise weights from the RIMO. detweights = {} for d in tod.detectors: net = tod.rimo[d].net fsample = tod.rimo[d].fsample detweights[d] = 1.0 / (fsample * net * net) if args.debug: with open("debug_planck_exchange_madam.txt", "w") as f: data.info(f) if do_dipole: # Simulate the dipole if args.dipole: dipomode = 'total' elif args.solsys_dipole: dipomode = 'solsys' else: dipomode = 'orbital' dipo = tp.OpDipolePlanck(args.freq, solsys_speed=args.solsys_speed, solsys_glon=args.solsys_glon, solsys_glat=args.solsys_glat, mode=dipomode, output='dipole', keep_quats=False) dipo.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Dipole") # Loop over Monte Carlos madam = None for mc in range(args.MC_start, args.MC_start + args.MC_count): out = "{}/{:05d}".format(args.out, mc) if comm.world_rank == 0: if not os.path.isdir(out): os.makedirs(out) # clear all noise data from the cache, so that we can generate # new noise timestreams. for ob in data.obs: ob['tod'].cache.clear("noise_.*") tod_name = 'signal' if do_dipole: adder = tp.OpCacheMath(in1=tod_name, in2='dipole', add=True, out='noise') adder.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Add dipole".format(mc)) tod_name = 'noise' # Simulate noise if not args.skip_noise: tod_name = 'noise' nse = toast.tod.OpSimNoise(out=tod_name, realization=mc, component=0, noise='noise_simu', rate=fsample) if comm.world_rank == 0: print('Simulating noise from {}'.format(args.noisefile_simu), flush=True) nse.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Noise simulation".format(mc)) # If we didn't add the dipole, we need to add the input # signal with the noise we just simulated if args.read_eff and not do_dipole: adder = tp.OpCacheMath(in1=tod_name, in2='signal', add=True, out=tod_name) adder.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Add input signal".format(mc)) # Make rings if args.make_rings: ringmaker = tp.OpRingMaker(args.nside_ring, nside, signal=tod_name, fileroot=args.ring_root, out=out, commonmask=args.obtmask, detmask=args.flagmask) ringmaker.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Ringmaking".format(mc)) # Apply calibration errors if args.decalibrate is not None: fn = args.decalibrate try: fn = fn.format(mc) except Exception: pass if comm.world_rank == 0: print('Decalibrating with {}'.format(fn), flush=True) decalibrator = tp.OpCalibPlanck(signal_in=tod_name, signal_out='noise', file_gain=fn, decalibrate=True) decalibrator.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Decalibrate".format(mc)) tod_name = 'noise' if args.calibrate is not None: fn = args.calibrate try: fn = fn.format(mc) except Exception: pass if comm.world_rank == 0: print('Calibrating with {}'.format(fn), flush=True) calibrator = tp.OpCalibPlanck(signal_in=tod_name, signal_out='noise', file_gain=fn) calibrator.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Calibrate".format(mc)) tod_name = 'noise' # Subtract the dipole and straylight if do_dipole: subtractor = tp.OpCacheMath(in1=tod_name, in2='dipole', subtract=True, out='noise') subtractor.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Subtract dipole".format(mc)) tod_name = 'noise' if do_fsl: subtractor = tp.OpCacheMath(in1=tod_name, in2='fsl', subtract=True, out='noise') subtractor.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Subtract straylight".format(mc)) tod_name = 'noise' # Make the map if not args.skip_madam: # Make maps if madam is None: try: madam = toast.todmap.OpMadam(params=pars, detweights=detweights, purge_tod=True, name=tod_name, apply_flags=False, name_out=None, noise='noise', mcmode=madam_mcmode, translate_timestamps=False) except Exception as e: raise Exception( '{:4} : ERROR: failed to initialize Madam: ' '{}'.format(comm.world_rank, e)) madam.params['path_output'] = out madam.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Mapmaking".format(mc)) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def apply_mappraiser( args, comm, data, params, signalname, noisename, time_comms=None, telescope_data=None, verbose=True, ): """ Use libmappraiser to run the ML map-making Args: time_comms (iterable) : Series of disjoint communicators that map, e.g., seasons and days. Each entry is a tuple of the form (`name`, `communicator`) telescope_data (iterable) : series of disjoint TOAST data objects. Each entry is tuple of the form (`name`, `data`). """ if comm.comm_world is None: raise RuntimeError("Mappraiser requires MPI") log = Logger.get() total_timer = Timer() total_timer.start() if comm.world_rank == 0 and verbose: log.info("Making maps") mappraiser = OpMappraiser( params= params, purge=True, name=signalname, noise_name = noisename, conserve_memory=args.conserve_memory, ) if time_comms is None: time_comms = [("all", comm.comm_world)] if telescope_data is None: telescope_data = [("all", data)] timer = Timer() for time_name, time_comm in time_comms: for tele_name, tele_data in telescope_data: if len(time_name.split("-")) == 3: # Special rules for daily maps if args.do_daymaps: continue if len(telescope_data) > 1 and tele_name == "all": # Skip daily maps over multiple telescopes continue timer.start() # N.B: code below is for Madam but may be useful to copy in Mappraiser # once we start doing multiple maps in one run # madam.params["file_root"] = "{}_telescope_{}_time_{}".format( # file_root, tele_name, time_name # ) # if time_comm == comm.comm_world: # madam.params["info"] = info # else: # # Cannot have verbose output from concurrent mapmaking # madam.params["info"] = 0 # if (time_comm is None or time_comm.rank == 0) and verbose: # log.info("Mapping {}".format(madam.params["file_root"])) mappraiser.exec(tele_data, time_comm) if time_comm is not None: time_comm.barrier() if comm.world_rank == 0 and verbose: timer.report_clear("Mapping {}_telescope_{}_time_{}".format( args.outpath, tele_name, time_name, )) if comm.comm_world is not None: comm.comm_world.barrier() total_timer.stop() if comm.world_rank == 0 and verbose: total_timer.report("Mappraiser total") return
# normally we would get the intervals from somewhere else, but since # the Exchange TOD already had to get that information, we can # get it from there. ob = {} ob["name"] = "mission" ob["id"] = 0 ob["tod"] = tod ob["intervals"] = tod.valid_intervals ob["baselines"] = None ob["noise"] = tod.noise data.obs.append(ob) if comm.world_rank == 0: timer.report_clear("Metadata queries") ring_starts = [t.first for t in tod.valid_intervals] ring_times = [t.start for t in tod.valid_intervals] ring_lens = [(t.last - t.first) for t in tod.valid_intervals] ring_offset = tod.globalfirst_ring for interval in tod.valid_intervals: if interval.last < tod.local_samples[0]: ring_offset += 1 ring_number = ring_offset - 1 globalfirst = tod.globalfirst ringtable = ringdb_table_name(args.freq)
def main(): env = Environment.get() log = Logger.get() gt = GlobalTimers.get() gt.start("toast_satellite_sim (total)") timer0 = Timer() timer0.start() mpiworld, procs, rank, comm = get_comm() args, comm, groupsize = parse_arguments(comm, procs) # Parse options tmr = Timer() tmr.start() if comm.world_rank == 0: os.makedirs(args.outdir, exist_ok=True) focalplane, gain, detweights = load_focalplane(args, comm) data = create_observations(args, comm, focalplane, groupsize) expand_pointing(args, comm, data) localpix, localsm, subnpix = get_submaps(args, comm, data) signalname = None skyname = simulate_sky_signal(args, comm, data, [focalplane], subnpix, localsm, "signal") if skyname is not None: signalname = skyname diponame = simulate_dipole(args, comm, data, "signal") if diponame is not None: signalname = diponame # Mapmaking. if not args.use_madam: if comm.world_rank == 0: log.info("Not using Madam, will only make a binned map") npp, zmap = init_binner(args, comm, data, detweights, subnpix=subnpix, localsm=localsm) # Loop over Monte Carlos firstmc = args.MC_start nmc = args.MC_count for mc in range(firstmc, firstmc + nmc): mctmr = Timer() mctmr.start() outpath = os.path.join(args.outdir, "mc_{:03d}".format(mc)) simulate_noise(args, comm, data, mc, "tot_signal", overwrite=True) # add sky signal add_signal(args, comm, data, "tot_signal", signalname) if gain is not None: timer = Timer() timer.start() op_apply_gain = OpApplyGain(gain, name="tot_signal") op_apply_gain.exec(data) if comm.world_rank == 0: timer.report_clear(" Apply gains {:04d}".format(mc)) if mc == firstmc: # For the first realization, optionally export the # timestream data. If we had observation intervals defined, # we could pass "use_interval=True" to the export operators, # which would ensure breaks in the exported data at # acceptable places. output_tidas(args, comm, data, "tot_signal") output_spt3g(args, comm, data, "tot_signal") apply_binner(args, comm, data, npp, zmap, detweights, outpath, "tot_signal") if comm.world_rank == 0: mctmr.report_clear(" Map-making {:04d}".format(mc)) else: # Initialize madam parameters madampars = setup_madam(args) # in debug mode, print out data distribution information if args.debug: handle = None if comm.world_rank == 0: handle = open(os.path.join(args.outdir, "distdata.txt"), "w") data.info(handle) if comm.world_rank == 0: handle.close() if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear("Dumping data distribution") # Loop over Monte Carlos firstmc = args.MC_start nmc = args.MC_count for mc in range(firstmc, firstmc + nmc): mctmr = Timer() mctmr.start() # create output directory for this realization outpath = os.path.join(args.outdir, "mc_{:03d}".format(mc)) simulate_noise(args, comm, data, mc, "tot_signal", overwrite=True) # add sky signal add_signal(args, comm, data, "tot_signal", signalname) if gain is not None: op_apply_gain = OpApplyGain(gain, name="tot_signal") op_apply_gain.exec(data) if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear(" Apply gains {:04d}".format(mc)) apply_madam(args, comm, data, madampars, outpath, detweights, "tot_signal") if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: mctmr.report_clear(" Map-making {:04d}".format(mc)) gt.stop_all() if comm.comm_world is not None: comm.comm_world.barrier() tmr.stop() tmr.clear() tmr.start() alltimers = gather_timers(comm=comm.comm_world) if comm.world_rank == 0: out = os.path.join(args.outdir, "timing") dump_timing(alltimers, out) tmr.stop() tmr.report("Gather and dump timing info") timer0.report_clear("toast_satellite_sim.py") return
def create_observations(args, comm, schedule): """Simulate constant elevation scans. Simulate constant elevation scans at "site" matching entries in "all_ces". Each operational day is assigned to a different process group to allow making day maps. """ timer = Timer() log = Logger.get() data = Data(comm) telescope = schedule.telescope site = telescope.site focalplane = telescope.focalplane all_ces = schedule.ceslist nces = len(all_ces) breaks = get_breaks(comm, all_ces, args) nbreak = len(breaks) groupdist = distribute_uniform(nces, comm.ngroups, breaks=breaks) group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] if comm.comm_group is not None: ndetrank = comm.comm_group.size else: ndetrank = 1 for ices in range(group_firstobs, group_firstobs + group_numobs): ces = all_ces[ices] totsamples = int((ces.stop_time - ces.start_time) * args.sample_rate) # create the single TOD for this observation try: tod = TODGround( comm.comm_group, focalplane.detquats, totsamples, detranks=ndetrank, firsttime=ces.start_time, rate=args.sample_rate, site_lon=site.lon, site_lat=site.lat, site_alt=site.alt, azmin=ces.azmin, azmax=ces.azmax, el=ces.el, scanrate=args.scan_rate, scan_accel=args.scan_accel, cosecant_modulation=args.scan_cosecant_modulate, CES_start=None, CES_stop=None, sun_angle_min=args.sun_angle_min, coord=args.coord, sampsizes=None, report_timing=args.debug, ) except RuntimeError as e: raise RuntimeError("Failed to create the CES scan: {}".format(e)) # Create the (single) observation ob = {} ob["name"] = "CES-{}-{}-{}".format(ces.name, ces.scan, ces.subscan) ob["tod"] = tod if len(tod.subscans) > 0: ob["intervals"] = tod.subscans else: raise RuntimeError("{} has no valid intervals".format(ob["name"])) ob["baselines"] = None ob["noise"] = focalplane.noise ob["id"] = int(ces.mjdstart * 10000) data.obs.append(ob) for ob in data.obs: tod = ob["tod"] tod.free_azel_quats() if comm.comm_world is None or comm.comm_group.rank == 0: log.info("Group # {:4} has {} observations.".format( comm.group, len(data.obs))) if len(data.obs) == 0: raise RuntimeError("Too many tasks. Every MPI task must " "be assigned to at least one observation.") if comm.world_rank == 0: timer.report_clear("Simulate scans") return data
def parse_arguments(comm): timer = Timer() timer.start() log = Logger.get() parser = argparse.ArgumentParser( description="Simulate ground-based boresight pointing. Simulate " "atmosphere and make maps for some number of noise Monte Carlos.", fromfile_prefix_chars="@", ) toast_tools.add_dist_args(parser) toast_tools.add_todground_args(parser) toast_tools.add_pointing_args(parser) toast_tools.add_polyfilter_args(parser) toast_tools.add_groundfilter_args(parser) toast_tools.add_atmosphere_args(parser) toast_tools.add_noise_args(parser) toast_tools.add_gainscrambler_args(parser) toast_tools.add_madam_args(parser) toast_tools.add_mapmaker_args(parser) toast_tools.add_filterbin_args(parser) toast_tools.add_sky_map_args(parser) toast_tools.add_sss_args(parser) toast_tools.add_tidas_args(parser) toast_tools.add_mc_args(parser) so_tools.add_corotator_args(parser) so_tools.add_time_constant_args(parser) so_tools.add_demodulation_args(parser) so_tools.add_h_n_args(parser) so_tools.add_crosslinking_args(parser) so_tools.add_cadence_map_args(parser) so_tools.add_hw_args(parser) so_tools.add_so_noise_args(parser) so_tools.add_pysm_args(parser) so_tools.add_export_args(parser) toast_tools.add_debug_args(parser) so_tools.add_import_args(parser) so_tools.add_sim_sso_args(parser) so_tools.add_flag_sso_args(parser) so_tools.add_sim_hwpss_args(parser) parser.add_argument( "--no-maps", required=False, default=False, action="store_true", help="Disable all mapmaking.", ) parser.add_argument("--outdir", required=False, default="out", help="Output directory") parser.add_argument( "--madam", required=False, action="store_true", help="Use libmadam for map-making", dest="use_madam", ) parser.add_argument( "--no-madam", required=False, action="store_false", help="Do not use libmadam for map-making [default]", dest="use_madam", ) parser.set_defaults(use_madam=True) try: args = parser.parse_args() except SystemExit as e: sys.exit() if len(args.bands.split(",")) != 1: # Multi frequency run. We don't support multiple copies of # scanned signal. if args.input_map: raise RuntimeError( "Multiple frequencies are not supported when scanning from a map" ) if args.weather is None: raise RuntimeError("You must provide a TOAST weather file") if comm.world_rank == 0: log.info("\n") log.info("All parameters:") for ag in vars(args): log.info("{} = {}".format(ag, getattr(args, ag))) log.info("\n") if args.group_size: comm = Comm(groupsize=args.group_size) if comm.world_rank == 0: if not os.path.isdir(args.outdir): try: os.makedirs(args.outdir) except FileExistsError: pass timer.report_clear("Parse arguments") return args, comm
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.comm_world.rank == 0: print('Running with {} processes at {}' ''.format(procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Planck Ringset making', fromfile_prefix_chars='@') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--nosingle', dest='nosingle', required=False, default=False, action='store_true', help='Do not compute single detector PSDs') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory ' 'for pointing') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--skymask', required=False, help='Pixel mask file') parser.add_argument('--skymap', required=False, help='Sky estimate file') parser.add_argument('--skypol', dest='skypol', required=False, default=False, action='store_true', help='Sky estimate is polarized') parser.add_argument('--no_spin_harmonics', dest='no_spin_harmonics', required=False, default=False, action='store_true', help='Do not include PSD bins with spin harmonics') parser.add_argument('--calibrate', required=False, help='Path to calibration file to calibrate with.') parser.add_argument('--calibrate_signal_estimate', dest='calibrate_signal_estimate', required=False, default=False, action='store_true', help='Calibrate ' 'the signal estimate using linear regression.') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, type=np.int, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, type=np.int, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, type=np.int, help='First ring to use') parser.add_argument('--ringlast', required=False, default=None, type=np.int, help='Last ring to use') parser.add_argument('--obtfirst', required=False, default=None, type=np.float, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, type=np.float, help='Last OBT to use') parser.add_argument('--out', required=False, default='.', help='Output directory') parser.add_argument('--nbin_psd', required=False, default=1000, type=np.int, help='Number of logarithmically ' 'distributed spectral bins to write.') parser.add_argument('--lagmax', required=False, default=100000, type=np.int, help='Maximum lag to evaluate for the ' 'autocovariance function [samples].') parser.add_argument('--stationary_period', required=False, default=86400., type=np.float, help='Length of a stationary interval [seconds].') # Dipole parameters dipogroup = parser.add_mutually_exclusive_group() dipogroup.add_argument('--dipole', dest='dipole', required=False, default=False, action='store_true', help='Simulate dipole') dipogroup.add_argument('--solsys_dipole', dest='solsys_dipole', required=False, default=False, action='store_true', help='Simulate solar system dipole') dipogroup.add_argument('--orbital_dipole', dest='orbital_dipole', required=False, default=False, action='store_true', help='Simulate orbital dipole') # Extra filter parser.add_argument('--filterfile', required=False, help='Extra filter file.') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.comm_world.rank == 0: print('All parameters:') print(args, flush=True) timer = Timer() timer.start() odrange = None if args.odfirst is not None and args.odlast is not None: odrange = (args.odfirst, args.odlast) ringrange = None if args.ringfirst is not None and args.ringlast is not None: ringrange = (args.ringfirst, args.ringlast) obtrange = None if args.obtfirst is not None and args.obtlast is not None: obtrange = (args.obtfirst, args.obtlast) detectors = None if args.dets is not None: detectors = re.split(',', args.dets) if args.nosingle and len(detectors) != 2: raise RuntimeError('You cannot skip the single detectors PSDs ' 'without multiple detectors.') # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) # Make output directory if not os.path.isdir(args.out) and comm.comm_world.rank == 0: os.mkdir(args.out) # create the TOD for this observation tod = tp.Exchange( comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, do_eff_cache=False, ) rimo = tod.rimo ob = {} ob['name'] = 'mission' ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise data.obs.append(ob) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Metadata queries") # Read the signal tod_name = 'signal' flags_name = 'flags' reader = tp.OpInputPlanck(signal_name=tod_name, flags_name=flags_name) if comm.comm_world.rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Reading") if args.calibrate is not None: fn = args.calibrate if comm.comm_world.rank == 0: print('Calibrating with {}'.format(fn), flush=True) calibrator = tp.OpCalibPlanck(signal_in=tod_name, signal_out=tod_name, file_gain=fn) calibrator.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Calibrate") # Optionally subtract the dipole do_dipole = (args.dipole or args.solsys_dipole or args.orbital_dipole) if do_dipole: if args.dipole: dipomode = 'total' elif args.solsys_dipole: dipomode = 'solsys' else: dipomode = 'orbital' dipo = tp.OpDipolePlanck(args.freq, mode=dipomode, output='dipole', keep_quats=True) dipo.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Dipole") subtractor = tp.OpCacheMath(in1=tod_name, in2='dipole', subtract=True, out=tod_name) if comm.comm_world.rank == 0: print('Subtracting dipole', flush=True) subtractor.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Dipole subtraction") # Optionally filter the signal apply_filter(args, data) timer.clear() # Estimate noise noise_estimator = tp.OpNoiseEstim( signal=tod_name, flags=flags_name, detmask=args.flagmask, commonmask=args.obtmask, maskfile=args.skymask, mapfile=args.skymap, out=args.out, rimo=rimo, pol=args.skypol, nbin_psd=args.nbin_psd, lagmax=args.lagmax, stationary_period=args.stationary_period, nosingle=args.nosingle, no_spin_harmonics=args.no_spin_harmonics, calibrate_signal_estimate=args.calibrate_signal_estimate) noise_estimator.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Noise estimation") gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def get_analytic_noise(args, comm, focalplane, verbose=True): """ Create a TOAST noise object. Create a noise object from the 1/f noise parameters contained in the focalplane database. Optionally add thermal common modes. """ timer = Timer() timer.start() detectors = sorted(focalplane.detector_data.keys()) fmins = {} fknees = {} alphas = {} NETs = {} rates = {} indices = {} for d in detectors: rates[d] = args.sample_rate fmins[d] = focalplane[d]["fmin"] fknees[d] = focalplane[d]["fknee"] alphas[d] = focalplane[d]["alpha"] NETs[d] = focalplane[d]["NET"] indices[d] = focalplane[d]["index"] ncommon = 0 coupling_strength_distributions = [] common_modes = [] if args.common_mode_noise: # Add an extra "virtual" detector for common mode noise for # every optics tube for common_mode in args.common_mode_noise.split(";"): ncommon += 1 try: fmin, fknee, alpha, net, center, width = np.array( common_mode.split(",")).astype(np.float64) except ValueError: fmin, fknee, alpha, net = np.array( common_mode.split(",")).astype(np.float64) center, width = 1, 0 coupling_strength_distributions.append([center, width]) hw = get_example() for itube, tube_slot in enumerate( sorted(hw.data["tube_slots"].keys())): d = "common_mode_{}_{}".format(ncommon - 1, tube_slot) detectors.append(d) common_modes.append(d) rates[d] = args.sample_rate fmins[d] = fmin fknees[d] = fknee alphas[d] = alpha NETs[d] = net indices[d] = ncommon * 100000 + itube noise = AnalyticNoise( rate=rates, fmin=fmins, detectors=detectors, fknee=fknees, alpha=alphas, NET=NETs, indices=indices, ) if args.common_mode_noise: mixmatrix = {} keys = set() if args.common_mode_only: detweight = 0 else: detweight = 1 for icommon in range(ncommon): # Update the mixing matrix in the noise operator center, width = coupling_strength_distributions[icommon] np.random.seed(1001 + icommon) couplings = center + np.random.randn(1000000) * width for det in focalplane.detector_data.keys(): if det not in mixmatrix: mixmatrix[det] = {det: detweight} keys.add(det) tube_slot = focalplane[det]["tube_slot"] common = "common_mode_{}_{}".format(icommon, tube_slot) index = focalplane[det]["index"] mixmatrix[det][common] = couplings[index] keys.add(common) # Add a diagonal entries, even if we wouldn't usually ask for # the common mode alone. for common in common_modes: mixmatrix[common] = {common: 1} # There should probably be an accessor method to update the # mixmatrix in the TOAST Noise object. if noise._mixmatrix is not None: raise RuntimeError("Did not expect non-empty mixing matrix") noise._mixmatrix = mixmatrix noise._keys = list(sorted(keys)) focalplane._noise = noise if comm.world_rank == 0 and verbose: timer.report_clear("Creating noise model") return noise
def create_observations(args, comm, focalplane, groupsize): timer = Timer() timer.start() if groupsize > len(focalplane.keys()): if comm.world_rank == 0: log.error("process group is too large for the number of detectors") comm.comm_world.Abort() # Detector information from the focalplane detectors = sorted(focalplane.keys()) detquats = {} detindx = None if "index" in focalplane[detectors[0]]: detindx = {} for d in detectors: detquats[d] = focalplane[d]["quat"] if detindx is not None: detindx[d] = focalplane[d]["index"] # Distribute the observations uniformly groupdist = distribute_uniform(args.obs_num, comm.ngroups) # Compute global time and sample ranges of all observations obsrange = regular_intervals( args.obs_num, args.start_time, 0, args.sample_rate, 3600 * args.obs_time_h, 3600 * args.gap_h, ) noise = get_analytic_noise(args, comm, focalplane) # The distributed timestream data data = Data(comm) # Every process group creates its observations group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] for ob in range(group_firstobs, group_firstobs + group_numobs): tod = TODSatellite( comm.comm_group, detquats, obsrange[ob].samples, coord=args.coord, firstsamp=obsrange[ob].first, firsttime=obsrange[ob].start, rate=args.sample_rate, spinperiod=args.spin_period_min, spinangle=args.spin_angle_deg, precperiod=args.prec_period_min, precangle=args.prec_angle_deg, detindx=detindx, detranks=comm.group_size, hwprpm=hwprpm, hwpstep=hwpstep, hwpsteptime=hwpsteptime, ) obs = {} obs["name"] = "science_{:05d}".format(ob) obs["tod"] = tod obs["intervals"] = None obs["baselines"] = None obs["noise"] = noise obs["id"] = ob data.obs.append(obs) if comm.world_rank == 0: timer.report_clear("Read parameters, compute data distribution") # Since we are simulating noise timestreams, we want # them to be contiguous and reproducible over the whole # observation. We distribute data by detector within an # observation, so ensure that our group size is not larger # than the number of detectors we have. # we set the precession axis now, which will trigger calculation # of the boresight pointing. for ob in range(group_numobs): curobs = data.obs[ob] tod = curobs["tod"] # Get the global sample offset from the original distribution of # intervals obsoffset = obsrange[group_firstobs + ob].first # Constantly slewing precession axis degday = 360.0 / 365.25 precquat = np.empty(4 * tod.local_samples[1], dtype=np.float64).reshape((-1, 4)) slew_precession_axis( precquat, firstsamp=(obsoffset + tod.local_samples[0]), samplerate=args.sample_rate, degday=degday, ) tod.set_prec_axis(qprec=precquat) del precquat if comm.world_rank == 0: timer.report_clear("Construct boresight pointing") return data
def parse_arguments(comm): timer = Timer() log = Logger.get() parser = argparse.ArgumentParser( description="Simulate ground-based boresight pointing. Simulate " "and map astrophysical signal.", fromfile_prefix_chars="@", ) add_dist_args(parser) add_debug_args(parser) add_todground_args(parser) add_pointing_args(parser) add_polyfilter_args(parser) add_groundfilter_args(parser) add_gainscrambler_args(parser) add_noise_args(parser) add_sky_map_args(parser) add_tidas_args(parser) parser.add_argument("--outdir", required=False, default="out", help="Output directory") add_madam_args(parser) add_binner_args(parser) parser.add_argument( "--madam", required=False, action="store_true", help="Use libmadam for map-making", dest="use_madam", ) parser.add_argument( "--no-madam", required=False, action="store_false", help="Do not use libmadam for map-making [default]", dest="use_madam", ) parser.set_defaults(use_madam=False) parser.add_argument( "--focalplane", required=False, default=None, help="Pickle file containing a dictionary of detector " "properties. The keys of this dict are the detector " "names, and each value is also a dictionary with keys " '"quat" (4 element ndarray), "fwhm" (float, arcmin), ' '"fknee" (float, Hz), "alpha" (float), and ' '"NET" (float). For optional plotting, the key "color"' " can specify a valid matplotlib color string.", ) try: args = parser.parse_args() except SystemExit: sys.exit(0) if args.tidas is not None: if not tidas_available: raise RuntimeError("TIDAS not found- cannot export") if comm.comm_world is None or comm.world_rank == 0: log.info("All parameters:") for ag in vars(args): log.info("{} = {}".format(ag, getattr(args, ag))) if args.group_size: comm = Comm(groupsize=args.group_size) if comm.comm_world is None or comm.comm_world.rank == 0: os.makedirs(args.outdir, exist_ok=True) if comm.comm_world is None or comm.world_rank == 0: timer.report_clear("Parsed parameters") return args, comm
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_ground_sim (total)") mpiworld, procs, rank, comm = get_comm() args, comm = parse_arguments(comm) # Initialize madam parameters madampars = setup_madam(args) # Load and broadcast the schedule file schedule = load_schedule(args, comm)[0] # load or simulate the focalplane detweights = load_focalplane(args, comm, schedule) # Create the TOAST data object to match the schedule. This will # include simulating the boresight pointing. data = create_observations(args, comm, schedule) # Expand boresight quaternions into detector pointing weights and # pixel numbers expand_pointing(args, comm, data) # Scan input map signalname = scan_sky_signal(args, comm, data, "signal") # Simulate noise if signalname is None: signalname = "signal" mc = 0 simulate_noise(args, comm, data, mc, signalname) # Set up objects to take copies of the TOD at appropriate times signalname_madam, sigcopy_madam, sigclear = setup_sigcopy( args, comm, signalname) npp, zmap = init_binner(args, comm, data, detweights) output_tidas(args, comm, data, signalname) outpath = setup_output(args, comm) # Make a copy of the signal for Madam copy_signal_madam(args, comm, data, sigcopy_madam) # Bin unprocessed signal for reference apply_binner(args, comm, data, npp, zmap, detweights, outpath, signalname) if args.apply_polyfilter or args.apply_groundfilter: # Filter signal apply_polyfilter(args, comm, data, signalname) apply_groundfilter(args, comm, data, signalname) # Bin the filtered signal apply_binner( args, comm, data, npp, zmap, detweights, outpath, signalname, prefix="filtered", ) data.obs[0]["tod"].cache.report() clear_signal(args, comm, data, sigclear) data.obs[0]["tod"].cache.report() # Now run Madam on the unprocessed copy of the signal if args.use_madam: apply_madam(args, comm, data, madampars, outpath, detweights, signalname_madam) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.outdir, "timing") dump_timing(alltimers, out) timer.report_clear("Gather and dump timing info") return
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_ground_sim (total)") timer0 = Timer() timer0.start() mpiworld, procs, rank, comm = get_comm() args, comm = parse_arguments(comm) # Initialize madam parameters madampars = setup_madam(args) # Load and broadcast the schedule file schedules = load_schedule(args, comm) # Load the weather and append to schedules load_weather(args, comm, schedules) # load or simulate the focalplane detweights = load_focalplanes(args, comm, schedules) # Create the TOAST data object to match the schedule. This will # include simulating the boresight pointing. data, telescope_data = create_observations(args, comm, schedules) # Split the communicator for day and season mapmaking time_comms = get_time_communicators(args, comm, data) # Expand boresight quaternions into detector pointing weights and # pixel numbers expand_pointing(args, comm, data) # Purge the pointing if we are NOT going to export the # data to a TIDAS volume if (args.tidas is None) and (args.spt3g is None): for ob in data.obs: tod = ob["tod"] tod.free_radec_quats() # Prepare auxiliary information for distributed map objects _, localsm, subnpix = get_submaps(args, comm, data) if args.pysm_model: focalplanes = [s.telescope.focalplane.detector_data for s in schedules] signalname = simulate_sky_signal(args, comm, data, focalplanes, subnpix, localsm, "signal") else: signalname = scan_sky_signal(args, comm, data, localsm, subnpix, "signal") # Set up objects to take copies of the TOD at appropriate times totalname, totalname_freq = setup_sigcopy(args) # Loop over Monte Carlos firstmc = args.MC_start nsimu = args.MC_count freqs = [float(freq) for freq in args.freq.split(",")] nfreq = len(freqs) for mc in range(firstmc, firstmc + nsimu): simulate_atmosphere(args, comm, data, mc, totalname) # Loop over frequencies with identical focal planes and identical # atmospheric noise. for ifreq, freq in enumerate(freqs): if comm.world_rank == 0: log.info("Processing frequency {}GHz {} / {}, MC = {}".format( freq, ifreq + 1, nfreq, mc)) # Make a copy of the atmosphere so we can scramble the gains and apply # frequency-dependent scaling. copy_signal(args, comm, data, totalname, totalname_freq) scale_atmosphere_by_frequency(args, comm, data, freq=freq, mc=mc, cache_name=totalname_freq) update_atmospheric_noise_weights(args, comm, data, freq, mc) # Add previously simulated sky signal to the atmospheric noise. add_signal(args, comm, data, totalname_freq, signalname, purge=(nsimu == 1)) mcoffset = ifreq * 1000000 simulate_noise(args, comm, data, mc + mcoffset, totalname_freq) simulate_sss(args, comm, data, mc + mcoffset, totalname_freq) scramble_gains(args, comm, data, mc + mcoffset, totalname_freq) if (mc == firstmc) and (ifreq == 0): # For the first realization and frequency, optionally # export the timestream data. output_tidas(args, comm, data, totalname) output_spt3g(args, comm, data, totalname) outpath = setup_output(args, comm, mc + mcoffset, freq) # Bin and destripe maps apply_madam( args, comm, data, madampars, outpath, detweights, totalname_freq, freq=freq, time_comms=time_comms, telescope_data=telescope_data, first_call=(mc == firstmc), ) if args.apply_polyfilter or args.apply_groundfilter: # Filter signal apply_polyfilter(args, comm, data, totalname_freq) apply_groundfilter(args, comm, data, totalname_freq) # Bin filtered maps apply_madam( args, comm, data, madampars, outpath, detweights, totalname_freq, freq=freq, time_comms=time_comms, telescope_data=telescope_data, first_call=False, extra_prefix="filtered", bin_only=True, ) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.outdir, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") timer0.report_clear("toast_ground_sim.py") return