def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_ground_sim (total)") mpiworld, procs, rank, comm = get_comm() args, comm = parse_arguments(comm) # Initialize madam parameters madampars = setup_madam(args) # Load and broadcast the schedule file schedule = load_schedule(args, comm)[0] # load or simulate the focalplane detweights = load_focalplane(args, comm, schedule) # Create the TOAST data object to match the schedule. This will # include simulating the boresight pointing. data = create_observations(args, comm, schedule) # Expand boresight quaternions into detector pointing weights and # pixel numbers expand_pointing(args, comm, data) # Prepare auxiliary information for distributed map objects localpix, localsm, subnpix = get_submaps(args, comm, data) # Scan input map signalname = scan_sky_signal(args, comm, data, localsm, subnpix, "signal") # Simulate noise if signalname is None: signalname = "signal" mc = 0 simulate_noise(args, comm, data, mc, signalname) # Set up objects to take copies of the TOD at appropriate times signalname_madam, sigcopy_madam, sigclear = setup_sigcopy( args, comm, signalname) npp, zmap = init_binner(args, comm, data, detweights, subnpix=subnpix, localsm=localsm) output_tidas(args, comm, data, signalname) outpath = setup_output(args, comm) # Make a copy of the signal for Madam copy_signal_madam(args, comm, data, sigcopy_madam) # Bin unprocessed signal for reference apply_binner(args, comm, data, npp, zmap, detweights, outpath, signalname) if args.apply_polyfilter or args.apply_groundfilter: # Filter signal apply_polyfilter(args, comm, data, signalname) apply_groundfilter(args, comm, data, signalname) # Bin the filtered signal apply_binner( args, comm, data, npp, zmap, detweights, outpath, signalname, prefix="filtered", ) data.obs[0]["tod"].cache.report() clear_signal(args, comm, data, sigclear) data.obs[0]["tod"].cache.report() # Now run Madam on the unprocessed copy of the signal if args.use_madam: apply_madam(args, comm, data, madampars, outpath, detweights, signalname_madam) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.outdir, "timing") dump_timing(alltimers, out) timer.report_clear("Gather and dump timing info") return
def create_observations(args, comm, focalplane, groupsize): timer = Timer() timer.start() log = Logger.get() if groupsize > len(focalplane.keys()): if comm.world_rank == 0: log.error("process group is too large for the number of detectors") comm.comm_world.Abort() # Detector information from the focalplane detectors = sorted(focalplane.keys()) detquats = {} detindx = None if "index" in focalplane[detectors[0]]: detindx = {} for d in detectors: detquats[d] = focalplane[d]["quat"] if detindx is not None: detindx[d] = focalplane[d]["index"] # Distribute the observations uniformly groupdist = distribute_uniform(args.obs_num, comm.ngroups) # Compute global time and sample ranges of all observations obsrange = regular_intervals( args.obs_num, args.start_time, 0, args.sample_rate, 3600 * args.obs_time_h, 3600 * args.gap_h, ) noise = pipeline_tools.get_analytic_noise(args, comm, focalplane) # The distributed timestream data data = Data(comm) # Every process group creates its observations group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] for ob in range(group_firstobs, group_firstobs + group_numobs): tod = TODSatellite( comm.comm_group, detquats, obsrange[ob].samples, coord=args.coord, firstsamp=obsrange[ob].first, firsttime=obsrange[ob].start, rate=args.sample_rate, spinperiod=args.spin_period_min, spinangle=args.spin_angle_deg, precperiod=args.prec_period_min, precangle=args.prec_angle_deg, detindx=detindx, detranks=comm.group_size, hwprpm=args.hwp_rpm, hwpstep=args.hwp_step_deg, hwpsteptime=args.hwp_step_time_s, ) obs = {} obs["name"] = "science_{:05d}".format(ob) obs["tod"] = tod obs["intervals"] = None obs["baselines"] = None obs["noise"] = noise obs["id"] = ob obs["focalplane"] = pipeline_tools.Focalplane(focalplane) data.obs.append(obs) if comm.world_rank == 0: timer.report_clear("Read parameters, compute data distribution") # Since we are simulating noise timestreams, we want # them to be contiguous and reproducible over the whole # observation. We distribute data by detector within an # observation, so ensure that our group size is not larger # than the number of detectors we have. # we set the precession axis now, which will trigger calculation # of the boresight pointing. for ob in range(group_numobs): curobs = data.obs[ob] tod = curobs["tod"] # Get the global sample offset from the original distribution of # intervals obsoffset = obsrange[group_firstobs + ob].first # Constantly slewing precession axis degday = 360.0 / 365.25 precquat = np.empty(4 * tod.local_samples[1], dtype=np.float64).reshape((-1, 4)) slew_precession_axis( precquat, firstsamp=(obsoffset + tod.local_samples[0]), samplerate=args.sample_rate, degday=degday, ) tod.set_prec_axis(qprec=precquat) del precquat if comm.world_rank == 0: timer.report_clear("Construct boresight pointing") return data
def _prepare(self): """ Examine the data object. """ log = Logger.get() timer = Timer() timer.start() nsamp = self._count_samples() # Determine the detectors and the pointing matrix non-zeros # from the first observation. Mappraiser will expect these to remain # unchanged across observations. tod = self._data.obs[0]["tod"] if self._dets is None: dets = tod.local_dets else: dets = [det for det in tod.local_dets if det in self._dets] ndet = len(dets) # We get the number of Non-zero pointing weights per pixel, from the # shape of the data from the first detector nnzname = "{}_{}".format(self._weights, dets[0]) nnz_full = tod.cache.reference(nnzname).shape[1] if nnz_full != 3: raise RuntimeError("OpMappraiser: Don't know how to make a map " "with nnz={}".format(nnz_full)) nnz = 3 nnz_stride = 1 else: nnz = nnz_full nnz_stride = 1 if "nside" not in self._params: raise RuntimeError( 'OpMappraiser: "nside" must be set in the parameter dictionary' ) nside = int(self._params["nside"]) # Inspect the valid intervals across all observations to # determine the number of samples per detector # N.B: Above comment is from OpMadam, for now # it only gives frequency binning of the PSDs psdfreqs = self._get_period_ranges(dets) self._comm.Barrier() if self._rank == 0 and self._verbose: timer.report_clear("Collect dataset dimensions") return ( dets, nsamp, ndet, nnz, nnz_full, nnz_stride, psdfreqs, nside, )
def create_observations(args, comm): timer = Timer() timer.start() nrange = 1 odranges = None if args.odfirst is not None and args.odlast is not None: odranges = [] firsts = [int(i) for i in str(args.odfirst).split(',')] lasts = [int(i) for i in str(args.odlast).split(',')] for odfirst, odlast in zip(firsts, lasts): odranges.append((odfirst, odlast)) nrange = len(odranges) ringranges = None if args.ringfirst is not None and args.ringlast is not None: ringranges = [] firsts = [int(i) for i in str(args.ringfirst).split(',')] lasts = [int(i) for i in str(args.ringlast).split(',')] for ringfirst, ringlast in zip(firsts, lasts): ringranges.append((ringfirst, ringlast)) nrange = len(ringranges) obtranges = None if args.obtfirst is not None and args.obtlast is not None: obtranges = [] firsts = [float(i) for i in str(args.obtfirst).split(',')] lasts = [float(i) for i in str(args.obtlast).split(',')] for obtfirst, obtlast in zip(firsts, lasts): obtranges.append((obtfirst, obtlast)) nrange = len(obtranges) if odranges is None: odranges = [None] * nrange if ringranges is None: ringranges = [None] * nrange if obtranges is None: obtranges = [None] * nrange detectors = None if args.dets is not None: detectors = re.split(',', args.dets) # create the TOD for this observation if args.noisefile != 'RIMO': do_eff_cache = True else: do_eff_cache = False tods = [] for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges): tods.append( tp.Exchange( comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, pntflagmask=args.pntflagmask, do_eff_cache=do_eff_cache, coord=args.coord, )) rimo = tods[0].rimo # Make output directory if comm.world_rank == 0: os.makedirs(args.out, exist_ok=True) if args.noisefile != 'RIMO': # We split MPI_COMM_WORLD into single process groups, each of # which is assigned one or more observations (rings) comm = Comm(groupsize=1) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = Data(comm) for iobs, tod in enumerate(tods): if args.noisefile != 'RIMO': # Use a toast helper method to optimally distribute rings between # processes. dist = distribute_discrete(tod.ringsizes, comm.world_size) my_first_ring, my_n_ring = dist[comm.world_rank] for my_ring in range(my_first_ring, my_first_ring + my_n_ring): ringtod = tp.Exchange.from_tod(tod, my_ring, comm.comm_group, noisefile=args.noisefile) ob = {} ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring) ob['id'] = ringtod.globalfirst_ring ob['tod'] = ringtod ob['intervals'] = ringtod.valid_intervals ob['baselines'] = None ob['noise'] = ringtod.noise data.obs.append(ob) else: ob = {} ob['name'] = 'observation{:04}'.format(iobs) ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise data.obs.append(ob) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Metadata queries") return data
def _synthesize_map(self, fn_cmb, there): """ Synthesize the stored alm expansion into a map and place the map in node-shared memory. """ timer = Timer() timer.start() if not there: # Use libsharp to perform the synthesis across the communicator if self._quickpolbeam is None: beam = hp.gauss_beam(fwhm=self._fwhm, lmax=self._lmax, pol=True) beam = beam[:, 0:3].copy() else: beam = np.array(hp.read_cl(self._quickpolbeam)) if beam.ndim == 1: beam = np.vstack([beam, beam, beam]) beam = beam[:, :self._lmax + 1].T.copy() almT = self._alm[0].reshape(1, 1, -1) self._alminfo.almxfl(almT, np.ascontiguousarray(beam[:, 0:1])) my_outmap = synthesis(self._grid, self._alminfo, almT, spin=0, comm=self._comm)[0] my_outmap = [my_outmap] if self._pol: almP = self._alm[1:3].reshape(1, 2, -1) self._alminfo.almxfl(almP, np.ascontiguousarray(beam[:, (1, 2)])) my_outmap.append( synthesis(self._grid, self._alminfo, almP, spin=2, comm=self._comm)[0]) # Discard the a_lm del self._alm my_outmap = np.vstack(my_outmap) my_pixels = self._dist_rings.local_pixels my_maptemp = np.zeros([self._nnz, self._npix], dtype=np.float) maptemp = np.zeros([self._nnz, self._npix], dtype=np.float) my_maptemp[:, my_pixels] = my_outmap self._comm.Reduce(my_maptemp, maptemp) del my_maptemp maptemp = hp.reorder(maptemp, r2n=True) timer.stop() if self._global_rank == 0: timer.report("synthesize CMB map") # Save the CMB map os.makedirs(CMBCACHE, exist_ok=True) header = [("fwhm", np.degrees(self._fwhm), "gaussian smoothing (deg)")] hp.write_map(fn_cmb, maptemp, extra_header=header, overwrite=True, nest=True) print("CMB map saved in {}".format(fn_cmb), flush=True) else: if self._global_rank == 0: print("Loading cached CMB map from {}".format(fn_cmb), flush=True) if self._rank == 0: maptemp = hp.read_map(fn_cmb, None, nest=True, verbose=False, dtype=np.float32) if not self._pol: maptemp = maptemp[0] else: maptemp = None self.mapsampler = MapSampler( None, pol=self._pol, comm=self._comm, preloaded_map=maptemp, nest=True, plug_holes=False, use_shmem=True, ) del maptemp return
def create_observations(args, comm, schedules): """ Create and distribute TOAST observations for every CES in schedules. Args: schedules (iterable) : a list of Schedule objects. """ log = Logger.get() timer = Timer() timer.start() data = Data(comm) # Loop over the schedules, distributing each schedule evenly across # the process groups. For now, we'll assume that each schedule has # the same number of operational days and the number of process groups # matches the number of operational days. Relaxing these constraints # will cause the season break to occur on different process groups # for different schedules and prevent splitting the communicator. for schedule in schedules: telescope = schedule.telescope all_ces = schedule.ceslist nces = len(all_ces) breaks = pipeline_tools.get_breaks(comm, all_ces, args) groupdist = distribute_uniform(nces, comm.ngroups, breaks=breaks) group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] for ices in range(group_firstobs, group_firstobs + group_numobs): obs = create_observation(args, comm, telescope, all_ces[ices]) data.obs.append(obs) if comm.comm_world is None or comm.comm_group.rank == 0: log.info("Group # {:4} has {} observations.".format(comm.group, len(data.obs))) if len(data.obs) == 0: raise RuntimeError( "Too many tasks. Every MPI task must " "be assigned to at least one observation." ) if comm.comm_world is not None: comm.comm_world.barrier() timer.stop() if comm.world_rank == 0: timer.report("Simulated scans") # Split the data object for each telescope for separate mapmaking. # We could also split by site. if len(schedules) > 1: telescope_data = data.split("telescope_name") if len(telescope_data) == 1: # Only one telescope available telescope_data = [] else: telescope_data = [] telescope_data.insert(0, ("all", data)) return data, telescope_data
def parse_arguments(comm): timer = Timer() timer.start() log = Logger.get() parser = argparse.ArgumentParser( description="Simulate ground-based boresight pointing. Simulate " "atmosphere and make maps for some number of noise Monte Carlos.", fromfile_prefix_chars="@", ) pipeline_tools.add_dist_args(parser) pipeline_tools.add_debug_args(parser) pipeline_tools.add_todground_args(parser) pipeline_tools.add_pointing_args(parser) pipeline_tools.add_polyfilter_args(parser) pipeline_tools.add_groundfilter_args(parser) pipeline_tools.add_atmosphere_args(parser) pipeline_tools.add_noise_args(parser) pipeline_tools.add_gainscrambler_args(parser) pipeline_tools.add_madam_args(parser) pipeline_tools.add_mapmaker_args(parser) pipeline_tools.add_sky_map_args(parser) pipeline_tools.add_pysm_args(parser) pipeline_tools.add_sss_args(parser) pipeline_tools.add_tidas_args(parser) pipeline_tools.add_spt3g_args(parser) pipeline_tools.add_mc_args(parser) parser.add_argument( "--outdir", required=False, default="out", help="Output directory" ) parser.add_argument( "--madam", required=False, action="store_true", help="Use libmadam for map-making", dest="use_madam", ) parser.add_argument( "--no-madam", required=False, action="store_false", help="Do not use libmadam for map-making [default]", dest="use_madam", ) parser.set_defaults(use_madam=False) parser.add_argument( "--focalplane", required=False, default=None, help="Pickle file containing a dictionary of detector " "properties. The keys of this dict are the detector " "names, and each value is also a dictionary with keys " '"quat" (4 element ndarray), "fwhm" (float, arcmin), ' '"fknee" (float, Hz), "alpha" (float), and ' '"NET" (float).', ) parser.add_argument( "--freq", required=True, help="Comma-separated list of frequencies with identical focal planes." " They override the bandpasses in the focalplane for the purpose of" " scaling the atmospheric signal but not for simulating the sky signal.", ) try: args = parser.parse_args() except SystemExit: sys.exit(0) if args.tidas is not None: if not tidas_available: raise RuntimeError("TIDAS not found- cannot export") if args.spt3g is not None: if not spt3g_available: raise RuntimeError("SPT3G not found- cannot export") if len(args.freq.split(",")) != 1: # Multi frequency run. We don't support multiple copies of # scanned signal. if args.input_map: raise RuntimeError( "Multiple frequencies are not supported when scanning from a map" ) if args.simulate_atmosphere and args.weather is None: raise RuntimeError("Cannot simulate atmosphere without a TOAST weather file") if comm.world_rank == 0: log.info("All parameters:") for ag in vars(args): log.info("{} = {}".format(ag, getattr(args, ag))) if args.group_size: comm = Comm(groupsize=args.group_size) if comm.world_rank == 0: os.makedirs(args.outdir, exist_ok=True) timer.stop() if comm.world_rank == 0: timer.report("Parsed parameters") return args, comm
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_so_sim (total)") timer0 = Timer() timer0.start() mpiworld, procs, rank, comm = toast_tools.get_comm() memreport("at the beginning of the pipeline", comm.comm_world) args, comm = parse_arguments(comm) # Load and broadcast the schedule file schedules = toast_tools.load_schedule(args, comm) # Load the weather and append to schedules toast_tools.load_weather(args, comm, schedules) # load or simulate the focalplane detweights = so_tools.load_focalplanes(args, comm, schedules) # Create the TOAST data object to match the schedule. This will # include simulating the boresight pointing. data, telescope_data = so_tools.create_observations(args, comm, schedules) memreport("after creating observations", comm.comm_world) # Optionally rewrite the noise PSD:s in each observation to include # elevation-dependence so_tools.get_elevation_noise(args, comm, data) totalname = "total" # Split the communicator for day and season mapmaking time_comms = toast_tools.get_time_communicators(args, comm, data) # Expand boresight quaternions into detector pointing weights and # pixel numbers toast_tools.expand_pointing(args, comm, data) memreport("after pointing", comm.comm_world) # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) madam = None for mc in range(firstmc, firstmc + nmc): timer_mc = Timer() timer_mc.start() outpath = setup_output(args, comm, mc) outprefix = args.map_prefix + "_filtered" if args.madam: outmap = os.path.join(outpath, outprefix + "_bmap.fits") else: outmap = os.path.join(outpath, outprefix + "_binned.fits") if os.path.isfile(outmap): if comm.world_rank == 0: log.info("{} exists, skipping".format(outmap)) continue if comm.world_rank == 0: log.info("Processing MC = {} into {}".format(mc, outmap)) # Ensure there is no stale signal in the cache toast.tod.OpCacheClear(totalname).exec(data) if args.pysm_model: if schedules is not None: focalplanes = [ s.telescope.focalplane.detector_data for s in schedules ] else: focalplanes = [telescope.focalplane.detector_data] so_tools.simulate_sky_signal(args, comm, data, focalplanes, totalname, mc=mc) else: toast_tools.scan_sky_signal(args, comm, data, totalname, mc=mc) memreport("after PySM", comm.comm_world) if args.apply_polyfilter or args.apply_groundfilter: # Filter signal toast_tools.apply_polyfilter(args, comm, data, totalname) toast_tools.apply_groundfilter(args, comm, data, totalname) memreport("after filter", comm.comm_world) # Bin maps timer_map = Timer() timer_map.start() if args.madam: if madam is None: madampars = {} madampars["temperature_only"] = False for name in [ "kfirst", "write_map", "write_matrix", "write_wcov", "write_hits" ]: madampars[name] = False madampars["write_binmap"] = True madampars["concatenate_messages"] = True madampars["allreduce"] = True madampars["nside_submap"] = args.nside_submap madampars["reassign_submaps"] = True madampars["pixlim_map"] = 1e-2 madampars["pixmode_map"] = 2 # Instead of fixed detector weights, we'll want to use scaled noise # PSD:s that include the atmospheric noise madampars["radiometers"] = True madampars["noise_weights_from_psd"] = True madampars["nside_map"] = args.nside madampars["fsample"] = args.sample_rate madampars["path_output"] = outpath madampars["file_root"] = outprefix if args.madam_concatenate_messages: # Collective communication is fast but requires memory madampars["concatenate_messages"] = True if args.madam_allreduce: # Every process will allocate a copy of every observed submap. madampars["allreduce"] = True else: # Every process will allocate complete send and receive buffers madampars["allreduce"] = False else: # Slow but memory-efficient point-to-point communication. Allocate # only enough memory to communicate with one process at a time. madampars["concatenate_messages"] = False madampars["allreduce"] = False madam = toast.todmap.OpMadam( params=madampars, detweights=detweights, name=totalname, common_flag_mask=args.common_flag_mask, purge_tod=True, mcmode=False, conserve_memory=args.madam_conserve_memory, ) else: madam.params["path_output"] = outpath madam.exec(data) del madam madam = None else: mapmaker = toast.todmap.OpMapMaker( nside=args.nside, nnz=3, name=totalname, outdir=outpath, outprefix=outprefix + "_", write_hits=False, zip_maps=False, write_wcov_inv=False, write_wcov=False, write_binned=True, write_destriped=False, write_rcond=False, rcond_limit=1e-3, baseline_length=None, common_flag_mask=args.common_flag_mask, ) mapmaker.exec(data) if comm.world_rank == 0: timer_map.report_clear("Bin map") memreport("after filter & bin", comm.comm_world) if comm.world_rank == 0: timer_mc.report_clear( "Monte Carlo iteration # {:05}".format(mc)) if comm.comm_world is not None: comm.comm_world.barrier() memreport("at the end of the pipeline", comm.comm_world) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if rank == 0: out = os.path.join(args.outdir, "timing") dump_timing(alltimers, out) timer.report_clear("Gather and dump timing info") if comm.world_rank == 0: timer0.report_clear("toast_so_tf.py pipeline") return
def parse_arguments(comm): timer = Timer() timer.start() log = Logger.get() parser = argparse.ArgumentParser( description="Simulate ground-based boresight pointing. Simulate " "atmosphere and make maps for some number of noise Monte Carlos.", fromfile_prefix_chars="@", ) toast_tools.add_dist_args(parser) toast_tools.add_todground_args(parser) toast_tools.add_pointing_args(parser) toast_tools.add_polyfilter_args(parser) toast_tools.add_groundfilter_args(parser) toast_tools.add_noise_args(parser) toast_tools.add_sky_map_args(parser) toast_tools.add_mc_args(parser) so_tools.add_hw_args(parser) so_tools.add_so_noise_args(parser) so_tools.add_pysm_args(parser) so_tools.add_export_args(parser) toast_tools.add_debug_args(parser) parser.add_argument("--outdir", required=False, default="out", help="Output directory") parser.add_argument("--map-prefix", required=False, default="toast", help="Output map prefix") parser.add_argument( "--madam", required=False, action="store_true", help="Use libmadam to bin the signal", dest="madam", ) parser.add_argument( "--no-madam", required=False, action="store_false", help="Do not use libMadam to bin the signal", dest="madam", ) parser.set_defaults(madam=False) parser.add_argument( "--madam-conserve-memory", required=False, action="store_true", help="Stage the Madam buffer packing", dest="madam_conserve_memory", ) parser.add_argument( "--no-madam-conserve-memory", required=False, action="store_false", help="Do not stage the Madam buffer packing", dest="madam_conserve_memory", ) parser.set_defaults(madam_conserve_memory=True) parser.add_argument( "--madam-allreduce", required=False, action="store_true", help="Use the allreduce communication pattern in Madam", dest="madam_allreduce", ) parser.add_argument( "--no-madam-allreduce", required=False, action="store_false", help="Do not use the allreduce communication pattern in Madam", dest="madam_allreduce", ) parser.set_defaults(madam_allreduce=False) parser.add_argument( "--madam-concatenate-messages", required=False, action="store_true", help="Use the alltoallv commucation pattern in Madam", dest="madam_concatenate_messages", ) parser.add_argument( "--no-madam-concatenate-messages", required=False, action="store_false", help="Use the point-to-point communication pattern in Madam", dest="madam_concatenate_messages", ) parser.set_defaults(madam_concatenate_messages=True) try: args = parser.parse_args() except SystemExit as e: sys.exit() if len(args.bands.split(",")) != 1: # Multi frequency run. We don't support multiple copies of # scanned signal. if args.input_map: raise RuntimeError( "Multiple frequencies are not supported when scanning from a map" ) if comm.world_rank == 0: log.info("\n") log.info("All parameters:") for ag in vars(args): log.info("{} = {}".format(ag, getattr(args, ag))) log.info("\n") if args.group_size: comm = Comm(groupsize=args.group_size) if comm.world_rank == 0: if not os.path.isdir(args.outdir): try: os.makedirs(args.outdir) except FileExistsError: pass timer.report_clear("Parse arguments") return args, comm
def simulate_sky_signal(args, comm, data, focalplanes, subnpix, localsm, signalname=None): """ Use PySM to simulate smoothed sky signal. """ log = Logger.get() timer = Timer() timer.start() # Convolve a signal TOD from PySM if comm.world_rank == 0: log.info("Simulating sky signal with PySM") map_dist = (None if comm is None else pysm.MapDistribution( nside=args.nside, mpi_comm=comm.comm_rank)) pysm_component_objects = [] pysm_model = [] for model_tag in args.pysm_model.split(","): if not model_tag.startswith("SO"): pysm_model.append(model_tag) else: if so_pysm_models is None: raise RuntimeError( "{} requires so_pysm_models".format(model_tag)) if model_tag == "SO_x1_cib": pysm_component_objects.append( so_pysm_models.WebSkyCIB( websky_version="0.3", interpolation_kind="linear", nside=args.nside, map_dist=map_dist, )) elif model_tag == "SO_x1_ksz": pysm_component_objects.append( so_pysm_models.WebSkySZ( version="0.3", nside=args.nside, map_dist=map_dist, sz_type="kinetic", )) elif model_tag == "SO_x1_tsz": pysm_component_objects.append( so_pysm_models.WebSkySZ( version="0.3", nside=args.nside, map_dist=map_dist, sz_type="thermal", )) elif model_tag.startswith("SO_x1_cmb"): lensed = "unlensed" not in model_tag include_solar_dipole = "solar" in model_tag pysm_component_objects.append( so_pysm_models.WebSkyCMBMap( websky_version="0.3", lensed=lensed, include_solar_dipole=include_solar_dipole, seed=1, nside=args.nside, map_dist=map_dist, )) else: pysm_component_objects.append( so_pysm_models.get_so_models(model_tag, args.nside, map_dist=map_dist)) if signalname is None: signalname = "pysmsignal" op_sim_pysm = OpSimPySM( comm=comm.comm_rank, out=signalname, pysm_model=pysm_model, pysm_component_objects=pysm_component_objects, focalplanes=focalplanes, nside=args.nside, subnpix=subnpix, localsm=localsm, apply_beam=args.pysm_apply_beam, coord="G", # setting G doesn't perform any rotation map_dist=map_dist, ) assert args.coord in "CQ", "Input SO models are always in Equatorial coordinates" op_sim_pysm.exec(data) if comm.comm_world is not None: comm.comm_world.barrier() timer.stop() if comm.world_rank == 0: timer.report("PySM") return signalname
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() if comm.world_rank == 0: print("Running with {} processes at {}".format( procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Simple dipole pipeline', fromfile_prefix_chars='@') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory for ' 'pointing') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--pntflagmask', required=False, default=0, type=np.int, help='Which OBT flag bits to raise for HCM maneuvers') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, help='First ring to use') parser.add_argument('--ringlast', required=False, default=None, help='Last ring to use') parser.add_argument('--obtfirst', required=False, default=None, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, help='Last OBT to use') parser.add_argument('--out', required=False, default='.', help='Output directory') # Dipole parameters dipogroup = parser.add_mutually_exclusive_group() dipogroup.add_argument('--dipole', dest='dipole', required=False, default=False, action='store_true', help='Simulate dipole') dipogroup.add_argument('--solsys-dipole', required=False, default=False, action='store_true', help='Simulate solar system dipole') dipogroup.add_argument('--orbital-dipole', required=False, default=False, action='store_true', help='Simulate orbital dipole') dipo_parameters_group = parser.add_argument_group('dipole_parameters') dipo_parameters_group.add_argument( '--solsys_speed', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_speed"], help='Solar system speed wrt. CMB rest frame in km/s. Default is ' 'Planck 2015 best fit value') dipo_parameters_group.add_argument( '--solsys-glon', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_glon"], help='Solar system velocity direction longitude in degrees') dipo_parameters_group.add_argument( '--solsys-glat', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_glat"], help='Solar system velocity direction latitude in degrees') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.world_rank == 0: print('All parameters:') print(args, flush=True) timer = Timer() timer.start() do_dipole = args.dipole or args.solsys_dipole or args.orbital_dipole if not do_dipole: raise RuntimeError( "You have to set dipole, solsys-dipole or orbital-dipole") nrange = 1 odranges = None if args.odfirst is not None and args.odlast is not None: odranges = [] firsts = [int(i) for i in str(args.odfirst).split(',')] lasts = [int(i) for i in str(args.odlast).split(',')] for odfirst, odlast in zip(firsts, lasts): odranges.append((odfirst, odlast)) nrange = len(odranges) ringranges = None if args.ringfirst is not None and args.ringlast is not None: ringranges = [] firsts = [int(i) for i in str(args.ringfirst).split(',')] lasts = [int(i) for i in str(args.ringlast).split(',')] for ringfirst, ringlast in zip(firsts, lasts): ringranges.append((ringfirst, ringlast)) nrange = len(ringranges) obtranges = None if args.obtfirst is not None and args.obtlast is not None: obtranges = [] firsts = [float(i) for i in str(args.obtfirst).split(',')] lasts = [float(i) for i in str(args.obtlast).split(',')] for obtfirst, obtlast in zip(firsts, lasts): obtranges.append((obtfirst, obtlast)) nrange = len(obtranges) if odranges is None: odranges = [None] * nrange if ringranges is None: ringranges = [None] * nrange if obtranges is None: obtranges = [None] * nrange detectors = None if args.dets is not None: detectors = re.split(',', args.dets) # create the TOD for this observation tods = [] for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges): # create the TOD for this observation tods.append( tp.Exchange(comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, pntflagmask=args.pntflagmask, do_eff_cache=False)) # Make output directory if not os.path.isdir(args.out) and comm.world_rank == 0: os.makedirs(args.out) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) for iobs, tod in enumerate(tods): ob = {} ob['name'] = 'observation{:04}'.format(iobs) ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise ob['noise_simu'] = tod.noise data.obs.append(ob) rimo = tods[0].rimo if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Metadata queries") # Always read the signal and flags, even if the signal is later # overwritten. There is no overhead for the signal because it is # interlaced with the flags. tod_name = 'signal' timestamps_name = 'timestamps' flags_name = 'flags' common_flags_name = 'common_flags' reader = tp.OpInputPlanck(signal_name=tod_name, flags_name=flags_name, timestamps_name=timestamps_name, commonflags_name=common_flags_name) if comm.world_rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Read") """ # Clear the signal eraser = tp.OpCacheMath(in1=tod_name, in2=0, multiply=True, out=tod_name) if comm.world_rank == 0: print('Erasing TOD', flush=True) eraser.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Erase") """ # make a planck Healpix pointing matrix mode = 'IQU' nside = 512 pointing = tp.OpPointingPlanck(nside=nside, mode=mode, RIMO=rimo, margin=0, apply_flags=True, keep_vel=do_dipole, keep_pos=False, keep_phase=False, keep_quats=do_dipole) pointing.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Pointing Matrix") flags_name = 'flags' common_flags_name = 'common_flags' # Simulate the dipole if args.dipole: dipomode = 'total' elif args.solsys_dipole: dipomode = 'solsys' else: dipomode = 'orbital' dipo = tp.OpDipolePlanck(args.freq, solsys_speed=args.solsys_speed, solsys_glon=args.solsys_glon, solsys_glat=args.solsys_glat, mode=dipomode, output='dipole', keep_quats=False, npipe_mode=True) dipo.exec(data) dipo = tp.OpDipolePlanck(args.freq, solsys_speed=args.solsys_speed, solsys_glon=args.solsys_glon, solsys_glat=args.solsys_glat, mode=dipomode, output='dipole4pi', keep_quats=False, npipe_mode=False, lfi_mode=False) dipo.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Dipole") # Write out the values in ASCII for iobs, obs in enumerate(data.obs): tod = obs["tod"] times = tod.local_times() velocity = tod.local_velocity() for det in tod.local_dets: quat = tod.local_pointing(det) angles = np.vstack(qarray.to_angles(quat)).T signal = tod.local_signal(det) dipole = tod.local_signal(det, "dipole") dipole4pi = tod.local_signal(det, "dipole4pi") fname_out = os.path.join( args.out, "{}_dipole.{}.{}.{}.txt".format(dipomode, comm.world_rank, iobs, det)) with open(fname_out, "w") as fout: for t, ang, vel, sig, dipo, dipo4pi in zip( times, angles, velocity, signal, dipole, dipole4pi): fout.write( (10 * " {}" + "\n").format(t, *ang, *vel, sig, dipo, dipo4pi)) print("{} : Wrote {}".format(comm.world_rank, fname_out)) if comm.world_rank == 0: timer.report_clear("Write dipole") gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def _stage_data( self, nsamp, ndet, nnz, nnz_full, nnz_stride, psdfreqs, detectors, nside, ): """ create Mappraiser-compatible buffers Collect the TOD into Mappraiser buffers. Process pixel weights Separate from the rest to reduce the memory high water mark When the user has set purge=True Moving data between toast and Mappraiser buffers has an overhead. We perform the operation in a staggered fashion to have the overhead only once per node. """ log = Logger.get() nodecomm = self._comm.Split_type(MPI.COMM_TYPE_SHARED, self._rank) # Check if the user has elected to stagger staging the data on each # node to avoid exhausting memory if self._conserve_memory: if self._conserve_memory == 1: nread = nodecomm.size else: nread = min(self._conserve_memory, nodecomm.size) else: nread = 1 self._comm.Barrier() timer_tot = Timer() timer_tot.start() # Stage time (Tpltz blocks in Mappraiser), it is never purged # so the staging is never stepped timer = Timer() # THIS STEP IS SKIPPED: we do not have timestamps, nor do we build Toeplitz blocks # from TOAST psds which comprise detector noise only - a psd fit is done when staging noise - #timer.start() #invtt_list = self._stage_time(detectors, nsamp, psdfreqs) #self._mappraiser_invtt = np.array([np.array(invtt_i, dtype= mappraiser.INVTT_TYPE) for invtt_i in invtt_list]) #del invtt_list #self._mappraiser_invtt = np.concatenate(self._mappraiser_invtt) #if self._verbose: # nodecomm.Barrier() # if self._rank == 0: # timer.report_clear("Stage time") #memreport("after staging time", self._comm) # DEBUG #count_caches( # self._data, self._comm, nodecomm, self._cache, "after staging time" #) # DEBUG # Stage signal. If signal is not being purged, staging is not stepped timer.start() signal_dtype, local_blocks_sizes = self._stage_signal( detectors, nsamp, ndet, nodecomm, nread) if self._verbose: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Stage signal") memreport("after staging signal", self._comm) # DEBUG count_caches(self._data, self._comm, nodecomm, self._cache, "after staging signal") # DEBUG # Stage noise. If noise is not being purged, staging is not stepped timer.start() invtt_list, noise_dtype = self._stage_noise(detectors, nsamp, ndet, nodecomm, nread) self._mappraiser_invtt = np.array([ np.array(invtt_i, dtype=mappraiser.INVTT_TYPE) for invtt_i in invtt_list ]) del invtt_list self._mappraiser_invtt = np.concatenate(self._mappraiser_invtt) if self._params["uniform_w"] == 1: self._mappraiser_invtt = np.ones_like(self._mappraiser_invtt) if self._verbose: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Stage noise") memreport("after staging noise", self._comm) # DEBUG count_caches(self._data, self._comm, nodecomm, self._cache, "after staging noise") # DEBUG # Stage pixels timer_step = Timer() timer_step.start() for iread in range(nread): nodecomm.Barrier() timer.start() if nodecomm.rank % nread == iread: pixels_dtype = self._stage_pixels(detectors, nsamp, ndet, nnz, nside) if self._verbose and nread > 1: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Stage pixels {} / {}".format( iread + 1, nread)) if self._verbose: nodecomm.Barrier() if self._rank == 0: timer_step.report_clear("Stage pixels") memreport("after staging pixels", self._comm) # DEBUG count_caches(self._data, self._comm, nodecomm, self._cache, "after staging pixels") # DEBUG # Stage pixel weights timer_step.start() weight_dtype = self._stage_pixweights( detectors, nsamp, ndet, nnz, nnz_full, nnz_stride, nodecomm, nread, ) if self._verbose: nodecomm.Barrier() if self._rank == 0: timer_step.report_clear("Stage pixel weights") memreport("after staging pixel weights", self._comm) # DEBUG count_caches(self._data, self._comm, nodecomm, self._cache, "after staging pixel weights") # DEBUG del nodecomm if self._rank == 0 and self._verbose: timer_tot.report_clear("Stage all data") # detweights is either a dictionary of weights specified at # construction time, or else we use uniform weighting. # N.B: This is essentially useless in current implementation detw = {} if self._detw is None: for idet, det in enumerate(detectors): detw[det] = 1.0 else: detw = self._detw detweights = np.zeros(ndet, dtype=np.float64) for idet, det in enumerate(detectors): detweights[idet] = detw[det] # Get global array of data sizes of the full communicator data_size_proc = np.array(self._comm.allgather( len(self._mappraiser_signal)), dtype=np.int32) # Get number of local observations nobsloc = len(self._data.obs) return data_size_proc, nobsloc, local_blocks_sizes, signal_dtype, noise_dtype, pixels_dtype, weight_dtype
def _stage_pixweights( self, detectors, nsamp, ndet, nnz, nnz_full, nnz_stride, nodecomm, nread, ): """Now collect the pixel weights """ log = Logger.get() timer = Timer() # Determine if we can purge the pixel weights and avoid keeping two # copies of the weights in memory purge = self._purge_weights or (nnz == nnz_full) if not purge: nread = 1 nodecomm = MPI.COMM_SELF for iread in range(nread): nodecomm.Barrier() timer.start() if nodecomm.rank % nread == iread: self._mappraiser_pixweights = self._cache.create( "pixweights", mappraiser.WEIGHT_TYPE, (nsamp * ndet * nnz, )) self._mappraiser_pixweights[:] = 0 global_offset = 0 for iobs, obs in enumerate(self._data.obs): tod = obs["tod"] for idet, det in enumerate(detectors): # get the pixels and weights for the valid intervals # from the cache weightsname = "{}_{}".format(self._weights, det) weights = tod.cache.reference(weightsname) weight_dtype = weights.dtype offset = global_offset nn = len(weights) dwslice = slice( (idet * nsamp + offset) * nnz, (idet * nsamp + offset + nn) * nnz, ) self._mappraiser_pixweights[dwslice] = weights.flatten( )[::nnz_stride] offset += nn del weights # Purge the weights but restore them from the Mappraiser # buffers when purge_weights=False. if purge: for idet, det in enumerate(detectors): weightsname = "{}_{}".format(self._weights, det) tod.cache.clear(pattern=weightsname) global_offset = offset if self._verbose and nread > 1: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Stage pixel weights {} / {}".format( iread + 1, nread)) return weight_dtype
def _stage_noise(self, detectors, nsamp, ndet, nodecomm, nread): """ Stage noise timestream (detector noise + atmosphere) """ log = Logger.get() timer = Timer() # Determine if we can purge the signal and avoid keeping two # copies in memory purge = self._noise_name is not None and self._purge_tod if not purge: nread = 1 nodecomm = MPI.COMM_SELF for iread in range(nread): nodecomm.Barrier() timer.start() if nodecomm.rank % nread == iread: self._mappraiser_noise = self._cache.create( "noise", mappraiser.SIGNAL_TYPE, (nsamp * ndet, )) if self._noise_name == None: self._mappraiser_noise = np.zeros_like( self._mappraiser_noise) invtt_list = [] for i in range(len(self._data.obs) * len(detectors)): invtt_list.append( np.ones(1)) #Must be used with lambda = 1 return invtt_list, self._mappraiser_noise.dtype self._mappraiser_noise[:] = np.nan global_offset = 0 invtt_list = [] # fknee_list = [] # fmin_list = [] # alpha_list = [] # logsigma2_list = [] for iobs, obs in enumerate(self._data.obs): tod = obs["tod"] for idet, det in enumerate(detectors): # Get the signal. noise = tod.local_signal(det, self._noise_name) # if self._rank ==0 and idet == 0: # print("|noise| = {}".format(np.sum(noise**2))) noise_dtype = noise.dtype offset = global_offset nn = len(noise) invtt = self._noise2invtt( noise, nn, idet ) #, logsigma2, alpha, fknee, fmin = self._noise2invtt(noise, nn, idet) invtt_list.append(invtt) # logsigma2_list.append(logsigma2) # alpha_list.append(alpha) # fknee_list.append(fknee) # fmin_list.append(fmin) dslice = slice(idet * nsamp + offset, idet * nsamp + offset + nn) self._mappraiser_noise[dslice] = noise offset += nn del noise # Purge only after all detectors are staged in case some are aliased # cache.clear() will not fail if the object was already # deleted as an alias if purge: for det in detectors: cachename = "{}_{}".format(self._noise_name, det) tod.cache.clear(cachename) global_offset = offset if self._verbose and nread > 1: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Stage noise {} / {}".format( iread + 1, nread)) # sendcounts = np.array(self._comm.gather(len(fknee_list), 0)) # # Fknee_list = None # Fmin_list = None # Alpha_list = None # Logsigma2_list = None # # if self._rank ==0: # Fknee_list = np.empty(sum(sendcounts)) # Fmin_list = np.empty(sum(sendcounts)) # Alpha_list = np.empty(sum(sendcounts)) # Logsigma2_list = np.empty(sum(sendcounts)) # # self._comm.Gatherv(np.array(fknee_list),(Fknee_list,sendcounts),0) # self._comm.Gatherv(np.array(fmin_list),(Fmin_list, sendcounts),0) # self._comm.Gatherv(np.array(alpha_list),(Alpha_list, sendcounts),0) # self._comm.Gatherv(np.array(logsigma2_list),(Logsigma2_list, sendcounts),0) # if self._rank ==0: # np.save("fknee.npy",Fknee_list) # np.save("fmin.npy", Fmin_list) # np.save("logsigma2.npy",Logsigma2_list) # np.save("alpha.npy",Alpha_list) return invtt_list, noise_dtype
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() memreport("at beginning of main", mpiworld) # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.world_rank == 0: print("Running with {} processes at {}".format( procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking', fromfile_prefix_chars='@') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--nside', required=False, type=np.int, default=512, help='Map resolution') parser.add_argument('--nside_cross', required=False, type=np.int, default=512, help='Destriping resolution') parser.add_argument('--debug', dest='debug', default=False, action='store_true', help='Write data distribution info to file') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir_in_diode0', required=False, default=None, help='Input Exchange Format File directory, ' 'LFI diode 0') parser.add_argument('--effdir_in_diode1', required=False, default=None, help='Input Exchange Format File directory, ' 'LFI diode 1') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory ' 'for pointing') parser.add_argument('--effdir_out', required=False, help='Output directory for destriped TOD') parser.add_argument('--effdir_out_diode0', required=False, help='Output directory for destriped TOD, LFI diode 0') parser.add_argument('--effdir_out_diode1', required=False, help='Output directory for destriped TOD, LFI diode 1') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--pntflagmask', required=False, default=0, type=np.int, help='Pointing flag mask') parser.add_argument('--bad_intervals', required=False, help='Path to bad interval file.') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, type=np.int, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, type=np.int, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, help='First ring to use (can be a list)') parser.add_argument('--ringlast', required=False, default=None, help='Last ring to use (can be a list)') parser.add_argument('--obtfirst', required=False, default=None, type=np.float, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, type=np.float, help='Last OBT to use') parser.add_argument('--madampar', required=False, default=None, help='Madam parameter file') parser.add_argument('--out', required=False, default='.', help='Output directory') parser.add_argument('--madam_prefix', required=False, help='map prefix') parser.add_argument('--split_mask', required=False, default=None, help='Intensity mask, non-zero pixels are not split.') parser.add_argument('--save_leakage_matrices', dest='save_leakage_matrices', default=False, action='store_true', help='Compile and write out the leakage projection ' 'matrices.') # noise parameters parser.add_argument('--noisefile', required=False, default='RIMO', help='Path to noise PSD files for noise filter. ' 'Tag DETECTOR will be replaced with detector name.') parser.add_argument('--static_noise', dest='static_noise', required=False, default=False, action='store_true', help='Assume constant noise PSD') parser.add_argument('--filterfile', required=False, help='Extra filter file.') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.comm_world.rank == 0: print('All parameters:') print(args, flush=True) timer = Timer() timer.start() nrange = 1 odranges = None if args.odfirst is not None and args.odlast is not None: odranges = [] firsts = [int(i) for i in str(args.odfirst).split(',')] lasts = [int(i) for i in str(args.odlast).split(',')] for odfirst, odlast in zip(firsts, lasts): odranges.append((odfirst, odlast)) nrange = len(odranges) ringranges = None if args.ringfirst is not None and args.ringlast is not None: ringranges = [] firsts = [int(i) for i in str(args.ringfirst).split(',')] lasts = [int(i) for i in str(args.ringlast).split(',')] for ringfirst, ringlast in zip(firsts, lasts): ringranges.append((ringfirst, ringlast)) nrange = len(ringranges) obtranges = None if args.obtfirst is not None and args.obtlast is not None: obtranges = [] firsts = [float(i) for i in str(args.obtfirst).split(',')] lasts = [float(i) for i in str(args.obtlast).split(',')] for obtfirst, obtlast in zip(firsts, lasts): obtranges.append((obtfirst, obtlast)) nrange = len(obtranges) if odranges is None: odranges = [None] * nrange if ringranges is None: ringranges = [None] * nrange if obtranges is None: obtranges = [None] * nrange detectors = None if args.dets is not None: detectors = re.split(',', args.dets) # create the TOD for this observation if args.noisefile != 'RIMO' and not args.static_noise: do_eff_cache = True else: do_eff_cache = False tods = [] if args.static_noise: noisefile = args.noisefile else: noisefile = 'RIMO' for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges): tods.append( tp.Exchange(comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_in_diode0=args.effdir_in_diode0, effdir_in_diode1=args.effdir_in_diode1, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, pntflagmask=args.pntflagmask, do_eff_cache=do_eff_cache, noisefile=noisefile)) rimo = tods[0].rimo # Make output directory if not os.path.isdir(args.out) and comm.comm_world.rank == 0: os.makedirs(args.out) # Read in madam parameter file # Allow more than one entry, gather into a list repeated_keys = ['detset', 'detset_nopol', 'survey'] pars = {} if comm.comm_world.rank == 0: pars['kfirst'] = False pars['temperature_only'] = True pars['base_first'] = 60.0 pars['nside_map'] = args.nside pars['nside_cross'] = min(args.nside, args.nside_cross) pars['nside_submap'] = 16 pars['write_map'] = False pars['write_binmap'] = True pars['write_matrix'] = False pars['write_wcov'] = False pars['write_hits'] = True pars['kfilter'] = False pars['info'] = 3 pars['pixlim_map'] = 1e-3 pars['pixlim_cross'] = 1e-3 if args.madampar: pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*') comment = re.compile(r'^#.*') with open(args.madampar, 'r') as f: for line in f: if not comment.match(line): result = pat.match(line) if result: key, value = result.group(1), result.group(2) if key in repeated_keys: if key not in pars: pars[key] = [] pars[key].append(value) else: pars[key] = value # Command line parameters override the ones in the madam parameter file if 'file_root' not in pars: pars['file_root'] = 'madam' if args.madam_prefix is not None: pars['file_root'] = args.madam_prefix sfreq = '{:03}'.format(args.freq) if sfreq not in pars['file_root']: pars['file_root'] += '_' + sfreq try: fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq] except Exception: fsample = 180.3737 pars['fsample'] = fsample pars['path_output'] = args.out if args.save_leakage_matrices: pars['write_leakmatrix'] = True pars = comm.comm_world.bcast(pars, root=0) if args.noisefile != 'RIMO': # We split MPI_COMM_WORLD into single process groups, each of # which is assigned one or more observations (rings) comm = toast.Comm(groupsize=1) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) for iobs, tod in enumerate(tods): if args.noisefile != 'RIMO' and not args.static_noise: # Use a toast helper method to optimally distribute rings between # processes. dist = toast.distribute_discrete(tod.ringsizes, comm.world_size) my_first_ring, my_n_ring = dist[comm.comm_world.rank] for my_ring in range(my_first_ring, my_first_ring + my_n_ring): ringtod = tp.Exchange.from_tod(tod, my_ring, comm.comm_group, noisefile=args.noisefile) ob = {} ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring) ob['id'] = ringtod.globalfirst_ring ob['tod'] = ringtod ob['intervals'] = ringtod.valid_intervals ob['baselines'] = None ob['noise'] = ringtod.noise data.obs.append(ob) else: ob = {} ob['name'] = 'observation{:04}'.format(iobs) ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise data.obs.append(ob) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Metadata queries") if args.effdir_out is not None or (args.effdir_out_diode0 is not None and args.effdir_out_diode1 is not None): do_output = True else: do_output = False # Read in the signal timer.clear() timer.start() reader = tp.OpInputPlanck(signal_name='signal', flags_name='flags') if comm.comm_world.rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Read") tod_name = 'signal' flags_name = 'flags' # Optionally filter the signal apply_filter(args, data) # Optionally flag bad intervals if args.bad_intervals is not None: timer = Timer() timer.start() flagger = tp.OpBadIntervals(path=args.bad_intervals) flagger.exec(data) timer.stop() if comm.comm_world.rank == 0: timer.report("Apply {}".format(args.bad_intervals)) # make a planck Healpix pointing matrix timer.clear() timer.start() mode = 'IQU' if pars['temperature_only'] == 'T': mode = 'I' nside = int(pars['nside_map']) pointing = tp.OpPointingPlanck(nside=nside, mode=mode, RIMO=rimo, margin=0, apply_flags=(not do_output), keep_vel=False, keep_pos=False, keep_phase=False, keep_quats=False) pointing.exec(data) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Pointing Matrix, mode = {}".format(mode)) for obs in data.obs: obs['tod'].purge_eff_cache() # for now, we pass in the noise weights from the RIMO. detweights = {} for d in tod.detectors: if d[-1] in '01' and d[-2] != '-': det = to_radiometer(d) else: det = d net = tod.rimo[det].net fsample = tod.rimo[det].fsample detweights[d] = 1.0 / (fsample * net * net) if do_output: name_out = 'madam_tod' else: name_out = None timer.clear() timer.start() try: madam = toast.todmap.OpMadam(name=tod_name, flag_name=flags_name, apply_flags=do_output, params=pars, detweights=detweights, purge=True, name_out=name_out, translate_timestamps=False) except Exception as e: raise Exception('{:4} : ERROR: failed to initialize Madam: {}'.format( comm.comm_world.rank, e)) madam.exec(data) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Madam") if do_output: timer = Timer() timer.start() writer = tp.OpOutputPlanck(signal_name='madam_tod', flags_name=None, commonflags_name=None, effdir_out=args.effdir_out, effdir_out_diode0=args.effdir_out_diode0, effdir_out_diode1=args.effdir_out_diode1) writer.exec(data) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Madam output") memreport("at end of main", mpiworld) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def get_hardware(args, comm, verbose=False): """ Get the hardware configuration, either from file or by simulating. Then trim it down to the bands that were selected. """ log = Logger.get() timer = Timer() timer.start() telescope = get_telescope(args, comm, verbose=verbose) if comm.world_rank == 0: if args.hardware: log.info("Loading hardware configuration from {}...".format( args.hardware)) if args.hardware.endswith(".pkl"): with open(args.hardware, "rb") as fin: hw = pickle.load(fin) else: hw = hardware.Hardware(args.hardware) timer.report_clear("Load {}".format(args.hardware)) else: log.info("Simulating default hardware configuration") hw = hardware.get_example() timer.report_clear("Get example hardware") hw.data["detectors"] = hardware.sim_telescope_detectors( hw, telescope.name) timer.report_clear("Get telescope detectors") # Construct a running index for all detectors across all # telescopes for independent noise realizations det_index = {} for idet, det in enumerate(sorted(hw.data["detectors"])): det_index[det] = idet match = {"band": args.bands.replace(",", "|")} if args.tubes is None: tubes = None else: tubes = args.tubes.split(",") # If one provides both telescopes and tubes, the tubes matching *either* # will be concatenated # hw = hw.select(telescopes=[telescope.name], tubes=tubes, match=match) hw = hw.select(tubes=tubes, match=match) ndetector = len(hw.data["detectors"]) if ndetector == 0: raise RuntimeError("No detectors match query: telescope={}, " "tubes={}, match={}".format( telescope.name, tubes, match)) if args.thinfp: # Only accept a fraction of the detectors for # testing and development thin_index = {} for idet, det in enumerate(sorted(hw.data["detectors"])): thin_index[det] = idet delete_detectors = [] for det_name in hw.data["detectors"].keys(): if (thin_index[det_name] // 2) % args.thinfp != 0: delete_detectors.append(det_name) for det_name in delete_detectors: del hw.data["detectors"][det_name] if args.radiusfp_deg: # Only accept detectors inside the given radius radius = np.radians(args.radiusfp_deg) # Comparing to the cosine of the radius avoids repeated # arccos evaluations cosradius = np.cos(radius) inside = radius > 0 delete_detectors = [] for det_name, det_data in hw.data["detectors"].items(): det_quat = det_data["quat"] vec = qa.rotate(det_quat, ZAXIS) rcos = np.dot(vec, ZAXIS) if (rcos < cosradius and inside) or (rcos > cosradius and not inside): delete_detectors.append(det_name) for det_name in delete_detectors: del hw.data["detectors"][det_name] ndetector = len(hw.data["detectors"]) log.info( "Telescope = {} tubes = {} bands = {}, thinfp = {} radiusfp = {} " "matches {} detectors".format( telescope.name, args.tubes, args.bands, args.thinfp, args.radiusfp_deg, ndetector, )) timer.report_clear("Trim detectors") else: hw = None det_index = None if comm.comm_world is not None: hw = comm.comm_world.bcast(hw) det_index = comm.comm_world.bcast(det_index) return hw, telescope, det_index
def get_hardware(args, comm, verbose=False): """ Get the hardware configuration, either from file or by simulating. Then trim it down to the bands that were selected. """ log = Logger.get() telescope = get_telescope(args, comm, verbose=verbose) timer = Timer() if comm.world_rank == 0: timer.start() if args.hardware: log.info("Loading hardware configuration from {}..." "".format(args.hardware)) hw = Hardware(args.hardware) timer.report_clear("Load hardware map") else: log.info("Simulating default hardware configuration") hw = get_example() hw.data["detectors"] = sim_telescope_detectors(hw, telescope.name) timer.report_clear("Simulate hardware map") # Construct a running index for all detectors across all # telescopes for independent noise realizations det_index = {} for idet, det in enumerate(sorted(hw.data["detectors"])): det_index[det] = idet match = {"band": args.bands.replace(",", "|")} tube_slots = None if args.wafer_slots is not None: match["wafer_slot"] = args.wafer_slots.split(",") elif args.tube_slots is not None: tube_slots = args.tube_slots.split(",") # If one provides both telescopes and tube_slots, the tube_slots matching *either* # will be concatenated #hw = hw.select(telescopes=[telescope.name], tube_slots=tube_slots, match=match) hw = hw.select(tube_slots=tube_slots, match=match) if args.thinfp: # Only accept a fraction of the detectors for # testing and development delete_detectors = [] for det_name in hw.data["detectors"].keys(): if (det_index[det_name] // 4) % args.thinfp != 0: delete_detectors.append(det_name) for det_name in delete_detectors: del hw.data["detectors"][det_name] ndetector = len(hw.data["detectors"]) if ndetector == 0: raise RuntimeError("No detectors match query: telescope={}, " "tube_slots={}, match={}".format( telescope.name, tube_slots, match)) log.info( f"Telescope = {telescope.name} tube_slots = {args.tube_slots}, " f"wafer_slots = {args.wafer_slots}, bands = {args.bands}, " f"thinfp = {args.thinfp} matches {ndetector} detectors") timer.report_clear("Select detectors") else: hw = None det_index = None if comm.comm_world is not None: hw = comm.comm_world.bcast(hw) det_index = comm.comm_world.bcast(det_index) if comm.world_rank == 0: timer.report_clear("Broadcast hardware map") return hw, telescope, det_index
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() if comm.comm_world.rank == 0: print("Running with {} processes at {}".format( procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Planck Ringset making', fromfile_prefix_chars='@') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--read_eff', dest='read_eff', default=False, action='store_true', help='Read and co-add the signal from effdir') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory for ' 'pointing') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, help='First ring to use') parser.add_argument('--ringlast', required=False, default=None, help='Last ring to use') parser.add_argument('--obtfirst', required=False, default=None, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, help='Last OBT to use') parser.add_argument('--out', required=False, default='.', help='Output directory') dipogroup = parser.add_mutually_exclusive_group() dipogroup.add_argument('--dipole', dest='dipole', required=False, default=False, action='store_true', help='Simulate dipole') dipogroup.add_argument('--solsys_dipole', dest='solsys_dipole', required=False, default=False, action='store_true', help='Simulate solar system dipole') dipogroup.add_argument('--orbital_dipole', dest='orbital_dipole', required=False, default=False, action='store_true', help='Simulate orbital dipole') # dipole parameters dipo_parameters_group = parser.add_argument_group('dipole_parameters') dipo_parameters_group.add_argument( '--solsys_speed', dest='solsys_speed', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_speed"], help='Solar system speed wrt. CMB rest frame in km/s. ' 'Default is Planck 2015 best fit value') dipo_parameters_group.add_argument( '--solsys_glon', dest='solsys_glon', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_glon"], help='Solar system velocity direction longitude in degrees') dipo_parameters_group.add_argument( '--solsys_glat', dest='solsys_glat', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_glat"], help='Solar system velocity direction latitude in degrees') # libconviqt parameters parser.add_argument('--lmax', required=False, default=1024, type=np.int, help='Simulation lmax') parser.add_argument('--fwhm', required=False, default=0.0, type=np.float, help='Sky fwhm [arcmin] to deconvolve') parser.add_argument('--beammmax', required=False, default=None, type=np.int, help='Beam mmax') parser.add_argument('--order', required=False, default=11, type=np.int, help='Iteration order') parser.add_argument('--pxx', required=False, default=False, action='store_true', help='Beams are in Pxx frame, not Dxx') parser.add_argument('--skyfile', required=False, default=None, help='Path to sky alm files. Tag DETECTOR will be ' 'replaced with detector name.') parser.add_argument('--beamfile', required=False, default=None, help='Path to beam alm files. Tag DETECTOR will be ' 'replaced with detector name.') parser.add_argument('--nopol', dest='nopol', default=False, action='store_true', help='Sky and beam should be treated unpolarized') # noise simulation parameters parser.add_argument('--add_noise', dest='add_noise', default=False, action='store_true', help='Simulate noise') parser.add_argument('--noisefile', required=False, default='RIMO', help='Path to noise PSD files for noise filter. ' 'Tag DETECTOR will be replaced with detector name.') parser.add_argument('--noisefile_simu', required=False, default='RIMO', help='Path to noise PSD files for noise simulation. ' 'Tag DETECTOR will be replaced with detector name.') parser.add_argument('--mc', required=False, default=0, type=np.int, help='Noise realization') # ringset parameters parser.add_argument('--nside_ring', required=False, default=128, type=np.int, help='Ringset resolution') parser.add_argument('--ring_root', required=False, default='ringset', help='Root filename for ringsets (setting to empty ' 'disables ringset output).') args = parser.parse_args() if comm.world_rank == 0: print('All parameters:') print(args, flush=True) timer = Timer() timer.start() nrange = 1 odranges = None if args.odfirst is not None and args.odlast is not None: odranges = [] firsts = [int(i) for i in str(args.odfirst).split(',')] lasts = [int(i) for i in str(args.odlast).split(',')] for odfirst, odlast in zip(firsts, lasts): odranges.append((odfirst, odlast)) nrange = len(odranges) ringranges = None if args.ringfirst is not None and args.ringlast is not None: ringranges = [] firsts = [int(i) for i in str(args.ringfirst).split(',')] lasts = [int(i) for i in str(args.ringlast).split(',')] for ringfirst, ringlast in zip(firsts, lasts): ringranges.append((ringfirst, ringlast)) nrange = len(ringranges) obtranges = None if args.obtfirst is not None and args.obtlast is not None: obtranges = [] firsts = [float(i) for i in str(args.obtfirst).split(',')] lasts = [float(i) for i in str(args.obtlast).split(',')] for obtfirst, obtlast in zip(firsts, lasts): obtranges.append((obtfirst, obtlast)) nrange = len(obtranges) if odranges is None: odranges = [None] * nrange if ringranges is None: ringranges = [None] * nrange if obtranges is None: obtranges = [None] * nrange detectors = None if args.dets is not None: detectors = re.split(',', args.dets) # Make output directory if comm.world_rank == 0: os.makedirs(args.out, exist_ok=True) do_dipole = args.dipole or args.solsys_dipole or args.orbital_dipole do_convolve = args.skyfile is not None and args.beamfile is not None do_noise = args.add_noise if not do_noise and args.noisefile_simu != 'RIMO': raise RuntimeError('Did you mean to simulate noise? add_noise = {} ' 'but noisefile_simu = {}'.format( args.add_noise, args.noisefile_simu)) if comm.world_rank == 0: print('read_eff = {}'.format(args.read_eff)) print('do_dipole = {}'.format(do_dipole)) print('solsys_speed = {}'.format(args.solsys_speed)) print('solsys_glon = {}'.format(args.solsys_glon)) print('solsys_glat = {}'.format(args.solsys_glat)) print('do_convolve = {}'.format(do_convolve)) print('do_noise = {}'.format(do_noise), flush=True) # create the TOD for the whole data span (loads ring database and caches # directory contents) if do_noise and args.noisefile_simu != 'RIMO': do_eff_cache = True else: do_eff_cache = False tods = [] for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges): tods.append( tp.Exchange(comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, do_eff_cache=do_eff_cache)) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Metadata queries") if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO': # We split MPI_COMM_WORLD into single process groups, each of # which is assigned one or more observations (rings) comm = Comm(groupsize=1) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = Data(comm) for tod in tods: if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO': # Use a toast helper method to optimally distribute rings between # processes. dist = distribute_discrete(tod.ringsizes, procs) my_first_ring, my_n_ring = dist[comm.world_rank] for my_ring in range(my_first_ring, my_first_ring + my_n_ring): ringtod = tp.Exchange.from_tod( tod, my_ring, comm.comm_group, noisefile=args.noisefile, noisefile_simu=args.noisefile_simu) ob = {} ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring) ob['id'] = ringtod.globalfirst_ring ob['tod'] = ringtod ob['intervals'] = ringtod.valid_intervals ob['baselines'] = None ob['noise'] = ringtod.noise ob['noise_simu'] = ringtod.noise_simu data.obs.append(ob) else: # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. ob = {} ob['name'] = 'mission' ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise data.obs.append(ob) rimo = tods[0].rimo fsample = rimo[detectors[0]].fsample if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Create observations") # make a planck Healpix pointing matrix mode = 'IQU' pointing = tp.OpPointingPlanck(nside=args.nside_ring, mode=mode, RIMO=rimo, margin=0, apply_flags=False, keep_vel=do_dipole, keep_pos=False, keep_phase=False, keep_quats=(do_dipole or do_convolve)) pointing.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Pointing Matrix") # Always read the signal because we always need the flags reader = tp.OpInputPlanck(signal_name='tod') if comm.world_rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) comm.comm_world.barrier() if comm.world_rank == 0: timer.report_clear("Read") tod_name = 'tod' # Clear the signal if we don't need it if not args.read_eff: eraser = tp.OpCacheMath(in1='tod', in2=0, multiply=True, out='tod') if comm.comm_world.rank == 0: print('Erasing TOD', flush=True) eraser.exec(data) comm.comm_world.barrier() if comm.world_rank == 0: timer.report_clear("Erase") if do_convolve: # simulate the TOD by convolving the sky with the beams detectordata = [] for det in tod.detectors: skyfile = args.skyfile.replace('DETECTOR', det) beamfile = args.beamfile.replace('DETECTOR', det) epsilon = rimo[det].epsilon # Getting the right polarization angle can be a sensitive matter. # Dxx beams are always defined without psi_uv or psi_pol rotation # but some Pxx beams may require psi_pol to be removed and psi_uv # left in. if args.pxx: # Beam is in the polarization basis. # No extra rotations are needed psipol = np.radians(rimo[det].psi_pol) else: # Beam is in the detector basis. Convolver needs to remove # the last rotation into the polarization sensitive frame. psipol = np.radians(rimo[det].psi_uv + rimo[det].psi_pol) detectordata.append((det, skyfile, beamfile, epsilon, psipol)) # always construct conviqt with dxx=True and modify the psipol # to produce the desired rotation. conviqt = tt.OpSimConviqt(args.lmax, args.beammmax, detectordata, pol=(not args.nopol), fwhm=args.fwhm, order=args.order, calibrate=True, dxx=True, out='tod', quat_name='quats', apply_flags=False) conviqt.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Convolution") tod_name = 'tod' if do_dipole: # Simulate the dipole if args.dipole: dipomode = 'total' elif args.solsys_dipole: dipomode = 'solsys' else: dipomode = 'orbital' dipo = tp.OpDipolePlanck(args.freq, solsys_speed=args.solsys_speed, solsys_glon=args.solsys_glon, solsys_glat=args.solsys_glat, mode=dipomode, output='tod', add_to_existing=args.read_eff) dipo.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Dipole") tod_name = 'tod' if do_noise: nse = tt.OpSimNoise(out='tod', realization=args.mc, component=0, noise='noise_simu', rate=fsample) nse.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Noise simulation") tod_name = 'tod' # for now, we pass in the noise weights from the RIMO. detweights = {} for d in tod.detectors: net = tod.rimo[d].net fsample = tod.rimo[d].fsample detweights[d] = 1.0 / (fsample * net * net) # Make rings ringmaker = tp.OpRingMaker(args.nside_ring, args.nside_ring, signal=tod_name, fileroot=args.ring_root, out=args.out, detmask=args.flagmask, commonmask=args.obtmask) ringmaker.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Ringmaking") gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_ground_sim (total)") timer0 = Timer() timer0.start() mpiworld, procs, rank, comm = pipeline_tools.get_comm() args, comm = parse_arguments(comm) if args.use_madam: # Initialize madam parameters madampars = pipeline_tools.setup_madam(args) # Load and broadcast the schedule file schedules = pipeline_tools.load_schedule(args, comm) # Load the weather and append to schedules pipeline_tools.load_weather(args, comm, schedules) # load or simulate the focalplane detweights = load_focalplanes(args, comm, schedules) # Create the TOAST data object to match the schedule. This will # include simulating the boresight pointing. data, telescope_data = create_observations(args, comm, schedules) # Split the communicator for day and season mapmaking time_comms = pipeline_tools.get_time_communicators(args, comm, data) # Expand boresight quaternions into detector pointing weights and # pixel numbers pipeline_tools.expand_pointing(args, comm, data) # Optionally rewrite the noise PSD:s in each observation to include # elevation-dependence pipeline_tools.get_elevation_noise(args, comm, data) # Purge the pointing if we are NOT going to export the # data to a TIDAS volume if (args.tidas is None) and (args.spt3g is None): for ob in data.obs: tod = ob["tod"] tod.free_radec_quats() # Prepare auxiliary information for distributed map objects if args.pysm_model: focalplanes = [s.telescope.focalplane.detector_data for s in schedules] signalname = pipeline_tools.simulate_sky_signal( args, comm, data, focalplanes, "signal" ) else: signalname = pipeline_tools.scan_sky_signal(args, comm, data, "signal") # Set up objects to take copies of the TOD at appropriate times totalname, totalname_freq = setup_sigcopy(args) # Loop over Monte Carlos firstmc = args.MC_start nsimu = args.MC_count freqs = [float(freq) for freq in args.freq.split(",")] nfreq = len(freqs) for mc in range(firstmc, firstmc + nsimu): pipeline_tools.simulate_atmosphere(args, comm, data, mc, totalname) # Loop over frequencies with identical focal planes and identical # atmospheric noise. for ifreq, freq in enumerate(freqs): if comm.world_rank == 0: log.info( "Processing frequency {}GHz {} / {}, MC = {}".format( freq, ifreq + 1, nfreq, mc ) ) # Make a copy of the atmosphere so we can scramble the gains and apply # frequency-dependent scaling. pipeline_tools.copy_signal(args, comm, data, totalname, totalname_freq) pipeline_tools.scale_atmosphere_by_frequency( args, comm, data, freq=freq, mc=mc, cache_name=totalname_freq ) pipeline_tools.update_atmospheric_noise_weights(args, comm, data, freq, mc) # Add previously simulated sky signal to the atmospheric noise. pipeline_tools.add_signal( args, comm, data, totalname_freq, signalname, purge=(nsimu == 1) ) mcoffset = ifreq * 1000000 pipeline_tools.simulate_noise( args, comm, data, mc + mcoffset, totalname_freq ) pipeline_tools.simulate_sss(args, comm, data, mc + mcoffset, totalname_freq) pipeline_tools.scramble_gains( args, comm, data, mc + mcoffset, totalname_freq ) if (mc == firstmc) and (ifreq == 0): # For the first realization and frequency, optionally # export the timestream data. pipeline_tools.output_tidas(args, comm, data, totalname) pipeline_tools.output_spt3g(args, comm, data, totalname) outpath = setup_output(args, comm, mc + mcoffset, freq) # Bin and destripe maps if args.use_madam: pipeline_tools.apply_madam( args, comm, data, madampars, outpath, detweights, totalname_freq, freq=freq, time_comms=time_comms, telescope_data=telescope_data, first_call=(mc == firstmc), ) else: pipeline_tools.apply_mapmaker( args, comm, data, outpath, totalname_freq, time_comms=time_comms, telescope_data=telescope_data, first_call=(mc == firstmc), ) if args.apply_polyfilter or args.apply_groundfilter: # Filter signal pipeline_tools.apply_polyfilter(args, comm, data, totalname_freq) pipeline_tools.apply_groundfilter(args, comm, data, totalname_freq) # Bin filtered maps if args.use_madam: pipeline_tools.apply_madam( args, comm, data, madampars, outpath, detweights, totalname_freq, freq=freq, time_comms=time_comms, telescope_data=telescope_data, first_call=False, extra_prefix="filtered", bin_only=True, ) else: pipeline_tools.apply_mapmaker( args, comm, data, outpath, totalname_freq, time_comms=time_comms, telescope_data=telescope_data, first_call=False, extra_prefix="filtered", bin_only=True, ) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.outdir, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") timer0.report_clear("toast_ground_sim.py") return
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_s4_sim (total)") timer0 = Timer() timer0.start() mpiworld, procs, rank, comm = toast_tools.get_comm() memreport("at the beginning of the pipeline", comm.comm_world) args, comm = parse_arguments(comm) # Initialize madam parameters madampars = toast_tools.setup_madam(args) # Load and broadcast the schedule file schedules = toast_tools.load_schedule(args, comm) # Load the weather and append to schedules toast_tools.load_weather(args, comm, schedules) # load or simulate the focalplane detweights = s4_tools.load_focalplanes(args, comm, schedules) # Create the TOAST data object to match the schedule. This will # include simulating the boresight pointing. data, telescope_data = s4_tools.create_observations(args, comm, schedules) memreport("after creating observations", comm.comm_world) # Optionally rewrite the noise PSD:s in each observation to include # elevation-dependence s4_tools.get_elevation_noise(args, comm, data) totalname = "total" # Split the communicator for day and season mapmaking time_comms = toast_tools.get_time_communicators(args, comm, data) # Expand boresight quaternions into detector pointing weights and # pixel numbers toast_tools.expand_pointing(args, comm, data) # Only purge the pointing if we are NOT going to export the # data to a TIDAS volume if args.tidas is None: for ob in data.obs: tod = ob["tod"] try: tod.free_radec_quats() except AttributeError: # These TOD objects do not have RA/Dec quaternions pass memreport("after pointing", comm.comm_world) # Prepare auxiliary information for distributed map objects memreport("after submaps", comm.comm_world) # Set up objects to take copies of the TOD at appropriate times if args.pysm_model: if schedules is not None: focalplanes = [ s.telescope.focalplane.detector_data for s in schedules ] else: focalplanes = [telescope.focalplane.detector_data] signalname = s4_tools.simulate_sky_signal(args, comm, data, focalplanes) else: signalname = toast_tools.scan_sky_signal(args, comm, data) memreport("after PySM", comm.comm_world) # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc + nmc): if comm.world_rank == 0: log.info("Processing MC = {}".format(mc)) # Uncomment to run with new TOAST #toast_tools.draw_weather(args, comm, data, mc) outpath = setup_output(args, comm, mc) if outputs_exist(args, comm, outpath): if comm.world_rank == 0: log.info("Outputs already exist, skipping.") continue toast.tod.OpCacheClear(totalname).exec(data) toast_tools.simulate_atmosphere(args, comm, data, mc, totalname) s4_tools.scale_atmosphere_by_bandpass(args, comm, data, totalname, mc) memreport("after atmosphere", comm.comm_world) # update_atmospheric_noise_weights(args, comm, data, freq, mc) toast_tools.add_signal(args, comm, data, totalname, signalname, purge=(mc == firstmc + nmc - 1)) memreport("after adding sky", comm.comm_world) toast_tools.simulate_noise(args, comm, data, mc, totalname) memreport("after simulating noise", comm.comm_world) toast_tools.simulate_sss(args, comm, data, mc, totalname) memreport("after simulating SSS", comm.comm_world) toast_tools.scramble_gains(args, comm, data, mc, totalname) if mc == firstmc: # For the first realization and frequency, optionally # export the timestream data. toast_tools.output_tidas(args, comm, data, totalname) memreport("after export", comm.comm_world) if args.no_maps: continue # Bin and destripe maps pairdiff(data, args, comm, totalname, mc == firstmc) if not args.skip_madam: toast_tools.apply_madam( args, comm, data, madampars, outpath, detweights, totalname, time_comms=time_comms, telescope_data=telescope_data, first_call=(mc == firstmc), ) memreport("after madam", comm.comm_world) if (args.filterbin_ground_order is not None or args.filterbin_poly_order is not None): toast_tools.apply_filterbin( args, comm, data, outpath, totalname, time_comms=time_comms, telescope_data=telescope_data, first_call=(mc == firstmc), ) if (args.apply_polyfilter or args.apply_polyfilter2D or args.apply_common_mode_filter or args.apply_groundfilter): # Filter signal toast_tools.apply_common_mode_filter(args, comm, data, totalname) toast_tools.apply_polyfilter2D(args, comm, data, totalname) toast_tools.apply_polyfilter(args, comm, data, totalname) toast_tools.apply_groundfilter(args, comm, data, totalname) memreport("after filter", comm.comm_world) # Bin maps toast_tools.apply_madam( args, comm, data, madampars, outpath, detweights, totalname, time_comms=time_comms, telescope_data=telescope_data, first_call=(args.skip_madam and mc == firstmc), extra_prefix="filtered", bin_only=(not args.skip_madam), ) memreport("after filter & bin", comm.comm_world) if comm.comm_world is not None: comm.comm_world.barrier() memreport("at the end of the pipeline", comm.comm_world) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if rank == 0: out = os.path.join(args.outdir, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") timer0.stop() if comm.world_rank == 0: timer0.report("toast_s4_sim.py pipeline") return
def main(): timer0 = Timer() timer0.start() log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() memreport("At start of pipeline", mpiworld) if comm.world_rank == 0: print("Running with {} processes at {}".format( procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking', fromfile_prefix_chars='@') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--debug', dest='debug', default=False, action='store_true', help='Write data distribution info to file') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory ' 'for pointing') parser.add_argument('--coord', default='G', help='Coordinate system, "G", "E" or "C"') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--pntflagmask', required=False, default=0, type=np.int, help='Pointing flag mask') parser.add_argument('--bad_intervals', required=False, help='Path to bad interval file.') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, type=np.int, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, type=np.int, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, help='First ring to use (can be a list)') parser.add_argument('--ringlast', required=False, default=None, help='Last ring to use (can be a list)') parser.add_argument('--obtfirst', required=False, default=None, type=np.float, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, type=np.float, help='Last OBT to use') parser.add_argument('--out', required=False, default='.', help='Output directory') parser.add_argument('--catalog', required=True, help='Target catalog file') parser.add_argument('--radius', required=True, type=np.float, help='Search radius about the source [arc min]') parser.add_argument('--mask', required=False, help='Mask defining region of the sky to accept') parser.add_argument('--bg', required=False, help='Background map to subtract') parser.add_argument('--recalib_bg', dest='recalib_bg', default=False, action='store_true', help='Recalibrate bg map for each ring.') parser.add_argument('--full_rings', dest='full_rings', default=False, action='store_true', help='Extract impacted rings entirely.') # noise parameters parser.add_argument('--noisefile', required=False, default='RIMO', help='Path to noise PSD files for noise filter. ' 'Tag DETECTOR will be replaced with detector name.') # Dipole parameters dipogroup = parser.add_mutually_exclusive_group() dipogroup.add_argument('--dipole', dest='dipole', required=False, default=False, action='store_true', help='Simulate dipole') dipogroup.add_argument('--solsys_dipole', dest='solsys_dipole', required=False, default=False, action='store_true', help='Simulate solar system dipole') dipogroup.add_argument('--orbital_dipole', dest='orbital_dipole', required=False, default=False, action='store_true', help='Simulate orbital dipole') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.world_rank == 0: print('All parameters:') print(args, flush=True) data = create_observations(args, comm) rimo = data.obs[0]["tod"].rimo memreport("After create observations", mpiworld) # Read in the signal timer = Timer() timer.start() reader = tp.OpInputPlanck(signal_name='signal', flags_name='flags') if comm.world_rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Reading") tod_name = 'signal' flags_name = 'flags' memreport("After read", mpiworld) # Optionally flag bad intervals if args.bad_intervals is not None: flagger = tp.OpBadIntervals(path=args.bad_intervals) flagger.exec(data) if comm.world_rank == 0: timer.report_clear("Applying {}".format(args.bad_intervals)) do_dipole = (args.dipole or args.solsys_dipole or args.orbital_dipole) # make a planck Healpix pointing matrix pointing = tp.OpPointingPlanck(nside=1024, mode='IQU', RIMO=rimo, margin=0, apply_flags=False, keep_vel=do_dipole, keep_pos=False, keep_phase=True, keep_quats=True) pointing.exec(data) memreport("After pointing", mpiworld) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Pointing Matrix") # Optionally subtract the dipole if do_dipole: if args.dipole: dipomode = 'total' elif args.solsys_dipole: dipomode = 'solsys' else: dipomode = 'orbital' dipo = tp.OpDipolePlanck(args.freq, mode=dipomode, output='dipole', keep_quats=True) dipo.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Dipole") subtractor = tp.OpCacheMath(in1=tod_name, in2='dipole', subtract=True, out=tod_name) if comm.comm_world.rank == 0: print('Subtracting dipole', flush=True) subtractor.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Dipole subtraction") memreport("After dipole", mpiworld) extract = tp.OpExtractPlanck(rimo, args.catalog, args.radius, mpiworld, common_flag_mask=args.obtmask, flag_mask=args.flagmask, maskfile=args.mask, bg=args.bg, full_rings=args.full_rings, recalibrate_bg=args.recalib_bg, out=args.out) extract.exec(data) memreport("After extract", mpiworld) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Extraction") gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.report_clear("Gather and dump timing info") timer0.report_clear("Full pipeline") return
def parse_arguments(comm): timer = Timer() timer.start() log = Logger.get() parser = argparse.ArgumentParser( description="Simulate ground-based boresight pointing. Simulate " "atmosphere and make maps for some number of noise Monte Carlos.", fromfile_prefix_chars="@", ) toast_tools.add_dist_args(parser) toast_tools.add_todground_args(parser) toast_tools.add_pointing_args(parser) toast_tools.add_polyfilter_args(parser) toast_tools.add_polyfilter2D_args(parser) toast_tools.add_common_mode_filter_args(parser) toast_tools.add_groundfilter_args(parser) toast_tools.add_atmosphere_args(parser) toast_tools.add_noise_args(parser) toast_tools.add_gainscrambler_args(parser) toast_tools.add_madam_args(parser) toast_tools.add_filterbin_args(parser) toast_tools.add_sky_map_args(parser) toast_tools.add_sss_args(parser) toast_tools.add_tidas_args(parser) toast_tools.add_mc_args(parser) s4_tools.add_hw_args(parser) s4_tools.add_s4_noise_args(parser) s4_tools.add_pysm_args(parser) toast_tools.add_debug_args(parser) parser.add_argument( "--no-maps", required=False, default=False, action="store_true", help="Disable all mapmaking.", ) parser.add_argument( "--skip-madam", required=False, default=False, action="store_true", help="Skip the first Madam call.", ) parser.add_argument( "--pairdiff", required=False, default=False, action="store_true", help="Pair-difference TOD and pointing.", ) parser.add_argument("--outdir", required=False, default="out", help="Output directory") try: args = parser.parse_args() except SystemExit as e: sys.exit() if len(args.bands.split(",")) != 1: # Multi frequency run. We don't support multiple copies of # scanned signal. if args.input_map: raise RuntimeError( "Multiple frequencies are not supported when scanning from a map" ) if args.simulate_atmosphere and args.weather is None: raise RuntimeError( "Cannot simulate atmosphere without a TOAST weather file") if comm.world_rank == 0: log.info("\n") log.info("All parameters:") for ag in vars(args): log.info("{} = {}".format(ag, getattr(args, ag))) log.info("\n") if args.group_size: comm = Comm(groupsize=args.group_size) if comm.world_rank == 0: os.makedirs(args.outdir, exist_ok=True) timer.report_clear("Parse arguments") return args, comm
def cache_effdirs(self, effdir_in, effdir_in_diode0, effdir_in_diode1, effdir_out, effdir_dark, effdir_pntg, effdir_fsl, extra_effdirs, effdir_flags): """ Cache the metadata so we don't need to look for files while reading and writing """ if effdir_in is not None and PATTERN_SEPARATOR in effdir_in: self.effdir_in, self.effdir_in_pattern = effdir_in.split( PATTERN_SEPARATOR) else: self.effdir_in, self.effdir_in_pattern = effdir_in, None self.effdir_in_diode0 = effdir_in_diode0 self.effdir_in_diode1 = effdir_in_diode1 if effdir_out is not None and PATTERN_SEPARATOR in effdir_out: self.effdir_out, self.effdir_out_pattern = effdir_out.split( PATTERN_SEPARATOR) else: self.effdir_out, self.effdir_out_pattern = effdir_out, None self.effdir_out = effdir_out if effdir_dark is not None: self.effdir_dark = effdir_dark else: self.effdir_dark = self.effdir_in if effdir_pntg is not None: self.effdir_pntg = effdir_pntg else: self.effdir_pntg = self.effdir_in self.effdir_fsl = effdir_fsl self.extra_effdirs = extra_effdirs if effdir_flags is None: self.effdir_flags = self.effdir_in self.effdir_flags_pattern = self.effdir_in_pattern else: if PATTERN_SEPARATOR in effdir_flags: (self.effdir_flags, self.effdir_flags_pattern ) = effdir_flags.split(PATTERN_SEPARATOR) else: (self.effdir_flags, self.effdir_flags_pattern) = effdir_flags, None if self.rank == 0: all_effdirs = [ self.effdir_in, self.effdir_out, self.effdir_pntg, self.effdir_dark, self.effdir_fsl, self.effdir_flags, self.effdir_in_diode0, self.effdir_in_diode1 ] if self.extra_effdirs is not None: for effdir in self.extra_effdirs: all_effdirs.append(effdir) for effdir in all_effdirs: if effdir is None: continue if effdir in filenames_cache: continue print('Building a list of files under {} ...'.format(effdir), end='', flush=True) timer = Timer() timer.start() filenames_cache[effdir] = sorted(list_files(effdir)) timer.stop() timer.report("List files") if self.comm is None: self.filenames = filenames_cache else: self.filenames = self.comm.bcast(filenames_cache, root=0) return
def scale_atmosphere_by_bandpass(args, comm, data, totalname, mc, verbose=False): """ Scale atmospheric fluctuations by bandpass. Assume that cached signal under totalname is pure atmosphere and scale the absorption coefficient according to the bandpass. If the focalplane is included in the observation and defines bandpasses for the detectors, the scaling is computed for each detector separately. """ if not args.simulate_atmosphere: return timer = Timer() log = Logger.get() if comm.world_rank == 0 and verbose: log.info("Scaling atmosphere by bandpass") timer.start() for obs in data.obs: tod = obs["tod"] todcomm = tod.mpicomm site_id = obs["site_id"] weather = obs["weather"] if "focalplane" in obs: focalplane = obs["focalplane"] else: focalplane = None start_time = obs["start_time"] weather.set(site_id, mc, start_time) altitude = obs["altitude"] air_temperature = weather.air_temperature surface_pressure = weather.surface_pressure pwv = weather.pwv # Use the entire processing group to sample the absorption # coefficient as a function of frequency freqmin = 0 freqmax = 1000 nfreq = 10001 freqstep = (freqmax - freqmin) / (nfreq - 1) if todcomm is None: nfreq_task = nfreq my_ifreq_min = 0 my_ifreq_max = nfreq else: nfreq_task = int(nfreq // todcomm.size) + 1 my_ifreq_min = nfreq_task * todcomm.rank my_ifreq_max = min(nfreq, nfreq_task * (todcomm.rank + 1)) my_nfreq = my_ifreq_max - my_ifreq_min if my_nfreq > 0: if atm_available_utils: my_freqs = freqmin + np.arange(my_ifreq_min, my_ifreq_max) * freqstep my_absorption = atm_absorption_coefficient_vec( altitude, air_temperature, surface_pressure, pwv, my_freqs[0], my_freqs[-1], my_nfreq, ) else: raise RuntimeError( "Atmosphere utilities from libaatm are not available") else: my_freqs = np.array([]) my_absorption = np.array([]) if todcomm is None: freqs = my_freqs absorption = my_absorption else: freqs = np.hstack(todcomm.allgather(my_freqs)) absorption = np.hstack(todcomm.allgather(my_absorption)) # loading = atm_atmospheric_loading(altitude, pwv, freq) for det in tod.local_dets: # Use detector bandpass from the focalplane center = focalplane[det]["bandcenter_ghz"] width = focalplane[det]["bandwidth_ghz"] nstep = 101 # Interpolate the absorption coefficient to do a top hat # integral across the bandpass det_freqs = np.linspace(center - width / 2, center + width / 2, nstep) absorption_det = np.mean(np.interp(det_freqs, freqs, absorption)) cachename = "{}_{}".format(totalname, det) ref = tod.cache.reference(cachename) ref *= absorption_det del ref if comm.comm_world is not None: comm.comm_world.barrier() timer.stop() if comm.world_rank == 0 and verbose: timer.report("Atmosphere scaling") return
def export_TOD(args, comm, data, totalname, schedules, other=None, verbose=True): if args.export is None: return log = Logger.get() timer = Timer() # Only import spt3g if we are writing out so3g files from spt3g import core as core3g from ..export import ToastExport path = os.path.abspath(args.export) key = args.export_key if key is not None: if key not in ALLOWED_KEYS: raise RuntimeError( f"Cannot export data, --export-key='{key}' not in {ALLOWED_KEYS}" ) prefix = "{}_{}".format(args.bands, key) det_groups = {} for obs in data.obs: for (det_name, det_data) in obs["focalplane"].items(): value = det_data[key] if value not in det_groups: det_groups[value] = [] det_groups[value].append(det_name) else: prefix = args.bands det_groups = None if comm.world_rank == 0 and verbose: log.info("Exporting data to directory tree at {}".format(path)) timer.start() export = ToastExport( path, prefix=prefix, use_intervals=True, cache_name=totalname, cache_copy=other, mask_flag_common=TODGround.TURNAROUND, filesize=2**30, units=core3g.G3TimestreamUnits.Tcmb, detgroups=det_groups, compress=args.compress, ) export.exec(data) if comm.comm_world is not None: comm.comm_world.Barrier() timer.stop() if comm.world_rank == 0 and verbose: timer.report("Wrote simulated data to {}:{}" "".format(path, "total")) return
def get_analytic_noise(args, comm, focalplane, verbose=True): """ Create a TOAST noise object. Create a noise object from the 1/f noise parameters contained in the focalplane database. """ timer = Timer() timer.start() detectors = sorted(focalplane.keys()) fmins = {} fknees = {} alphas = {} NETs = {} rates = {} indices = {} for d in detectors: rates[d] = args.sample_rate fmins[d] = focalplane[d]["fmin"] fknees[d] = focalplane[d]["fknee"] alphas[d] = focalplane[d]["alpha"] NETs[d] = focalplane[d]["NET"] indices[d] = focalplane[d]["index"] if args.common_mode_noise: # Add an extra "virtual" detector for common mode noise for # every optics tube fmin, fknee, alpha, net = np.array( args.common_mode_noise.split(",")).astype(np.float64) hw = hardware.get_example() for itube, tube in enumerate(sorted(hw.data["tubes"].keys())): d = "common_mode_{}".format(tube) detectors.append(d) rates[d] = args.sample_rate fmins[d] = fmin fknees[d] = fknee alphas[d] = alpha NETs[d] = net indices[d] = 100000 + itube noise = AnalyticNoise( rate=rates, fmin=fmins, detectors=detectors, fknee=fknees, alpha=alphas, NET=NETs, indices=indices, ) if args.common_mode_noise: # Update the mixing matrix in the noise operator mixmatrix = {} keys = set() for det in focalplane.keys(): tube = focalplane[det]["tube"] common = "common_mode_{}".format(tube) mixmatrix[det] = {det: 1, common: 1} keys.add(det) keys.add(common) # There should probably be an accessor method to update the # mixmatrix in the TOAST Noise object. if noise._mixmatrix is not None: raise RuntimeError("Did not expect non-empty mixing matrix") noise._mixmatrix = mixmatrix noise._keys = list(sorted(keys)) timer.stop() if comm.world_rank == 0 and verbose: timer.report("Creating noise model") return noise
def main(): env = Environment.get() log = Logger.get() gt = GlobalTimers.get() gt.start("toast_satellite_sim (total)") timer0 = Timer() timer0.start() mpiworld, procs, rank, comm = pipeline_tools.get_comm() args, comm, groupsize = parse_arguments(comm, procs) # Parse options tmr = Timer() tmr.start() if comm.world_rank == 0: os.makedirs(args.outdir, exist_ok=True) focalplane, gain, detweights = load_focalplane(args, comm) if comm.world_rank == 0: tmr.report_clear("Load focalplane") data = create_observations(args, comm, focalplane, groupsize) if comm.world_rank == 0: tmr.report_clear("Create observations") pipeline_tools.expand_pointing(args, comm, data) if comm.world_rank == 0: tmr.report_clear("Expand pointing") signalname = None skyname = pipeline_tools.simulate_sky_signal(args, comm, data, [focalplane], "signal") if skyname is not None: signalname = skyname if comm.world_rank == 0: tmr.report_clear("Simulate sky signal") skyname = pipeline_tools.apply_conviqt(args, comm, data, "signal") if skyname is not None: signalname = skyname if comm.world_rank == 0: tmr.report_clear("Apply beam convolution") diponame = pipeline_tools.simulate_dipole(args, comm, data, "signal") if diponame is not None: signalname = diponame if comm.world_rank == 0: tmr.report_clear("Simulate dipole") # in debug mode, print out data distribution information if args.debug: handle = None if comm.world_rank == 0: handle = open(os.path.join(args.outdir, "distdata.txt"), "w") data.info(handle) if comm.world_rank == 0: handle.close() if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear("Dumping data distribution") # in debug mode, print out data distribution information if args.debug: handle = None if comm.world_rank == 0: handle = open(os.path.join(args.outdir, "distdata.txt"), "w") data.info(handle) if comm.world_rank == 0: handle.close() if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear("Dumping data distribution") # Mapmaking. if args.use_madam: # Initialize madam parameters madampars = pipeline_tools.setup_madam(args) if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear("Initialize madam map-making") # Loop over Monte Carlos firstmc = args.MC_start nmc = args.MC_count for mc in range(firstmc, firstmc + nmc): mctmr = Timer() mctmr.start() # create output directory for this realization outpath = os.path.join(args.outdir, "mc_{:03d}".format(mc)) pipeline_tools.simulate_noise(args, comm, data, mc, "tot_signal", overwrite=True) if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear(" Simulate noise {:04d}".format(mc)) # add sky signal pipeline_tools.add_signal(args, comm, data, "tot_signal", signalname) if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear(" Add sky signal {:04d}".format(mc)) if gain is not None: op_apply_gain = OpApplyGain(gain, name="tot_signal") op_apply_gain.exec(data) if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear(" Apply gains {:04d}".format(mc)) if mc == firstmc: # For the first realization, optionally export the # timestream data. If we had observation intervals defined, # we could pass "use_interval=True" to the export operators, # which would ensure breaks in the exported data at # acceptable places. pipeline_tools.output_tidas(args, comm, data, "tot_signal") pipeline_tools.output_spt3g(args, comm, data, "tot_signal") if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear(" Write TOD snapshot {:04d}".format(mc)) if args.use_madam: pipeline_tools.apply_madam(args, comm, data, madampars, outpath, detweights, "tot_signal") else: pipeline_tools.apply_mapmaker(args, comm, data, outpath, "tot_signal") if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: tmr.report_clear(" Map-making {:04d}".format(mc)) if comm.comm_world is not None: comm.comm_world.barrier() if comm.world_rank == 0: mctmr.report_clear(" Monte Carlo loop {:04d}".format(mc)) gt.stop_all() if comm.comm_world is not None: comm.comm_world.barrier() tmr.stop() tmr.clear() tmr.start() alltimers = gather_timers(comm=comm.comm_world) if comm.world_rank == 0: out = os.path.join(args.outdir, "timing") dump_timing(alltimers, out) tmr.stop() tmr.report("Gather and dump timing info") timer0.report_clear("toast_satellite_sim.py") return
def _unstage_data( self, nsamp, nnz, nnz_full, detectors, signal_type, noise_type, pixels_dtype, nside, weight_dtype, ): """ Clear Mappraiser buffers, [restore pointing into TOAST caches-> not done currently]. """ log = Logger.get() # self._mappraiser_timestamps = None # self._cache.destroy("timestamps") if self._conserve_memory: nodecomm = self._comm.Split_type(MPI.COMM_TYPE_SHARED, self._rank) nread = nodecomm.size else: nodecomm = MPI.COMM_SELF nread = 1 self._comm.Barrier() timer_tot = Timer() timer_tot.start() for iread in range(nread): timer_step = Timer() timer_step.start() timer = Timer() timer.start() if nodecomm.rank % nread == iread: self._unstage_signal(detectors, nsamp, signal_type) if self._verbose: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Unstage signal {} / {}".format( iread + 1, nread)) if nodecomm.rank % nread == iread: self._unstage_noise(detectors, nsamp, noise_type) if self._verbose: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Unstage noise {} / {}".format( iread + 1, nread)) if nodecomm.rank % nread == iread: self._unstage_pixels(detectors, nsamp, pixels_dtype, nside) if self._verbose: nodecomm.Barrier() if self._rank == 0: timer.report_clear("Unstage pixels {} / {}".format( iread + 1, nread)) if nodecomm.rank % nread == iread: self._unstage_pixweights(detectors, nsamp, weight_dtype, nnz, nnz_full) nodecomm.Barrier() if self._verbose and self._rank == 0: timer.report_clear("Unstage pixel weights {} / {}".format( iread + 1, nread)) if self._rank == 0 and self._verbose and nread > 1: timer_step.report_clear("Unstage data {} / {}".format( iread + 1, nread)) self._comm.Barrier() if self._rank == 0 and self._verbose: timer_tot.report_clear("Unstage all data") del nodecomm return