def load_focalplane(args, comm): """Load focalplane information """ timer = Timer() gain = None fp = None if comm.world_rank == 0: if args.focalplane is None: # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0]) fake["fwhm"] = 30.0 fake["fknee"] = 0.0 fake["fmin"] = 1.0e-5 fake["alpha"] = 1.0 fake["NET"] = 1.0 fake["polangle_deg"] = 0 fake["color"] = "r" fp = {} fp["bore"] = fake else: with open(args.focalplane, "rb") as p: fp = pickle.load(p) if args.gain is not None: gain = {} with fits.open(args.gain) as f: gain["TIME"] = np.array(f["TIME"].data["TIME"]) for i_det, det_name in f["DETECTORS"].data["DETECTORS"]: gain[det_name] = np.array(f["GAINS"].data[i_det, :]) if comm.comm_world is not None: if args.gain is not None: gain = comm.comm_world.bcast(gain, root=0) fp = comm.comm_world.bcast(fp, root=0) timer.stop() if comm.world_rank == 0: timer.report("Create focalplane ({} dets)".format(len(fp.keys()))) if args.debug: if comm.world_rank == 0: outfile = os.path.join(args.outdir, "focalplane.png") set_backend() dquats = {x: fp[x]["quat"] for x in fp.keys()} dfwhm = {x: fp[x]["fwhm"] for x in fp.keys()} plot_focalplane(dquats, 10.0, 10.0, outfile, fwhm=dfwhm) # For purposes of this simulation, we use detector noise # weights based on the NET (white noise level). If the destriping # baseline is too long, this will not be the best choice. detweights = {} for d in fp.keys(): net = fp[d]["NET"] detweights[d] = 1.0 / (args.sample_rate * net * net) return fp, gain, detweights
def plot( self, width: float = 4., height: float = 4., outfile: Optional[Path] = None, facecolor: Optional[Dict[str, Union[str, Tuple[float, float, float]]]] = None, polcolor: Optional[Dict[str, Union[str, Tuple[float, float, float]]]] = None, ): df = self.dataframe if polcolor is None: data = self.data kinds = set(name[-1] for name in data) n_kinds = len(kinds) if n_kinds == 2: logger.info( f'Found detector names ending in 2 characters, treat them as unqiue polarizations: {kinds}' ) colors = dict(zip(kinds, sns.color_palette("husl", 2))) polcolor = {name: colors[name[-1]] for name in data} else: logger.warn( 'Cannot guess the polarization of detectors, drawing with no polarization color. Define polcolor explicitly to fix this.' ) return plot_focalplane( df.quat, width, height, outfile, fwhm=df.fwhm_arcmin, facecolor=facecolor, polcolor=polcolor, labels=DictValueEqualsKey(), )
def main(): parser = argparse.ArgumentParser( description="Simulate fake hexagonal focalplane.", fromfile_prefix_chars='@') parser.add_argument("--minpix", required=False, type=int, default=100, help="minimum number of pixels to use") parser.add_argument("--out", required=False, default="fp_fake", help="Root name of output pickle file") parser.add_argument("--fwhm", required=False, type=float, default=5.0, help="beam FWHM in arcmin") parser.add_argument("--fwhm_sigma", required=False, type=float, default=0, help="Relative beam FWHM distribution width") parser.add_argument("--fov", required=False, type=float, default=5.0, help="Field of View in degrees") parser.add_argument("--psd_fknee", required=False, type=float, default=0.05, help="Detector noise model f_knee in Hz") parser.add_argument("--psd_NET", required=False, type=float, default=60.0e-6, help="Detector noise model NET in K*sqrt(sec)") parser.add_argument("--psd_alpha", required=False, type=float, default=1.0, help="Detector noise model slope") parser.add_argument("--psd_fmin", required=False, type=float, default=1.0e-5, help="Detector noise model f_min in Hz") parser.add_argument("--bandcenter_ghz", required=False, type=float, help="Band center frequency [GHz]") parser.add_argument("--bandcenter_sigma", required=False, type=float, default=0, help="Relative band center distribution width") parser.add_argument("--bandwidth_ghz", required=False, type=float, help="Bandwidth [GHz]") parser.add_argument("--bandwidth_sigma", required=False, type=float, default=0, help="Relative bandwidth distribution width") parser.add_argument("--random_seed", required=False, type=np.int, default=123456, help="Random number generator seed for randomized " "detector parameters") args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) # Guard against being called with multiple processes if MPI.COMM_WORLD.rank == 0: # Make one big hexagon layout at the center of the focalplane. # Compute the number of pixels that is at least the number requested. test = args.minpix - 1 nrings = 0 while (test - 6 * nrings) > 0: test -= 6 * nrings nrings += 1 npix = 1 for r in range(1, nrings + 1): npix += 6 * r print("using {} pixels ({} detectors)".format(npix, npix * 2)) # Translate the field-of-view into distance between flag sides angwidth = args.fov * np.cos(30 * degree) Apol = tt.hex_pol_angles_qu(npix, offset=0.0) Bpol = tt.hex_pol_angles_qu(npix, offset=90.0) Adets = tt.hex_layout(npix, angwidth, "fake_", "A", Apol) Bdets = tt.hex_layout(npix, angwidth, "fake_", "B", Bpol) dets = Adets.copy() dets.update(Bdets) np.random.seed(args.random_seed) for indx, d in enumerate(sorted(dets.keys())): dets[d]["fknee"] = args.psd_fknee dets[d]["fmin"] = args.psd_fmin dets[d]["alpha"] = args.psd_alpha dets[d]["NET"] = args.psd_NET # This is in degrees, but the input is in arcmin. dets[d]["fwhm_deg"] = (args.fwhm / 60.0) \ * (1 + np.random.randn()*args.fwhm_sigma) # This is a fixed value, in arcmin. dets[d]["fwhm"] = args.fwhm if args.bandcenter_ghz: dets[d]["bandcenter_ghz"] \ = args.bandcenter_ghz * (1+np.random.randn()*args.bandcenter_sigma) if args.bandwidth_ghz: dets[d]["bandwidth_ghz"] \ = args.bandwidth_ghz * (1+np.random.randn()*args.bandwidth_sigma) dets[d]["index"] = indx outfile = "{}_{}".format(args.out, npix) qdets = {x: y["quat"] for x, y in dets.items()} beams = {x: (60.0 * y["fwhm_deg"]) for x, y in dets.items()} tt.plot_focalplane(qdets, args.fov, args.fov, "{}.png".format(outfile), fwhm=beams) with open("{}.pkl".format(outfile), "wb") as p: pickle.dump(dets, p) return
import sys import pickle import toast.tod as tt from toast.vis import set_backend focalplane_filename = sys.argv[1] with open(focalplane_filename, "rb") as p: fp = pickle.load(p) outfile = focalplane_filename.replace("pkl", "png") set_backend() fp_onlyquat = { det: each_fp["quat"] for det, each_fp in fp.items() } tt.plot_focalplane(fp_onlyquat, 10.0, 10.0, outfile)
def load_fp(args, comm): start = MPI.Wtime() autotimer = timing.auto_timer() fp = None # Load focalplane information nullquat = np.array([0, 0, 0, 1], dtype=np.float64) if comm.comm_world.rank == 0: if args.fp is None: # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake['quat'] = nullquat fake['fwhm'] = 30.0 fake['fknee'] = 0.0 fake['fmin'] = 1e-9 fake['alpha'] = 1.0 fake['NET'] = 1.0 fake['color'] = 'r' fp = {} # Second detector at 22.5 degree polarization angle fp['bore1'] = fake fake2 = {} zrot = qa.rotation(ZAXIS, 22.5 * degree) fake2['quat'] = qa.mult(fake['quat'], zrot) fake2['fwhm'] = 30.0 fake2['fknee'] = 0.0 fake2['fmin'] = 1e-9 fake2['alpha'] = 1.0 fake2['NET'] = 1.0 fake2['color'] = 'r' fp['bore2'] = fake2 # Third detector at 45 degree polarization angle fake3 = {} zrot = qa.rotation(ZAXIS, 45 * degree) fake3['quat'] = qa.mult(fake['quat'], zrot) fake3['fwhm'] = 30.0 fake3['fknee'] = 0.0 fake3['fmin'] = 1e-9 fake3['alpha'] = 1.0 fake3['NET'] = 1.0 fake3['color'] = 'r' fp['bore3'] = fake3 # Fourth detector at 67.5 degree polarization angle fake4 = {} zrot = qa.rotation(ZAXIS, 67.5 * degree) fake4['quat'] = qa.mult(fake['quat'], zrot) fake4['fwhm'] = 30.0 fake4['fknee'] = 0.0 fake4['fmin'] = 1e-9 fake4['alpha'] = 1.0 fake4['NET'] = 1.0 fake4['color'] = 'r' fp['bore4'] = fake4 else: with open(args.fp, 'rb') as p: fp = pickle.load(p) fp = comm.comm_world.bcast(fp, root=0) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print('Create focalplane: {:.2f} seconds'.format(stop - start), flush=args.flush) start = stop if args.debug: if comm.comm_world.rank == 0: outfile = '{}/focalplane.png'.format(args.outdir) tt.plot_focalplane(fp, 6, 6, outfile) detectors = sorted(fp.keys()) detweights = {} for d in detectors: net = fp[d]['NET'] detweights[d] = 1.0 / (args.samplerate * net * net) return fp, detweights
def load_fp(args, comm): start = MPI.Wtime() autotimer = timing.auto_timer() fp = None # Load focalplane information nullquat = np.array([0,0,0,1], dtype=np.float64) if comm.comm_world.rank == 0: if args.fp is None: # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake['quat'] = nullquat fake['fwhm'] = 30.0 fake['fknee'] = 0.0 fake['fmin'] = 1e-9 fake['alpha'] = 1.0 fake['NET'] = 1.0 fake['color'] = 'r' fp = {} # Second detector at 22.5 degree polarization angle fp['bore1'] = fake fake2 = {} zrot = qa.rotation(ZAXIS, 22.5*degree) fake2['quat'] = qa.mult(fake['quat'], zrot) fake2['fwhm'] = 30.0 fake2['fknee'] = 0.0 fake2['fmin'] = 1e-9 fake2['alpha'] = 1.0 fake2['NET'] = 1.0 fake2['color'] = 'r' fp['bore2'] = fake2 # Third detector at 45 degree polarization angle fake3 = {} zrot = qa.rotation(ZAXIS, 45*degree) fake3['quat'] = qa.mult(fake['quat'], zrot) fake3['fwhm'] = 30.0 fake3['fknee'] = 0.0 fake3['fmin'] = 1e-9 fake3['alpha'] = 1.0 fake3['NET'] = 1.0 fake3['color'] = 'r' fp['bore3'] = fake3 # Fourth detector at 67.5 degree polarization angle fake4 = {} zrot = qa.rotation(ZAXIS, 67.5*degree) fake4['quat'] = qa.mult(fake['quat'], zrot) fake4['fwhm'] = 30.0 fake4['fknee'] = 0.0 fake4['fmin'] = 1e-9 fake4['alpha'] = 1.0 fake4['NET'] = 1.0 fake4['color'] = 'r' fp['bore4'] = fake4 else: with open(args.fp, 'rb') as p: fp = pickle.load(p) fp = comm.comm_world.bcast(fp, root=0) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print('Create focalplane: {:.2f} seconds'.format(stop-start), flush=args.flush) start = stop if args.debug: if comm.comm_world.rank == 0: outfile = '{}/focalplane.png'.format(args.outdir) tt.plot_focalplane(fp, 6, 6, outfile) detectors = sorted(fp.keys()) detweights = {} for d in detectors: net = fp[d]['NET'] detweights[d] = 1.0 / (args.samplerate * net * net) return fp, detweights
def load_focalplane(args, comm, schedule): focalplane = None # Load focalplane information if comm.comm_world is None or comm.comm_world.rank == 0: if args.focalplane is None: detector_data = {} ZAXIS = np.array([0, 0, 1.0]) # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake["quat"] = np.array([0, 0, 0, 1.0]) fake["fwhm"] = 30.0 fake["fknee"] = 0.0 fake["fmin"] = 1e-9 fake["alpha"] = 1.0 fake["NET"] = 1.0 fake["color"] = "r" detector_data["bore1"] = fake # Second detector at 22.5 degree polarization angle fake2 = {} zrot = qa.rotation(ZAXIS, np.radians(22.5)) fake2["quat"] = qa.mult(fake["quat"], zrot) fake2["fwhm"] = 30.0 fake2["fknee"] = 0.0 fake2["fmin"] = 1e-9 fake2["alpha"] = 1.0 fake2["NET"] = 1.0 fake2["color"] = "r" detector_data["bore2"] = fake2 # Third detector at 45 degree polarization angle fake3 = {} zrot = qa.rotation(ZAXIS, np.radians(45)) fake3["quat"] = qa.mult(fake["quat"], zrot) fake3["fwhm"] = 30.0 fake3["fknee"] = 0.0 fake3["fmin"] = 1e-9 fake3["alpha"] = 1.0 fake3["NET"] = 1.0 fake3["color"] = "r" detector_data["bore3"] = fake3 # Fourth detector at 67.5 degree polarization angle fake4 = {} zrot = qa.rotation(ZAXIS, np.radians(67.5)) fake4["quat"] = qa.mult(fake["quat"], zrot) fake4["fwhm"] = 30.0 fake4["fknee"] = 0.0 fake4["fmin"] = 1e-9 fake4["alpha"] = 1.0 fake4["NET"] = 1.0 fake4["color"] = "r" detector_data["bore4"] = fake4 focalplane = Focalplane(detector_data=detector_data, sample_rate=args.sample_rate) else: focalplane = Focalplane(fname_pickle=args.focalplane, sample_rate=args.sample_rate) if comm.comm_world is not None: focalplane = comm.comm_world.bcast(focalplane, root=0) if args.debug: if comm.comm_world is None or comm.comm_world.rank == 0: outfile = "{}/focalplane.png".format(args.outdir) plot_focalplane(focalplane, 6, 6, outfile) schedule.telescope.focalplane = focalplane detweights = focalplane.detweights return detweights
def main(): # We are going to group our processes in a single group. This is fine # if we have fewer processes than detectors. Otherwise we should group # them in a reasonable size that is smaller than the number of detectors # and which divides evenly into the total number of processes. comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=MPI.COMM_WORLD.size) # Make a fake focalplane. Plot it just for fun (don't waste time on this # for very large runs though). fp = fake_focalplane() if comm.comm_world.rank == 0: outfile = "custom_example_focalplane.png" set_backend() tt.plot_focalplane(fp, 6.0, 6.0, outfile) # Read in 2 boresight files borefiles = [ "../data/custom_example_boresight_1.txt", "../data/custom_example_boresight_2.txt" ] # Set up the distributed data rate = 100.0 data = create_observations(comm, rate, fp, borefiles) # Configure the healpix pixelization we will use for map-making and # also the "submap" resolution, which sets granularity of the locally # stored pieces of the sky. map_nside = 512 map_npix = 12 * map_nside**2 sub_nside = 4 sub_npix = 12 * sub_nside**2 # Compute a pointing matrix with healpix pixels and weights. pointing = tt.OpPointingHpix(nside=map_nside, nest=True, mode="IQU", pixels="pixels", weights="weights") pointing.exec(data) # Compute the locally hit submaps local_submaps = pixel_dist(data, sub_npix) # Sources of simulated data: scan from a symmetric beam convolved sky # and then add some simulated noise. signalmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3, dtype=np.float64, submap=sub_npix, local=local_submaps) signalmap.read_healpix_fits("../data/custom_example_sky.fits") scanmap = tt.OpSimScan(distmap=signalmap, pixels='pixels', weights='weights', out="sim") scanmap.exec(data) nse = tt.OpSimNoise(out="sim", realization=0) nse.exec(data) # Accumulate the hits and inverse diagonal pixel covariance, as well as the # noise weighted map. Here we simply use inverse noise weighting. detweights = {} for d in fp.keys(): net = fp[d]["NET"] detweights[d] = 1.0 / (rate * net * net) invnpp = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=6, dtype=np.float64, submap=sub_npix, local=local_submaps) hits = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=1, dtype=np.int64, submap=sub_npix, local=local_submaps) zmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3, dtype=np.float64, submap=sub_npix, local=local_submaps) invnpp.data.fill(0.0) hits.data.fill(0) zmap.data.fill(0.0) build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits, zmap=zmap, name="sim") build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() zmap.allreduce() # Write these products out hits.write_healpix_fits("custom_example_hits.fits") invnpp.write_healpix_fits("custom_example_invnpp.fits") zmap.write_healpix_fits("custom_example_zmap.fits") # Invert the covariance and write tm.covariance_invert(invnpp, 1.0e-3) invnpp.write_healpix_fits("custom_example_npp.fits") # Apply covariance to make the binned map tm.covariance_apply(invnpp, zmap) zmap.write_healpix_fits("custom_example_binned.fits") MPI.Finalize() return
def main(): if MPI.COMM_WORLD.rank == 0: print("Running with {} processes".format(MPI.COMM_WORLD.size), flush=True) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description="Read existing data and make a simple map.", fromfile_prefix_chars="@", ) parser.add_argument( "--groupsize", required=False, type=int, default=0, help="size of processor groups used to distribute " "observations", ) parser.add_argument( "--hwprpm", required=False, type=float, default=0.0, help="The rate (in RPM) of the HWP rotation", ) parser.add_argument( "--samplerate", required=False, default=100.0, type=np.float, help="Detector sample rate (Hz)", ) parser.add_argument("--outdir", required=False, default="out", help="Output directory") parser.add_argument("--nside", required=False, type=int, default=64, help="Healpix NSIDE") parser.add_argument( "--subnside", required=False, type=int, default=8, help="Distributed pixel sub-map NSIDE", ) parser.add_argument("--coord", required=False, default="E", help="Sky coordinate system [C,E,G]") parser.add_argument( "--baseline", required=False, type=float, default=60.0, help="Destriping baseline length (seconds)", ) parser.add_argument( "--noisefilter", required=False, default=False, action="store_true", help="Destripe with the noise filter enabled", ) parser.add_argument( "--madam", required=False, default=False, action="store_true", help="If specified, use libmadam for map-making", ) parser.add_argument("--madampar", required=False, default=None, help="Madam parameter file") parser.add_argument( "--polyorder", required=False, type=int, help="Polynomial order for the polyfilter", ) parser.add_argument( "--wbin_ground", required=False, type=float, help="Ground template bin width [degrees]", ) parser.add_argument( "--flush", required=False, default=False, action="store_true", help="Flush every print statement.", ) parser.add_argument("--tidas", required=False, default=None, help="Input TIDAS volume") parser.add_argument("--tidas_detgroup", required=False, default=None, help="TIDAS detector group") parser.add_argument("--spt3g", required=False, default=None, help="Input SPT3G data directory") parser.add_argument( "--spt3g_prefix", required=False, default=None, help="SPT3G data frame file prefix", ) parser.add_argument( "--common_flag_mask", required=False, default=0, type=np.uint8, help="Common flag mask", ) parser.add_argument( "--debug", required=False, default=False, action="store_true", help="Write data distribution info and focalplane plot", ) args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) # args = parser.parse_args(sys.argv) autotimer = timing.auto_timer("@{}".format(timing.FILE())) if (args.tidas is not None) and (args.spt3g is not None): raise RuntimeError("Cannot read two datasets!") if (args.tidas is None) and (args.spt3g is None): raise RuntimeError("No dataset specified!") if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot load") if args.spt3g is not None: if not tt.spt3g_available: raise RuntimeError("SPT3G not found- cannot load") groupsize = args.groupsize if groupsize == 0: groupsize = MPI.COMM_WORLD.size # Pixelization nside = args.nside npix = 12 * args.nside * args.nside subnside = args.subnside if subnside > nside: subnside = nside subnpix = 12 * subnside * subnside # This is the 2-level toast communicator. if MPI.COMM_WORLD.size % groupsize != 0: if MPI.COMM_WORLD.rank == 0: print( "WARNING: process groupsize does not evenly divide into " "total number of processes", flush=True, ) comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize) # Create output directory mtime = MPI.Wtime() if comm.comm_world.rank == 0: if not os.path.isdir(args.outdir): os.makedirs(args.outdir) mtime = elapsed(comm.comm_world, mtime, "Creating output directory") # The distributed timestream data data = None if args.tidas is not None: if args.tidas_detgroup is None: raise RuntimeError("you must specify the detector group") data = tds.load_tidas( comm, comm.group_size, args.tidas, "r", args.tidas_detgroup, tds.TODTidas, group_dets=args.tidas_detgroup, distintervals="chunks", ) if args.spt3g is not None: if args.spt3g_prefix is None: raise RuntimeError("you must specify the frame file prefix") data = s3g.load_spt3g( comm, comm.group_size, args.spt3g, args.spt3g_prefix, s3g.obsweight_spt3g, s3g.TOD3G, ) mtime = elapsed(comm.comm_world, mtime, "Distribute data") # In debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open("{}_distdata.txt".format(args.outdir), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() mtime = elapsed(comm.comm_world, mtime, "Dumping debug data distribution") if comm.comm_world.rank == 0: outfile = "{}_focalplane.png".format(args.outdir) set_backend() # Just plot the dets from the first TOD temptod = data.obs[0]["tod"] # FIXME: change this once we store det info in the metadata. dfwhm = {x: 10.0 for x in temptod.detectors} tt.plot_focalplane(temptod.detoffset(), 10.0, 10.0, outfile, fwhm=dfwhm) comm.comm_world.barrier() mtime = elapsed(comm.comm_world, mtime, "Plotting debug focalplane") # Compute pointing matrix pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU", hwprpm=args.hwprpm) pointing.exec(data) mtime = elapsed(comm.comm_world, mtime, "Expand pointing") # Mapmaking. # FIXME: We potentially have a different noise model for every # observation. We need to have both spt3g and tidas format Noise # classes which read the information from disk. Then the mapmaking # operators need to get these noise weights from each observation. detweights = {d: 1.0 for d in data.obs[0]["tod"].detectors} if not args.madam: if comm.comm_world.rank == 0: print("Not using Madam, will only make a binned map!", flush=True) # Filter data if desired if args.polyorder: polyfilter = tt.OpPolyFilter( order=args.polyorder, common_flag_mask=args.common_flag_mask) polyfilter.exec(data) mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering") if args.wbin_ground: groundfilter = tt.OpGroundFilter( wbin=args.wbin_ground, common_flag_mask=args.common_flag_mask) groundfilter.exec(data) mtime = elapsed(comm.comm_world, mtime, "Ground template filtering") # Compute pixel space distribution lc = tm.OpLocalPixels() localpix = lc.exec(data) if localpix is None: raise RuntimeError( "Process {} has no hit pixels. Perhaps there are fewer " "detectors than processes in the group?".format( comm.comm_world.rank)) localsm = np.unique(np.floor_divide(localpix, subnpix)) mtime = elapsed(comm.comm_world, mtime, "Compute local submaps") # construct distributed maps to store the covariance, # noise weighted map, and hits mtime = MPI.Wtime() invnpp = tm.DistPixels( comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm, ) hits = tm.DistPixels( comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm, ) zmap = tm.DistPixels( comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm, ) # compute the hits and covariance. invnpp.data.fill(0.0) hits.data.fill(0) build_invnpp = tm.OpAccumDiag( detweights=detweights, invnpp=invnpp, hits=hits, common_flag_mask=args.common_flag_mask, ) build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1") hits.write_healpix_fits("{}_hits.fits".format(args.outdir)) invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir)) mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1") # invert it tm.covariance_invert(invnpp, 1.0e-3) mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1") invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir)) mtime = elapsed(comm.comm_world, mtime, "Writing N_pp") zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag(zmap=zmap, detweights=detweights, common_flag_mask=args.common_flag_mask) build_zmap.exec(data) zmap.allreduce() mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map") tm.covariance_apply(invnpp, zmap) mtime = elapsed(comm.comm_world, mtime, "Computing binned map") zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits")) mtime = elapsed(comm.comm_world, mtime, "Writing binned map") else: # Set up MADAM map making. pars = {} pars["temperature_only"] = "F" pars["force_pol"] = "T" pars["kfirst"] = "T" pars["concatenate_messages"] = "T" pars["write_map"] = "T" pars["write_binmap"] = "T" pars["write_matrix"] = "T" pars["write_wcov"] = "T" pars["write_hits"] = "T" pars["nside_cross"] = nside // 2 pars["nside_submap"] = subnside if args.madampar is not None: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars["base_first"] = args.baseline pars["nside_map"] = nside if args.noisefilter: pars["kfilter"] = "T" else: pars["kfilter"] = "F" pars["fsample"] = args.samplerate madam = tm.OpMadam(params=pars, detweights=detweights, common_flag_mask=args.common_flag_mask) madam.exec(data) mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking") comm.comm_world.barrier() stop = MPI.Wtime() dur = stop - global_start if comm.comm_world.rank == 0: print("Total Time: {:.2f} seconds".format(dur), flush=True) return
dets.update(Bdets) np.random.seed(args.random_seed) for indx, d in enumerate(sorted(dets.keys())): dets[d]["fknee"] = args.psd_fknee dets[d]["fmin"] = args.psd_fmin dets[d]["alpha"] = args.psd_alpha dets[d]["NET"] = args.psd_NET dets[d]["fwhm_deg"] = fwhm \ * (1 + np.random.randn()*args.fwhm_sigma) dets[d]["fwhm"] = dets[d]["fwhm_deg"] # Support legacy code if args.bandcenter_ghz: dets[d]["bandcenter_ghz"] \ = args.bandcenter_ghz * (1+np.random.randn()*args.bandcenter_sigma) if args.bandwidth_ghz: dets[d]["bandwidth_ghz"] \ = args.bandwidth_ghz * (1+np.random.randn()*args.bandwidth_sigma) dets[d]["index"] = indx outfile = "{}_{}".format(args.out, npix) tt.plot_focalplane(dets, args.fov, args.fov, "{}.png".format(outfile)) with open("{}.pkl".format(outfile), "wb") as p: pickle.dump(dets, p) tman = timing.timing_manager() tman.report() tman.clear()
def main(): if MPI.COMM_WORLD.rank == 0: print("Running with {} processes".format(MPI.COMM_WORLD.size), flush=True) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description="Simulate satellite " "boresight pointing and make a noise map.", fromfile_prefix_chars="@" ) parser.add_argument( "--groupsize", required=False, type=int, default=0, help="size of processor groups used to distribute observations" ) parser.add_argument( "--samplerate", required=False, type=float, default=40.0, help="Detector sample rate (Hz)" ) parser.add_argument( "--starttime", required=False, type=float, default=0.0, help="The overall start time of the simulation" ) parser.add_argument( "--spinperiod", required=False, type=float, default=10.0, help="The period (in minutes) of the rotation about the " "spin axis" ) parser.add_argument( "--spinangle", required=False, type=float, default=30.0, help="The opening angle (in degrees) of the boresight " "from the spin axis" ) parser.add_argument( "--precperiod", required=False, type=float, default=50.0, help="The period (in minutes) of the rotation about the " "precession axis" ) parser.add_argument( "--precangle", required=False, type=float, default=65.0, help="The opening angle (in degrees) of the spin axis " "from the precession axis" ) parser.add_argument( "--hwprpm", required=False, type=float, default=0.0, help="The rate (in RPM) of the HWP rotation" ) parser.add_argument( "--hwpstep", required=False, default=None, help="For stepped HWP, the angle in degrees of each step" ) parser.add_argument( "--hwpsteptime", required=False, type=float, default=0.0, help="For stepped HWP, the the time in seconds between " "steps" ) parser.add_argument( "--obs", required=False, type=float, default=1.0, help="Number of hours in one science observation" ) parser.add_argument( "--gap", required=False, type=float, default=0.0, help="Cooler cycle time in hours between science obs" ) parser.add_argument( "--numobs", required=False, type=int, default=1, help="Number of complete observations" ) parser.add_argument( "--outdir", required=False, default="out", help="Output directory" ) parser.add_argument( "--debug", required=False, default=False, action="store_true", help="Write diagnostics" ) parser.add_argument( "--nside", required=False, type=int, default=64, help="Healpix NSIDE" ) parser.add_argument( "--subnside", required=False, type=int, default=4, help="Distributed pixel sub-map NSIDE" ) parser.add_argument( "--baseline", required=False, type=float, default=60.0, help="Destriping baseline length (seconds)" ) parser.add_argument( "--noisefilter", required=False, default=False, action="store_true", help="Destripe with the noise filter enabled" ) parser.add_argument( "--madam", required=False, default=False, action="store_true", help="If specified, use libmadam for map-making" ) parser.add_argument( "--madampar", required=False, default=None, help="Madam parameter file" ) parser.add_argument('--flush', required=False, default=False, action='store_true', help='Flush every print statement.') parser.add_argument( "--MC_start", required=False, type=int, default=0, help="First Monte Carlo noise realization" ) parser.add_argument( "--MC_count", required=False, type=int, default=1, help="Number of Monte Carlo noise realizations" ) parser.add_argument( "--fp", required=False, default=None, help="Pickle file containing a dictionary of detector properties. " "The keys of this dict are the detector names, and each value is also " "a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" " "(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" " "(float). For optional plotting, the key \"color\" can specify a " "valid matplotlib color string." ) parser.add_argument('--tidas', required=False, default=None, help='Output TIDAS export path') parser.add_argument('--input_map', required=False, help='Input map for signal') parser.add_argument('--input_pysm_model', required=False, help='Comma separated models for on-the-fly PySM ' 'simulation, e.g. s3,d6,f1,a2"') parser.add_argument('--apply_beam', required=False, action='store_true', help='Apply beam convolution to input map with gaussian ' 'beam parameters defined in focalplane') parser.add_argument('--input_dipole', required=False, help='Simulate dipole, possible values are ' 'total, orbital, solar') parser.add_argument('--input_dipole_solar_speed_kms', required=False, help='Solar system speed [km/s]', type=float, default=369.0) parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False, help='Solar system speed galactic latitude [degrees]', type=float, default=48.26) parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False, help='Solar system speed galactic longitude[degrees]', type=float, default=263.99) args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) autotimer = timing.auto_timer("@{}".format(timing.FILE())) if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot export") groupsize = args.groupsize if groupsize == 0: groupsize = MPI.COMM_WORLD.size # This is the 2-level toast communicator. if MPI.COMM_WORLD.size % groupsize != 0: if MPI.COMM_WORLD.rank == 0: print("WARNING: process groupsize does not evenly divide into " "total number of processes", flush=True) comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize) # get options hwpstep = None if args.hwpstep is not None: hwpstep = float(args.hwpstep) npix = 12 * args.nside * args.nside subnside = args.subnside if subnside > args.nside: subnside = args.nside subnpix = 12 * subnside * subnside start = MPI.Wtime() fp = None # Load focalplane information if comm.comm_world.rank == 0: if args.fp is None: # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0]) fake["fwhm"] = 30.0 fake["fknee"] = 0.0 fake["alpha"] = 1.0 fake["NET"] = 1.0 fake["color"] = "r" fp = {} fp["bore"] = fake else: with open(args.fp, "rb") as p: fp = pickle.load(p) fp = comm.comm_world.bcast(fp, root=0) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Create focalplane: {:.2f} seconds".format(stop-start), flush=True) start = stop if args.debug: if comm.comm_world.rank == 0: outfile = "{}_focalplane.png".format(args.outdir) set_backend() tt.plot_focalplane(fp, 10.0, 10.0, outfile) # Since we are simulating noise timestreams, we want # them to be contiguous and reproducible over the whole # observation. We distribute data by detector within an # observation, so ensure that our group size is not larger # than the number of detectors we have. if groupsize > len(fp.keys()): if comm.comm_world.rank == 0: print("process group is too large for the number of detectors", flush=True) comm.comm_world.Abort() # Detector information from the focalplane detectors = sorted(fp.keys()) detquats = {} detindx = None if "index" in fp[detectors[0]]: detindx = {} for d in detectors: detquats[d] = fp[d]["quat"] if detindx is not None: detindx[d] = fp[d]["index"] # Distribute the observations uniformly groupdist = toast.distribute_uniform(args.numobs, comm.ngroups) # Compute global time and sample ranges of all observations obsrange = tt.regular_intervals(args.numobs, args.starttime, 0, args.samplerate, 3600*args.obs, 3600*args.gap) # Create the noise model used for all observations fmin = {} fknee = {} alpha = {} NET = {} rates = {} for d in detectors: rates[d] = args.samplerate fmin[d] = fp[d]["fmin"] fknee[d] = fp[d]["fknee"] alpha[d] = fp[d]["alpha"] NET[d] = fp[d]["NET"] noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors, fknee=fknee, alpha=alpha, NET=NET) mem_counter = tt.OpMemoryCounter() # The distributed timestream data data = toast.Data(comm) # Every process group creates its observations group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] for ob in range(group_firstobs, group_firstobs + group_numobs): tod = tt.TODSatellite( comm.comm_group, detquats, obsrange[ob].samples, firsttime=obsrange[ob].start, rate=args.samplerate, spinperiod=args.spinperiod, spinangle=args.spinangle, precperiod=args.precperiod, precangle=args.precangle, detindx=detindx, detranks=comm.group_size ) obs = {} obs["name"] = "science_{:05d}".format(ob) obs["tod"] = tod obs["intervals"] = None obs["baselines"] = None obs["noise"] = noise obs["id"] = ob data.obs.append(obs) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Read parameters, compute data distribution: " "{:.2f} seconds".format(stop-start), flush=True) start = stop # we set the precession axis now, which will trigger calculation # of the boresight pointing. for ob in range(group_numobs): curobs = data.obs[ob] tod = curobs["tod"] # Get the global sample offset from the original distribution of # intervals obsoffset = obsrange[group_firstobs + ob].first # Constantly slewing precession axis degday = 360.0 / 365.25 precquat = tt.slew_precession_axis(nsim=tod.local_samples[1], firstsamp=obsoffset, samplerate=args.samplerate, degday=degday) tod.set_prec_axis(qprec=precquat) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Construct boresight pointing: " "{:.2f} seconds".format(stop-start), flush=True) start = stop # make a Healpix pointing matrix. pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU", hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime) pointing.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Pointing generation took {:.3f} s".format(elapsed), flush=True) start = stop localpix, localsm, subnpix = get_submaps(args, comm, data) signalname = "signal" if args.input_pysm_model: simulate_sky_signal(args, comm, data, mem_counter, [fp], subnpix, localsm, signalname=signalname) if args.input_dipole: print("Simulating dipole") op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole, solar_speed=args.input_dipole_solar_speed_kms, solar_gal_lat=args.input_dipole_solar_gal_lat_deg, solar_gal_lon=args.input_dipole_solar_gal_lon_deg, out=signalname, keep_quats=True, keep_vel=False, subtract=False, coord="G", freq=0, # we could use frequency for quadrupole correction flag_mask=255, common_flag_mask=255) op_sim_dipole.exec(data) # Mapmaking. For purposes of this simulation, we use detector noise # weights based on the NET (white noise level). If the destriping # baseline is too long, this will not be the best choice. detweights = {} for d in detectors: net = fp[d]["NET"] detweights[d] = 1.0 / (args.samplerate * net * net) if not args.madam: if comm.comm_world.rank == 0: print("Not using Madam, will only make a binned map!", flush=True) # get locally hit pixels lc = tm.OpLocalPixels() localpix = lc.exec(data) # find the locally hit submaps. localsm = np.unique(np.floor_divide(localpix, subnpix)) # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) # compute the hits and covariance once, since the pointing and noise # weights are fixed. invnpp.data.fill(0.0) hits.data.fill(0) build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits) build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop hits.write_healpix_fits("{}_hits.fits".format(args.outdir)) invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop # invert it tm.covariance_invert(invnpp, 1.0e-3) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Inverting N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Writing N_pp took {:.3f} s".format(elapsed), flush=True) start = stop # in debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open("{}_distdata.txt".format(args.outdir), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Dumping debug data distribution took " "{:.3f} s".format(elapsed), flush=True) start = stop mcstart = start # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc+nmc): # create output directory for this realization outpath = "{}_{:03d}".format(args.outdir, mc) if comm.comm_world.rank == 0: if not os.path.isdir(outpath): os.makedirs(outpath) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Creating output dir {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop # clear all signal data from the cache, so that we can generate # new noise timestreams. tod.cache.clear("tot_signal_.*") # simulate noise nse = tt.OpSimNoise(out="tot_signal", realization=mc) nse.exec(data) # add sky signal add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname) if mc == firstmc: # For the first realization, optionally export the # timestream data to a TIDAS volume. if args.tidas is not None: from toast.tod.tidas import OpTidasExport tidas_path = os.path.abspath(args.tidas) export = OpTidasExport(tidas_path, name="tot_signal") export.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Noise simulation {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal", detweights=detweights) build_zmap.exec(data) zmap.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Building noise weighted map {:04d} took {:.3f} s".format( mc, elapsed), flush=True) start = stop tm.covariance_apply(invnpp, zmap) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Computing binned map {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop zmap.write_healpix_fits(os.path.join(outpath, "binned.fits")) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Writing binned map {:04d} took {:.3f} s".format(mc, elapsed), flush=True) elapsed = stop - mcstart if comm.comm_world.rank == 0: print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop else: # Set up MADAM map making. pars = {} cross = args.nside // 2 pars[ "temperature_only" ] = "F" pars[ "force_pol" ] = "T" pars[ "kfirst" ] = "T" pars[ "concatenate_messages" ] = "T" pars[ "write_map" ] = "T" pars[ "write_binmap" ] = "T" pars[ "write_matrix" ] = "T" pars[ "write_wcov" ] = "T" pars[ "write_hits" ] = "T" pars[ "nside_cross" ] = cross pars[ "nside_submap" ] = subnside if args.madampar is not None: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars[ "base_first" ] = args.baseline pars[ "nside_map" ] = args.nside if args.noisefilter: pars[ "kfilter" ] = "T" else: pars[ "kfilter" ] = "F" pars[ "fsample" ] = args.samplerate # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc+nmc): # clear all total signal data from the cache, so that we can generate # new noise timestreams. tod.cache.clear("tot_signal_.*") # simulate noise nse = tt.OpSimNoise(out="tot_signal", realization=mc) nse.exec(data) # add sky signal add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Noise simulation took {:.3f} s".format(elapsed), flush=True) start = stop # create output directory for this realization pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc) if comm.comm_world.rank == 0: if not os.path.isdir(pars["path_output"]): os.makedirs(pars["path_output"]) # in debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open(os.path.join(pars["path_output"], "distdata.txt"), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() madam = tm.OpMadam(params=pars, detweights=detweights, name="tot_signal") madam.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Mapmaking took {:.3f} s".format(elapsed), flush=True) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - global_start if comm.comm_world.rank == 0: print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main(): if MPI.COMM_WORLD.rank == 0: print("Running with {} processes".format(MPI.COMM_WORLD.size), flush=True) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description="Simulate satellite " "boresight pointing and make a noise map.", fromfile_prefix_chars="@" ) parser.add_argument( "--groupsize", required=False, type=int, default=0, help="size of processor groups used to distribute observations" ) parser.add_argument( "--samplerate", required=False, type=float, default=40.0, help="Detector sample rate (Hz)" ) parser.add_argument( "--starttime", required=False, type=float, default=0.0, help="The overall start time of the simulation" ) parser.add_argument( "--spinperiod", required=False, type=float, default=10.0, help="The period (in minutes) of the rotation about the " "spin axis" ) parser.add_argument( "--spinangle", required=False, type=float, default=30.0, help="The opening angle (in degrees) of the boresight " "from the spin axis" ) parser.add_argument( "--precperiod", required=False, type=float, default=50.0, help="The period (in minutes) of the rotation about the " "precession axis" ) parser.add_argument( "--precangle", required=False, type=float, default=65.0, help="The opening angle (in degrees) of the spin axis " "from the precession axis" ) parser.add_argument( "--hwprpm", required=False, type=float, default=0.0, help="The rate (in RPM) of the HWP rotation" ) parser.add_argument( "--hwpstep", required=False, default=None, help="For stepped HWP, the angle in degrees of each step" ) parser.add_argument( "--hwpsteptime", required=False, type=float, default=0.0, help="For stepped HWP, the the time in seconds between " "steps" ) parser.add_argument( "--obs", required=False, type=float, default=1.0, help="Number of hours in one science observation" ) parser.add_argument( "--gap", required=False, type=float, default=0.0, help="Cooler cycle time in hours between science obs" ) parser.add_argument( "--numobs", required=False, type=int, default=1, help="Number of complete observations" ) parser.add_argument( "--outdir", required=False, default="out", help="Output directory" ) parser.add_argument( "--debug", required=False, default=False, action="store_true", help="Write diagnostics" ) parser.add_argument( "--nside", required=False, type=int, default=64, help="Healpix NSIDE" ) parser.add_argument( "--subnside", required=False, type=int, default=4, help="Distributed pixel sub-map NSIDE" ) parser.add_argument('--coord', required=False, default='E', help='Sky coordinate system [C,E,G]') parser.add_argument( "--baseline", required=False, type=float, default=60.0, help="Destriping baseline length (seconds)" ) parser.add_argument( "--noisefilter", required=False, default=False, action="store_true", help="Destripe with the noise filter enabled" ) parser.add_argument( "--madam", required=False, default=False, action="store_true", help="If specified, use libmadam for map-making" ) parser.add_argument( "--madampar", required=False, default=None, help="Madam parameter file" ) parser.add_argument('--flush', required=False, default=False, action='store_true', help='Flush every print statement.') parser.add_argument( "--MC_start", required=False, type=int, default=0, help="First Monte Carlo noise realization" ) parser.add_argument( "--MC_count", required=False, type=int, default=1, help="Number of Monte Carlo noise realizations" ) parser.add_argument( "--fp", required=False, default=None, help="Pickle file containing a dictionary of detector properties. " "The keys of this dict are the detector names, and each value is also " "a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" " "(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" " "(float). For optional plotting, the key \"color\" can specify a " "valid matplotlib color string." ) parser.add_argument( "--gain", required=False, default=None, help= "Calibrate the input timelines with a set of gains from a" "FITS file containing 3 extensions:" "HDU named DETECTORS : table with list of detector names in a column named DETECTORS" "HDU named TIME: table with common timestamps column named TIME" "HDU named GAINS: 2D image of floats with one row per detector and one column per value.") parser.add_argument('--tidas', required=False, default=None, help='Output TIDAS export path') parser.add_argument('--spt3g', required=False, default=None, help='Output SPT3G export path') parser.add_argument('--input_map', required=False, help='Input map for signal') parser.add_argument('--input_pysm_model', required=False, help='Comma separated models for on-the-fly PySM ' 'simulation, e.g. s3,d6,f1,a2"') parser.add_argument('--input_pysm_precomputed_cmb_K_CMB', required=False, help='Precomputed CMB map for PySM in K_CMB' 'it overrides any model defined in input_pysm_model"') parser.add_argument('--apply_beam', required=False, action='store_true', help='Apply beam convolution to input map with gaussian ' 'beam parameters defined in focalplane') parser.add_argument('--input_dipole', required=False, help='Simulate dipole, possible values are ' 'total, orbital, solar') parser.add_argument('--input_dipole_solar_speed_kms', required=False, help='Solar system speed [km/s]', type=float, default=369.0) parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False, help='Solar system speed galactic latitude [degrees]', type=float, default=48.26) parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False, help='Solar system speed galactic longitude[degrees]', type=float, default=263.99) args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) autotimer = timing.auto_timer("@{}".format(timing.FILE())) if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot export") if args.spt3g is not None: if not tt.spt3g_available: raise RuntimeError("SPT3G not found- cannot export") groupsize = args.groupsize if groupsize == 0: groupsize = MPI.COMM_WORLD.size # This is the 2-level toast communicator. if MPI.COMM_WORLD.size % groupsize != 0: if MPI.COMM_WORLD.rank == 0: print("WARNING: process groupsize does not evenly divide into " "total number of processes", flush=True) comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize) # get options hwpstep = None if args.hwpstep is not None: hwpstep = float(args.hwpstep) npix = 12 * args.nside * args.nside subnside = args.subnside if subnside > args.nside: subnside = args.nside subnpix = 12 * subnside * subnside start = MPI.Wtime() fp = None gain = None # Load focalplane information if comm.comm_world.rank == 0: if args.fp is None: # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0]) fake["fwhm"] = 30.0 fake["fknee"] = 0.0 fake["alpha"] = 1.0 fake["NET"] = 1.0 fake["color"] = "r" fp = {} fp["bore"] = fake else: with open(args.fp, "rb") as p: fp = pickle.load(p) if args.gain is not None: gain = {} with fits.open(args.gain) as f: gain["TIME"] = np.array(f["TIME"].data["TIME"]) for i_det, det_name in f["DETECTORS"].data["DETECTORS"]: gain[det_name] = np.array(f["GAINS"].data[i_det, :]) if args.gain is not None: gain = comm.comm_world.bcast(gain, root=0) fp = comm.comm_world.bcast(fp, root=0) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Create focalplane ({} dets): {:.2f} seconds"\ .format(len(fp.keys()), stop-start), flush=True) start = stop if args.debug: if comm.comm_world.rank == 0: outfile = "{}_focalplane.png".format(args.outdir) set_backend() dquats = { x : fp[x]["quat"] for x in fp.keys() } dfwhm = { x : fp[x]["fwhm"] for x in fp.keys() } tt.plot_focalplane(dquats, 10.0, 10.0, outfile, fwhm=dfwhm) # Since we are simulating noise timestreams, we want # them to be contiguous and reproducible over the whole # observation. We distribute data by detector within an # observation, so ensure that our group size is not larger # than the number of detectors we have. if groupsize > len(fp.keys()): if comm.comm_world.rank == 0: print("process group is too large for the number of detectors", flush=True) comm.comm_world.Abort() # Detector information from the focalplane detectors = sorted(fp.keys()) detquats = {} detindx = None if "index" in fp[detectors[0]]: detindx = {} for d in detectors: detquats[d] = fp[d]["quat"] if detindx is not None: detindx[d] = fp[d]["index"] # Distribute the observations uniformly groupdist = toast.distribute_uniform(args.numobs, comm.ngroups) # Compute global time and sample ranges of all observations obsrange = tt.regular_intervals(args.numobs, args.starttime, 0, args.samplerate, 3600*args.obs, 3600*args.gap) # Create the noise model used for all observations fmin = {} fknee = {} alpha = {} NET = {} rates = {} for d in detectors: rates[d] = args.samplerate fmin[d] = fp[d]["fmin"] fknee[d] = fp[d]["fknee"] alpha[d] = fp[d]["alpha"] NET[d] = fp[d]["NET"] noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors, fknee=fknee, alpha=alpha, NET=NET) mem_counter = tt.OpMemoryCounter() # The distributed timestream data data = toast.Data(comm) # Every process group creates its observations group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] for ob in range(group_firstobs, group_firstobs + group_numobs): tod = tt.TODSatellite( comm.comm_group, detquats, obsrange[ob].samples, coord=args.coord, firstsamp=obsrange[ob].first, firsttime=obsrange[ob].start, rate=args.samplerate, spinperiod=args.spinperiod, spinangle=args.spinangle, precperiod=args.precperiod, precangle=args.precangle, detindx=detindx, detranks=comm.group_size ) obs = {} obs["name"] = "science_{:05d}".format(ob) obs["tod"] = tod obs["intervals"] = None obs["baselines"] = None obs["noise"] = noise obs["id"] = ob data.obs.append(obs) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Read parameters, compute data distribution: " "{:.2f} seconds".format(stop-start), flush=True) start = stop # we set the precession axis now, which will trigger calculation # of the boresight pointing. for ob in range(group_numobs): curobs = data.obs[ob] tod = curobs["tod"] # Get the global sample offset from the original distribution of # intervals obsoffset = obsrange[group_firstobs + ob].first # Constantly slewing precession axis degday = 360.0 / 365.25 precquat = np.empty(4*tod.local_samples[1], dtype=np.float64).reshape((-1,4)) tt.slew_precession_axis(precquat, firstsamp=(obsoffset + tod.local_samples[0]), samplerate=args.samplerate, degday=degday) tod.set_prec_axis(qprec=precquat) del precquat stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Construct boresight pointing: " "{:.2f} seconds".format(stop-start), flush=True) start = stop # make a Healpix pointing matrix. pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU", hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime) pointing.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Pointing generation took {:.3f} s".format(elapsed), flush=True) start = stop localpix, localsm, subnpix = get_submaps(args, comm, data) signalname = "signal" has_signal = False if args.input_pysm_model: has_signal = True simulate_sky_signal(args, comm, data, mem_counter, [fp], subnpix, localsm, signalname=signalname) if args.input_dipole: print("Simulating dipole") has_signal = True op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole, solar_speed=args.input_dipole_solar_speed_kms, solar_gal_lat=args.input_dipole_solar_gal_lat_deg, solar_gal_lon=args.input_dipole_solar_gal_lon_deg, out=signalname, keep_quats=False, keep_vel=False, subtract=False, coord=args.coord, freq=0, # we could use frequency for quadrupole correction flag_mask=255, common_flag_mask=255) op_sim_dipole.exec(data) del op_sim_dipole # Mapmaking. For purposes of this simulation, we use detector noise # weights based on the NET (white noise level). If the destriping # baseline is too long, this will not be the best choice. detweights = {} for d in detectors: net = fp[d]["NET"] detweights[d] = 1.0 / (args.samplerate * net * net) if not args.madam: if comm.comm_world.rank == 0: print("Not using Madam, will only make a binned map!", flush=True) # get locally hit pixels lc = tm.OpLocalPixels() localpix = lc.exec(data) # find the locally hit submaps. localsm = np.unique(np.floor_divide(localpix, subnpix)) # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) # compute the hits and covariance once, since the pointing and noise # weights are fixed. invnpp.data.fill(0.0) hits.data.fill(0) build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits) build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop hits.write_healpix_fits("{}_hits.fits".format(args.outdir)) invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop # invert it tm.covariance_invert(invnpp, 1.0e-3) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Inverting N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Writing N_pp took {:.3f} s".format(elapsed), flush=True) start = stop # in debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open("{}_distdata.txt".format(args.outdir), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Dumping debug data distribution took " "{:.3f} s".format(elapsed), flush=True) start = stop mcstart = start # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc+nmc): # create output directory for this realization outpath = "{}_{:03d}".format(args.outdir, mc) if comm.comm_world.rank == 0: if not os.path.isdir(outpath): os.makedirs(outpath) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Creating output dir {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop # clear all signal data from the cache, so that we can generate # new noise timestreams. tod.cache.clear("tot_signal_.*") # simulate noise nse = tt.OpSimNoise(out="tot_signal", realization=mc) nse.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Noise simulation {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop # add sky signal if has_signal: add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Add sky signal {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop if gain is not None: op_apply_gain = tt.OpApplyGain(gain, name="tot_signal") op_apply_gain.exec(data) if mc == firstmc: # For the first realization, optionally export the # timestream data. If we had observation intervals defined, # we could pass "use_interval=True" to the export operators, # which would ensure breaks in the exported data at # acceptable places. if args.tidas is not None: tidas_path = os.path.abspath(args.tidas) export = OpTidasExport(tidas_path, TODTidas, backend="hdf5", use_todchunks=True, create_opts={"group_dets":"sim"}, ctor_opts={"group_dets":"sim"}, cache_name="tot_signal") export.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Tidas export took {:.3f} s"\ .format(elapsed), flush=True) start = stop if args.spt3g is not None: spt3g_path = os.path.abspath(args.spt3g) export = Op3GExport(spt3g_path, TOD3G, use_todchunks=True, export_opts={"prefix" : "sim"}, cache_name="tot_signal") export.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" SPT3G export took {:.3f} s"\ .format(elapsed), flush=True) start = stop zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal", detweights=detweights) build_zmap.exec(data) zmap.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Building noise weighted map {:04d} took {:.3f} s".format( mc, elapsed), flush=True) start = stop tm.covariance_apply(invnpp, zmap) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Computing binned map {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop zmap.write_healpix_fits(os.path.join(outpath, "binned.fits")) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Writing binned map {:04d} took {:.3f} s".format(mc, elapsed), flush=True) elapsed = stop - mcstart if comm.comm_world.rank == 0: print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop else: # Set up MADAM map making. pars = {} cross = args.nside // 2 pars[ "temperature_only" ] = "F" pars[ "force_pol" ] = "T" pars[ "kfirst" ] = "T" pars[ "concatenate_messages" ] = "T" pars[ "write_map" ] = "T" pars[ "write_binmap" ] = "T" pars[ "write_matrix" ] = "T" pars[ "write_wcov" ] = "T" pars[ "write_hits" ] = "T" pars[ "nside_cross" ] = cross pars[ "nside_submap" ] = subnside if args.madampar is not None: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars[ "base_first" ] = args.baseline pars[ "nside_map" ] = args.nside if args.noisefilter: pars[ "kfilter" ] = "T" else: pars[ "kfilter" ] = "F" pars[ "fsample" ] = args.samplerate # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc+nmc): # clear all total signal data from the cache, so that we can generate # new noise timestreams. for obs in data.obs: tod = obs['tod'] tod.cache.clear("tot_signal_.*") # simulate noise nse = tt.OpSimNoise(out="tot_signal", realization=mc) nse.exec(data) # add sky signal if has_signal: add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname) if gain is not None: op_apply_gain = tt.OpApplyGain(gain, name="tot_signal") op_apply_gain.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Noise simulation took {:.3f} s".format(elapsed), flush=True) start = stop # create output directory for this realization pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc) if comm.comm_world.rank == 0: if not os.path.isdir(pars["path_output"]): os.makedirs(pars["path_output"]) # in debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open(os.path.join(pars["path_output"], "distdata.txt"), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() madam = tm.OpMadam(params=pars, detweights=detweights, name="tot_signal") madam.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Mapmaking took {:.3f} s".format(elapsed), flush=True) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - global_start if comm.comm_world.rank == 0: print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main(): if MPI.COMM_WORLD.rank == 0: print("Running with {} processes".format(MPI.COMM_WORLD.size), flush=True) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description="Read existing data and make a simple map.", fromfile_prefix_chars="@") parser.add_argument("--groupsize", required=False, type=int, default=0, help="size of processor groups used to distribute "\ "observations") parser.add_argument("--hwprpm", required=False, type=float, default=0.0, help="The rate (in RPM) of the HWP rotation") parser.add_argument('--samplerate', required=False, default=100.0, type=np.float, help='Detector sample rate (Hz)') parser.add_argument("--outdir", required=False, default="out", help="Output directory") parser.add_argument("--nside", required=False, type=int, default=64, help="Healpix NSIDE") parser.add_argument("--subnside", required=False, type=int, default=8, help="Distributed pixel sub-map NSIDE") parser.add_argument("--coord", required=False, default="E", help="Sky coordinate system [C,E,G]") parser.add_argument("--baseline", required=False, type=float, default=60.0, help="Destriping baseline length (seconds)") parser.add_argument("--noisefilter", required=False, default=False, action="store_true", help="Destripe with the noise filter enabled") parser.add_argument("--madam", required=False, default=False, action="store_true", help="If specified, use libmadam for map-making") parser.add_argument("--madampar", required=False, default=None, help="Madam parameter file") parser.add_argument("--polyorder", required=False, type=int, help="Polynomial order for the polyfilter") parser.add_argument("--wbin_ground", required=False, type=float, help="Ground template bin width [degrees]") parser.add_argument("--flush", required=False, default=False, action="store_true", help="Flush every print statement.") parser.add_argument("--tidas", required=False, default=None, help="Input TIDAS volume") parser.add_argument("--tidas_detgroup", required=False, default=None, help="TIDAS detector group") parser.add_argument("--spt3g", required=False, default=None, help="Input SPT3G data directory") parser.add_argument("--spt3g_prefix", required=False, default=None, help="SPT3G data frame file prefix") parser.add_argument("--common_flag_mask", required=False, default=0, type=np.uint8, help="Common flag mask") parser.add_argument("--debug", required=False, default=False, action="store_true", help="Write data distribution info and focalplane plot") args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) #args = parser.parse_args(sys.argv) autotimer = timing.auto_timer("@{}".format(timing.FILE())) if (args.tidas is not None) and (args.spt3g is not None): raise RuntimeError("Cannot read two datasets!") if (args.tidas is None) and (args.spt3g is None): raise RuntimeError("No dataset specified!") if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot load") if args.spt3g is not None: if not tt.spt3g_available: raise RuntimeError("SPT3G not found- cannot load") groupsize = args.groupsize if groupsize == 0: groupsize = MPI.COMM_WORLD.size # Pixelization nside = args.nside npix = 12 * args.nside * args.nside subnside = args.subnside if subnside > nside: subnside = nside subnpix = 12 * subnside * subnside # This is the 2-level toast communicator. if MPI.COMM_WORLD.size % groupsize != 0: if MPI.COMM_WORLD.rank == 0: print("WARNING: process groupsize does not evenly divide into " "total number of processes", flush=True) comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize) # Create output directory mtime = MPI.Wtime() if comm.comm_world.rank == 0: if not os.path.isdir(args.outdir): os.makedirs(args.outdir) mtime = elapsed(comm.comm_world, mtime, "Creating output directory") # The distributed timestream data data = None if args.tidas is not None: if args.tidas_detgroup is None: raise RuntimeError("you must specify the detector group") data = tds.load_tidas(comm, comm.group_size, args.tidas, "r", args.tidas_detgroup, tds.TODTidas, group_dets=args.tidas_detgroup, distintervals="chunks") if args.spt3g is not None: if args.spt3g_prefix is None: raise RuntimeError("you must specify the frame file prefix") data = s3g.load_spt3g(comm, comm.group_size, args.spt3g, args.spt3g_prefix, s3g.obsweight_spt3g, s3g.TOD3G) mtime = elapsed(comm.comm_world, mtime, "Distribute data") # In debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open("{}_distdata.txt".format(args.outdir), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() mtime = elapsed(comm.comm_world, mtime, "Dumping debug data distribution") if comm.comm_world.rank == 0: outfile = "{}_focalplane.png".format(args.outdir) set_backend() # Just plot the dets from the first TOD temptod = data.obs[0]["tod"] # FIXME: change this once we store det info in the metadata. dfwhm = { x : 10.0 for x in temptod.detectors } tt.plot_focalplane(temptod.detoffset(), 10.0, 10.0, outfile, fwhm=dfwhm) comm.comm_world.barrier() mtime = elapsed(comm.comm_world, mtime, "Plotting debug focalplane") # Compute pointing matrix pointing = tt.OpPointingHpix( nside=args.nside, nest=True, mode="IQU", hwprpm=args.hwprpm) pointing.exec(data) mtime = elapsed(comm.comm_world, mtime, "Expand pointing") # Mapmaking. # FIXME: We potentially have a different noise model for every # observation. We need to have both spt3g and tidas format Noise # classes which read the information from disk. Then the mapmaking # operators need to get these noise weights from each observation. detweights = { d : 1.0 for d in data.obs[0]["tod"].detectors } if not args.madam: if comm.comm_world.rank == 0: print("Not using Madam, will only make a binned map!", flush=True) # Filter data if desired if args.polyorder: polyfilter = tt.OpPolyFilter( order=args.polyorder, common_flag_mask=args.common_flag_mask) polyfilter.exec(data) mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering") if args.wbin_ground: groundfilter = tt.OpGroundFilter( wbin=args.wbin_ground, common_flag_mask=args.common_flag_mask) groundfilter.exec(data) mtime = elapsed(comm.comm_world, mtime, "Ground template filtering") # Compute pixel space distribution lc = tm.OpLocalPixels() localpix = lc.exec(data) if localpix is None: raise RuntimeError( "Process {} has no hit pixels. Perhaps there are fewer " "detectors than processes in the group?".format( comm.comm_world.rank)) localsm = np.unique(np.floor_divide(localpix, subnpix)) mtime = elapsed(comm.comm_world, mtime, "Compute local submaps") # construct distributed maps to store the covariance, # noise weighted map, and hits mtime = MPI.Wtime() invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) # compute the hits and covariance. invnpp.data.fill(0.0) hits.data.fill(0) build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits, common_flag_mask=args.common_flag_mask) build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1") hits.write_healpix_fits("{}_hits.fits".format(args.outdir)) invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir)) mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1") # invert it tm.covariance_invert(invnpp, 1.0e-3) mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1") invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir)) mtime = elapsed(comm.comm_world, mtime, "Writing N_pp") zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag(zmap=zmap, detweights=detweights, common_flag_mask=args.common_flag_mask) build_zmap.exec(data) zmap.allreduce() mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map") tm.covariance_apply(invnpp, zmap) mtime = elapsed(comm.comm_world, mtime, "Computing binned map") zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits")) mtime = elapsed(comm.comm_world, mtime, "Writing binned map") else: # Set up MADAM map making. pars = {} pars[ "temperature_only" ] = "F" pars[ "force_pol" ] = "T" pars[ "kfirst" ] = "T" pars[ "concatenate_messages" ] = "T" pars[ "write_map" ] = "T" pars[ "write_binmap" ] = "T" pars[ "write_matrix" ] = "T" pars[ "write_wcov" ] = "T" pars[ "write_hits" ] = "T" pars[ "nside_cross" ] = nside // 2 pars[ "nside_submap" ] = subnside if args.madampar is not None: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars[ "base_first" ] = args.baseline pars[ "nside_map" ] = nside if args.noisefilter: pars[ "kfilter" ] = "T" else: pars[ "kfilter" ] = "F" pars[ "fsample" ] = args.samplerate madam = tm.OpMadam(params=pars, detweights=detweights, common_flag_mask=args.common_flag_mask) madam.exec(data) mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking") comm.comm_world.barrier() stop = MPI.Wtime() dur = stop - global_start if comm.comm_world.rank == 0: print("Total Time: {:.2f} seconds".format(dur), flush=True) return
def main(): parser = argparse.ArgumentParser( description="Simulate fake hexagonal focalplane.", fromfile_prefix_chars='@') parser.add_argument( "--minpix", required=False, type=int, default=100, help="minimum number of pixels to use" ) parser.add_argument( "--out", required=False, default="fp_fake", help="Root name of output pickle file" ) parser.add_argument( "--fwhm", required=False, type=float, default=5.0, help="beam FWHM in arcmin" ) parser.add_argument( "--fwhm_sigma", required=False, type=float, default=0, help="Relative beam FWHM distribution width" ) parser.add_argument( "--fov", required=False, type=float, default=5.0, help="Field of View in degrees" ) parser.add_argument( "--psd_fknee", required=False, type=float, default=0.05, help="Detector noise model f_knee in Hz" ) parser.add_argument( "--psd_NET", required=False, type=float, default=60.0e-6, help="Detector noise model NET in K*sqrt(sec)" ) parser.add_argument( "--psd_alpha", required=False, type=float, default=1.0, help="Detector noise model slope" ) parser.add_argument( "--psd_fmin", required=False, type=float, default=1.0e-5, help="Detector noise model f_min in Hz" ) parser.add_argument( "--bandcenter_ghz", required=False, type=float, help="Band center frequency [GHz]" ) parser.add_argument( "--bandcenter_sigma", required=False, type=float, default=0, help="Relative band center distribution width" ) parser.add_argument( "--bandwidth_ghz", required=False, type=float, help="Bandwidth [GHz]" ) parser.add_argument( "--bandwidth_sigma", required=False, type=float, default=0, help="Relative bandwidth distribution width" ) parser.add_argument( "--random_seed", required=False, type=np.int, default=123456, help="Random number generator seed for randomized " "detector parameters" ) args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) # Guard against being called with multiple processes if MPI.COMM_WORLD.rank == 0: # Make one big hexagon layout at the center of the focalplane. # Compute the number of pixels that is at least the number requested. test = args.minpix - 1 nrings = 0 while (test - 6 * nrings) > 0: test -= 6 * nrings nrings += 1 npix = 1 for r in range(1, nrings+1): npix += 6 * r print("using {} pixels ({} detectors)".format(npix, npix*2)) # Translate the field-of-view into distance between flag sides angwidth = args.fov * np.cos(30 * degree) Apol = tt.hex_pol_angles_qu(npix, offset=0.0) Bpol = tt.hex_pol_angles_qu(npix, offset=90.0) Adets = tt.hex_layout(npix, angwidth, "fake_", "A", Apol) Bdets = tt.hex_layout(npix, angwidth, "fake_", "B", Bpol) dets = Adets.copy() dets.update(Bdets) np.random.seed(args.random_seed) for indx, d in enumerate(sorted(dets.keys())): dets[d]["fknee"] = args.psd_fknee dets[d]["fmin"] = args.psd_fmin dets[d]["alpha"] = args.psd_alpha dets[d]["NET"] = args.psd_NET # This is in degrees, but the input is in arcmin. dets[d]["fwhm_deg"] = (args.fwhm / 60.0) \ * (1 + np.random.randn()*args.fwhm_sigma) # This is a fixed value, in arcmin. dets[d]["fwhm"] = args.fwhm if args.bandcenter_ghz: dets[d]["bandcenter_ghz"] \ = args.bandcenter_ghz * (1+np.random.randn()*args.bandcenter_sigma) if args.bandwidth_ghz: dets[d]["bandwidth_ghz"] \ = args.bandwidth_ghz * (1+np.random.randn()*args.bandwidth_sigma) dets[d]["index"] = indx outfile = "{}_{}".format(args.out, npix) qdets = { x : y["quat"] for x, y in dets.items() } beams = { x : (60.0*y["fwhm_deg"]) for x, y in dets.items() } tt.plot_focalplane(qdets, args.fov, args.fov, "{}.png".format(outfile), fwhm=beams) with open("{}.pkl".format(outfile), "wb") as p: pickle.dump(dets, p) return