def main(): # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() # Create an argparse and add custom arguments parser = argparse.ArgumentParser(description=“...") parser.add_argument('--groupsize', required=False, type=np.int, help='Size of a process group assigned to a CES') # pass the argparse object to timing module which will add timing # arguments and return "parse.parse_args() result after handling # the timing specific options args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) # create the primary auto timer for the entire script autotimer = timing.auto_timer(timing.FILE())
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() if comm.world_rank == 0: print("Running with {} processes at {}".format( procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking', fromfile_prefix_chars='@') parser.add_argument('--skip_madam', dest='skip_madam', default=False, action='store_true', help='D not make maps with Madam.') parser.add_argument('--skip_noise', dest='skip_noise', default=False, action='store_true', help='Do not add simulated noise to the TOD.') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--debug', dest='debug', default=False, action='store_true', help='Write data distribution info to file') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir2', required=False, help='Additional input Exchange Format File directory') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory for ' 'pointing') parser.add_argument('--effdir_fsl', required=False, help='Input Exchange Format File directory for ' 'straylight') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--pntflagmask', required=False, default=0, type=np.int, help='Which OBT flag bits to raise for HCM maneuvers') parser.add_argument('--bad_intervals', required=False, help='Path to bad interval file.') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, help='First ring to use') parser.add_argument('--ringlast', required=False, default=None, help='Last ring to use') parser.add_argument('--obtfirst', required=False, default=None, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, help='Last OBT to use') parser.add_argument('--read_eff', dest='read_eff', default=False, action='store_true', help='Read and co-add the signal from effdir') parser.add_argument('--decalibrate', required=False, help='Path to calibration file to decalibrate with. ' 'You can use python string formatting, assuming ' '.format(mc)') parser.add_argument('--calibrate', required=False, help='Path to calibration file to calibrate with. ' 'You can use python string formatting, assuming ' '.format(mc)') parser.add_argument('--madampar', required=False, default=None, help='Madam parameter file') parser.add_argument('--nside', required=False, default=None, type=np.int, help='Madam resolution') parser.add_argument('--out', required=False, default='.', help='Output directory') parser.add_argument('--madam_prefix', required=False, help='map prefix') parser.add_argument('--make_rings', dest='make_rings', default=False, action='store_true', help='Compile ringsets.') parser.add_argument('--nside_ring', required=False, default=128, type=np.int, help='Ringset resolution') parser.add_argument('--ring_root', required=False, default='ringset', help='Root filename for ringsets (setting to empty ' 'disables ringset output).') parser.add_argument('--MC_start', required=False, default=0, type=np.int, help='First Monte Carlo noise realization') parser.add_argument('--MC_count', required=False, default=1, type=np.int, help='Number of Monte Carlo noise realizations') # noise parameters parser.add_argument('--noisefile', required=False, default='RIMO', help='Path to noise PSD files for noise filter. ' 'Tag DETECTOR will be replaced with detector name.') parser.add_argument('--noisefile_simu', required=False, default='RIMO', help='Path to noise PSD files for noise simulation. ' 'Tag DETECTOR will be replaced with detector name.') # Dipole parameters dipogroup = parser.add_mutually_exclusive_group() dipogroup.add_argument('--dipole', dest='dipole', required=False, default=False, action='store_true', help='Simulate dipole') dipogroup.add_argument('--solsys_dipole', dest='solsys_dipole', required=False, default=False, action='store_true', help='Simulate solar system dipole') dipogroup.add_argument('--orbital_dipole', dest='orbital_dipole', required=False, default=False, action='store_true', help='Simulate orbital dipole') dipo_parameters_group = parser.add_argument_group('dipole_parameters') dipo_parameters_group.add_argument( '--solsys_speed', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_speed"], help='Solar system speed wrt. CMB rest frame in km/s. Default is ' 'Planck 2015 best fit value') dipo_parameters_group.add_argument( '--solsys_glon', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_glon"], help='Solar system velocity direction longitude in degrees') dipo_parameters_group.add_argument( '--solsys_glat', required=False, type=np.float, default=DEFAULT_PARAMETERS["solsys_glat"], help='Solar system velocity direction latitude in degrees') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.world_rank == 0: print('All parameters:') print(args, flush=True) if args.MC_count < 1: raise RuntimeError('MC_count = {} < 1. Nothing done.' ''.format(args.MC_count)) timer = Timer() timer.start() nrange = 1 odranges = None if args.odfirst is not None and args.odlast is not None: odranges = [] firsts = [int(i) for i in str(args.odfirst).split(',')] lasts = [int(i) for i in str(args.odlast).split(',')] for odfirst, odlast in zip(firsts, lasts): odranges.append((odfirst, odlast)) nrange = len(odranges) ringranges = None if args.ringfirst is not None and args.ringlast is not None: ringranges = [] firsts = [int(i) for i in str(args.ringfirst).split(',')] lasts = [int(i) for i in str(args.ringlast).split(',')] for ringfirst, ringlast in zip(firsts, lasts): ringranges.append((ringfirst, ringlast)) nrange = len(ringranges) obtranges = None if args.obtfirst is not None and args.obtlast is not None: obtranges = [] firsts = [float(i) for i in str(args.obtfirst).split(',')] lasts = [float(i) for i in str(args.obtlast).split(',')] for obtfirst, obtlast in zip(firsts, lasts): obtranges.append((obtfirst, obtlast)) nrange = len(obtranges) if odranges is None: odranges = [None] * nrange if ringranges is None: ringranges = [None] * nrange if obtranges is None: obtranges = [None] * nrange detectors = None if args.dets is not None: detectors = re.split(',', args.dets) # create the TOD for this observation if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO': do_eff_cache = True else: do_eff_cache = False tods = [] for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges): # create the TOD for this observation tods.append( tp.Exchange(comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, extra_effdirs=[args.effdir2, args.effdir_fsl], effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, pntflagmask=args.pntflagmask, do_eff_cache=do_eff_cache)) # Make output directory if not os.path.isdir(args.out) and comm.world_rank == 0: os.makedirs(args.out) # Read in madam parameter file # Allow more than one entry, gather into a list repeated_keys = ['detset', 'detset_nopol', 'survey'] pars = {} if comm.world_rank == 0: pars['kfirst'] = False pars['temperature_only'] = True pars['base_first'] = 60.0 pars['nside_submap'] = 16 pars['write_map'] = False pars['write_binmap'] = True pars['write_matrix'] = False pars['write_wcov'] = False pars['write_hits'] = True pars['kfilter'] = False pars['info'] = 3 if args.madampar: pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*') comment = re.compile(r'^#.*') with open(args.madampar, 'r') as f: for line in f: if not comment.match(line): result = pat.match(line) if result: key, value = result.group(1), result.group(2) if key in repeated_keys: if key not in pars: pars[key] = [] pars[key].append(value) else: pars[key] = value # Command line parameters override the ones in the madam parameter file if 'file_root' not in pars: pars['file_root'] = 'madam' if args.madam_prefix is not None: pars['file_root'] = args.madam_prefix sfreq = '{:03}'.format(args.freq) if sfreq not in pars['file_root']: pars['file_root'] += '_' + sfreq try: fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq] except Exception: fsample = 180.3737 pars['fsample'] = fsample pars['path_output'] = args.out pars = comm.comm_world.bcast(pars, root=0) madam_mcmode = True if 'nsubchunk' in pars and int(pars['nsubchunk']) > 1: madam_mcmode = False if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO': # We split MPI_COMM_WORLD into single process groups, each of # which is assigned one or more observations (rings) comm = toast.Comm(groupsize=1) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) for iobs, tod in enumerate(tods): if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO': # Use a toast helper method to optimally distribute rings between # processes. dist = distribute_discrete(tod.ringsizes, comm.world_size) my_first_ring, my_n_ring = dist[comm.world_rank] for my_ring in range(my_first_ring, my_first_ring + my_n_ring): ringtod = tp.Exchange.from_tod( tod, my_ring, comm.comm_group, noisefile=args.noisefile, noisefile_simu=args.noisefile_simu) ob = {} ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring) ob['id'] = ringtod.globalfirst_ring ob['tod'] = ringtod ob['intervals'] = ringtod.valid_intervals ob['baselines'] = None ob['noise'] = ringtod.noise ob['noise_simu'] = ringtod.noise_simu data.obs.append(ob) else: ob = {} ob['name'] = 'observation{:04}'.format(iobs) ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise ob['noise_simu'] = tod.noise data.obs.append(ob) rimo = tods[0].rimo if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Metadata queries") # Always read the signal and flags, even if the signal is later # overwritten. There is no overhead for the signal because it is # interlaced with the flags. tod_name = 'signal' timestamps_name = 'timestamps' flags_name = 'flags' common_flags_name = 'common_flags' reader = tp.OpInputPlanck(signal_name=tod_name, flags_name=flags_name, timestamps_name=timestamps_name, commonflags_name=common_flags_name) if comm.world_rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Read") # Clear the signal if we don't need it if not args.read_eff: eraser = tp.OpCacheMath(in1=tod_name, in2=0, multiply=True, out=tod_name) if comm.world_rank == 0: print('Erasing TOD', flush=True) eraser.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Erase") # Optionally flag bad intervals if args.bad_intervals is not None: flagger = tp.OpBadIntervals(path=args.bad_intervals) flagger.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Apply {}".format(args.bad_intervals)) # Now read an optional second TOD to add with the first if args.effdir2 is not None: # Read the extra TOD and add it to the first one reader = tp.OpInputPlanck(signal_name='tod2', flags_name=None, timestamps_name=None, commonflags_name=None, effdir=args.effdir2) if comm.world_rank == 0: print('Reading extra TOD from {}'.format(args.effdir2), flush=True) reader.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: print("Reading took {:.3f} s".format(elapsed), flush=True) adder = tp.OpCacheMath(in1=tod_name, in2='signal2', add=True, out=tod_name) if comm.world_rank == 0: print('Adding TODs', flush=True) adder.exec(data) # Erase the extra cache object for ob in data.obs: tod = ob['tod'] tod.cache.clear('signal2_.*') if args.effdir_fsl is not None: # Read the straylight signal into the tod cache under # "fsl_<detector>" reader = tp.OpInputPlanck(signal_name='fsl', flags_name=None, timestamps_name=None, commonflags_name=None, effdir=args.effdir_fsl) if comm.world_rank == 0: print('Reading straylight signal from {}'.format(args.effdir_fsl), flush=True) reader.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Read FSL") do_fsl = True else: do_fsl = False # make a planck Healpix pointing matrix mode = 'IQU' if pars['temperature_only'] == 'T': mode = 'I' if args.nside is None: if 'nside_map' in pars: nside = int(pars['nside_map']) else: raise RuntimeError( 'Nside must be set either in the Madam parameter file or on ' 'the command line') else: nside = args.nside pars['nside_map'] = nside if 'nside_cross' not in pars or pars['nside_cross'] > pars['nside_map']: pars['nside_cross'] = pars['nside_map'] do_dipole = args.dipole or args.solsys_dipole or args.orbital_dipole pointing = tp.OpPointingPlanck(nside=nside, mode=mode, RIMO=rimo, margin=0, apply_flags=True, keep_vel=do_dipole, keep_pos=False, keep_phase=False, keep_quats=do_dipole) pointing.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Pointing Matrix") flags_name = 'flags' common_flags_name = 'common_flags' # for now, we pass in the noise weights from the RIMO. detweights = {} for d in tod.detectors: net = tod.rimo[d].net fsample = tod.rimo[d].fsample detweights[d] = 1.0 / (fsample * net * net) if args.debug: with open("debug_planck_exchange_madam.txt", "w") as f: data.info(f) if do_dipole: # Simulate the dipole if args.dipole: dipomode = 'total' elif args.solsys_dipole: dipomode = 'solsys' else: dipomode = 'orbital' dipo = tp.OpDipolePlanck(args.freq, solsys_speed=args.solsys_speed, solsys_glon=args.solsys_glon, solsys_glat=args.solsys_glat, mode=dipomode, output='dipole', keep_quats=False) dipo.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("Dipole") # Loop over Monte Carlos madam = None for mc in range(args.MC_start, args.MC_start + args.MC_count): out = "{}/{:05d}".format(args.out, mc) if comm.world_rank == 0: if not os.path.isdir(out): os.makedirs(out) # clear all noise data from the cache, so that we can generate # new noise timestreams. for ob in data.obs: ob['tod'].cache.clear("noise_.*") tod_name = 'signal' if do_dipole: adder = tp.OpCacheMath(in1=tod_name, in2='dipole', add=True, out='noise') adder.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Add dipole".format(mc)) tod_name = 'noise' # Simulate noise if not args.skip_noise: tod_name = 'noise' nse = toast.tod.OpSimNoise(out=tod_name, realization=mc, component=0, noise='noise_simu', rate=fsample) if comm.world_rank == 0: print('Simulating noise from {}'.format(args.noisefile_simu), flush=True) nse.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Noise simulation".format(mc)) # If we didn't add the dipole, we need to add the input # signal with the noise we just simulated if args.read_eff and not do_dipole: adder = tp.OpCacheMath(in1=tod_name, in2='signal', add=True, out=tod_name) adder.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Add input signal".format(mc)) # Make rings if args.make_rings: ringmaker = tp.OpRingMaker(args.nside_ring, nside, signal=tod_name, fileroot=args.ring_root, out=out, commonmask=args.obtmask, detmask=args.flagmask) ringmaker.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Ringmaking".format(mc)) # Apply calibration errors if args.decalibrate is not None: fn = args.decalibrate try: fn = fn.format(mc) except Exception: pass if comm.world_rank == 0: print('Decalibrating with {}'.format(fn), flush=True) decalibrator = tp.OpCalibPlanck(signal_in=tod_name, signal_out='noise', file_gain=fn, decalibrate=True) decalibrator.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Decalibrate".format(mc)) tod_name = 'noise' if args.calibrate is not None: fn = args.calibrate try: fn = fn.format(mc) except Exception: pass if comm.world_rank == 0: print('Calibrating with {}'.format(fn), flush=True) calibrator = tp.OpCalibPlanck(signal_in=tod_name, signal_out='noise', file_gain=fn) calibrator.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Calibrate".format(mc)) tod_name = 'noise' # Subtract the dipole and straylight if do_dipole: subtractor = tp.OpCacheMath(in1=tod_name, in2='dipole', subtract=True, out='noise') subtractor.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Subtract dipole".format(mc)) tod_name = 'noise' if do_fsl: subtractor = tp.OpCacheMath(in1=tod_name, in2='fsl', subtract=True, out='noise') subtractor.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Subtract straylight".format(mc)) tod_name = 'noise' # Make the map if not args.skip_madam: # Make maps if madam is None: try: madam = toast.todmap.OpMadam(params=pars, detweights=detweights, purge_tod=True, name=tod_name, apply_flags=False, name_out=None, noise='noise', mcmode=madam_mcmode, translate_timestamps=False) except Exception as e: raise Exception( '{:4} : ERROR: failed to initialize Madam: ' '{}'.format(comm.world_rank, e)) madam.params['path_output'] = out madam.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.world_rank == 0: timer.report_clear("MC {}: Mapmaking".format(mc)) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
if cworld.rank == 0: print("Computing binned map took {:.3f} s".format(elapsed), flush=True) start = stop if cworld.rank == 0: print("Writing binned map ...", flush=True) zmap.write_healpix_fits(os.path.join(out, "binned.fits")) if cworld.rank == 0: print("Binned map done", flush=True) return # Our toast communicator- use the default for now, which is one # process group spanning all processes. comm = toast.Comm() if comm.world_rank == 0: print("Simulating all detector properties...", flush=True) # First, get the list of detectors we want to use # (Eventually we would load this from disk. Here we simulate it.) hw = get_example() dets = sim_telescope_detectors(hw, "LAT") hw.data["detectors"] = dets if comm.world_rank == 0: print("Selecting detectors...", flush=True) # Downselect to just 10 pixels on one wafer #small_hw = hw.select(match={"wafer_slot": "41", "pixel": "00."}) #small_hw = hw.select(match={"wafer_slot": "41"}) small_hw = hw.select(match={"wafer_slot": "40"})
def parse_arguments(comm): parser = argparse.ArgumentParser( description="Simulate ground-based boresight pointing. Simulate " "and map astrophysical signal.", fromfile_prefix_chars='@') parser.add_argument('--groupsize', required=False, type=np.int, help='Size of a process group assigned to a CES') parser.add_argument('--timezone', required=False, type=np.int, default=0, help='Offset to apply to MJD to separate days [hours]') parser.add_argument('--coord', required=False, default='C', help='Sky coordinate system [C,E,G]') parser.add_argument('--schedule', required=True, help='CES schedule file from toast_ground_schedule.py') parser.add_argument('--samplerate', required=False, default=100.0, type=np.float, help='Detector sample rate (Hz)') parser.add_argument('--scanrate', required=False, default=1.0, type=np.float, help='Scanning rate [deg / s]') parser.add_argument('--scan_accel', required=False, default=1.0, type=np.float, help='Scanning rate change [deg / s^2]') parser.add_argument('--sun_angle_min', required=False, default=30.0, type=np.float, help='Minimum azimuthal distance between the Sun and ' 'the bore sight [deg]') parser.add_argument('--polyorder', required=False, type=np.int, help='Polynomial order for the polyfilter') parser.add_argument('--wbin_ground', required=False, type=np.float, help='Ground template bin width [degrees]') parser.add_argument('--gain_sigma', required=False, type=np.float, help='Gain error distribution') parser.add_argument('--hwprpm', required=False, default=0.0, type=np.float, help='The rate (in RPM) of the HWP rotation') parser.add_argument('--hwpstep', required=False, default=None, help='For stepped HWP, the angle in degrees ' 'of each step') parser.add_argument('--hwpsteptime', required=False, default=0.0, type=np.float, help='For stepped HWP, the the time in seconds ' 'between steps') parser.add_argument('--input_map', required=False, help='Input map for signal') parser.add_argument('--skip_bin', required=False, default=False, action='store_true', help='Disable binning the map.') parser.add_argument('--skip_hits', required=False, default=False, action='store_true', help='Do not save the 3x3 matrices and hitmaps') parser.add_argument('--fp_radius', required=False, default=1, type=np.float, help='Focal plane radius assumed in the atmospheric ' 'simulation.') parser.add_argument('--outdir', required=False, default='out', help='Output directory') parser.add_argument('--zip', required=False, default=False, action='store_true', help='Compress the output fits files') parser.add_argument('--debug', required=False, default=False, action='store_true', help='Write diagnostics') parser.add_argument('--flush', required=False, default=False, action='store_true', help='Flush every print statement.') parser.add_argument('--nside', required=False, default=512, type=np.int, help='Healpix NSIDE') parser.add_argument('--madam_iter_max', required=False, default=1000, type=np.int, help='Maximum number of CG iterations in Madam') parser.add_argument('--madam_baseline_length', required=False, default=10000.0, type=np.float, help='Destriping baseline length (seconds)') parser.add_argument('--madam_baseline_order', required=False, default=0, type=np.int, help='Destriping baseline polynomial order') parser.add_argument('--madam_noisefilter', required=False, default=False, action='store_true', help='Destripe with the noise filter enabled') parser.add_argument('--madam', required=False, default=False, action='store_true', help='If specified, use libmadam for map-making') parser.add_argument('--madampar', required=False, default=None, help='Madam parameter file') parser.add_argument('--madam_allreduce', required=False, default=False, action='store_true', help='Use allreduce communication in Madam') parser.add_argument('--common_flag_mask', required=False, default=1, type=np.uint8, help='Common flag mask') parser.add_argument( '--fp', required=False, default=None, help='Pickle file containing a dictionary of detector ' 'properties. The keys of this dict are the detector ' 'names, and each value is also a dictionary with keys ' '"quat" (4 element ndarray), "fwhm" (float, arcmin), ' '"fknee" (float, Hz), "alpha" (float), and ' '"NET" (float). For optional plotting, the key "color"' ' can specify a valid matplotlib color string.') parser.add_argument('--tidas', required=False, default=None, help='Output TIDAS export path') args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot export") if comm.comm_world.rank == 0: print('\nAll parameters:') print(args, flush=args.flush) print('') if args.groupsize: comm = toast.Comm(groupsize=args.groupsize) if comm.comm_world.rank == 0: if not os.path.isdir(args.outdir): try: os.makedirs(args.outdir) except FileExistsError: pass return args, comm
def main(): # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.comm_world.rank == 0: print('Running with {} processes at {}'.format(comm.comm_world.size, str(datetime.now())), flush=True) global_timer = timing.simple_timer('Total time') global_timer.start() args, comm = parse_arguments(comm) autotimer = timing.auto_timer("@{}".format(timing.FILE())) # Load and broadcast the schedule file site, all_ces = load_schedule(args, comm) # load or simulate the focalplane fp, detweights = load_fp(args, comm) # Create the TOAST data object to match the schedule. This will # include simulating the boresight pointing. data = create_observations(args, comm, fp, all_ces, site) # Expand boresight quaternions into detector pointing weights and # pixel numbers expand_pointing(args, comm, data) # Prepare auxiliary information for distributed map objects localpix, localsm, subnpix = get_submaps(args, comm, data) # Scan input map signalname = scan_signal(args, comm, data, localsm, subnpix) # Set up objects to take copies of the TOD at appropriate times signalname_madam, sigcopy_madam, sigclear \ = setup_sigcopy(args, comm, signalname) common_flag_name = None flag_name = None invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name \ = build_npp(args, comm, data, localsm, subnpix, detweights, flag_name, common_flag_name) madampars = setup_madam(args, comm) output_tidas(args, comm, data, signalname, common_flag_name, flag_name) outpath = setup_output(args, comm) # Make a copy of the signal for Madam copy_signal_madam(args, comm, data, sigcopy_madam) # Bin unprocessed signal for reference bin_maps(args, comm, data, 'binned', zmap, invnpp, zmap_group, invnpp_group, detweights, signalname, flag_name, common_flag_name, outpath) # Filter signal apply_polyfilter(args, comm, data, signalname) apply_groundfilter(args, comm, data, signalname) # Bin the filtered signal if args.polyorder or args.wbin_ground: bin_maps(args, comm, data, 'filtered', zmap, invnpp, zmap_group, invnpp_group, detweights, signalname, flag_name, common_flag_name, outpath) clear_signal(args, comm, data, sigclear) # Now run Madam on the unprocessed copy of the signal apply_madam(args, comm, data, madampars, outpath, detweights, signalname_madam, flag_name, common_flag_name) comm.comm_world.barrier() global_timer.stop() if comm.comm_world.rank == 0: global_timer.report()
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.comm_world.rank == 0: print('Running with {} processes at {}' ''.format(procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Planck Ringset making', fromfile_prefix_chars='@') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--nosingle', dest='nosingle', required=False, default=False, action='store_true', help='Do not compute single detector PSDs') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory ' 'for pointing') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--skymask', required=False, help='Pixel mask file') parser.add_argument('--skymap', required=False, help='Sky estimate file') parser.add_argument('--skypol', dest='skypol', required=False, default=False, action='store_true', help='Sky estimate is polarized') parser.add_argument('--no_spin_harmonics', dest='no_spin_harmonics', required=False, default=False, action='store_true', help='Do not include PSD bins with spin harmonics') parser.add_argument('--calibrate', required=False, help='Path to calibration file to calibrate with.') parser.add_argument('--calibrate_signal_estimate', dest='calibrate_signal_estimate', required=False, default=False, action='store_true', help='Calibrate ' 'the signal estimate using linear regression.') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, type=np.int, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, type=np.int, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, type=np.int, help='First ring to use') parser.add_argument('--ringlast', required=False, default=None, type=np.int, help='Last ring to use') parser.add_argument('--obtfirst', required=False, default=None, type=np.float, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, type=np.float, help='Last OBT to use') parser.add_argument('--out', required=False, default='.', help='Output directory') parser.add_argument('--nbin_psd', required=False, default=1000, type=np.int, help='Number of logarithmically ' 'distributed spectral bins to write.') parser.add_argument('--lagmax', required=False, default=100000, type=np.int, help='Maximum lag to evaluate for the ' 'autocovariance function [samples].') parser.add_argument('--stationary_period', required=False, default=86400., type=np.float, help='Length of a stationary interval [seconds].') # Dipole parameters dipogroup = parser.add_mutually_exclusive_group() dipogroup.add_argument('--dipole', dest='dipole', required=False, default=False, action='store_true', help='Simulate dipole') dipogroup.add_argument('--solsys_dipole', dest='solsys_dipole', required=False, default=False, action='store_true', help='Simulate solar system dipole') dipogroup.add_argument('--orbital_dipole', dest='orbital_dipole', required=False, default=False, action='store_true', help='Simulate orbital dipole') # Extra filter parser.add_argument('--filterfile', required=False, help='Extra filter file.') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.comm_world.rank == 0: print('All parameters:') print(args, flush=True) timer = Timer() timer.start() odrange = None if args.odfirst is not None and args.odlast is not None: odrange = (args.odfirst, args.odlast) ringrange = None if args.ringfirst is not None and args.ringlast is not None: ringrange = (args.ringfirst, args.ringlast) obtrange = None if args.obtfirst is not None and args.obtlast is not None: obtrange = (args.obtfirst, args.obtlast) detectors = None if args.dets is not None: detectors = re.split(',', args.dets) if args.nosingle and len(detectors) != 2: raise RuntimeError('You cannot skip the single detectors PSDs ' 'without multiple detectors.') # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) # Make output directory if not os.path.isdir(args.out) and comm.comm_world.rank == 0: os.mkdir(args.out) # create the TOD for this observation tod = tp.Exchange( comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, do_eff_cache=False, ) rimo = tod.rimo ob = {} ob['name'] = 'mission' ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise data.obs.append(ob) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Metadata queries") # Read the signal tod_name = 'signal' flags_name = 'flags' reader = tp.OpInputPlanck(signal_name=tod_name, flags_name=flags_name) if comm.comm_world.rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Reading") if args.calibrate is not None: fn = args.calibrate if comm.comm_world.rank == 0: print('Calibrating with {}'.format(fn), flush=True) calibrator = tp.OpCalibPlanck(signal_in=tod_name, signal_out=tod_name, file_gain=fn) calibrator.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Calibrate") # Optionally subtract the dipole do_dipole = (args.dipole or args.solsys_dipole or args.orbital_dipole) if do_dipole: if args.dipole: dipomode = 'total' elif args.solsys_dipole: dipomode = 'solsys' else: dipomode = 'orbital' dipo = tp.OpDipolePlanck(args.freq, mode=dipomode, output='dipole', keep_quats=True) dipo.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Dipole") subtractor = tp.OpCacheMath(in1=tod_name, in2='dipole', subtract=True, out=tod_name) if comm.comm_world.rank == 0: print('Subtracting dipole', flush=True) subtractor.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Dipole subtraction") # Optionally filter the signal apply_filter(args, data) timer.clear() # Estimate noise noise_estimator = tp.OpNoiseEstim( signal=tod_name, flags=flags_name, detmask=args.flagmask, commonmask=args.obtmask, maskfile=args.skymask, mapfile=args.skymap, out=args.out, rimo=rimo, pol=args.skypol, nbin_psd=args.nbin_psd, lagmax=args.lagmax, stationary_period=args.stationary_period, nosingle=args.nosingle, no_spin_harmonics=args.no_spin_harmonics, calibrate_signal_estimate=args.calibrate_signal_estimate) noise_estimator.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Noise estimation") gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def main(): # We are going to group our processes in a single group. This is fine # if we have fewer processes than detectors. Otherwise we should group # them in a reasonable size that is smaller than the number of detectors # and which divides evenly into the total number of processes. comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=MPI.COMM_WORLD.size) # Make a fake focalplane. Plot it just for fun (don't waste time on this # for very large runs though). fp = fake_focalplane() if comm.comm_world.rank == 0: outfile = "custom_example_focalplane.png" set_backend() tt.plot_focalplane(fp, 6.0, 6.0, outfile) # Read in 2 boresight files borefiles = [ "../data/custom_example_boresight_1.txt", "../data/custom_example_boresight_2.txt" ] # Set up the distributed data rate = 100.0 data = create_observations(comm, rate, fp, borefiles) # Configure the healpix pixelization we will use for map-making and # also the "submap" resolution, which sets granularity of the locally # stored pieces of the sky. map_nside = 512 map_npix = 12 * map_nside**2 sub_nside = 4 sub_npix = 12 * sub_nside**2 # Compute a pointing matrix with healpix pixels and weights. pointing = tt.OpPointingHpix(nside=map_nside, nest=True, mode="IQU", pixels="pixels", weights="weights") pointing.exec(data) # Compute the locally hit submaps local_submaps = pixel_dist(data, sub_npix) # Sources of simulated data: scan from a symmetric beam convolved sky # and then add some simulated noise. signalmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3, dtype=np.float64, submap=sub_npix, local=local_submaps) signalmap.read_healpix_fits("../data/custom_example_sky.fits") scanmap = tt.OpSimScan(distmap=signalmap, pixels='pixels', weights='weights', out="sim") scanmap.exec(data) nse = tt.OpSimNoise(out="sim", realization=0) nse.exec(data) # Accumulate the hits and inverse diagonal pixel covariance, as well as the # noise weighted map. Here we simply use inverse noise weighting. detweights = {} for d in fp.keys(): net = fp[d]["NET"] detweights[d] = 1.0 / (rate * net * net) invnpp = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=6, dtype=np.float64, submap=sub_npix, local=local_submaps) hits = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=1, dtype=np.int64, submap=sub_npix, local=local_submaps) zmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3, dtype=np.float64, submap=sub_npix, local=local_submaps) invnpp.data.fill(0.0) hits.data.fill(0) zmap.data.fill(0.0) build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits, zmap=zmap, name="sim") build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() zmap.allreduce() # Write these products out hits.write_healpix_fits("custom_example_hits.fits") invnpp.write_healpix_fits("custom_example_invnpp.fits") zmap.write_healpix_fits("custom_example_zmap.fits") # Invert the covariance and write tm.covariance_invert(invnpp, 1.0e-3) invnpp.write_healpix_fits("custom_example_npp.fits") # Apply covariance to make the binned map tm.covariance_apply(invnpp, zmap) zmap.write_healpix_fits("custom_example_binned.fits") MPI.Finalize() return
def setUp(self): fixture_name = os.path.splitext(os.path.basename(__file__))[0] if not toast_available: print( "toast cannot be imported ({})- skipping unit test".format( toast_import_error), flush=True, ) return self.comm, self.procs, self.rank = get_world() self.outdir = create_outdir(fixture_name, comm=self.comm) toastcomm = toast.Comm(world=self.comm) self.data = toast.Data(toastcomm) # Focalplane hwfull = get_example() dets = sim_telescope_detectors(hwfull, "SAT4") hwfull.data["detectors"] = dets hw = hwfull.select(match={ "wafer_slot": "w42", "band": "SAT_f030", "pixel": "00[01]" }) # print(hw.data["detectors"], flush=True) detquats = {k: v["quat"] for k, v in hw.data["detectors"].items()} # Samples per observation self.totsamp = 10000 # Scan properties self.site_lon = '-67:47:10' self.site_lat = '-22:57:30' self.site_alt = 5200. self.coord = 'C' self.azmin = 45 self.azmax = 55 self.el = 60 self.scanrate = 1.0 self.scan_accel = 0.1 self.CES_start = None # Noise properties self.rate = 100.0 self.NET = 1e-3 # 1 mK NET self.epsilon = 0.0 self.fmin = 1.0e-5 self.alpha = 1.0 self.fknee = 0.05 for ob in range(3): ftime = (self.totsamp / self.rate) * ob + 1564015655.88 tod = TODGround(self.data.comm.comm_group, detquats, self.totsamp, detranks=self.data.comm.group_size, firsttime=ftime, rate=self.rate, site_lon=self.site_lon, site_lat=self.site_lat, site_alt=self.site_alt, azmin=self.azmin, azmax=self.azmax, el=self.el, coord=self.coord, scanrate=self.scanrate, scan_accel=self.scan_accel, CES_start=self.CES_start) # Analytic noise model detnames = list(detquats.keys()) drate = {x: self.rate for x in detnames} dfmin = {x: self.fmin for x in detnames} dfknee = {x: self.fknee for x in detnames} dalpha = {x: self.alpha for x in detnames} dnet = {x: self.NET for x in detnames} nse = AnalyticNoise(rate=drate, fmin=dfmin, detectors=detnames, fknee=dfknee, alpha=dalpha, NET=dnet) # Single observation obs = dict() obs["tod"] = tod obs["noise"] = nse obs["id"] = 12345 obs["intervals"] = tod.subscans obs["site"] = "SimonsObs" obs["telescope"] = "SAT4" obs["site_id"] = 1 obs["telescope_id"] = 4 obs["fpradius"] = 5.0 obs["start_time"] = ftime obs["altitude"] = self.site_alt obs["name"] = "test_{:02}".format(ob) # Add a focalplane dictionary with just the detector index focalplane = {} for idet, det in enumerate(detnames): focalplane[det] = {"index": idet} obs["focalplane"] = focalplane # Add the observation to the dataset self.data.obs.append(obs) nse = toast.tod.OpSimNoise(out="signal", realization=0) nse.exec(self.data) return
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.world_rank == 0: print("Running with {} processes at {}".format( procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser( description='Accumulate polarization moments', fromfile_prefix_chars='@') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--nside', required=False, type=np.int, default=512, help='Map resolution') parser.add_argument('--smax', required=False, type=np.int, default=6, help='Highest moment') parser.add_argument('--debug', dest='debug', default=False, action='store_true', help='Write data distribution info to file') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir_in_diode0', required=False, default=None, help='Input Exchange Format File directory, ' 'LFI diode 0') parser.add_argument('--effdir_in_diode1', required=False, default=None, help='Input Exchange Format File directory, ' 'LFI diode 1') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory ' 'for pointing') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--pntflagmask', required=False, default=0, type=np.int, help='Pointing flag mask') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, type=np.int, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, type=np.int, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, help='First ring to use (can be a list)') parser.add_argument('--ringlast', required=False, default=None, help='Last ring to use (can be a list)') parser.add_argument('--obtfirst', required=False, default=None, type=np.float, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, type=np.float, help='Last OBT to use') parser.add_argument('--out', required=False, default='.', help='Output directory') parser.add_argument('--prefix', required=False, default='spins', help='map prefix') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.world_rank == 0: print('All parameters:') print(args, flush=True) timer = Timer() timer.start() nrange = 1 odranges = None if args.odfirst is not None and args.odlast is not None: odranges = [] firsts = [int(i) for i in str(args.odfirst).split(',')] lasts = [int(i) for i in str(args.odlast).split(',')] for odfirst, odlast in zip(firsts, lasts): odranges.append((odfirst, odlast)) nrange = len(odranges) ringranges = None if args.ringfirst is not None and args.ringlast is not None: ringranges = [] firsts = [int(i) for i in str(args.ringfirst).split(',')] lasts = [int(i) for i in str(args.ringlast).split(',')] for ringfirst, ringlast in zip(firsts, lasts): ringranges.append((ringfirst, ringlast)) nrange = len(ringranges) obtranges = None if args.obtfirst is not None and args.obtlast is not None: obtranges = [] firsts = [float(i) for i in str(args.obtfirst).split(',')] lasts = [float(i) for i in str(args.obtlast).split(',')] for obtfirst, obtlast in zip(firsts, lasts): obtranges.append((obtfirst, obtlast)) nrange = len(obtranges) if odranges is None: odranges = [None] * nrange if ringranges is None: ringranges = [None] * nrange if obtranges is None: obtranges = [None] * nrange detectors = None if args.dets is not None: detectors = re.split(',', args.dets) # create the TOD for this observation tods = [] for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges): tods.append( tp.Exchange(comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_in_diode0=args.effdir_in_diode0, effdir_in_diode1=args.effdir_in_diode1, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, pntflagmask=args.pntflagmask, do_eff_cache=False, noisefile='RIMO')) rimo = tods[0].rimo # Make output directory if not os.path.isdir(args.out) and comm.comm_world.rank == 0: os.makedirs(args.out) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) for iobs, tod in enumerate(tods): ob = {} ob['name'] = 'observation{:04}'.format(iobs) ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise data.obs.append(ob) if mpiworld is not None: mpiworld.barrier() if comm.comm_world.rank == 0: timer.report_clear("Metadata queries") # Accumulate and save the moment maps polmoments = tp.OpPolMomentsPlanck(nside=args.nside, RIMO=rimo, margin=0, keep_vel=False, keep_pos=False, keep_phase=False, keep_quats=False, smax=args.smax, prefix=os.path.join( args.out, args.prefix)) polmoments.exec(data) if mpiworld is not None: mpiworld.barrier() if comm.comm_world.rank == 0: timer.report_clear("Accumulate moment maps") gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def main(): if MPI.COMM_WORLD.rank == 0: print("Running with {} processes".format(MPI.COMM_WORLD.size), flush=True) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description="Simulate satellite " "boresight pointing and make a noise map.", fromfile_prefix_chars="@" ) parser.add_argument( "--groupsize", required=False, type=int, default=0, help="size of processor groups used to distribute observations" ) parser.add_argument( "--samplerate", required=False, type=float, default=40.0, help="Detector sample rate (Hz)" ) parser.add_argument( "--starttime", required=False, type=float, default=0.0, help="The overall start time of the simulation" ) parser.add_argument( "--spinperiod", required=False, type=float, default=10.0, help="The period (in minutes) of the rotation about the " "spin axis" ) parser.add_argument( "--spinangle", required=False, type=float, default=30.0, help="The opening angle (in degrees) of the boresight " "from the spin axis" ) parser.add_argument( "--precperiod", required=False, type=float, default=50.0, help="The period (in minutes) of the rotation about the " "precession axis" ) parser.add_argument( "--precangle", required=False, type=float, default=65.0, help="The opening angle (in degrees) of the spin axis " "from the precession axis" ) parser.add_argument( "--hwprpm", required=False, type=float, default=0.0, help="The rate (in RPM) of the HWP rotation" ) parser.add_argument( "--hwpstep", required=False, default=None, help="For stepped HWP, the angle in degrees of each step" ) parser.add_argument( "--hwpsteptime", required=False, type=float, default=0.0, help="For stepped HWP, the the time in seconds between " "steps" ) parser.add_argument( "--obs", required=False, type=float, default=1.0, help="Number of hours in one science observation" ) parser.add_argument( "--gap", required=False, type=float, default=0.0, help="Cooler cycle time in hours between science obs" ) parser.add_argument( "--numobs", required=False, type=int, default=1, help="Number of complete observations" ) parser.add_argument( "--outdir", required=False, default="out", help="Output directory" ) parser.add_argument( "--debug", required=False, default=False, action="store_true", help="Write diagnostics" ) parser.add_argument( "--nside", required=False, type=int, default=64, help="Healpix NSIDE" ) parser.add_argument( "--subnside", required=False, type=int, default=4, help="Distributed pixel sub-map NSIDE" ) parser.add_argument( "--baseline", required=False, type=float, default=60.0, help="Destriping baseline length (seconds)" ) parser.add_argument( "--noisefilter", required=False, default=False, action="store_true", help="Destripe with the noise filter enabled" ) parser.add_argument( "--madam", required=False, default=False, action="store_true", help="If specified, use libmadam for map-making" ) parser.add_argument( "--madampar", required=False, default=None, help="Madam parameter file" ) parser.add_argument('--flush', required=False, default=False, action='store_true', help='Flush every print statement.') parser.add_argument( "--MC_start", required=False, type=int, default=0, help="First Monte Carlo noise realization" ) parser.add_argument( "--MC_count", required=False, type=int, default=1, help="Number of Monte Carlo noise realizations" ) parser.add_argument( "--fp", required=False, default=None, help="Pickle file containing a dictionary of detector properties. " "The keys of this dict are the detector names, and each value is also " "a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" " "(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" " "(float). For optional plotting, the key \"color\" can specify a " "valid matplotlib color string." ) parser.add_argument('--tidas', required=False, default=None, help='Output TIDAS export path') parser.add_argument('--input_map', required=False, help='Input map for signal') parser.add_argument('--input_pysm_model', required=False, help='Comma separated models for on-the-fly PySM ' 'simulation, e.g. s3,d6,f1,a2"') parser.add_argument('--apply_beam', required=False, action='store_true', help='Apply beam convolution to input map with gaussian ' 'beam parameters defined in focalplane') parser.add_argument('--input_dipole', required=False, help='Simulate dipole, possible values are ' 'total, orbital, solar') parser.add_argument('--input_dipole_solar_speed_kms', required=False, help='Solar system speed [km/s]', type=float, default=369.0) parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False, help='Solar system speed galactic latitude [degrees]', type=float, default=48.26) parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False, help='Solar system speed galactic longitude[degrees]', type=float, default=263.99) args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) autotimer = timing.auto_timer("@{}".format(timing.FILE())) if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot export") groupsize = args.groupsize if groupsize == 0: groupsize = MPI.COMM_WORLD.size # This is the 2-level toast communicator. if MPI.COMM_WORLD.size % groupsize != 0: if MPI.COMM_WORLD.rank == 0: print("WARNING: process groupsize does not evenly divide into " "total number of processes", flush=True) comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize) # get options hwpstep = None if args.hwpstep is not None: hwpstep = float(args.hwpstep) npix = 12 * args.nside * args.nside subnside = args.subnside if subnside > args.nside: subnside = args.nside subnpix = 12 * subnside * subnside start = MPI.Wtime() fp = None # Load focalplane information if comm.comm_world.rank == 0: if args.fp is None: # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0]) fake["fwhm"] = 30.0 fake["fknee"] = 0.0 fake["alpha"] = 1.0 fake["NET"] = 1.0 fake["color"] = "r" fp = {} fp["bore"] = fake else: with open(args.fp, "rb") as p: fp = pickle.load(p) fp = comm.comm_world.bcast(fp, root=0) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Create focalplane: {:.2f} seconds".format(stop-start), flush=True) start = stop if args.debug: if comm.comm_world.rank == 0: outfile = "{}_focalplane.png".format(args.outdir) set_backend() tt.plot_focalplane(fp, 10.0, 10.0, outfile) # Since we are simulating noise timestreams, we want # them to be contiguous and reproducible over the whole # observation. We distribute data by detector within an # observation, so ensure that our group size is not larger # than the number of detectors we have. if groupsize > len(fp.keys()): if comm.comm_world.rank == 0: print("process group is too large for the number of detectors", flush=True) comm.comm_world.Abort() # Detector information from the focalplane detectors = sorted(fp.keys()) detquats = {} detindx = None if "index" in fp[detectors[0]]: detindx = {} for d in detectors: detquats[d] = fp[d]["quat"] if detindx is not None: detindx[d] = fp[d]["index"] # Distribute the observations uniformly groupdist = toast.distribute_uniform(args.numobs, comm.ngroups) # Compute global time and sample ranges of all observations obsrange = tt.regular_intervals(args.numobs, args.starttime, 0, args.samplerate, 3600*args.obs, 3600*args.gap) # Create the noise model used for all observations fmin = {} fknee = {} alpha = {} NET = {} rates = {} for d in detectors: rates[d] = args.samplerate fmin[d] = fp[d]["fmin"] fknee[d] = fp[d]["fknee"] alpha[d] = fp[d]["alpha"] NET[d] = fp[d]["NET"] noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors, fknee=fknee, alpha=alpha, NET=NET) mem_counter = tt.OpMemoryCounter() # The distributed timestream data data = toast.Data(comm) # Every process group creates its observations group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] for ob in range(group_firstobs, group_firstobs + group_numobs): tod = tt.TODSatellite( comm.comm_group, detquats, obsrange[ob].samples, firsttime=obsrange[ob].start, rate=args.samplerate, spinperiod=args.spinperiod, spinangle=args.spinangle, precperiod=args.precperiod, precangle=args.precangle, detindx=detindx, detranks=comm.group_size ) obs = {} obs["name"] = "science_{:05d}".format(ob) obs["tod"] = tod obs["intervals"] = None obs["baselines"] = None obs["noise"] = noise obs["id"] = ob data.obs.append(obs) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Read parameters, compute data distribution: " "{:.2f} seconds".format(stop-start), flush=True) start = stop # we set the precession axis now, which will trigger calculation # of the boresight pointing. for ob in range(group_numobs): curobs = data.obs[ob] tod = curobs["tod"] # Get the global sample offset from the original distribution of # intervals obsoffset = obsrange[group_firstobs + ob].first # Constantly slewing precession axis degday = 360.0 / 365.25 precquat = tt.slew_precession_axis(nsim=tod.local_samples[1], firstsamp=obsoffset, samplerate=args.samplerate, degday=degday) tod.set_prec_axis(qprec=precquat) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Construct boresight pointing: " "{:.2f} seconds".format(stop-start), flush=True) start = stop # make a Healpix pointing matrix. pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU", hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime) pointing.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Pointing generation took {:.3f} s".format(elapsed), flush=True) start = stop localpix, localsm, subnpix = get_submaps(args, comm, data) signalname = "signal" if args.input_pysm_model: simulate_sky_signal(args, comm, data, mem_counter, [fp], subnpix, localsm, signalname=signalname) if args.input_dipole: print("Simulating dipole") op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole, solar_speed=args.input_dipole_solar_speed_kms, solar_gal_lat=args.input_dipole_solar_gal_lat_deg, solar_gal_lon=args.input_dipole_solar_gal_lon_deg, out=signalname, keep_quats=True, keep_vel=False, subtract=False, coord="G", freq=0, # we could use frequency for quadrupole correction flag_mask=255, common_flag_mask=255) op_sim_dipole.exec(data) # Mapmaking. For purposes of this simulation, we use detector noise # weights based on the NET (white noise level). If the destriping # baseline is too long, this will not be the best choice. detweights = {} for d in detectors: net = fp[d]["NET"] detweights[d] = 1.0 / (args.samplerate * net * net) if not args.madam: if comm.comm_world.rank == 0: print("Not using Madam, will only make a binned map!", flush=True) # get locally hit pixels lc = tm.OpLocalPixels() localpix = lc.exec(data) # find the locally hit submaps. localsm = np.unique(np.floor_divide(localpix, subnpix)) # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) # compute the hits and covariance once, since the pointing and noise # weights are fixed. invnpp.data.fill(0.0) hits.data.fill(0) build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits) build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop hits.write_healpix_fits("{}_hits.fits".format(args.outdir)) invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop # invert it tm.covariance_invert(invnpp, 1.0e-3) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Inverting N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Writing N_pp took {:.3f} s".format(elapsed), flush=True) start = stop # in debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open("{}_distdata.txt".format(args.outdir), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Dumping debug data distribution took " "{:.3f} s".format(elapsed), flush=True) start = stop mcstart = start # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc+nmc): # create output directory for this realization outpath = "{}_{:03d}".format(args.outdir, mc) if comm.comm_world.rank == 0: if not os.path.isdir(outpath): os.makedirs(outpath) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Creating output dir {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop # clear all signal data from the cache, so that we can generate # new noise timestreams. tod.cache.clear("tot_signal_.*") # simulate noise nse = tt.OpSimNoise(out="tot_signal", realization=mc) nse.exec(data) # add sky signal add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname) if mc == firstmc: # For the first realization, optionally export the # timestream data to a TIDAS volume. if args.tidas is not None: from toast.tod.tidas import OpTidasExport tidas_path = os.path.abspath(args.tidas) export = OpTidasExport(tidas_path, name="tot_signal") export.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Noise simulation {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal", detweights=detweights) build_zmap.exec(data) zmap.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Building noise weighted map {:04d} took {:.3f} s".format( mc, elapsed), flush=True) start = stop tm.covariance_apply(invnpp, zmap) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Computing binned map {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop zmap.write_healpix_fits(os.path.join(outpath, "binned.fits")) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Writing binned map {:04d} took {:.3f} s".format(mc, elapsed), flush=True) elapsed = stop - mcstart if comm.comm_world.rank == 0: print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop else: # Set up MADAM map making. pars = {} cross = args.nside // 2 pars[ "temperature_only" ] = "F" pars[ "force_pol" ] = "T" pars[ "kfirst" ] = "T" pars[ "concatenate_messages" ] = "T" pars[ "write_map" ] = "T" pars[ "write_binmap" ] = "T" pars[ "write_matrix" ] = "T" pars[ "write_wcov" ] = "T" pars[ "write_hits" ] = "T" pars[ "nside_cross" ] = cross pars[ "nside_submap" ] = subnside if args.madampar is not None: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars[ "base_first" ] = args.baseline pars[ "nside_map" ] = args.nside if args.noisefilter: pars[ "kfilter" ] = "T" else: pars[ "kfilter" ] = "F" pars[ "fsample" ] = args.samplerate # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc+nmc): # clear all total signal data from the cache, so that we can generate # new noise timestreams. tod.cache.clear("tot_signal_.*") # simulate noise nse = tt.OpSimNoise(out="tot_signal", realization=mc) nse.exec(data) # add sky signal add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Noise simulation took {:.3f} s".format(elapsed), flush=True) start = stop # create output directory for this realization pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc) if comm.comm_world.rank == 0: if not os.path.isdir(pars["path_output"]): os.makedirs(pars["path_output"]) # in debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open(os.path.join(pars["path_output"], "distdata.txt"), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() madam = tm.OpMadam(params=pars, detweights=detweights, name="tot_signal") madam.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Mapmaking took {:.3f} s".format(elapsed), flush=True) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - global_start if comm.comm_world.rank == 0: print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main(parameter_file, outdir): 'The main function' # Set up the logging system logformat = '[%(asctime)s %(levelname)s] %(message)s' log.basicConfig(level=log.INFO, format=logformat) # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.comm_world.rank == 0: log.info("running with %d processes", comm.comm_world.size) # Read the parameter file conf = None if comm.comm_world.rank == 0: configuration_parser = ConfigParser() configuration_parser.read(parameter_file) conf = build_configuration_obj(configuration_parser) log.info('configuration read from file "%s"', parameter_file) conf = comm.comm_world.bcast(conf, root=0) if outdir != '.': if not os.path.isdir(outdir): os.mkdir(outdir) # construct the list of intervals which split the whole observation time intervals = tt.regular_intervals(n=int(conf.num_obs), start=0.0, first=0, rate=conf.sample_rate_Hz, duration=3600*float(conf.hours_per_obs), gap=3600*float(conf.gap_width_hours)) detquats = {} # type: Dict[str, Any] for det in conf.detectors: detquats[det.name] = det.quat # Create the noise model for this observation rate = {} # type: Dict[str, float] fmin = {} # type: Dict[str, float] fknee = {} # type: Dict[str, float] alpha = {} # type: Dict[str, float] net = {} # type: Dict[str, float] for det in conf.detectors: rate[det.name] = conf.sample_rate_Hz fmin[det.name] = 1.0 / (3600.0 * conf.hours_per_obs * conf.num_obs) fknee[det.name] = det.fknee_Hz alpha[det.name] = det.alpha net[det.name] = det.net noise = tt.AnalyticNoise(rate=rate, fmin=fmin, detectors=[x.name for x in conf.detectors], fknee=fknee, alpha=alpha, NET=net) # The distributed time stream data data = toast.Data(comm) intsamp = None if len(intervals) == 1: intsamp = intervals[0].last - intervals[0].first + 1 else: # include also the samples in the gap intsamp = intervals[1].first - intervals[0].first intchunk = int(intsamp / conf.chunks_per_obs) distint = [] # type: List[int] for _ in range(conf.chunks_per_obs - 1): distint.append(intchunk) distint.append(intsamp - (intchunk * (conf.chunks_per_obs - 1))) distsizes = [] for _ in intervals: distsizes.extend(distint) totsamples = np.sum(distsizes) # create the single TOD for this observation tod = tt.TODSatellite( mpicomm=comm.comm_group, detectors=detquats, samples=totsamples, firsttime=0.0, rate=conf.sample_rate_Hz, spinperiod=conf.spin_period_min, spinangle=conf.spin_angle_deg, precperiod=conf.prec_period_min, precangle=conf.prec_angle_deg, sizes=distsizes) # Create the (single) observation ob = {} # type: Dict[str, Any] ob['id'] = 0 ob['name'] = 'mission' ob['tod'] = tod ob['intervals'] = intervals ob['noise'] = noise data.obs.append(ob) precquat = tt.slew_precession_axis(nsim=tod.local_samples[1], firstsamp=tod.local_samples[0], samplerate=conf.sample_rate_Hz, degday=DEG_ANGLE_PER_DAY) # we set the precession axis now, which will trigger calculation # of the boresight pointing. tod.set_prec_axis(qprec=precquat) # simulate noise nse = tt.OpSimNoise(out='noise', stream=0, realization=0) nse.exec(data) comm.comm_world.barrier() # make a Healpix pointing matrix. By setting purge_pntg=True, # we purge the detector quaternion pointing to save memory. # If we ever change this pipeline in a way that needs this # pointing at a later stage, we need to set this to False # and run at higher concurrency. pointing = tt.OpPointingHpix(nside=conf.nside, nest=False, mode='IQU', hwprpm=conf.hwp_rpm, hwpstep=None, hwpsteptime=None) pointing.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: log.info('reading map "%s"', conf.sky_map) foreground_map = healpy.read_map(conf.sky_map, field=(conf.i_column, conf.q_column, conf.u_column), verbose=False) for det in conf.detectors: if comm.comm_world.rank == 0: log.info('processing detector "%s"', det.name) times = tod.read_times() flags, glflags = tod.read_flags(detector=det.name) pixels = tod.cache.reference('pixels_{0}'.format(det.name)) theta, phi = healpy.pix2ang(conf.nside, pixels) vect = healpy.pix2vec(conf.nside, pixels) weights = tod.cache.reference('weights_{0}'.format(det.name)) psi = 0.5 * np.arctan2(weights[:, 2], weights[:, 1]) psi += np.deg2rad(det.polarisation_angle_deg) noise = tod.cache.reference('noise_{0}'.format(det.name)) # Estimate the total TOD using the formula # P = I + Q cos 2psi + U sin 2psi # where Q and U are expressed in the reference frame of the detector sky_i, sky_q, sky_u = [healpy.get_interp_val(foreground_map[x], theta, phi) for x in (0, 1, 2)] det_q = sky_q * np.cos(2. * psi) + sky_u * np.sin(2. * psi) det_u = -sky_q * np.sin(2. * psi) + sky_u * np.cos(2. * psi) foreground_tod = sky_i + det_q * np.cos(2. * psi) + det_u * np.sin(2. * psi) dipole_tod = dip.get_dipole_temperature(np.column_stack(vect).T) total_tod = det.gain * (foreground_tod + dipole_tod + noise) hdu = fits.BinTableHDU.from_columns([fits.Column(name='TIME', format='D', unit='s', array=times), fits.Column(name='THETA', format='E', unit='rad', array=theta), fits.Column(name='PHI', format='E', unit='rad', array=phi), fits.Column(name='PSI', format='E', unit='rad', array=psi), fits.Column(name='FGTOD', format='E', unit='K', array=foreground_tod), fits.Column(name='DIPTOD', format='E', unit='K', array=dipole_tod), fits.Column(name='NOISETOD', format='E', unit='K', array=noise), fits.Column(name='TOTALTOD', format='E', unit='V', array=total_tod), fits.Column(name='FLAGS', format='I', unit='', array=flags), fits.Column(name='GLFLAGS', format='I', unit='', array=glflags)]) hdu.header['COMMENT'] = 'Angles are expressed in the Ecliptic system' hdu.header['FIRSTT'] = (times[0], 'Time of the first sample [s]') hdu.header['LASTT'] = (times[-1], 'Time of the last sample [s]') hdu.header['GAIN'] = (det.gain, 'Detector gain [V/K]') hdu.header['NAME'] = (det.name, 'Name of the detector') hdu.header['VERSION'] = (__version__, 'Version of the code used to create ' 'this file') hdu.header['net'] = (det.net, 'net') hdu.header['ALPHA'] = (det.alpha, 'Slope of the 1/f noise') hdu.header['FKNEE'] = (det.fknee_Hz, 'Knee frequency of the 1/f noise [Hz]') output_file = os.path.join(outdir, ('tod_{0}_mpi{1:04d}.fits' .format(det.name, comm.comm_world.rank))) # Save a copy of the parameter file into the primary HDU of the file, # so that it will be always possible to know the value of the input parameters # used to create it prim_hdu = fits.PrimaryHDU(data=conf.param_file_contents) hdu_list = fits.HDUList([prim_hdu, hdu]) hdu_list.writeto(output_file, clobber=True) if comm.comm_world.rank == 0: log.info('pointings have been created successfully')
def main(): # Our toast communicator- use the default for now, which is one # process group spanning all processes. comm = toast.Comm() parser = argparse.ArgumentParser( description="Make a Healpix map of a given observation ", fromfile_prefix_chars="@", ) parser.add_argument("obsdir", help="Input directory for all observations") parser.add_argument("--obs", required=False, help="Comma-separated list of observations") parser.add_argument( "--nside", required=False, type=np.int, default=1024, help="Map resolution", ) toast_tools.add_polyfilter_args(parser) args = parser.parse_args() print("Loading data from {}".format(args.obsdir), flush=True) if args.obs is None: paths = glob.glob(args.obsdir + "/*") obs = [os.path.basename(path) for path in paths] print("Found {} observations: {}".format(len(obs), obs)) else: obs = [] for path in args.obs.split(","): obs.append(path) print("Mapping {} observations: {}".format(len(obs), obs)) if comm.world_rank == 0: print("Loading data from disk...", flush=True) # Load our selected data data = load_data(args.obsdir, obs=obs, comm=comm) # Everybody look at their data my_world_rank = data.comm.world_rank my_group_rank = data.comm.group_rank for ob in data.obs: tod = ob["tod"] my_dets = tod.local_dets my_first_samp, my_nsamp = tod.local_samples msg = "proc {} with group rank {} has {} dets for samples {} - {}".format( my_world_rank, my_group_rank, len(my_dets), my_first_samp, my_first_samp + my_nsamp - 1) print(msg, flush=True) if comm.world_rank == 0: # Plot some local data from the first observation import matplotlib.pyplot as plt ob = data.obs[0] tod = ob["tod"] boloname = tod.local_dets[0] bolodata = tod.cache.reference("signal_{}".format(boloname)) fig = plt.figure() plt.plot(np.arange(len(bolodata)), bolodata) plt.savefig("bolo_{}.png".format(boloname)) del bolodata # Construct a pointing matrix pointing = toast.todmap.OpPointingHpix(nside=args.nside, nest=True, mode="IQU", pixels="pixels", weights="weights") pointing.exec(data) # Filter signal if args.apply_polyfilter: toast_tools.apply_polyfilter(args, comm, data, "signal") # Bin the TOD to a map mapmaker = toast.todmap.OpMapMaker( nside=args.nside, nnz=3, name="signal", pixels="pixels", intervals=None, baseline_length=None, use_noise_prior=False, outdir=".", ) mapmaker.exec(data) # Plot if comm.world_rank == 0: # Plot the hit map import healpy as hp import matplotlib.pyplot as plt plt.figure(figsize=[18, 12]) hits = hp.read_map("hits.fits") imax = np.argmax(hits) lon, lat = hp.pix2ang(hp.get_nside(hits), imax, lonlat=True) hits[hits == 0] = hp.UNSEEN hp.mollview(hits, xsize=1200, sub=[2, 2, 1], title="hits") hp.gnomview( hits, rot=(lon, lat), xsize=800, reso=1.0, sub=[2, 2, 2], title="hits, lon={:.2f}deg, lat={:.2f}deg".format(lon, lat), ) binned = hp.read_map("binned.fits") binned[binned == 0] = hp.UNSEEN plt.savefig("binned.png") hp.mollview(binned, xsize=1200, sub=[2, 2, 3], title="binned") hp.gnomview( binned, rot=(lon, lat), xsize=800, reso=1.0, sub=[2, 2, 4], title="binned, lon={:.2f}deg, lat={:.2f}deg".format(lon, lat), ) fname = "{}.png".format(os.path.basename(args.obsdir)) plt.savefig(fname) print("Plot saved in", fname)
def setUp(self): fixture_name = os.path.splitext(os.path.basename(__file__))[0] if not toast_available: print("toast cannot be imported- skipping unit tests", flush=True) return self.comm, self.procs, self.rank = get_world() self.outdir = create_outdir(fixture_name, comm=self.comm) toastcomm = toast.Comm() self.data = toast.Data(toastcomm) # Focalplane hwfull = get_example() dets = sim_telescope_detectors(hwfull, "SAT4") hwfull.data["detectors"] = dets hw = hwfull.select(match={ "wafer_slot": "w42", "band": "f030", "pixel": "00[01]" }) print(hw.data["detectors"], flush=True) detquats = {k: v["quat"] for k, v in hw.data["detectors"].items()} # Samples per observation self.totsamp = 10000 # Pixelization nside = 512 self.sim_nside = nside self.map_nside = nside # Scan properties self.site_lon = '-67:47:10' self.site_lat = '-22:57:30' self.site_alt = 5200. self.coord = 'C' self.azmin = 45 self.azmax = 55 self.el = 60 self.scanrate = 1.0 self.scan_accel = 0.1 self.CES_start = None # Noise properties self.rate = 100.0 self.NET = 5.0 self.epsilon = 0.0 self.fmin = 1.0e-5 self.alpha = 1.0 self.fknee = 0.05 tod = TODGround(self.data.comm.comm_group, detquats, self.totsamp, detranks=self.data.comm.group_size, firsttime=0.0, rate=self.rate, site_lon=self.site_lon, site_lat=self.site_lat, site_alt=self.site_alt, azmin=self.azmin, azmax=self.azmax, el=self.el, coord=self.coord, scanrate=self.scanrate, scan_accel=self.scan_accel, CES_start=self.CES_start) # Analytic noise model detnames = list(detquats.keys()) drate = {x: self.rate for x in detnames} dfmin = {x: self.fmin for x in detnames} dfknee = {x: self.fknee for x in detnames} dalpha = {x: self.alpha for x in detnames} dnet = {x: self.NET for x in detnames} nse = AnalyticNoise(rate=drate, fmin=dfmin, detectors=detnames, fknee=dfknee, alpha=dalpha, NET=dnet) # Single observation obs = dict() obs["tod"] = tod obs["noise"] = nse obs["id"] = 12345 obs["intervals"] = tod.subscans obs["site"] = "SimonsObs" obs["telescope"] = "SAT4" obs["site_id"] = 1 obs["telescope_id"] = 4 obs["fpradius"] = 5.0 obs["start_time"] = 0 obs["altitude"] = self.site_alt obs["name"] = "test" # Add the observation to the dataset self.data.obs.append(obs) return
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() memreport("at beginning of main", mpiworld) # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.world_rank == 0: print("Running with {} processes at {}".format( procs, str(datetime.datetime.now()))) parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking', fromfile_prefix_chars='@') parser.add_argument('--rimo', required=True, help='RIMO file') parser.add_argument('--freq', required=True, type=np.int, help='Frequency') parser.add_argument('--nside', required=False, type=np.int, default=512, help='Map resolution') parser.add_argument('--nside_cross', required=False, type=np.int, default=512, help='Destriping resolution') parser.add_argument('--debug', dest='debug', default=False, action='store_true', help='Write data distribution info to file') parser.add_argument('--dets', required=False, default=None, help='Detector list (comma separated)') parser.add_argument('--effdir', required=True, help='Input Exchange Format File directory') parser.add_argument('--effdir_in_diode0', required=False, default=None, help='Input Exchange Format File directory, ' 'LFI diode 0') parser.add_argument('--effdir_in_diode1', required=False, default=None, help='Input Exchange Format File directory, ' 'LFI diode 1') parser.add_argument('--effdir_pntg', required=False, help='Input Exchange Format File directory ' 'for pointing') parser.add_argument('--effdir_out', required=False, help='Output directory for destriped TOD') parser.add_argument('--effdir_out_diode0', required=False, help='Output directory for destriped TOD, LFI diode 0') parser.add_argument('--effdir_out_diode1', required=False, help='Output directory for destriped TOD, LFI diode 1') parser.add_argument('--obtmask', required=False, default=1, type=np.int, help='OBT flag mask') parser.add_argument('--flagmask', required=False, default=1, type=np.int, help='Quality flag mask') parser.add_argument('--pntflagmask', required=False, default=0, type=np.int, help='Pointing flag mask') parser.add_argument('--bad_intervals', required=False, help='Path to bad interval file.') parser.add_argument('--ringdb', required=True, help='Ring DB file') parser.add_argument('--odfirst', required=False, default=None, type=np.int, help='First OD to use') parser.add_argument('--odlast', required=False, default=None, type=np.int, help='Last OD to use') parser.add_argument('--ringfirst', required=False, default=None, help='First ring to use (can be a list)') parser.add_argument('--ringlast', required=False, default=None, help='Last ring to use (can be a list)') parser.add_argument('--obtfirst', required=False, default=None, type=np.float, help='First OBT to use') parser.add_argument('--obtlast', required=False, default=None, type=np.float, help='Last OBT to use') parser.add_argument('--madampar', required=False, default=None, help='Madam parameter file') parser.add_argument('--out', required=False, default='.', help='Output directory') parser.add_argument('--madam_prefix', required=False, help='map prefix') parser.add_argument('--split_mask', required=False, default=None, help='Intensity mask, non-zero pixels are not split.') parser.add_argument('--save_leakage_matrices', dest='save_leakage_matrices', default=False, action='store_true', help='Compile and write out the leakage projection ' 'matrices.') # noise parameters parser.add_argument('--noisefile', required=False, default='RIMO', help='Path to noise PSD files for noise filter. ' 'Tag DETECTOR will be replaced with detector name.') parser.add_argument('--static_noise', dest='static_noise', required=False, default=False, action='store_true', help='Assume constant noise PSD') parser.add_argument('--filterfile', required=False, help='Extra filter file.') try: args = parser.parse_args() except SystemExit: sys.exit(0) if comm.comm_world.rank == 0: print('All parameters:') print(args, flush=True) timer = Timer() timer.start() nrange = 1 odranges = None if args.odfirst is not None and args.odlast is not None: odranges = [] firsts = [int(i) for i in str(args.odfirst).split(',')] lasts = [int(i) for i in str(args.odlast).split(',')] for odfirst, odlast in zip(firsts, lasts): odranges.append((odfirst, odlast)) nrange = len(odranges) ringranges = None if args.ringfirst is not None and args.ringlast is not None: ringranges = [] firsts = [int(i) for i in str(args.ringfirst).split(',')] lasts = [int(i) for i in str(args.ringlast).split(',')] for ringfirst, ringlast in zip(firsts, lasts): ringranges.append((ringfirst, ringlast)) nrange = len(ringranges) obtranges = None if args.obtfirst is not None and args.obtlast is not None: obtranges = [] firsts = [float(i) for i in str(args.obtfirst).split(',')] lasts = [float(i) for i in str(args.obtlast).split(',')] for obtfirst, obtlast in zip(firsts, lasts): obtranges.append((obtfirst, obtlast)) nrange = len(obtranges) if odranges is None: odranges = [None] * nrange if ringranges is None: ringranges = [None] * nrange if obtranges is None: obtranges = [None] * nrange detectors = None if args.dets is not None: detectors = re.split(',', args.dets) # create the TOD for this observation if args.noisefile != 'RIMO' and not args.static_noise: do_eff_cache = True else: do_eff_cache = False tods = [] if args.static_noise: noisefile = args.noisefile else: noisefile = 'RIMO' for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges): tods.append( tp.Exchange(comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_in_diode0=args.effdir_in_diode0, effdir_in_diode1=args.effdir_in_diode1, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, pntflagmask=args.pntflagmask, do_eff_cache=do_eff_cache, noisefile=noisefile)) rimo = tods[0].rimo # Make output directory if not os.path.isdir(args.out) and comm.comm_world.rank == 0: os.makedirs(args.out) # Read in madam parameter file # Allow more than one entry, gather into a list repeated_keys = ['detset', 'detset_nopol', 'survey'] pars = {} if comm.comm_world.rank == 0: pars['kfirst'] = False pars['temperature_only'] = True pars['base_first'] = 60.0 pars['nside_map'] = args.nside pars['nside_cross'] = min(args.nside, args.nside_cross) pars['nside_submap'] = 16 pars['write_map'] = False pars['write_binmap'] = True pars['write_matrix'] = False pars['write_wcov'] = False pars['write_hits'] = True pars['kfilter'] = False pars['info'] = 3 pars['pixlim_map'] = 1e-3 pars['pixlim_cross'] = 1e-3 if args.madampar: pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*') comment = re.compile(r'^#.*') with open(args.madampar, 'r') as f: for line in f: if not comment.match(line): result = pat.match(line) if result: key, value = result.group(1), result.group(2) if key in repeated_keys: if key not in pars: pars[key] = [] pars[key].append(value) else: pars[key] = value # Command line parameters override the ones in the madam parameter file if 'file_root' not in pars: pars['file_root'] = 'madam' if args.madam_prefix is not None: pars['file_root'] = args.madam_prefix sfreq = '{:03}'.format(args.freq) if sfreq not in pars['file_root']: pars['file_root'] += '_' + sfreq try: fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq] except Exception: fsample = 180.3737 pars['fsample'] = fsample pars['path_output'] = args.out if args.save_leakage_matrices: pars['write_leakmatrix'] = True pars = comm.comm_world.bcast(pars, root=0) if args.noisefile != 'RIMO': # We split MPI_COMM_WORLD into single process groups, each of # which is assigned one or more observations (rings) comm = toast.Comm(groupsize=1) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) for iobs, tod in enumerate(tods): if args.noisefile != 'RIMO' and not args.static_noise: # Use a toast helper method to optimally distribute rings between # processes. dist = toast.distribute_discrete(tod.ringsizes, comm.world_size) my_first_ring, my_n_ring = dist[comm.comm_world.rank] for my_ring in range(my_first_ring, my_first_ring + my_n_ring): ringtod = tp.Exchange.from_tod(tod, my_ring, comm.comm_group, noisefile=args.noisefile) ob = {} ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring) ob['id'] = ringtod.globalfirst_ring ob['tod'] = ringtod ob['intervals'] = ringtod.valid_intervals ob['baselines'] = None ob['noise'] = ringtod.noise data.obs.append(ob) else: ob = {} ob['name'] = 'observation{:04}'.format(iobs) ob['id'] = 0 ob['tod'] = tod ob['intervals'] = tod.valid_intervals ob['baselines'] = None ob['noise'] = tod.noise data.obs.append(ob) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Metadata queries") if args.effdir_out is not None or (args.effdir_out_diode0 is not None and args.effdir_out_diode1 is not None): do_output = True else: do_output = False # Read in the signal timer.clear() timer.start() reader = tp.OpInputPlanck(signal_name='signal', flags_name='flags') if comm.comm_world.rank == 0: print('Reading input signal from {}'.format(args.effdir), flush=True) reader.exec(data) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Read") tod_name = 'signal' flags_name = 'flags' # Optionally filter the signal apply_filter(args, data) # Optionally flag bad intervals if args.bad_intervals is not None: timer = Timer() timer.start() flagger = tp.OpBadIntervals(path=args.bad_intervals) flagger.exec(data) timer.stop() if comm.comm_world.rank == 0: timer.report("Apply {}".format(args.bad_intervals)) # make a planck Healpix pointing matrix timer.clear() timer.start() mode = 'IQU' if pars['temperature_only'] == 'T': mode = 'I' nside = int(pars['nside_map']) pointing = tp.OpPointingPlanck(nside=nside, mode=mode, RIMO=rimo, margin=0, apply_flags=(not do_output), keep_vel=False, keep_pos=False, keep_phase=False, keep_quats=False) pointing.exec(data) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Pointing Matrix, mode = {}".format(mode)) for obs in data.obs: obs['tod'].purge_eff_cache() # for now, we pass in the noise weights from the RIMO. detweights = {} for d in tod.detectors: if d[-1] in '01' and d[-2] != '-': det = to_radiometer(d) else: det = d net = tod.rimo[det].net fsample = tod.rimo[det].fsample detweights[d] = 1.0 / (fsample * net * net) if do_output: name_out = 'madam_tod' else: name_out = None timer.clear() timer.start() try: madam = toast.todmap.OpMadam(name=tod_name, flag_name=flags_name, apply_flags=do_output, params=pars, detweights=detweights, purge=True, name_out=name_out, translate_timestamps=False) except Exception as e: raise Exception('{:4} : ERROR: failed to initialize Madam: {}'.format( comm.comm_world.rank, e)) madam.exec(data) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Madam") if do_output: timer = Timer() timer.start() writer = tp.OpOutputPlanck(signal_name='madam_tod', flags_name=None, commonflags_name=None, effdir_out=args.effdir_out, effdir_out_diode0=args.effdir_out_diode0, effdir_out_diode1=args.effdir_out_diode1) writer.exec(data) comm.comm_world.barrier() timer.stop() if comm.comm_world.rank == 0: timer.report("Madam output") memreport("at end of main", mpiworld) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def main(): log = Logger.get() gt = GlobalTimers.get() gt.start("toast_planck_reduce (total)") mpiworld, procs, rank, comm = get_comm() # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.comm_world.rank == 0: print( "Running with {} processes at {}".format( procs, str(datetime.datetime.now()) ) ) parser = argparse.ArgumentParser( description="Simple on-the-fly signal convolution + MADAM Mapmaking", fromfile_prefix_chars="@", ) parser.add_argument("--lmax", required=True, type=np.int, help="Simulation lmax") parser.add_argument( "--fwhm", required=True, type=np.float, help="Sky fwhm [arcmin] to deconvolve" ) parser.add_argument("--beammmax", required=True, type=np.int, help="Beam mmax") parser.add_argument("--order", default=11, type=np.int, help="Iteration order") parser.add_argument( "--pxx", required=False, default=False, action="store_true", help="Beams are in Pxx frame, not Dxx", ) parser.add_argument( "--normalize", required=False, default=False, action="store_true", help="Normalize the beams", ) parser.add_argument( "--skyfile", required=True, help="Path to sky alm files. Tag DETECTOR will be " "replaced with detector name.", ) parser.add_argument( "--remove_monopole", required=False, default=False, action="store_true", help="Remove the sky monopole before convolution", ) parser.add_argument( "--remove_dipole", required=False, default=False, action="store_true", help="Remove the sky dipole before convolution", ) parser.add_argument( "--beamfile", required=True, help="Path to beam alm files. Tag DETECTOR will be " "replaced with detector name.", ) parser.add_argument("--rimo", required=True, help="RIMO file") parser.add_argument("--freq", required=True, type=np.int, help="Frequency") parser.add_argument( "--dets", required=False, default=None, help="Detector list (comma separated)" ) parser.add_argument( "--effdir", required=True, help="Input Exchange Format File directory" ) parser.add_argument( "--effdir_pntg", required=False, help="Input Exchange Format File directory " "for pointing", ) parser.add_argument( "--effdir_out", required=False, help="Output directory for convolved TOD" ) parser.add_argument( "--obtmask", required=False, default=1, type=np.int, help="OBT flag mask" ) parser.add_argument( "--flagmask", required=False, default=1, type=np.int, help="Quality flag mask" ) parser.add_argument("--ringdb", required=True, help="Ring DB file") parser.add_argument( "--odfirst", required=False, default=None, type=np.int, help="First OD to use" ) parser.add_argument( "--odlast", required=False, default=None, type=np.int, help="Last OD to use" ) parser.add_argument( "--ringfirst", required=False, default=None, type=np.int, help="First ring to use", ) parser.add_argument( "--ringlast", required=False, default=None, type=np.int, help="Last ring to use" ) parser.add_argument( "--obtfirst", required=False, default=None, type=np.float, help="First OBT to use", ) parser.add_argument( "--obtlast", required=False, default=None, type=np.float, help="Last OBT to use" ) parser.add_argument("--madam_prefix", required=False, help="map prefix") parser.add_argument( "--madampar", required=False, default=None, help="Madam parameter file" ) parser.add_argument( "--obtmask_madam", required=False, type=np.int, help="OBT flag mask for Madam" ) parser.add_argument( "--flagmask_madam", required=False, type=np.int, help="Quality flag mask for Madam", ) parser.add_argument( "--skip_madam", required=False, default=False, action="store_true", help="Do not run Madam on the convolved timelines", ) parser.add_argument("--out", required=False, default=".", help="Output directory") try: args = parser.parse_args() except SystemExit: sys.exit(0) timer = Timer() timer.start() odrange = None if args.odfirst is not None and args.odlast is not None: odrange = (args.odfirst, args.odlast) ringrange = None if args.ringfirst is not None and args.ringlast is not None: ringrange = (args.ringfirst, args.ringlast) obtrange = None if args.obtfirst is not None and args.obtlast is not None: obtrange = (args.obtfirst, args.obtlast) detectors = None if args.dets is not None: detectors = re.split(",", args.dets) # This is the distributed data, consisting of one or # more observations, each distributed over a communicator. data = toast.Data(comm) # Ensure output directory exists if not os.path.isdir(args.out) and comm.comm_world.rank == 0: os.makedirs(args.out) # Read in madam parameter file # Allow more than one entry, gather into a list repeated_keys = ["detset", "detset_nopol", "survey"] pars = {} if comm.comm_world.rank == 0: pars["kfirst"] = False pars["temperature_only"] = True pars["base_first"] = 60.0 pars["nside_map"] = 512 pars["nside_cross"] = 512 pars["nside_submap"] = 16 pars["write_map"] = False pars["write_binmap"] = True pars["write_matrix"] = False pars["write_wcov"] = False pars["write_hits"] = True pars["kfilter"] = False pars["info"] = 3 if args.madampar: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if not comment.match(line): result = pat.match(line) if result: key, value = result.group(1), result.group(2) if key in repeated_keys: if key not in pars: pars[key] = [] pars[key].append(value) else: pars[key] = value # Command line parameters override the ones in the madam parameter file if "file_root" not in pars: pars["file_root"] = "madam" if args.madam_prefix is not None: pars["file_root"] = args.madam_prefix sfreq = "{:03}".format(args.freq) if sfreq not in pars["file_root"]: pars["file_root"] += "_" + sfreq try: fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq] except Exception: fsample = 180.3737 pars["fsample"] = fsample pars["path_output"] = args.out print("All parameters:") print(args, flush=True) pars = comm.comm_world.bcast(pars, root=0) memreport("after parameters", MPI.COMM_WORLD) # madam only supports a single observation. Normally # we would have multiple observations with some subset # assigned to each process group. # create the TOD for this observation tod = tp.Exchange( comm=comm.comm_group, detectors=detectors, ringdb=args.ringdb, effdir_in=args.effdir, effdir_pntg=args.effdir_pntg, obt_range=obtrange, ring_range=ringrange, od_range=odrange, freq=args.freq, RIMO=args.rimo, obtmask=args.obtmask, flagmask=args.flagmask, do_eff_cache=False, ) # normally we would get the intervals from somewhere else, but since # the Exchange TOD already had to get that information, we can # get it from there. ob = {} ob["name"] = "mission" ob["id"] = 0 ob["tod"] = tod ob["intervals"] = tod.valid_intervals ob["baselines"] = None ob["noise"] = tod.noise # Add the bare minimum focal plane information for the conviqt operator focalplane = {} for det in tod.detectors: if args.pxx: # Beam is in the polarization basis. # No extra rotations are needed psipol = tod.rimo[det].psi_pol else: # Beam is in the detector basis. Convolver needs to remove # the last rotation into the polarization sensitive frame. psipol = tod.rimo[det].psi_uv + tod.rimo[det].psi_pol focalplane[det] = { "pol_leakage" : tod.rimo[det].epsilon, "pol_angle_deg" : psipol, } ob["focalplane"] = focalplane data.obs.append(ob) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Metadata queries") loader = tp.OpInputPlanck( commonflags_name="common_flags", flags_name="flags", margin=0 ) loader.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Data read and cache") tod.cache.report() memreport("after loading", mpiworld) # make a planck Healpix pointing matrix mode = "IQU" if pars["temperature_only"] == "T": mode = "I" nside = int(pars["nside_map"]) pointing = tp.OpPointingPlanck( nside=nside, mode=mode, RIMO=tod.RIMO, margin=0, apply_flags=False, keep_vel=False, keep_pos=False, keep_phase=False, keep_quats=True, ) pointing.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Pointing Matrix took, mode = {}".format(mode)) memreport("after pointing", mpiworld) # simulate the TOD by convolving the sky with the beams if comm.comm_world.rank == 0: print("Convolving TOD", flush=True) for pattern in args.beamfile.split(","): skyfiles = {} beamfiles = {} for det in tod.detectors: freq = "{:03}".format(tp.utilities.det2freq(det)) if "LFI" in det: psmdet = "{}_{}".format(freq, det[3:]) if det.endswith("M"): arm = "y" else: arm = "x" graspdet = "{}_{}_{}".format(freq[1:], det[3:5], arm) else: psmdet = det.replace("-", "_") graspdet = det skyfile = ( args.skyfile.replace("FREQ", freq) .replace("PSMDETECTOR", psmdet) .replace("DETECTOR", det) ) skyfiles[det] = skyfile beamfile = pattern.replace("GRASPDETECTOR", graspdet).replace( "DETECTOR", det ) beamfiles[det] = beamfile if comm.comm_world.rank == 0: print("Convolving {} with {}".format(skyfile, beamfile), flush=True) conviqt = OpSimConviqt( comm.comm_world, skyfiles, beamfiles, lmax=args.lmax, beammmax=args.beammmax, pol=True, fwhm=args.fwhm, order=args.order, calibrate=True, dxx=True, out="conviqt_tod", apply_flags=False, remove_monopole=args.remove_monopole, remove_dipole=args.remove_dipole, verbosity=1, normalize_beam=args.normalize, ) conviqt.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Convolution") memreport("after conviqt", mpiworld) if args.effdir_out is not None: if comm.comm_world.rank == 0: print("Writing TOD", flush=True) tod.set_effdir_out(args.effdir_out, None) writer = tp.OpOutputPlanck( signal_name="conviqt_tod", flags_name="flags", commonflags_name="common_flags", ) writer.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Conviqt output") memreport("after writing", mpiworld) # for now, we pass in the noise weights from the RIMO. detweights = {} for d in tod.detectors: net = tod.rimo[d].net fsample = tod.rimo[d].fsample detweights[d] = 1.0 / (fsample * net * net) if not args.skip_madam: if comm.comm_world.rank == 0: print("Calling Madam", flush=True) try: if args.obtmask_madam is None: obtmask = args.obtmask else: obtmask = args.obtmask_madam if args.flagmask_madam is None: flagmask = args.flagmask else: flagmask = args.flagmask_madam madam = OpMadam( params=pars, detweights=detweights, name="conviqt_tod", flag_name="flags", purge=True, name_out="madam_tod", common_flag_mask=obtmask, flag_mask=flagmask, ) except Exception as e: raise Exception( "{:4} : ERROR: failed to initialize Madam: {}".format( comm.comm_world.rank, e ) ) madam.exec(data) comm.comm_world.barrier() if comm.comm_world.rank == 0: timer.report_clear("Madam took {:.3f} s") memreport("after madam", mpiworld) gt.stop_all() if mpiworld is not None: mpiworld.barrier() timer = Timer() timer.start() alltimers = gather_timers(comm=mpiworld) if comm.world_rank == 0: out = os.path.join(args.out, "timing") dump_timing(alltimers, out) timer.stop() timer.report("Gather and dump timing info") return
def setUp(self): fixture_name = os.path.splitext(os.path.basename(__file__))[0] if not toast_available: print("toast cannot be imported- skipping unit tests", flush=True) return self.outdir = None if MPI.COMM_WORLD.rank == 0: self.outdir = create_outdir(fixture_name) self.outdir = MPI.COMM_WORLD.bcast(self.outdir, root=0) toastcomm = toast.Comm() self.data = toast.Data(toastcomm) # Focalplane hwfull = get_example() dets = sim_telescope_detectors(hwfull, "SAT3") hwfull.data["detectors"] = dets hw = hwfull.select(match={ "wafer": "42", "band": "LF1", "pixel": "00[01]" }) # print(hw.data["detectors"], flush=True) detquats = {k: v["quat"] for k, v in hw.data["detectors"].items()} # File dump size in bytes (1MB) self.dumpsize = 2**20 # Samples per observation self.totsamp = 10000 # Pixelization nside = 512 self.sim_nside = nside self.map_nside = nside # Scan properties self.site_lon = '-67:47:10' self.site_lat = '-22:57:30' self.site_alt = 5200. self.coord = 'C' self.azmin = 45 self.azmax = 55 self.el = 60 self.scanrate = 1.0 self.scan_accel = 0.1 self.CES_start = None # Noise properties self.rate = 100.0 self.NET = 1e-3 # 1 mK NET self.epsilon = 0.0 self.fmin = 1.0e-5 self.alpha = 1.0 self.fknee = 0.05 for ob in range(3): ftime = (self.totsamp / self.rate) * ob + 1564015655.88 tod = TODGround(self.data.comm.comm_group, detquats, self.totsamp, detranks=self.data.comm.group_size, firsttime=ftime, rate=self.rate, site_lon=self.site_lon, site_lat=self.site_lat, site_alt=self.site_alt, azmin=self.azmin, azmax=self.azmax, el=self.el, coord=self.coord, scanrate=self.scanrate, scan_accel=self.scan_accel, CES_start=self.CES_start) # Analytic noise model detnames = list(detquats.keys()) drate = {x: self.rate for x in detnames} dfmin = {x: self.fmin for x in detnames} dfknee = {x: self.fknee for x in detnames} dalpha = {x: self.alpha for x in detnames} dnet = {x: self.NET for x in detnames} nse = AnalyticNoise(rate=drate, fmin=dfmin, detectors=detnames, fknee=dfknee, alpha=dalpha, NET=dnet) # Single observation obs = dict() obs["tod"] = tod obs["noise"] = nse obs["id"] = 12345 obs["intervals"] = tod.subscans obs["site"] = "SimonsObs" obs["telescope"] = "SAT3" obs["site_id"] = 1 obs["telescope_id"] = 4 obs["fpradius"] = 5.0 obs["start_time"] = ftime obs["altitude"] = self.site_alt obs["name"] = "test_{:02}".format(ob) # Add the observation to the dataset self.data.obs.append(obs) # Simulate some noise into multiple cache prefixes. This is used # to test the export of multiple timestream flavors. nse = toast.tod.OpSimNoise(out="signal", realization=0) nse.exec(self.data) nse = toast.tod.OpSimNoise(out="component1", realization=0) nse.exec(self.data) nse = toast.tod.OpSimNoise(out="component2", realization=0) nse.exec(self.data) return
def main(): if MPI.COMM_WORLD.rank == 0: print("Running with {} processes".format(MPI.COMM_WORLD.size), flush=True) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description="Read existing data and make a simple map.", fromfile_prefix_chars="@", ) parser.add_argument( "--groupsize", required=False, type=int, default=0, help="size of processor groups used to distribute " "observations", ) parser.add_argument( "--hwprpm", required=False, type=float, default=0.0, help="The rate (in RPM) of the HWP rotation", ) parser.add_argument( "--samplerate", required=False, default=100.0, type=np.float, help="Detector sample rate (Hz)", ) parser.add_argument("--outdir", required=False, default="out", help="Output directory") parser.add_argument("--nside", required=False, type=int, default=64, help="Healpix NSIDE") parser.add_argument( "--subnside", required=False, type=int, default=8, help="Distributed pixel sub-map NSIDE", ) parser.add_argument("--coord", required=False, default="E", help="Sky coordinate system [C,E,G]") parser.add_argument( "--baseline", required=False, type=float, default=60.0, help="Destriping baseline length (seconds)", ) parser.add_argument( "--noisefilter", required=False, default=False, action="store_true", help="Destripe with the noise filter enabled", ) parser.add_argument( "--madam", required=False, default=False, action="store_true", help="If specified, use libmadam for map-making", ) parser.add_argument("--madampar", required=False, default=None, help="Madam parameter file") parser.add_argument( "--polyorder", required=False, type=int, help="Polynomial order for the polyfilter", ) parser.add_argument( "--wbin_ground", required=False, type=float, help="Ground template bin width [degrees]", ) parser.add_argument( "--flush", required=False, default=False, action="store_true", help="Flush every print statement.", ) parser.add_argument("--tidas", required=False, default=None, help="Input TIDAS volume") parser.add_argument("--tidas_detgroup", required=False, default=None, help="TIDAS detector group") parser.add_argument("--spt3g", required=False, default=None, help="Input SPT3G data directory") parser.add_argument( "--spt3g_prefix", required=False, default=None, help="SPT3G data frame file prefix", ) parser.add_argument( "--common_flag_mask", required=False, default=0, type=np.uint8, help="Common flag mask", ) parser.add_argument( "--debug", required=False, default=False, action="store_true", help="Write data distribution info and focalplane plot", ) args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) # args = parser.parse_args(sys.argv) autotimer = timing.auto_timer("@{}".format(timing.FILE())) if (args.tidas is not None) and (args.spt3g is not None): raise RuntimeError("Cannot read two datasets!") if (args.tidas is None) and (args.spt3g is None): raise RuntimeError("No dataset specified!") if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot load") if args.spt3g is not None: if not tt.spt3g_available: raise RuntimeError("SPT3G not found- cannot load") groupsize = args.groupsize if groupsize == 0: groupsize = MPI.COMM_WORLD.size # Pixelization nside = args.nside npix = 12 * args.nside * args.nside subnside = args.subnside if subnside > nside: subnside = nside subnpix = 12 * subnside * subnside # This is the 2-level toast communicator. if MPI.COMM_WORLD.size % groupsize != 0: if MPI.COMM_WORLD.rank == 0: print( "WARNING: process groupsize does not evenly divide into " "total number of processes", flush=True, ) comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize) # Create output directory mtime = MPI.Wtime() if comm.comm_world.rank == 0: if not os.path.isdir(args.outdir): os.makedirs(args.outdir) mtime = elapsed(comm.comm_world, mtime, "Creating output directory") # The distributed timestream data data = None if args.tidas is not None: if args.tidas_detgroup is None: raise RuntimeError("you must specify the detector group") data = tds.load_tidas( comm, comm.group_size, args.tidas, "r", args.tidas_detgroup, tds.TODTidas, group_dets=args.tidas_detgroup, distintervals="chunks", ) if args.spt3g is not None: if args.spt3g_prefix is None: raise RuntimeError("you must specify the frame file prefix") data = s3g.load_spt3g( comm, comm.group_size, args.spt3g, args.spt3g_prefix, s3g.obsweight_spt3g, s3g.TOD3G, ) mtime = elapsed(comm.comm_world, mtime, "Distribute data") # In debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open("{}_distdata.txt".format(args.outdir), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() mtime = elapsed(comm.comm_world, mtime, "Dumping debug data distribution") if comm.comm_world.rank == 0: outfile = "{}_focalplane.png".format(args.outdir) set_backend() # Just plot the dets from the first TOD temptod = data.obs[0]["tod"] # FIXME: change this once we store det info in the metadata. dfwhm = {x: 10.0 for x in temptod.detectors} tt.plot_focalplane(temptod.detoffset(), 10.0, 10.0, outfile, fwhm=dfwhm) comm.comm_world.barrier() mtime = elapsed(comm.comm_world, mtime, "Plotting debug focalplane") # Compute pointing matrix pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU", hwprpm=args.hwprpm) pointing.exec(data) mtime = elapsed(comm.comm_world, mtime, "Expand pointing") # Mapmaking. # FIXME: We potentially have a different noise model for every # observation. We need to have both spt3g and tidas format Noise # classes which read the information from disk. Then the mapmaking # operators need to get these noise weights from each observation. detweights = {d: 1.0 for d in data.obs[0]["tod"].detectors} if not args.madam: if comm.comm_world.rank == 0: print("Not using Madam, will only make a binned map!", flush=True) # Filter data if desired if args.polyorder: polyfilter = tt.OpPolyFilter( order=args.polyorder, common_flag_mask=args.common_flag_mask) polyfilter.exec(data) mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering") if args.wbin_ground: groundfilter = tt.OpGroundFilter( wbin=args.wbin_ground, common_flag_mask=args.common_flag_mask) groundfilter.exec(data) mtime = elapsed(comm.comm_world, mtime, "Ground template filtering") # Compute pixel space distribution lc = tm.OpLocalPixels() localpix = lc.exec(data) if localpix is None: raise RuntimeError( "Process {} has no hit pixels. Perhaps there are fewer " "detectors than processes in the group?".format( comm.comm_world.rank)) localsm = np.unique(np.floor_divide(localpix, subnpix)) mtime = elapsed(comm.comm_world, mtime, "Compute local submaps") # construct distributed maps to store the covariance, # noise weighted map, and hits mtime = MPI.Wtime() invnpp = tm.DistPixels( comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm, ) hits = tm.DistPixels( comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm, ) zmap = tm.DistPixels( comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm, ) # compute the hits and covariance. invnpp.data.fill(0.0) hits.data.fill(0) build_invnpp = tm.OpAccumDiag( detweights=detweights, invnpp=invnpp, hits=hits, common_flag_mask=args.common_flag_mask, ) build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1") hits.write_healpix_fits("{}_hits.fits".format(args.outdir)) invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir)) mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1") # invert it tm.covariance_invert(invnpp, 1.0e-3) mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1") invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir)) mtime = elapsed(comm.comm_world, mtime, "Writing N_pp") zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag(zmap=zmap, detweights=detweights, common_flag_mask=args.common_flag_mask) build_zmap.exec(data) zmap.allreduce() mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map") tm.covariance_apply(invnpp, zmap) mtime = elapsed(comm.comm_world, mtime, "Computing binned map") zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits")) mtime = elapsed(comm.comm_world, mtime, "Writing binned map") else: # Set up MADAM map making. pars = {} pars["temperature_only"] = "F" pars["force_pol"] = "T" pars["kfirst"] = "T" pars["concatenate_messages"] = "T" pars["write_map"] = "T" pars["write_binmap"] = "T" pars["write_matrix"] = "T" pars["write_wcov"] = "T" pars["write_hits"] = "T" pars["nside_cross"] = nside // 2 pars["nside_submap"] = subnside if args.madampar is not None: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars["base_first"] = args.baseline pars["nside_map"] = nside if args.noisefilter: pars["kfilter"] = "T" else: pars["kfilter"] = "F" pars["fsample"] = args.samplerate madam = tm.OpMadam(params=pars, detweights=detweights, common_flag_mask=args.common_flag_mask) madam.exec(data) mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking") comm.comm_world.barrier() stop = MPI.Wtime() dur = stop - global_start if comm.comm_world.rank == 0: print("Total Time: {:.2f} seconds".format(dur), flush=True) return