def main():
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
# Create an argparse and add custom arguments
parser = argparse.ArgumentParser(description=“...")
parser.add_argument('--groupsize',
required=False, type=np.int,
help='Size of a process group assigned to a CES')
# pass the argparse object to timing module which will add timing
# arguments and return "parse.parse_args() result after handling
# the timing specific options
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
# create the primary auto timer for the entire script
autotimer = timing.auto_timer(timing.FILE())
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking',
fromfile_prefix_chars='@')
parser.add_argument('--skip_madam',
dest='skip_madam',
default=False,
action='store_true',
help='D not make maps with Madam.')
parser.add_argument('--skip_noise',
dest='skip_noise',
default=False,
action='store_true',
help='Do not add simulated noise to the TOD.')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help='Write data distribution info to file')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir2',
required=False,
help='Additional input Exchange Format File directory')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory for '
'pointing')
parser.add_argument('--effdir_fsl',
required=False,
help='Input Exchange Format File directory for '
'straylight')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Which OBT flag bits to raise for HCM maneuvers')
parser.add_argument('--bad_intervals',
required=False,
help='Path to bad interval file.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use')
parser.add_argument('--obtfirst',
required=False,
default=None,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
help='Last OBT to use')
parser.add_argument('--read_eff',
dest='read_eff',
default=False,
action='store_true',
help='Read and co-add the signal from effdir')
parser.add_argument('--decalibrate',
required=False,
help='Path to calibration file to decalibrate with. '
'You can use python string formatting, assuming '
'.format(mc)')
parser.add_argument('--calibrate',
required=False,
help='Path to calibration file to calibrate with. '
'You can use python string formatting, assuming '
'.format(mc)')
parser.add_argument('--madampar',
required=False,
default=None,
help='Madam parameter file')
parser.add_argument('--nside',
required=False,
default=None,
type=np.int,
help='Madam resolution')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--madam_prefix', required=False, help='map prefix')
parser.add_argument('--make_rings',
dest='make_rings',
default=False,
action='store_true',
help='Compile ringsets.')
parser.add_argument('--nside_ring',
required=False,
default=128,
type=np.int,
help='Ringset resolution')
parser.add_argument('--ring_root',
required=False,
default='ringset',
help='Root filename for ringsets (setting to empty '
'disables ringset output).')
parser.add_argument('--MC_start',
required=False,
default=0,
type=np.int,
help='First Monte Carlo noise realization')
parser.add_argument('--MC_count',
required=False,
default=1,
type=np.int,
help='Number of Monte Carlo noise realizations')
# noise parameters
parser.add_argument('--noisefile',
required=False,
default='RIMO',
help='Path to noise PSD files for noise filter. '
'Tag DETECTOR will be replaced with detector name.')
parser.add_argument('--noisefile_simu',
required=False,
default='RIMO',
help='Path to noise PSD files for noise simulation. '
'Tag DETECTOR will be replaced with detector name.')
# Dipole parameters
dipogroup = parser.add_mutually_exclusive_group()
dipogroup.add_argument('--dipole',
dest='dipole',
required=False,
default=False,
action='store_true',
help='Simulate dipole')
dipogroup.add_argument('--solsys_dipole',
dest='solsys_dipole',
required=False,
default=False,
action='store_true',
help='Simulate solar system dipole')
dipogroup.add_argument('--orbital_dipole',
dest='orbital_dipole',
required=False,
default=False,
action='store_true',
help='Simulate orbital dipole')
dipo_parameters_group = parser.add_argument_group('dipole_parameters')
dipo_parameters_group.add_argument(
'--solsys_speed',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_speed"],
help='Solar system speed wrt. CMB rest frame in km/s. Default is '
'Planck 2015 best fit value')
dipo_parameters_group.add_argument(
'--solsys_glon',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_glon"],
help='Solar system velocity direction longitude in degrees')
dipo_parameters_group.add_argument(
'--solsys_glat',
required=False,
type=np.float,
default=DEFAULT_PARAMETERS["solsys_glat"],
help='Solar system velocity direction latitude in degrees')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.world_rank == 0:
print('All parameters:')
print(args, flush=True)
if args.MC_count < 1:
raise RuntimeError('MC_count = {} < 1. Nothing done.'
''.format(args.MC_count))
timer = Timer()
timer.start()
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# create the TOD for this observation
if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO':
do_eff_cache = True
else:
do_eff_cache = False
tods = []
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
# create the TOD for this observation
tods.append(
tp.Exchange(comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
extra_effdirs=[args.effdir2, args.effdir_fsl],
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
pntflagmask=args.pntflagmask,
do_eff_cache=do_eff_cache))
# Make output directory
if not os.path.isdir(args.out) and comm.world_rank == 0:
os.makedirs(args.out)
# Read in madam parameter file
# Allow more than one entry, gather into a list
repeated_keys = ['detset', 'detset_nopol', 'survey']
pars = {}
if comm.world_rank == 0:
pars['kfirst'] = False
pars['temperature_only'] = True
pars['base_first'] = 60.0
pars['nside_submap'] = 16
pars['write_map'] = False
pars['write_binmap'] = True
pars['write_matrix'] = False
pars['write_wcov'] = False
pars['write_hits'] = True
pars['kfilter'] = False
pars['info'] = 3
if args.madampar:
pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*')
comment = re.compile(r'^#.*')
with open(args.madampar, 'r') as f:
for line in f:
if not comment.match(line):
result = pat.match(line)
if result:
key, value = result.group(1), result.group(2)
if key in repeated_keys:
if key not in pars:
pars[key] = []
pars[key].append(value)
else:
pars[key] = value
# Command line parameters override the ones in the madam parameter file
if 'file_root' not in pars:
pars['file_root'] = 'madam'
if args.madam_prefix is not None:
pars['file_root'] = args.madam_prefix
sfreq = '{:03}'.format(args.freq)
if sfreq not in pars['file_root']:
pars['file_root'] += '_' + sfreq
try:
fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq]
except Exception:
fsample = 180.3737
pars['fsample'] = fsample
pars['path_output'] = args.out
pars = comm.comm_world.bcast(pars, root=0)
madam_mcmode = True
if 'nsubchunk' in pars and int(pars['nsubchunk']) > 1:
madam_mcmode = False
if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO':
# We split MPI_COMM_WORLD into single process groups, each of
# which is assigned one or more observations (rings)
comm = toast.Comm(groupsize=1)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
for iobs, tod in enumerate(tods):
if args.noisefile != 'RIMO' or args.noisefile_simu != 'RIMO':
# Use a toast helper method to optimally distribute rings between
# processes.
dist = distribute_discrete(tod.ringsizes, comm.world_size)
my_first_ring, my_n_ring = dist[comm.world_rank]
for my_ring in range(my_first_ring, my_first_ring + my_n_ring):
ringtod = tp.Exchange.from_tod(
tod,
my_ring,
comm.comm_group,
noisefile=args.noisefile,
noisefile_simu=args.noisefile_simu)
ob = {}
ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring)
ob['id'] = ringtod.globalfirst_ring
ob['tod'] = ringtod
ob['intervals'] = ringtod.valid_intervals
ob['baselines'] = None
ob['noise'] = ringtod.noise
ob['noise_simu'] = ringtod.noise_simu
data.obs.append(ob)
else:
ob = {}
ob['name'] = 'observation{:04}'.format(iobs)
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
ob['noise_simu'] = tod.noise
data.obs.append(ob)
rimo = tods[0].rimo
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Metadata queries")
# Always read the signal and flags, even if the signal is later
# overwritten. There is no overhead for the signal because it is
# interlaced with the flags.
tod_name = 'signal'
timestamps_name = 'timestamps'
flags_name = 'flags'
common_flags_name = 'common_flags'
reader = tp.OpInputPlanck(signal_name=tod_name,
flags_name=flags_name,
timestamps_name=timestamps_name,
commonflags_name=common_flags_name)
if comm.world_rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Read")
# Clear the signal if we don't need it
if not args.read_eff:
eraser = tp.OpCacheMath(in1=tod_name,
in2=0,
multiply=True,
out=tod_name)
if comm.world_rank == 0:
print('Erasing TOD', flush=True)
eraser.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Erase")
# Optionally flag bad intervals
if args.bad_intervals is not None:
flagger = tp.OpBadIntervals(path=args.bad_intervals)
flagger.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Apply {}".format(args.bad_intervals))
# Now read an optional second TOD to add with the first
if args.effdir2 is not None:
# Read the extra TOD and add it to the first one
reader = tp.OpInputPlanck(signal_name='tod2',
flags_name=None,
timestamps_name=None,
commonflags_name=None,
effdir=args.effdir2)
if comm.world_rank == 0:
print('Reading extra TOD from {}'.format(args.effdir2), flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
print("Reading took {:.3f} s".format(elapsed), flush=True)
adder = tp.OpCacheMath(in1=tod_name,
in2='signal2',
add=True,
out=tod_name)
if comm.world_rank == 0:
print('Adding TODs', flush=True)
adder.exec(data)
# Erase the extra cache object
for ob in data.obs:
tod = ob['tod']
tod.cache.clear('signal2_.*')
if args.effdir_fsl is not None:
# Read the straylight signal into the tod cache under
# "fsl_<detector>"
reader = tp.OpInputPlanck(signal_name='fsl',
flags_name=None,
timestamps_name=None,
commonflags_name=None,
effdir=args.effdir_fsl)
if comm.world_rank == 0:
print('Reading straylight signal from {}'.format(args.effdir_fsl),
flush=True)
reader.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Read FSL")
do_fsl = True
else:
do_fsl = False
# make a planck Healpix pointing matrix
mode = 'IQU'
if pars['temperature_only'] == 'T':
mode = 'I'
if args.nside is None:
if 'nside_map' in pars:
nside = int(pars['nside_map'])
else:
raise RuntimeError(
'Nside must be set either in the Madam parameter file or on '
'the command line')
else:
nside = args.nside
pars['nside_map'] = nside
if 'nside_cross' not in pars or pars['nside_cross'] > pars['nside_map']:
pars['nside_cross'] = pars['nside_map']
do_dipole = args.dipole or args.solsys_dipole or args.orbital_dipole
pointing = tp.OpPointingPlanck(nside=nside,
mode=mode,
RIMO=rimo,
margin=0,
apply_flags=True,
keep_vel=do_dipole,
keep_pos=False,
keep_phase=False,
keep_quats=do_dipole)
pointing.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Pointing Matrix")
flags_name = 'flags'
common_flags_name = 'common_flags'
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
net = tod.rimo[d].net
fsample = tod.rimo[d].fsample
detweights[d] = 1.0 / (fsample * net * net)
if args.debug:
with open("debug_planck_exchange_madam.txt", "w") as f:
data.info(f)
if do_dipole:
# Simulate the dipole
if args.dipole:
dipomode = 'total'
elif args.solsys_dipole:
dipomode = 'solsys'
else:
dipomode = 'orbital'
dipo = tp.OpDipolePlanck(args.freq,
solsys_speed=args.solsys_speed,
solsys_glon=args.solsys_glon,
solsys_glat=args.solsys_glat,
mode=dipomode,
output='dipole',
keep_quats=False)
dipo.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("Dipole")
# Loop over Monte Carlos
madam = None
for mc in range(args.MC_start, args.MC_start + args.MC_count):
out = "{}/{:05d}".format(args.out, mc)
if comm.world_rank == 0:
if not os.path.isdir(out):
os.makedirs(out)
# clear all noise data from the cache, so that we can generate
# new noise timestreams.
for ob in data.obs:
ob['tod'].cache.clear("noise_.*")
tod_name = 'signal'
if do_dipole:
adder = tp.OpCacheMath(in1=tod_name,
in2='dipole',
add=True,
out='noise')
adder.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Add dipole".format(mc))
tod_name = 'noise'
# Simulate noise
if not args.skip_noise:
tod_name = 'noise'
nse = toast.tod.OpSimNoise(out=tod_name,
realization=mc,
component=0,
noise='noise_simu',
rate=fsample)
if comm.world_rank == 0:
print('Simulating noise from {}'.format(args.noisefile_simu),
flush=True)
nse.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Noise simulation".format(mc))
# If we didn't add the dipole, we need to add the input
# signal with the noise we just simulated
if args.read_eff and not do_dipole:
adder = tp.OpCacheMath(in1=tod_name,
in2='signal',
add=True,
out=tod_name)
adder.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Add input signal".format(mc))
# Make rings
if args.make_rings:
ringmaker = tp.OpRingMaker(args.nside_ring,
nside,
signal=tod_name,
fileroot=args.ring_root,
out=out,
commonmask=args.obtmask,
detmask=args.flagmask)
ringmaker.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Ringmaking".format(mc))
# Apply calibration errors
if args.decalibrate is not None:
fn = args.decalibrate
try:
fn = fn.format(mc)
except Exception:
pass
if comm.world_rank == 0:
print('Decalibrating with {}'.format(fn), flush=True)
decalibrator = tp.OpCalibPlanck(signal_in=tod_name,
signal_out='noise',
file_gain=fn,
decalibrate=True)
decalibrator.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Decalibrate".format(mc))
tod_name = 'noise'
if args.calibrate is not None:
fn = args.calibrate
try:
fn = fn.format(mc)
except Exception:
pass
if comm.world_rank == 0:
print('Calibrating with {}'.format(fn), flush=True)
calibrator = tp.OpCalibPlanck(signal_in=tod_name,
signal_out='noise',
file_gain=fn)
calibrator.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Calibrate".format(mc))
tod_name = 'noise'
# Subtract the dipole and straylight
if do_dipole:
subtractor = tp.OpCacheMath(in1=tod_name,
in2='dipole',
subtract=True,
out='noise')
subtractor.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Subtract dipole".format(mc))
tod_name = 'noise'
if do_fsl:
subtractor = tp.OpCacheMath(in1=tod_name,
in2='fsl',
subtract=True,
out='noise')
subtractor.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Subtract straylight".format(mc))
tod_name = 'noise'
# Make the map
if not args.skip_madam:
# Make maps
if madam is None:
try:
madam = toast.todmap.OpMadam(params=pars,
detweights=detweights,
purge_tod=True,
name=tod_name,
apply_flags=False,
name_out=None,
noise='noise',
mcmode=madam_mcmode,
translate_timestamps=False)
except Exception as e:
raise Exception(
'{:4} : ERROR: failed to initialize Madam: '
'{}'.format(comm.world_rank, e))
madam.params['path_output'] = out
madam.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.world_rank == 0:
timer.report_clear("MC {}: Mapmaking".format(mc))
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
if cworld.rank == 0:
print("Computing binned map took {:.3f} s".format(elapsed), flush=True)
start = stop
if cworld.rank == 0:
print("Writing binned map ...", flush=True)
zmap.write_healpix_fits(os.path.join(out, "binned.fits"))
if cworld.rank == 0:
print("Binned map done", flush=True)
return
# Our toast communicator- use the default for now, which is one
# process group spanning all processes.
comm = toast.Comm()
if comm.world_rank == 0:
print("Simulating all detector properties...", flush=True)
# First, get the list of detectors we want to use
# (Eventually we would load this from disk. Here we simulate it.)
hw = get_example()
dets = sim_telescope_detectors(hw, "LAT")
hw.data["detectors"] = dets
if comm.world_rank == 0:
print("Selecting detectors...", flush=True)
# Downselect to just 10 pixels on one wafer
#small_hw = hw.select(match={"wafer_slot": "41", "pixel": "00."})
#small_hw = hw.select(match={"wafer_slot": "41"})
small_hw = hw.select(match={"wafer_slot": "40"})
def parse_arguments(comm):
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"and map astrophysical signal.",
fromfile_prefix_chars='@')
parser.add_argument('--groupsize',
required=False,
type=np.int,
help='Size of a process group assigned to a CES')
parser.add_argument('--timezone',
required=False,
type=np.int,
default=0,
help='Offset to apply to MJD to separate days [hours]')
parser.add_argument('--coord',
required=False,
default='C',
help='Sky coordinate system [C,E,G]')
parser.add_argument('--schedule',
required=True,
help='CES schedule file from toast_ground_schedule.py')
parser.add_argument('--samplerate',
required=False,
default=100.0,
type=np.float,
help='Detector sample rate (Hz)')
parser.add_argument('--scanrate',
required=False,
default=1.0,
type=np.float,
help='Scanning rate [deg / s]')
parser.add_argument('--scan_accel',
required=False,
default=1.0,
type=np.float,
help='Scanning rate change [deg / s^2]')
parser.add_argument('--sun_angle_min',
required=False,
default=30.0,
type=np.float,
help='Minimum azimuthal distance between the Sun and '
'the bore sight [deg]')
parser.add_argument('--polyorder',
required=False,
type=np.int,
help='Polynomial order for the polyfilter')
parser.add_argument('--wbin_ground',
required=False,
type=np.float,
help='Ground template bin width [degrees]')
parser.add_argument('--gain_sigma',
required=False,
type=np.float,
help='Gain error distribution')
parser.add_argument('--hwprpm',
required=False,
default=0.0,
type=np.float,
help='The rate (in RPM) of the HWP rotation')
parser.add_argument('--hwpstep',
required=False,
default=None,
help='For stepped HWP, the angle in degrees '
'of each step')
parser.add_argument('--hwpsteptime',
required=False,
default=0.0,
type=np.float,
help='For stepped HWP, the the time in seconds '
'between steps')
parser.add_argument('--input_map',
required=False,
help='Input map for signal')
parser.add_argument('--skip_bin',
required=False,
default=False,
action='store_true',
help='Disable binning the map.')
parser.add_argument('--skip_hits',
required=False,
default=False,
action='store_true',
help='Do not save the 3x3 matrices and hitmaps')
parser.add_argument('--fp_radius',
required=False,
default=1,
type=np.float,
help='Focal plane radius assumed in the atmospheric '
'simulation.')
parser.add_argument('--outdir',
required=False,
default='out',
help='Output directory')
parser.add_argument('--zip',
required=False,
default=False,
action='store_true',
help='Compress the output fits files')
parser.add_argument('--debug',
required=False,
default=False,
action='store_true',
help='Write diagnostics')
parser.add_argument('--flush',
required=False,
default=False,
action='store_true',
help='Flush every print statement.')
parser.add_argument('--nside',
required=False,
default=512,
type=np.int,
help='Healpix NSIDE')
parser.add_argument('--madam_iter_max',
required=False,
default=1000,
type=np.int,
help='Maximum number of CG iterations in Madam')
parser.add_argument('--madam_baseline_length',
required=False,
default=10000.0,
type=np.float,
help='Destriping baseline length (seconds)')
parser.add_argument('--madam_baseline_order',
required=False,
default=0,
type=np.int,
help='Destriping baseline polynomial order')
parser.add_argument('--madam_noisefilter',
required=False,
default=False,
action='store_true',
help='Destripe with the noise filter enabled')
parser.add_argument('--madam',
required=False,
default=False,
action='store_true',
help='If specified, use libmadam for map-making')
parser.add_argument('--madampar',
required=False,
default=None,
help='Madam parameter file')
parser.add_argument('--madam_allreduce',
required=False,
default=False,
action='store_true',
help='Use allreduce communication in Madam')
parser.add_argument('--common_flag_mask',
required=False,
default=1,
type=np.uint8,
help='Common flag mask')
parser.add_argument(
'--fp',
required=False,
default=None,
help='Pickle file containing a dictionary of detector '
'properties. The keys of this dict are the detector '
'names, and each value is also a dictionary with keys '
'"quat" (4 element ndarray), "fwhm" (float, arcmin), '
'"fknee" (float, Hz), "alpha" (float), and '
'"NET" (float). For optional plotting, the key "color"'
' can specify a valid matplotlib color string.')
parser.add_argument('--tidas',
required=False,
default=None,
help='Output TIDAS export path')
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
if comm.comm_world.rank == 0:
print('\nAll parameters:')
print(args, flush=args.flush)
print('')
if args.groupsize:
comm = toast.Comm(groupsize=args.groupsize)
if comm.comm_world.rank == 0:
if not os.path.isdir(args.outdir):
try:
os.makedirs(args.outdir)
except FileExistsError:
pass
return args, comm
def main():
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
print('Running with {} processes at {}'.format(comm.comm_world.size,
str(datetime.now())),
flush=True)
global_timer = timing.simple_timer('Total time')
global_timer.start()
args, comm = parse_arguments(comm)
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
# Load and broadcast the schedule file
site, all_ces = load_schedule(args, comm)
# load or simulate the focalplane
fp, detweights = load_fp(args, comm)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data = create_observations(args, comm, fp, all_ces, site)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
expand_pointing(args, comm, data)
# Prepare auxiliary information for distributed map objects
localpix, localsm, subnpix = get_submaps(args, comm, data)
# Scan input map
signalname = scan_signal(args, comm, data, localsm, subnpix)
# Set up objects to take copies of the TOD at appropriate times
signalname_madam, sigcopy_madam, sigclear \
= setup_sigcopy(args, comm, signalname)
common_flag_name = None
flag_name = None
invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name \
= build_npp(args, comm, data, localsm, subnpix, detweights,
flag_name, common_flag_name)
madampars = setup_madam(args, comm)
output_tidas(args, comm, data, signalname, common_flag_name, flag_name)
outpath = setup_output(args, comm)
# Make a copy of the signal for Madam
copy_signal_madam(args, comm, data, sigcopy_madam)
# Bin unprocessed signal for reference
bin_maps(args, comm, data, 'binned', zmap, invnpp, zmap_group,
invnpp_group, detweights, signalname, flag_name, common_flag_name,
outpath)
# Filter signal
apply_polyfilter(args, comm, data, signalname)
apply_groundfilter(args, comm, data, signalname)
# Bin the filtered signal
if args.polyorder or args.wbin_ground:
bin_maps(args, comm, data, 'filtered', zmap, invnpp, zmap_group,
invnpp_group, detweights, signalname, flag_name,
common_flag_name, outpath)
clear_signal(args, comm, data, sigclear)
# Now run Madam on the unprocessed copy of the signal
apply_madam(args, comm, data, madampars, outpath, detweights,
signalname_madam, flag_name, common_flag_name)
comm.comm_world.barrier()
global_timer.stop()
if comm.comm_world.rank == 0:
global_timer.report()
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
print('Running with {} processes at {}'
''.format(procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Planck Ringset making',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--nosingle',
dest='nosingle',
required=False,
default=False,
action='store_true',
help='Do not compute single detector PSDs')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory '
'for pointing')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--skymask', required=False, help='Pixel mask file')
parser.add_argument('--skymap', required=False, help='Sky estimate file')
parser.add_argument('--skypol',
dest='skypol',
required=False,
default=False,
action='store_true',
help='Sky estimate is polarized')
parser.add_argument('--no_spin_harmonics',
dest='no_spin_harmonics',
required=False,
default=False,
action='store_true',
help='Do not include PSD bins with spin harmonics')
parser.add_argument('--calibrate',
required=False,
help='Path to calibration file to calibrate with.')
parser.add_argument('--calibrate_signal_estimate',
dest='calibrate_signal_estimate',
required=False,
default=False,
action='store_true',
help='Calibrate '
'the signal estimate using linear regression.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
type=np.int,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
type=np.int,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
type=np.int,
help='First ring to use')
parser.add_argument('--ringlast',
required=False,
default=None,
type=np.int,
help='Last ring to use')
parser.add_argument('--obtfirst',
required=False,
default=None,
type=np.float,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
type=np.float,
help='Last OBT to use')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--nbin_psd',
required=False,
default=1000,
type=np.int,
help='Number of logarithmically '
'distributed spectral bins to write.')
parser.add_argument('--lagmax',
required=False,
default=100000,
type=np.int,
help='Maximum lag to evaluate for the '
'autocovariance function [samples].')
parser.add_argument('--stationary_period',
required=False,
default=86400.,
type=np.float,
help='Length of a stationary interval [seconds].')
# Dipole parameters
dipogroup = parser.add_mutually_exclusive_group()
dipogroup.add_argument('--dipole',
dest='dipole',
required=False,
default=False,
action='store_true',
help='Simulate dipole')
dipogroup.add_argument('--solsys_dipole',
dest='solsys_dipole',
required=False,
default=False,
action='store_true',
help='Simulate solar system dipole')
dipogroup.add_argument('--orbital_dipole',
dest='orbital_dipole',
required=False,
default=False,
action='store_true',
help='Simulate orbital dipole')
# Extra filter
parser.add_argument('--filterfile',
required=False,
help='Extra filter file.')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.comm_world.rank == 0:
print('All parameters:')
print(args, flush=True)
timer = Timer()
timer.start()
odrange = None
if args.odfirst is not None and args.odlast is not None:
odrange = (args.odfirst, args.odlast)
ringrange = None
if args.ringfirst is not None and args.ringlast is not None:
ringrange = (args.ringfirst, args.ringlast)
obtrange = None
if args.obtfirst is not None and args.obtlast is not None:
obtrange = (args.obtfirst, args.obtlast)
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
if args.nosingle and len(detectors) != 2:
raise RuntimeError('You cannot skip the single detectors PSDs '
'without multiple detectors.')
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
# Make output directory
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.mkdir(args.out)
# create the TOD for this observation
tod = tp.Exchange(
comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
do_eff_cache=False,
)
rimo = tod.rimo
ob = {}
ob['name'] = 'mission'
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
data.obs.append(ob)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Metadata queries")
# Read the signal
tod_name = 'signal'
flags_name = 'flags'
reader = tp.OpInputPlanck(signal_name=tod_name, flags_name=flags_name)
if comm.comm_world.rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Reading")
if args.calibrate is not None:
fn = args.calibrate
if comm.comm_world.rank == 0:
print('Calibrating with {}'.format(fn), flush=True)
calibrator = tp.OpCalibPlanck(signal_in=tod_name,
signal_out=tod_name,
file_gain=fn)
calibrator.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Calibrate")
# Optionally subtract the dipole
do_dipole = (args.dipole or args.solsys_dipole or args.orbital_dipole)
if do_dipole:
if args.dipole:
dipomode = 'total'
elif args.solsys_dipole:
dipomode = 'solsys'
else:
dipomode = 'orbital'
dipo = tp.OpDipolePlanck(args.freq,
mode=dipomode,
output='dipole',
keep_quats=True)
dipo.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Dipole")
subtractor = tp.OpCacheMath(in1=tod_name,
in2='dipole',
subtract=True,
out=tod_name)
if comm.comm_world.rank == 0:
print('Subtracting dipole', flush=True)
subtractor.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Dipole subtraction")
# Optionally filter the signal
apply_filter(args, data)
timer.clear()
# Estimate noise
noise_estimator = tp.OpNoiseEstim(
signal=tod_name,
flags=flags_name,
detmask=args.flagmask,
commonmask=args.obtmask,
maskfile=args.skymask,
mapfile=args.skymap,
out=args.out,
rimo=rimo,
pol=args.skypol,
nbin_psd=args.nbin_psd,
lagmax=args.lagmax,
stationary_period=args.stationary_period,
nosingle=args.nosingle,
no_spin_harmonics=args.no_spin_harmonics,
calibrate_signal_estimate=args.calibrate_signal_estimate)
noise_estimator.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Noise estimation")
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
# We are going to group our processes in a single group. This is fine
# if we have fewer processes than detectors. Otherwise we should group
# them in a reasonable size that is smaller than the number of detectors
# and which divides evenly into the total number of processes.
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=MPI.COMM_WORLD.size)
# Make a fake focalplane. Plot it just for fun (don't waste time on this
# for very large runs though).
fp = fake_focalplane()
if comm.comm_world.rank == 0:
outfile = "custom_example_focalplane.png"
set_backend()
tt.plot_focalplane(fp, 6.0, 6.0, outfile)
# Read in 2 boresight files
borefiles = [
"../data/custom_example_boresight_1.txt",
"../data/custom_example_boresight_2.txt"
]
# Set up the distributed data
rate = 100.0
data = create_observations(comm, rate, fp, borefiles)
# Configure the healpix pixelization we will use for map-making and
# also the "submap" resolution, which sets granularity of the locally
# stored pieces of the sky.
map_nside = 512
map_npix = 12 * map_nside**2
sub_nside = 4
sub_npix = 12 * sub_nside**2
# Compute a pointing matrix with healpix pixels and weights.
pointing = tt.OpPointingHpix(nside=map_nside, nest=True, mode="IQU",
pixels="pixels", weights="weights")
pointing.exec(data)
# Compute the locally hit submaps
local_submaps = pixel_dist(data, sub_npix)
# Sources of simulated data: scan from a symmetric beam convolved sky
# and then add some simulated noise.
signalmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3,
dtype=np.float64, submap=sub_npix, local=local_submaps)
signalmap.read_healpix_fits("../data/custom_example_sky.fits")
scanmap = tt.OpSimScan(distmap=signalmap, pixels='pixels',
weights='weights', out="sim")
scanmap.exec(data)
nse = tt.OpSimNoise(out="sim", realization=0)
nse.exec(data)
# Accumulate the hits and inverse diagonal pixel covariance, as well as the
# noise weighted map. Here we simply use inverse noise weighting.
detweights = {}
for d in fp.keys():
net = fp[d]["NET"]
detweights[d] = 1.0 / (rate * net * net)
invnpp = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=6,
dtype=np.float64, submap=sub_npix, local=local_submaps)
hits = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=1,
dtype=np.int64, submap=sub_npix, local=local_submaps)
zmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3,
dtype=np.float64, submap=sub_npix, local=local_submaps)
invnpp.data.fill(0.0)
hits.data.fill(0)
zmap.data.fill(0.0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits, zmap=zmap, name="sim")
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
zmap.allreduce()
# Write these products out
hits.write_healpix_fits("custom_example_hits.fits")
invnpp.write_healpix_fits("custom_example_invnpp.fits")
zmap.write_healpix_fits("custom_example_zmap.fits")
# Invert the covariance and write
tm.covariance_invert(invnpp, 1.0e-3)
invnpp.write_healpix_fits("custom_example_npp.fits")
# Apply covariance to make the binned map
tm.covariance_apply(invnpp, zmap)
zmap.write_healpix_fits("custom_example_binned.fits")
MPI.Finalize()
return
def setUp(self):
fixture_name = os.path.splitext(os.path.basename(__file__))[0]
if not toast_available:
print(
"toast cannot be imported ({})- skipping unit test".format(
toast_import_error),
flush=True,
)
return
self.comm, self.procs, self.rank = get_world()
self.outdir = create_outdir(fixture_name, comm=self.comm)
toastcomm = toast.Comm(world=self.comm)
self.data = toast.Data(toastcomm)
# Focalplane
hwfull = get_example()
dets = sim_telescope_detectors(hwfull, "SAT4")
hwfull.data["detectors"] = dets
hw = hwfull.select(match={
"wafer_slot": "w42",
"band": "SAT_f030",
"pixel": "00[01]"
})
# print(hw.data["detectors"], flush=True)
detquats = {k: v["quat"] for k, v in hw.data["detectors"].items()}
# Samples per observation
self.totsamp = 10000
# Scan properties
self.site_lon = '-67:47:10'
self.site_lat = '-22:57:30'
self.site_alt = 5200.
self.coord = 'C'
self.azmin = 45
self.azmax = 55
self.el = 60
self.scanrate = 1.0
self.scan_accel = 0.1
self.CES_start = None
# Noise properties
self.rate = 100.0
self.NET = 1e-3 # 1 mK NET
self.epsilon = 0.0
self.fmin = 1.0e-5
self.alpha = 1.0
self.fknee = 0.05
for ob in range(3):
ftime = (self.totsamp / self.rate) * ob + 1564015655.88
tod = TODGround(self.data.comm.comm_group,
detquats,
self.totsamp,
detranks=self.data.comm.group_size,
firsttime=ftime,
rate=self.rate,
site_lon=self.site_lon,
site_lat=self.site_lat,
site_alt=self.site_alt,
azmin=self.azmin,
azmax=self.azmax,
el=self.el,
coord=self.coord,
scanrate=self.scanrate,
scan_accel=self.scan_accel,
CES_start=self.CES_start)
# Analytic noise model
detnames = list(detquats.keys())
drate = {x: self.rate for x in detnames}
dfmin = {x: self.fmin for x in detnames}
dfknee = {x: self.fknee for x in detnames}
dalpha = {x: self.alpha for x in detnames}
dnet = {x: self.NET for x in detnames}
nse = AnalyticNoise(rate=drate,
fmin=dfmin,
detectors=detnames,
fknee=dfknee,
alpha=dalpha,
NET=dnet)
# Single observation
obs = dict()
obs["tod"] = tod
obs["noise"] = nse
obs["id"] = 12345
obs["intervals"] = tod.subscans
obs["site"] = "SimonsObs"
obs["telescope"] = "SAT4"
obs["site_id"] = 1
obs["telescope_id"] = 4
obs["fpradius"] = 5.0
obs["start_time"] = ftime
obs["altitude"] = self.site_alt
obs["name"] = "test_{:02}".format(ob)
# Add a focalplane dictionary with just the detector index
focalplane = {}
for idet, det in enumerate(detnames):
focalplane[det] = {"index": idet}
obs["focalplane"] = focalplane
# Add the observation to the dataset
self.data.obs.append(obs)
nse = toast.tod.OpSimNoise(out="signal", realization=0)
nse.exec(self.data)
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(
description='Accumulate polarization moments',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--nside',
required=False,
type=np.int,
default=512,
help='Map resolution')
parser.add_argument('--smax',
required=False,
type=np.int,
default=6,
help='Highest moment')
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help='Write data distribution info to file')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_in_diode0',
required=False,
default=None,
help='Input Exchange Format File directory, '
'LFI diode 0')
parser.add_argument('--effdir_in_diode1',
required=False,
default=None,
help='Input Exchange Format File directory, '
'LFI diode 1')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory '
'for pointing')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Pointing flag mask')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
type=np.int,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
type=np.int,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use (can be a list)')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use (can be a list)')
parser.add_argument('--obtfirst',
required=False,
default=None,
type=np.float,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
type=np.float,
help='Last OBT to use')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--prefix',
required=False,
default='spins',
help='map prefix')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.world_rank == 0:
print('All parameters:')
print(args, flush=True)
timer = Timer()
timer.start()
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# create the TOD for this observation
tods = []
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
tods.append(
tp.Exchange(comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_in_diode0=args.effdir_in_diode0,
effdir_in_diode1=args.effdir_in_diode1,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
pntflagmask=args.pntflagmask,
do_eff_cache=False,
noisefile='RIMO'))
rimo = tods[0].rimo
# Make output directory
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.makedirs(args.out)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
for iobs, tod in enumerate(tods):
ob = {}
ob['name'] = 'observation{:04}'.format(iobs)
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
data.obs.append(ob)
if mpiworld is not None:
mpiworld.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Metadata queries")
# Accumulate and save the moment maps
polmoments = tp.OpPolMomentsPlanck(nside=args.nside,
RIMO=rimo,
margin=0,
keep_vel=False,
keep_pos=False,
keep_phase=False,
keep_quats=False,
smax=args.smax,
prefix=os.path.join(
args.out, args.prefix))
polmoments.exec(data)
if mpiworld is not None:
mpiworld.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Accumulate moment maps")
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description="Simulate satellite "
"boresight pointing and make a noise map.", fromfile_prefix_chars="@" )
parser.add_argument( "--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute observations" )
parser.add_argument( "--samplerate", required=False, type=float,
default=40.0, help="Detector sample rate (Hz)" )
parser.add_argument( "--starttime", required=False, type=float,
default=0.0, help="The overall start time of the simulation" )
parser.add_argument( "--spinperiod", required=False, type=float,
default=10.0, help="The period (in minutes) of the rotation about the "
"spin axis" )
parser.add_argument( "--spinangle", required=False, type=float,
default=30.0, help="The opening angle (in degrees) of the boresight "
"from the spin axis" )
parser.add_argument( "--precperiod", required=False, type=float,
default=50.0, help="The period (in minutes) of the rotation about the "
"precession axis" )
parser.add_argument( "--precangle", required=False, type=float,
default=65.0, help="The opening angle (in degrees) of the spin axis "
"from the precession axis" )
parser.add_argument( "--hwprpm", required=False, type=float,
default=0.0, help="The rate (in RPM) of the HWP rotation" )
parser.add_argument( "--hwpstep", required=False, default=None,
help="For stepped HWP, the angle in degrees of each step" )
parser.add_argument( "--hwpsteptime", required=False, type=float,
default=0.0, help="For stepped HWP, the the time in seconds between "
"steps" )
parser.add_argument( "--obs", required=False, type=float, default=1.0,
help="Number of hours in one science observation" )
parser.add_argument( "--gap", required=False, type=float, default=0.0,
help="Cooler cycle time in hours between science obs" )
parser.add_argument( "--numobs", required=False, type=int, default=1,
help="Number of complete observations" )
parser.add_argument( "--outdir", required=False, default="out",
help="Output directory" )
parser.add_argument( "--debug", required=False, default=False,
action="store_true", help="Write diagnostics" )
parser.add_argument( "--nside", required=False, type=int, default=64,
help="Healpix NSIDE" )
parser.add_argument( "--subnside", required=False, type=int, default=4,
help="Distributed pixel sub-map NSIDE" )
parser.add_argument( "--baseline", required=False, type=float,
default=60.0, help="Destriping baseline length (seconds)" )
parser.add_argument( "--noisefilter", required=False, default=False,
action="store_true", help="Destripe with the noise filter enabled" )
parser.add_argument( "--madam", required=False, default=False,
action="store_true", help="If specified, use libmadam for map-making" )
parser.add_argument( "--madampar", required=False, default=None,
help="Madam parameter file" )
parser.add_argument('--flush',
required=False, default=False, action='store_true',
help='Flush every print statement.')
parser.add_argument( "--MC_start", required=False, type=int, default=0,
help="First Monte Carlo noise realization" )
parser.add_argument( "--MC_count", required=False, type=int, default=1,
help="Number of Monte Carlo noise realizations" )
parser.add_argument( "--fp", required=False, default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
"a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" "
"(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" "
"(float). For optional plotting, the key \"color\" can specify a "
"valid matplotlib color string." )
parser.add_argument('--tidas',
required=False, default=None,
help='Output TIDAS export path')
parser.add_argument('--input_map', required=False,
help='Input map for signal')
parser.add_argument('--input_pysm_model', required=False,
help='Comma separated models for on-the-fly PySM '
'simulation, e.g. s3,d6,f1,a2"')
parser.add_argument('--apply_beam', required=False, action='store_true',
help='Apply beam convolution to input map with gaussian '
'beam parameters defined in focalplane')
parser.add_argument('--input_dipole', required=False,
help='Simulate dipole, possible values are '
'total, orbital, solar')
parser.add_argument('--input_dipole_solar_speed_kms', required=False,
help='Solar system speed [km/s]', type=float,
default=369.0)
parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False,
help='Solar system speed galactic latitude [degrees]',
type=float, default=48.26)
parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False,
help='Solar system speed galactic longitude[degrees]',
type=float, default=263.99)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# get options
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > args.nside:
subnside = args.nside
subnpix = 12 * subnside * subnside
start = MPI.Wtime()
fp = None
# Load focalplane information
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.fp, "rb") as p:
fp = pickle.load(p)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Create focalplane: {:.2f} seconds".format(stop-start),
flush=True)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
tt.plot_focalplane(fp, 10.0, 10.0, outfile)
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
if groupsize > len(fp.keys()):
if comm.comm_world.rank == 0:
print("process group is too large for the number of detectors",
flush=True)
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(fp.keys())
detquats = {}
detindx = None
if "index" in fp[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = fp[d]["quat"]
if detindx is not None:
detindx[d] = fp[d]["index"]
# Distribute the observations uniformly
groupdist = toast.distribute_uniform(args.numobs, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = tt.regular_intervals(args.numobs, args.starttime, 0,
args.samplerate, 3600*args.obs, 3600*args.gap)
# Create the noise model used for all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]["fmin"]
fknee[d] = fp[d]["fknee"]
alpha[d] = fp[d]["alpha"]
NET[d] = fp[d]["NET"]
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
mem_counter = tt.OpMemoryCounter()
# The distributed timestream data
data = toast.Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = tt.TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
firsttime=obsrange[ob].start,
rate=args.samplerate,
spinperiod=args.spinperiod,
spinangle=args.spinangle,
precperiod=args.precperiod,
precangle=args.precangle,
detindx=detindx,
detranks=comm.group_size
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Read parameters, compute data distribution: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = tt.slew_precession_axis(nsim=tod.local_samples[1],
firstsamp=obsoffset, samplerate=args.samplerate,
degday=degday)
tod.set_prec_axis(qprec=precquat)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Construct boresight pointing: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# make a Healpix pointing matrix.
pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime)
pointing.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Pointing generation took {:.3f} s".format(elapsed), flush=True)
start = stop
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = "signal"
if args.input_pysm_model:
simulate_sky_signal(args, comm, data, mem_counter,
[fp], subnpix, localsm, signalname=signalname)
if args.input_dipole:
print("Simulating dipole")
op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole,
solar_speed=args.input_dipole_solar_speed_kms,
solar_gal_lat=args.input_dipole_solar_gal_lat_deg,
solar_gal_lon=args.input_dipole_solar_gal_lon_deg,
out=signalname,
keep_quats=True,
keep_vel=False,
subtract=False,
coord="G",
freq=0, # we could use frequency for quadrupole correction
flag_mask=255, common_flag_mask=255)
op_sim_dipole.exec(data)
# Mapmaking. For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in detectors:
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.samplerate * net * net)
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing N_pp took {:.3f} s".format(elapsed),
flush=True)
start = stop
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Dumping debug data distribution took "
"{:.3f} s".format(elapsed), flush=True)
start = stop
mcstart = start
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# create output directory for this realization
outpath = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(outpath):
os.makedirs(outpath)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Creating output dir {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# clear all signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
if mc == firstmc:
# For the first realization, optionally export the
# timestream data to a TIDAS volume.
if args.tidas is not None:
from toast.tod.tidas import OpTidasExport
tidas_path = os.path.abspath(args.tidas)
export = OpTidasExport(tidas_path, name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Noise simulation {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal",
detweights=detweights)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Building noise weighted map {:04d} took {:.3f} s".format(
mc, elapsed), flush=True)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Computing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.write_healpix_fits(os.path.join(outpath, "binned.fits"))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Writing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
elapsed = stop - mcstart
if comm.comm_world.rank == 0:
print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed),
flush=True)
start = stop
else:
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = cross
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = args.nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# clear all total signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Noise simulation took {:.3f} s".format(elapsed),
flush=True)
start = stop
# create output directory for this realization
pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(pars["path_output"]):
os.makedirs(pars["path_output"])
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open(os.path.join(pars["path_output"],
"distdata.txt"), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
madam = tm.OpMadam(params=pars, detweights=detweights,
name="tot_signal")
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Mapmaking took {:.3f} s".format(elapsed), flush=True)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main(parameter_file, outdir):
'The main function'
# Set up the logging system
logformat = '[%(asctime)s %(levelname)s] %(message)s'
log.basicConfig(level=log.INFO,
format=logformat)
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
log.info("running with %d processes", comm.comm_world.size)
# Read the parameter file
conf = None
if comm.comm_world.rank == 0:
configuration_parser = ConfigParser()
configuration_parser.read(parameter_file)
conf = build_configuration_obj(configuration_parser)
log.info('configuration read from file "%s"', parameter_file)
conf = comm.comm_world.bcast(conf, root=0)
if outdir != '.':
if not os.path.isdir(outdir):
os.mkdir(outdir)
# construct the list of intervals which split the whole observation time
intervals = tt.regular_intervals(n=int(conf.num_obs),
start=0.0,
first=0,
rate=conf.sample_rate_Hz,
duration=3600*float(conf.hours_per_obs),
gap=3600*float(conf.gap_width_hours))
detquats = {} # type: Dict[str, Any]
for det in conf.detectors:
detquats[det.name] = det.quat
# Create the noise model for this observation
rate = {} # type: Dict[str, float]
fmin = {} # type: Dict[str, float]
fknee = {} # type: Dict[str, float]
alpha = {} # type: Dict[str, float]
net = {} # type: Dict[str, float]
for det in conf.detectors:
rate[det.name] = conf.sample_rate_Hz
fmin[det.name] = 1.0 / (3600.0 * conf.hours_per_obs * conf.num_obs)
fknee[det.name] = det.fknee_Hz
alpha[det.name] = det.alpha
net[det.name] = det.net
noise = tt.AnalyticNoise(rate=rate,
fmin=fmin,
detectors=[x.name for x in conf.detectors],
fknee=fknee,
alpha=alpha,
NET=net)
# The distributed time stream data
data = toast.Data(comm)
intsamp = None
if len(intervals) == 1:
intsamp = intervals[0].last - intervals[0].first + 1
else:
# include also the samples in the gap
intsamp = intervals[1].first - intervals[0].first
intchunk = int(intsamp / conf.chunks_per_obs)
distint = [] # type: List[int]
for _ in range(conf.chunks_per_obs - 1):
distint.append(intchunk)
distint.append(intsamp - (intchunk * (conf.chunks_per_obs - 1)))
distsizes = []
for _ in intervals:
distsizes.extend(distint)
totsamples = np.sum(distsizes)
# create the single TOD for this observation
tod = tt.TODSatellite(
mpicomm=comm.comm_group,
detectors=detquats,
samples=totsamples,
firsttime=0.0,
rate=conf.sample_rate_Hz,
spinperiod=conf.spin_period_min,
spinangle=conf.spin_angle_deg,
precperiod=conf.prec_period_min,
precangle=conf.prec_angle_deg,
sizes=distsizes)
# Create the (single) observation
ob = {} # type: Dict[str, Any]
ob['id'] = 0
ob['name'] = 'mission'
ob['tod'] = tod
ob['intervals'] = intervals
ob['noise'] = noise
data.obs.append(ob)
precquat = tt.slew_precession_axis(nsim=tod.local_samples[1],
firstsamp=tod.local_samples[0],
samplerate=conf.sample_rate_Hz,
degday=DEG_ANGLE_PER_DAY)
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
tod.set_prec_axis(qprec=precquat)
# simulate noise
nse = tt.OpSimNoise(out='noise', stream=0, realization=0)
nse.exec(data)
comm.comm_world.barrier()
# make a Healpix pointing matrix. By setting purge_pntg=True,
# we purge the detector quaternion pointing to save memory.
# If we ever change this pipeline in a way that needs this
# pointing at a later stage, we need to set this to False
# and run at higher concurrency.
pointing = tt.OpPointingHpix(nside=conf.nside,
nest=False,
mode='IQU',
hwprpm=conf.hwp_rpm,
hwpstep=None,
hwpsteptime=None)
pointing.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
log.info('reading map "%s"', conf.sky_map)
foreground_map = healpy.read_map(conf.sky_map,
field=(conf.i_column, conf.q_column, conf.u_column),
verbose=False)
for det in conf.detectors:
if comm.comm_world.rank == 0:
log.info('processing detector "%s"', det.name)
times = tod.read_times()
flags, glflags = tod.read_flags(detector=det.name)
pixels = tod.cache.reference('pixels_{0}'.format(det.name))
theta, phi = healpy.pix2ang(conf.nside, pixels)
vect = healpy.pix2vec(conf.nside, pixels)
weights = tod.cache.reference('weights_{0}'.format(det.name))
psi = 0.5 * np.arctan2(weights[:, 2], weights[:, 1])
psi += np.deg2rad(det.polarisation_angle_deg)
noise = tod.cache.reference('noise_{0}'.format(det.name))
# Estimate the total TOD using the formula
# P = I + Q cos 2psi + U sin 2psi
# where Q and U are expressed in the reference frame of the detector
sky_i, sky_q, sky_u = [healpy.get_interp_val(foreground_map[x], theta, phi)
for x in (0, 1, 2)]
det_q = sky_q * np.cos(2. * psi) + sky_u * np.sin(2. * psi)
det_u = -sky_q * np.sin(2. * psi) + sky_u * np.cos(2. * psi)
foreground_tod = sky_i + det_q * np.cos(2. * psi) + det_u * np.sin(2. * psi)
dipole_tod = dip.get_dipole_temperature(np.column_stack(vect).T)
total_tod = det.gain * (foreground_tod + dipole_tod + noise)
hdu = fits.BinTableHDU.from_columns([fits.Column(name='TIME',
format='D',
unit='s',
array=times),
fits.Column(name='THETA',
format='E',
unit='rad',
array=theta),
fits.Column(name='PHI',
format='E',
unit='rad',
array=phi),
fits.Column(name='PSI',
format='E',
unit='rad',
array=psi),
fits.Column(name='FGTOD',
format='E',
unit='K',
array=foreground_tod),
fits.Column(name='DIPTOD',
format='E',
unit='K',
array=dipole_tod),
fits.Column(name='NOISETOD',
format='E',
unit='K',
array=noise),
fits.Column(name='TOTALTOD',
format='E',
unit='V',
array=total_tod),
fits.Column(name='FLAGS',
format='I',
unit='',
array=flags),
fits.Column(name='GLFLAGS',
format='I',
unit='',
array=glflags)])
hdu.header['COMMENT'] = 'Angles are expressed in the Ecliptic system'
hdu.header['FIRSTT'] = (times[0], 'Time of the first sample [s]')
hdu.header['LASTT'] = (times[-1], 'Time of the last sample [s]')
hdu.header['GAIN'] = (det.gain, 'Detector gain [V/K]')
hdu.header['NAME'] = (det.name, 'Name of the detector')
hdu.header['VERSION'] = (__version__,
'Version of the code used to create '
'this file')
hdu.header['net'] = (det.net, 'net')
hdu.header['ALPHA'] = (det.alpha, 'Slope of the 1/f noise')
hdu.header['FKNEE'] = (det.fknee_Hz,
'Knee frequency of the 1/f noise [Hz]')
output_file = os.path.join(outdir,
('tod_{0}_mpi{1:04d}.fits'
.format(det.name, comm.comm_world.rank)))
# Save a copy of the parameter file into the primary HDU of the file,
# so that it will be always possible to know the value of the input parameters
# used to create it
prim_hdu = fits.PrimaryHDU(data=conf.param_file_contents)
hdu_list = fits.HDUList([prim_hdu, hdu])
hdu_list.writeto(output_file, clobber=True)
if comm.comm_world.rank == 0:
log.info('pointings have been created successfully')
def main():
# Our toast communicator- use the default for now, which is one
# process group spanning all processes.
comm = toast.Comm()
parser = argparse.ArgumentParser(
description="Make a Healpix map of a given observation ",
fromfile_prefix_chars="@",
)
parser.add_argument("obsdir", help="Input directory for all observations")
parser.add_argument("--obs",
required=False,
help="Comma-separated list of observations")
parser.add_argument(
"--nside",
required=False,
type=np.int,
default=1024,
help="Map resolution",
)
toast_tools.add_polyfilter_args(parser)
args = parser.parse_args()
print("Loading data from {}".format(args.obsdir), flush=True)
if args.obs is None:
paths = glob.glob(args.obsdir + "/*")
obs = [os.path.basename(path) for path in paths]
print("Found {} observations: {}".format(len(obs), obs))
else:
obs = []
for path in args.obs.split(","):
obs.append(path)
print("Mapping {} observations: {}".format(len(obs), obs))
if comm.world_rank == 0:
print("Loading data from disk...", flush=True)
# Load our selected data
data = load_data(args.obsdir, obs=obs, comm=comm)
# Everybody look at their data
my_world_rank = data.comm.world_rank
my_group_rank = data.comm.group_rank
for ob in data.obs:
tod = ob["tod"]
my_dets = tod.local_dets
my_first_samp, my_nsamp = tod.local_samples
msg = "proc {} with group rank {} has {} dets for samples {} - {}".format(
my_world_rank, my_group_rank, len(my_dets), my_first_samp,
my_first_samp + my_nsamp - 1)
print(msg, flush=True)
if comm.world_rank == 0:
# Plot some local data from the first observation
import matplotlib.pyplot as plt
ob = data.obs[0]
tod = ob["tod"]
boloname = tod.local_dets[0]
bolodata = tod.cache.reference("signal_{}".format(boloname))
fig = plt.figure()
plt.plot(np.arange(len(bolodata)), bolodata)
plt.savefig("bolo_{}.png".format(boloname))
del bolodata
# Construct a pointing matrix
pointing = toast.todmap.OpPointingHpix(nside=args.nside,
nest=True,
mode="IQU",
pixels="pixels",
weights="weights")
pointing.exec(data)
# Filter signal
if args.apply_polyfilter:
toast_tools.apply_polyfilter(args, comm, data, "signal")
# Bin the TOD to a map
mapmaker = toast.todmap.OpMapMaker(
nside=args.nside,
nnz=3,
name="signal",
pixels="pixels",
intervals=None,
baseline_length=None,
use_noise_prior=False,
outdir=".",
)
mapmaker.exec(data)
# Plot
if comm.world_rank == 0:
# Plot the hit map
import healpy as hp
import matplotlib.pyplot as plt
plt.figure(figsize=[18, 12])
hits = hp.read_map("hits.fits")
imax = np.argmax(hits)
lon, lat = hp.pix2ang(hp.get_nside(hits), imax, lonlat=True)
hits[hits == 0] = hp.UNSEEN
hp.mollview(hits, xsize=1200, sub=[2, 2, 1], title="hits")
hp.gnomview(
hits,
rot=(lon, lat),
xsize=800,
reso=1.0,
sub=[2, 2, 2],
title="hits, lon={:.2f}deg, lat={:.2f}deg".format(lon, lat),
)
binned = hp.read_map("binned.fits")
binned[binned == 0] = hp.UNSEEN
plt.savefig("binned.png")
hp.mollview(binned, xsize=1200, sub=[2, 2, 3], title="binned")
hp.gnomview(
binned,
rot=(lon, lat),
xsize=800,
reso=1.0,
sub=[2, 2, 4],
title="binned, lon={:.2f}deg, lat={:.2f}deg".format(lon, lat),
)
fname = "{}.png".format(os.path.basename(args.obsdir))
plt.savefig(fname)
print("Plot saved in", fname)
def setUp(self):
fixture_name = os.path.splitext(os.path.basename(__file__))[0]
if not toast_available:
print("toast cannot be imported- skipping unit tests", flush=True)
return
self.comm, self.procs, self.rank = get_world()
self.outdir = create_outdir(fixture_name, comm=self.comm)
toastcomm = toast.Comm()
self.data = toast.Data(toastcomm)
# Focalplane
hwfull = get_example()
dets = sim_telescope_detectors(hwfull, "SAT4")
hwfull.data["detectors"] = dets
hw = hwfull.select(match={
"wafer_slot": "w42",
"band": "f030",
"pixel": "00[01]"
})
print(hw.data["detectors"], flush=True)
detquats = {k: v["quat"] for k, v in hw.data["detectors"].items()}
# Samples per observation
self.totsamp = 10000
# Pixelization
nside = 512
self.sim_nside = nside
self.map_nside = nside
# Scan properties
self.site_lon = '-67:47:10'
self.site_lat = '-22:57:30'
self.site_alt = 5200.
self.coord = 'C'
self.azmin = 45
self.azmax = 55
self.el = 60
self.scanrate = 1.0
self.scan_accel = 0.1
self.CES_start = None
# Noise properties
self.rate = 100.0
self.NET = 5.0
self.epsilon = 0.0
self.fmin = 1.0e-5
self.alpha = 1.0
self.fknee = 0.05
tod = TODGround(self.data.comm.comm_group,
detquats,
self.totsamp,
detranks=self.data.comm.group_size,
firsttime=0.0,
rate=self.rate,
site_lon=self.site_lon,
site_lat=self.site_lat,
site_alt=self.site_alt,
azmin=self.azmin,
azmax=self.azmax,
el=self.el,
coord=self.coord,
scanrate=self.scanrate,
scan_accel=self.scan_accel,
CES_start=self.CES_start)
# Analytic noise model
detnames = list(detquats.keys())
drate = {x: self.rate for x in detnames}
dfmin = {x: self.fmin for x in detnames}
dfknee = {x: self.fknee for x in detnames}
dalpha = {x: self.alpha for x in detnames}
dnet = {x: self.NET for x in detnames}
nse = AnalyticNoise(rate=drate,
fmin=dfmin,
detectors=detnames,
fknee=dfknee,
alpha=dalpha,
NET=dnet)
# Single observation
obs = dict()
obs["tod"] = tod
obs["noise"] = nse
obs["id"] = 12345
obs["intervals"] = tod.subscans
obs["site"] = "SimonsObs"
obs["telescope"] = "SAT4"
obs["site_id"] = 1
obs["telescope_id"] = 4
obs["fpradius"] = 5.0
obs["start_time"] = 0
obs["altitude"] = self.site_alt
obs["name"] = "test"
# Add the observation to the dataset
self.data.obs.append(obs)
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
memreport("at beginning of main", mpiworld)
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.world_rank == 0:
print("Running with {} processes at {}".format(
procs, str(datetime.datetime.now())))
parser = argparse.ArgumentParser(description='Simple MADAM Mapmaking',
fromfile_prefix_chars='@')
parser.add_argument('--rimo', required=True, help='RIMO file')
parser.add_argument('--freq', required=True, type=np.int, help='Frequency')
parser.add_argument('--nside',
required=False,
type=np.int,
default=512,
help='Map resolution')
parser.add_argument('--nside_cross',
required=False,
type=np.int,
default=512,
help='Destriping resolution')
parser.add_argument('--debug',
dest='debug',
default=False,
action='store_true',
help='Write data distribution info to file')
parser.add_argument('--dets',
required=False,
default=None,
help='Detector list (comma separated)')
parser.add_argument('--effdir',
required=True,
help='Input Exchange Format File directory')
parser.add_argument('--effdir_in_diode0',
required=False,
default=None,
help='Input Exchange Format File directory, '
'LFI diode 0')
parser.add_argument('--effdir_in_diode1',
required=False,
default=None,
help='Input Exchange Format File directory, '
'LFI diode 1')
parser.add_argument('--effdir_pntg',
required=False,
help='Input Exchange Format File directory '
'for pointing')
parser.add_argument('--effdir_out',
required=False,
help='Output directory for destriped TOD')
parser.add_argument('--effdir_out_diode0',
required=False,
help='Output directory for destriped TOD, LFI diode 0')
parser.add_argument('--effdir_out_diode1',
required=False,
help='Output directory for destriped TOD, LFI diode 1')
parser.add_argument('--obtmask',
required=False,
default=1,
type=np.int,
help='OBT flag mask')
parser.add_argument('--flagmask',
required=False,
default=1,
type=np.int,
help='Quality flag mask')
parser.add_argument('--pntflagmask',
required=False,
default=0,
type=np.int,
help='Pointing flag mask')
parser.add_argument('--bad_intervals',
required=False,
help='Path to bad interval file.')
parser.add_argument('--ringdb', required=True, help='Ring DB file')
parser.add_argument('--odfirst',
required=False,
default=None,
type=np.int,
help='First OD to use')
parser.add_argument('--odlast',
required=False,
default=None,
type=np.int,
help='Last OD to use')
parser.add_argument('--ringfirst',
required=False,
default=None,
help='First ring to use (can be a list)')
parser.add_argument('--ringlast',
required=False,
default=None,
help='Last ring to use (can be a list)')
parser.add_argument('--obtfirst',
required=False,
default=None,
type=np.float,
help='First OBT to use')
parser.add_argument('--obtlast',
required=False,
default=None,
type=np.float,
help='Last OBT to use')
parser.add_argument('--madampar',
required=False,
default=None,
help='Madam parameter file')
parser.add_argument('--out',
required=False,
default='.',
help='Output directory')
parser.add_argument('--madam_prefix', required=False, help='map prefix')
parser.add_argument('--split_mask',
required=False,
default=None,
help='Intensity mask, non-zero pixels are not split.')
parser.add_argument('--save_leakage_matrices',
dest='save_leakage_matrices',
default=False,
action='store_true',
help='Compile and write out the leakage projection '
'matrices.')
# noise parameters
parser.add_argument('--noisefile',
required=False,
default='RIMO',
help='Path to noise PSD files for noise filter. '
'Tag DETECTOR will be replaced with detector name.')
parser.add_argument('--static_noise',
dest='static_noise',
required=False,
default=False,
action='store_true',
help='Assume constant noise PSD')
parser.add_argument('--filterfile',
required=False,
help='Extra filter file.')
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if comm.comm_world.rank == 0:
print('All parameters:')
print(args, flush=True)
timer = Timer()
timer.start()
nrange = 1
odranges = None
if args.odfirst is not None and args.odlast is not None:
odranges = []
firsts = [int(i) for i in str(args.odfirst).split(',')]
lasts = [int(i) for i in str(args.odlast).split(',')]
for odfirst, odlast in zip(firsts, lasts):
odranges.append((odfirst, odlast))
nrange = len(odranges)
ringranges = None
if args.ringfirst is not None and args.ringlast is not None:
ringranges = []
firsts = [int(i) for i in str(args.ringfirst).split(',')]
lasts = [int(i) for i in str(args.ringlast).split(',')]
for ringfirst, ringlast in zip(firsts, lasts):
ringranges.append((ringfirst, ringlast))
nrange = len(ringranges)
obtranges = None
if args.obtfirst is not None and args.obtlast is not None:
obtranges = []
firsts = [float(i) for i in str(args.obtfirst).split(',')]
lasts = [float(i) for i in str(args.obtlast).split(',')]
for obtfirst, obtlast in zip(firsts, lasts):
obtranges.append((obtfirst, obtlast))
nrange = len(obtranges)
if odranges is None:
odranges = [None] * nrange
if ringranges is None:
ringranges = [None] * nrange
if obtranges is None:
obtranges = [None] * nrange
detectors = None
if args.dets is not None:
detectors = re.split(',', args.dets)
# create the TOD for this observation
if args.noisefile != 'RIMO' and not args.static_noise:
do_eff_cache = True
else:
do_eff_cache = False
tods = []
if args.static_noise:
noisefile = args.noisefile
else:
noisefile = 'RIMO'
for obtrange, ringrange, odrange in zip(obtranges, ringranges, odranges):
tods.append(
tp.Exchange(comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_in_diode0=args.effdir_in_diode0,
effdir_in_diode1=args.effdir_in_diode1,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
pntflagmask=args.pntflagmask,
do_eff_cache=do_eff_cache,
noisefile=noisefile))
rimo = tods[0].rimo
# Make output directory
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.makedirs(args.out)
# Read in madam parameter file
# Allow more than one entry, gather into a list
repeated_keys = ['detset', 'detset_nopol', 'survey']
pars = {}
if comm.comm_world.rank == 0:
pars['kfirst'] = False
pars['temperature_only'] = True
pars['base_first'] = 60.0
pars['nside_map'] = args.nside
pars['nside_cross'] = min(args.nside, args.nside_cross)
pars['nside_submap'] = 16
pars['write_map'] = False
pars['write_binmap'] = True
pars['write_matrix'] = False
pars['write_wcov'] = False
pars['write_hits'] = True
pars['kfilter'] = False
pars['info'] = 3
pars['pixlim_map'] = 1e-3
pars['pixlim_cross'] = 1e-3
if args.madampar:
pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*')
comment = re.compile(r'^#.*')
with open(args.madampar, 'r') as f:
for line in f:
if not comment.match(line):
result = pat.match(line)
if result:
key, value = result.group(1), result.group(2)
if key in repeated_keys:
if key not in pars:
pars[key] = []
pars[key].append(value)
else:
pars[key] = value
# Command line parameters override the ones in the madam parameter file
if 'file_root' not in pars:
pars['file_root'] = 'madam'
if args.madam_prefix is not None:
pars['file_root'] = args.madam_prefix
sfreq = '{:03}'.format(args.freq)
if sfreq not in pars['file_root']:
pars['file_root'] += '_' + sfreq
try:
fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq]
except Exception:
fsample = 180.3737
pars['fsample'] = fsample
pars['path_output'] = args.out
if args.save_leakage_matrices:
pars['write_leakmatrix'] = True
pars = comm.comm_world.bcast(pars, root=0)
if args.noisefile != 'RIMO':
# We split MPI_COMM_WORLD into single process groups, each of
# which is assigned one or more observations (rings)
comm = toast.Comm(groupsize=1)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
for iobs, tod in enumerate(tods):
if args.noisefile != 'RIMO' and not args.static_noise:
# Use a toast helper method to optimally distribute rings between
# processes.
dist = toast.distribute_discrete(tod.ringsizes, comm.world_size)
my_first_ring, my_n_ring = dist[comm.comm_world.rank]
for my_ring in range(my_first_ring, my_first_ring + my_n_ring):
ringtod = tp.Exchange.from_tod(tod,
my_ring,
comm.comm_group,
noisefile=args.noisefile)
ob = {}
ob['name'] = 'ring{:05}'.format(ringtod.globalfirst_ring)
ob['id'] = ringtod.globalfirst_ring
ob['tod'] = ringtod
ob['intervals'] = ringtod.valid_intervals
ob['baselines'] = None
ob['noise'] = ringtod.noise
data.obs.append(ob)
else:
ob = {}
ob['name'] = 'observation{:04}'.format(iobs)
ob['id'] = 0
ob['tod'] = tod
ob['intervals'] = tod.valid_intervals
ob['baselines'] = None
ob['noise'] = tod.noise
data.obs.append(ob)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Metadata queries")
if args.effdir_out is not None or (args.effdir_out_diode0 is not None
and args.effdir_out_diode1 is not None):
do_output = True
else:
do_output = False
# Read in the signal
timer.clear()
timer.start()
reader = tp.OpInputPlanck(signal_name='signal', flags_name='flags')
if comm.comm_world.rank == 0:
print('Reading input signal from {}'.format(args.effdir), flush=True)
reader.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Read")
tod_name = 'signal'
flags_name = 'flags'
# Optionally filter the signal
apply_filter(args, data)
# Optionally flag bad intervals
if args.bad_intervals is not None:
timer = Timer()
timer.start()
flagger = tp.OpBadIntervals(path=args.bad_intervals)
flagger.exec(data)
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Apply {}".format(args.bad_intervals))
# make a planck Healpix pointing matrix
timer.clear()
timer.start()
mode = 'IQU'
if pars['temperature_only'] == 'T':
mode = 'I'
nside = int(pars['nside_map'])
pointing = tp.OpPointingPlanck(nside=nside,
mode=mode,
RIMO=rimo,
margin=0,
apply_flags=(not do_output),
keep_vel=False,
keep_pos=False,
keep_phase=False,
keep_quats=False)
pointing.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Pointing Matrix, mode = {}".format(mode))
for obs in data.obs:
obs['tod'].purge_eff_cache()
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
if d[-1] in '01' and d[-2] != '-':
det = to_radiometer(d)
else:
det = d
net = tod.rimo[det].net
fsample = tod.rimo[det].fsample
detweights[d] = 1.0 / (fsample * net * net)
if do_output:
name_out = 'madam_tod'
else:
name_out = None
timer.clear()
timer.start()
try:
madam = toast.todmap.OpMadam(name=tod_name,
flag_name=flags_name,
apply_flags=do_output,
params=pars,
detweights=detweights,
purge=True,
name_out=name_out,
translate_timestamps=False)
except Exception as e:
raise Exception('{:4} : ERROR: failed to initialize Madam: {}'.format(
comm.comm_world.rank, e))
madam.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Madam")
if do_output:
timer = Timer()
timer.start()
writer = tp.OpOutputPlanck(signal_name='madam_tod',
flags_name=None,
commonflags_name=None,
effdir_out=args.effdir_out,
effdir_out_diode0=args.effdir_out_diode0,
effdir_out_diode1=args.effdir_out_diode1)
writer.exec(data)
comm.comm_world.barrier()
timer.stop()
if comm.comm_world.rank == 0:
timer.report("Madam output")
memreport("at end of main", mpiworld)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def main():
log = Logger.get()
gt = GlobalTimers.get()
gt.start("toast_planck_reduce (total)")
mpiworld, procs, rank, comm = get_comm()
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
print(
"Running with {} processes at {}".format(
procs, str(datetime.datetime.now())
)
)
parser = argparse.ArgumentParser(
description="Simple on-the-fly signal convolution + MADAM Mapmaking",
fromfile_prefix_chars="@",
)
parser.add_argument("--lmax", required=True, type=np.int, help="Simulation lmax")
parser.add_argument(
"--fwhm", required=True, type=np.float, help="Sky fwhm [arcmin] to deconvolve"
)
parser.add_argument("--beammmax", required=True, type=np.int, help="Beam mmax")
parser.add_argument("--order", default=11, type=np.int, help="Iteration order")
parser.add_argument(
"--pxx",
required=False,
default=False,
action="store_true",
help="Beams are in Pxx frame, not Dxx",
)
parser.add_argument(
"--normalize",
required=False,
default=False,
action="store_true",
help="Normalize the beams",
)
parser.add_argument(
"--skyfile",
required=True,
help="Path to sky alm files. Tag DETECTOR will be "
"replaced with detector name.",
)
parser.add_argument(
"--remove_monopole",
required=False,
default=False,
action="store_true",
help="Remove the sky monopole before convolution",
)
parser.add_argument(
"--remove_dipole",
required=False,
default=False,
action="store_true",
help="Remove the sky dipole before convolution",
)
parser.add_argument(
"--beamfile",
required=True,
help="Path to beam alm files. Tag DETECTOR will be "
"replaced with detector name.",
)
parser.add_argument("--rimo", required=True, help="RIMO file")
parser.add_argument("--freq", required=True, type=np.int, help="Frequency")
parser.add_argument(
"--dets", required=False, default=None, help="Detector list (comma separated)"
)
parser.add_argument(
"--effdir", required=True, help="Input Exchange Format File directory"
)
parser.add_argument(
"--effdir_pntg",
required=False,
help="Input Exchange Format File directory " "for pointing",
)
parser.add_argument(
"--effdir_out", required=False, help="Output directory for convolved TOD"
)
parser.add_argument(
"--obtmask", required=False, default=1, type=np.int, help="OBT flag mask"
)
parser.add_argument(
"--flagmask", required=False, default=1, type=np.int, help="Quality flag mask"
)
parser.add_argument("--ringdb", required=True, help="Ring DB file")
parser.add_argument(
"--odfirst", required=False, default=None, type=np.int, help="First OD to use"
)
parser.add_argument(
"--odlast", required=False, default=None, type=np.int, help="Last OD to use"
)
parser.add_argument(
"--ringfirst",
required=False,
default=None,
type=np.int,
help="First ring to use",
)
parser.add_argument(
"--ringlast", required=False, default=None, type=np.int, help="Last ring to use"
)
parser.add_argument(
"--obtfirst",
required=False,
default=None,
type=np.float,
help="First OBT to use",
)
parser.add_argument(
"--obtlast", required=False, default=None, type=np.float, help="Last OBT to use"
)
parser.add_argument("--madam_prefix", required=False, help="map prefix")
parser.add_argument(
"--madampar", required=False, default=None, help="Madam parameter file"
)
parser.add_argument(
"--obtmask_madam", required=False, type=np.int, help="OBT flag mask for Madam"
)
parser.add_argument(
"--flagmask_madam",
required=False,
type=np.int,
help="Quality flag mask for Madam",
)
parser.add_argument(
"--skip_madam",
required=False,
default=False,
action="store_true",
help="Do not run Madam on the convolved timelines",
)
parser.add_argument("--out", required=False, default=".", help="Output directory")
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
timer = Timer()
timer.start()
odrange = None
if args.odfirst is not None and args.odlast is not None:
odrange = (args.odfirst, args.odlast)
ringrange = None
if args.ringfirst is not None and args.ringlast is not None:
ringrange = (args.ringfirst, args.ringlast)
obtrange = None
if args.obtfirst is not None and args.obtlast is not None:
obtrange = (args.obtfirst, args.obtlast)
detectors = None
if args.dets is not None:
detectors = re.split(",", args.dets)
# This is the distributed data, consisting of one or
# more observations, each distributed over a communicator.
data = toast.Data(comm)
# Ensure output directory exists
if not os.path.isdir(args.out) and comm.comm_world.rank == 0:
os.makedirs(args.out)
# Read in madam parameter file
# Allow more than one entry, gather into a list
repeated_keys = ["detset", "detset_nopol", "survey"]
pars = {}
if comm.comm_world.rank == 0:
pars["kfirst"] = False
pars["temperature_only"] = True
pars["base_first"] = 60.0
pars["nside_map"] = 512
pars["nside_cross"] = 512
pars["nside_submap"] = 16
pars["write_map"] = False
pars["write_binmap"] = True
pars["write_matrix"] = False
pars["write_wcov"] = False
pars["write_hits"] = True
pars["kfilter"] = False
pars["info"] = 3
if args.madampar:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if not comment.match(line):
result = pat.match(line)
if result:
key, value = result.group(1), result.group(2)
if key in repeated_keys:
if key not in pars:
pars[key] = []
pars[key].append(value)
else:
pars[key] = value
# Command line parameters override the ones in the madam parameter file
if "file_root" not in pars:
pars["file_root"] = "madam"
if args.madam_prefix is not None:
pars["file_root"] = args.madam_prefix
sfreq = "{:03}".format(args.freq)
if sfreq not in pars["file_root"]:
pars["file_root"] += "_" + sfreq
try:
fsample = {30: 32.51, 44: 46.55, 70: 78.77}[args.freq]
except Exception:
fsample = 180.3737
pars["fsample"] = fsample
pars["path_output"] = args.out
print("All parameters:")
print(args, flush=True)
pars = comm.comm_world.bcast(pars, root=0)
memreport("after parameters", MPI.COMM_WORLD)
# madam only supports a single observation. Normally
# we would have multiple observations with some subset
# assigned to each process group.
# create the TOD for this observation
tod = tp.Exchange(
comm=comm.comm_group,
detectors=detectors,
ringdb=args.ringdb,
effdir_in=args.effdir,
effdir_pntg=args.effdir_pntg,
obt_range=obtrange,
ring_range=ringrange,
od_range=odrange,
freq=args.freq,
RIMO=args.rimo,
obtmask=args.obtmask,
flagmask=args.flagmask,
do_eff_cache=False,
)
# normally we would get the intervals from somewhere else, but since
# the Exchange TOD already had to get that information, we can
# get it from there.
ob = {}
ob["name"] = "mission"
ob["id"] = 0
ob["tod"] = tod
ob["intervals"] = tod.valid_intervals
ob["baselines"] = None
ob["noise"] = tod.noise
# Add the bare minimum focal plane information for the conviqt operator
focalplane = {}
for det in tod.detectors:
if args.pxx:
# Beam is in the polarization basis.
# No extra rotations are needed
psipol = tod.rimo[det].psi_pol
else:
# Beam is in the detector basis. Convolver needs to remove
# the last rotation into the polarization sensitive frame.
psipol = tod.rimo[det].psi_uv + tod.rimo[det].psi_pol
focalplane[det] = {
"pol_leakage" : tod.rimo[det].epsilon,
"pol_angle_deg" : psipol,
}
ob["focalplane"] = focalplane
data.obs.append(ob)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Metadata queries")
loader = tp.OpInputPlanck(
commonflags_name="common_flags", flags_name="flags", margin=0
)
loader.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Data read and cache")
tod.cache.report()
memreport("after loading", mpiworld)
# make a planck Healpix pointing matrix
mode = "IQU"
if pars["temperature_only"] == "T":
mode = "I"
nside = int(pars["nside_map"])
pointing = tp.OpPointingPlanck(
nside=nside,
mode=mode,
RIMO=tod.RIMO,
margin=0,
apply_flags=False,
keep_vel=False,
keep_pos=False,
keep_phase=False,
keep_quats=True,
)
pointing.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Pointing Matrix took, mode = {}".format(mode))
memreport("after pointing", mpiworld)
# simulate the TOD by convolving the sky with the beams
if comm.comm_world.rank == 0:
print("Convolving TOD", flush=True)
for pattern in args.beamfile.split(","):
skyfiles = {}
beamfiles = {}
for det in tod.detectors:
freq = "{:03}".format(tp.utilities.det2freq(det))
if "LFI" in det:
psmdet = "{}_{}".format(freq, det[3:])
if det.endswith("M"):
arm = "y"
else:
arm = "x"
graspdet = "{}_{}_{}".format(freq[1:], det[3:5], arm)
else:
psmdet = det.replace("-", "_")
graspdet = det
skyfile = (
args.skyfile.replace("FREQ", freq)
.replace("PSMDETECTOR", psmdet)
.replace("DETECTOR", det)
)
skyfiles[det] = skyfile
beamfile = pattern.replace("GRASPDETECTOR", graspdet).replace(
"DETECTOR", det
)
beamfiles[det] = beamfile
if comm.comm_world.rank == 0:
print("Convolving {} with {}".format(skyfile, beamfile), flush=True)
conviqt = OpSimConviqt(
comm.comm_world,
skyfiles,
beamfiles,
lmax=args.lmax,
beammmax=args.beammmax,
pol=True,
fwhm=args.fwhm,
order=args.order,
calibrate=True,
dxx=True,
out="conviqt_tod",
apply_flags=False,
remove_monopole=args.remove_monopole,
remove_dipole=args.remove_dipole,
verbosity=1,
normalize_beam=args.normalize,
)
conviqt.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Convolution")
memreport("after conviqt", mpiworld)
if args.effdir_out is not None:
if comm.comm_world.rank == 0:
print("Writing TOD", flush=True)
tod.set_effdir_out(args.effdir_out, None)
writer = tp.OpOutputPlanck(
signal_name="conviqt_tod",
flags_name="flags",
commonflags_name="common_flags",
)
writer.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Conviqt output")
memreport("after writing", mpiworld)
# for now, we pass in the noise weights from the RIMO.
detweights = {}
for d in tod.detectors:
net = tod.rimo[d].net
fsample = tod.rimo[d].fsample
detweights[d] = 1.0 / (fsample * net * net)
if not args.skip_madam:
if comm.comm_world.rank == 0:
print("Calling Madam", flush=True)
try:
if args.obtmask_madam is None:
obtmask = args.obtmask
else:
obtmask = args.obtmask_madam
if args.flagmask_madam is None:
flagmask = args.flagmask
else:
flagmask = args.flagmask_madam
madam = OpMadam(
params=pars,
detweights=detweights,
name="conviqt_tod",
flag_name="flags",
purge=True,
name_out="madam_tod",
common_flag_mask=obtmask,
flag_mask=flagmask,
)
except Exception as e:
raise Exception(
"{:4} : ERROR: failed to initialize Madam: {}".format(
comm.comm_world.rank, e
)
)
madam.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
timer.report_clear("Madam took {:.3f} s")
memreport("after madam", mpiworld)
gt.stop_all()
if mpiworld is not None:
mpiworld.barrier()
timer = Timer()
timer.start()
alltimers = gather_timers(comm=mpiworld)
if comm.world_rank == 0:
out = os.path.join(args.out, "timing")
dump_timing(alltimers, out)
timer.stop()
timer.report("Gather and dump timing info")
return
def setUp(self):
fixture_name = os.path.splitext(os.path.basename(__file__))[0]
if not toast_available:
print("toast cannot be imported- skipping unit tests", flush=True)
return
self.outdir = None
if MPI.COMM_WORLD.rank == 0:
self.outdir = create_outdir(fixture_name)
self.outdir = MPI.COMM_WORLD.bcast(self.outdir, root=0)
toastcomm = toast.Comm()
self.data = toast.Data(toastcomm)
# Focalplane
hwfull = get_example()
dets = sim_telescope_detectors(hwfull, "SAT3")
hwfull.data["detectors"] = dets
hw = hwfull.select(match={
"wafer": "42",
"band": "LF1",
"pixel": "00[01]"
})
# print(hw.data["detectors"], flush=True)
detquats = {k: v["quat"] for k, v in hw.data["detectors"].items()}
# File dump size in bytes (1MB)
self.dumpsize = 2**20
# Samples per observation
self.totsamp = 10000
# Pixelization
nside = 512
self.sim_nside = nside
self.map_nside = nside
# Scan properties
self.site_lon = '-67:47:10'
self.site_lat = '-22:57:30'
self.site_alt = 5200.
self.coord = 'C'
self.azmin = 45
self.azmax = 55
self.el = 60
self.scanrate = 1.0
self.scan_accel = 0.1
self.CES_start = None
# Noise properties
self.rate = 100.0
self.NET = 1e-3 # 1 mK NET
self.epsilon = 0.0
self.fmin = 1.0e-5
self.alpha = 1.0
self.fknee = 0.05
for ob in range(3):
ftime = (self.totsamp / self.rate) * ob + 1564015655.88
tod = TODGround(self.data.comm.comm_group,
detquats,
self.totsamp,
detranks=self.data.comm.group_size,
firsttime=ftime,
rate=self.rate,
site_lon=self.site_lon,
site_lat=self.site_lat,
site_alt=self.site_alt,
azmin=self.azmin,
azmax=self.azmax,
el=self.el,
coord=self.coord,
scanrate=self.scanrate,
scan_accel=self.scan_accel,
CES_start=self.CES_start)
# Analytic noise model
detnames = list(detquats.keys())
drate = {x: self.rate for x in detnames}
dfmin = {x: self.fmin for x in detnames}
dfknee = {x: self.fknee for x in detnames}
dalpha = {x: self.alpha for x in detnames}
dnet = {x: self.NET for x in detnames}
nse = AnalyticNoise(rate=drate,
fmin=dfmin,
detectors=detnames,
fknee=dfknee,
alpha=dalpha,
NET=dnet)
# Single observation
obs = dict()
obs["tod"] = tod
obs["noise"] = nse
obs["id"] = 12345
obs["intervals"] = tod.subscans
obs["site"] = "SimonsObs"
obs["telescope"] = "SAT3"
obs["site_id"] = 1
obs["telescope_id"] = 4
obs["fpradius"] = 5.0
obs["start_time"] = ftime
obs["altitude"] = self.site_alt
obs["name"] = "test_{:02}".format(ob)
# Add the observation to the dataset
self.data.obs.append(obs)
# Simulate some noise into multiple cache prefixes. This is used
# to test the export of multiple timestream flavors.
nse = toast.tod.OpSimNoise(out="signal", realization=0)
nse.exec(self.data)
nse = toast.tod.OpSimNoise(out="component1", realization=0)
nse.exec(self.data)
nse = toast.tod.OpSimNoise(out="component2", realization=0)
nse.exec(self.data)
return
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser(
description="Read existing data and make a simple map.",
fromfile_prefix_chars="@",
)
parser.add_argument(
"--groupsize",
required=False,
type=int,
default=0,
help="size of processor groups used to distribute "
"observations",
)
parser.add_argument(
"--hwprpm",
required=False,
type=float,
default=0.0,
help="The rate (in RPM) of the HWP rotation",
)
parser.add_argument(
"--samplerate",
required=False,
default=100.0,
type=np.float,
help="Detector sample rate (Hz)",
)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument("--nside",
required=False,
type=int,
default=64,
help="Healpix NSIDE")
parser.add_argument(
"--subnside",
required=False,
type=int,
default=8,
help="Distributed pixel sub-map NSIDE",
)
parser.add_argument("--coord",
required=False,
default="E",
help="Sky coordinate system [C,E,G]")
parser.add_argument(
"--baseline",
required=False,
type=float,
default=60.0,
help="Destriping baseline length (seconds)",
)
parser.add_argument(
"--noisefilter",
required=False,
default=False,
action="store_true",
help="Destripe with the noise filter enabled",
)
parser.add_argument(
"--madam",
required=False,
default=False,
action="store_true",
help="If specified, use libmadam for map-making",
)
parser.add_argument("--madampar",
required=False,
default=None,
help="Madam parameter file")
parser.add_argument(
"--polyorder",
required=False,
type=int,
help="Polynomial order for the polyfilter",
)
parser.add_argument(
"--wbin_ground",
required=False,
type=float,
help="Ground template bin width [degrees]",
)
parser.add_argument(
"--flush",
required=False,
default=False,
action="store_true",
help="Flush every print statement.",
)
parser.add_argument("--tidas",
required=False,
default=None,
help="Input TIDAS volume")
parser.add_argument("--tidas_detgroup",
required=False,
default=None,
help="TIDAS detector group")
parser.add_argument("--spt3g",
required=False,
default=None,
help="Input SPT3G data directory")
parser.add_argument(
"--spt3g_prefix",
required=False,
default=None,
help="SPT3G data frame file prefix",
)
parser.add_argument(
"--common_flag_mask",
required=False,
default=0,
type=np.uint8,
help="Common flag mask",
)
parser.add_argument(
"--debug",
required=False,
default=False,
action="store_true",
help="Write data distribution info and focalplane plot",
)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
# args = parser.parse_args(sys.argv)
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if (args.tidas is not None) and (args.spt3g is not None):
raise RuntimeError("Cannot read two datasets!")
if (args.tidas is None) and (args.spt3g is None):
raise RuntimeError("No dataset specified!")
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot load")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot load")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# Pixelization
nside = args.nside
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > nside:
subnside = nside
subnpix = 12 * subnside * subnside
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print(
"WARNING: process groupsize does not evenly divide into "
"total number of processes",
flush=True,
)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# Create output directory
mtime = MPI.Wtime()
if comm.comm_world.rank == 0:
if not os.path.isdir(args.outdir):
os.makedirs(args.outdir)
mtime = elapsed(comm.comm_world, mtime, "Creating output directory")
# The distributed timestream data
data = None
if args.tidas is not None:
if args.tidas_detgroup is None:
raise RuntimeError("you must specify the detector group")
data = tds.load_tidas(
comm,
comm.group_size,
args.tidas,
"r",
args.tidas_detgroup,
tds.TODTidas,
group_dets=args.tidas_detgroup,
distintervals="chunks",
)
if args.spt3g is not None:
if args.spt3g_prefix is None:
raise RuntimeError("you must specify the frame file prefix")
data = s3g.load_spt3g(
comm,
comm.group_size,
args.spt3g,
args.spt3g_prefix,
s3g.obsweight_spt3g,
s3g.TOD3G,
)
mtime = elapsed(comm.comm_world, mtime, "Distribute data")
# In debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
mtime = elapsed(comm.comm_world, mtime,
"Dumping debug data distribution")
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
# Just plot the dets from the first TOD
temptod = data.obs[0]["tod"]
# FIXME: change this once we store det info in the metadata.
dfwhm = {x: 10.0 for x in temptod.detectors}
tt.plot_focalplane(temptod.detoffset(),
10.0,
10.0,
outfile,
fwhm=dfwhm)
comm.comm_world.barrier()
mtime = elapsed(comm.comm_world, mtime, "Plotting debug focalplane")
# Compute pointing matrix
pointing = tt.OpPointingHpix(nside=args.nside,
nest=True,
mode="IQU",
hwprpm=args.hwprpm)
pointing.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Expand pointing")
# Mapmaking.
# FIXME: We potentially have a different noise model for every
# observation. We need to have both spt3g and tidas format Noise
# classes which read the information from disk. Then the mapmaking
# operators need to get these noise weights from each observation.
detweights = {d: 1.0 for d in data.obs[0]["tod"].detectors}
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# Filter data if desired
if args.polyorder:
polyfilter = tt.OpPolyFilter(
order=args.polyorder, common_flag_mask=args.common_flag_mask)
polyfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering")
if args.wbin_ground:
groundfilter = tt.OpGroundFilter(
wbin=args.wbin_ground, common_flag_mask=args.common_flag_mask)
groundfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime,
"Ground template filtering")
# Compute pixel space distribution
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
if localpix is None:
raise RuntimeError(
"Process {} has no hit pixels. Perhaps there are fewer "
"detectors than processes in the group?".format(
comm.comm_world.rank))
localsm = np.unique(np.floor_divide(localpix, subnpix))
mtime = elapsed(comm.comm_world, mtime, "Compute local submaps")
# construct distributed maps to store the covariance,
# noise weighted map, and hits
mtime = MPI.Wtime()
invnpp = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=6,
dtype=np.float64,
submap=subnpix,
local=localsm,
)
hits = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=1,
dtype=np.int64,
submap=subnpix,
local=localsm,
)
zmap = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=3,
dtype=np.float64,
submap=subnpix,
local=localsm,
)
# compute the hits and covariance.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(
detweights=detweights,
invnpp=invnpp,
hits=hits,
common_flag_mask=args.common_flag_mask,
)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1")
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1")
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1")
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing N_pp")
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap,
detweights=detweights,
common_flag_mask=args.common_flag_mask)
build_zmap.exec(data)
zmap.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map")
tm.covariance_apply(invnpp, zmap)
mtime = elapsed(comm.comm_world, mtime, "Computing binned map")
zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits"))
mtime = elapsed(comm.comm_world, mtime, "Writing binned map")
else:
# Set up MADAM map making.
pars = {}
pars["temperature_only"] = "F"
pars["force_pol"] = "T"
pars["kfirst"] = "T"
pars["concatenate_messages"] = "T"
pars["write_map"] = "T"
pars["write_binmap"] = "T"
pars["write_matrix"] = "T"
pars["write_wcov"] = "T"
pars["write_hits"] = "T"
pars["nside_cross"] = nside // 2
pars["nside_submap"] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars["base_first"] = args.baseline
pars["nside_map"] = nside
if args.noisefilter:
pars["kfilter"] = "T"
else:
pars["kfilter"] = "F"
pars["fsample"] = args.samplerate
madam = tm.OpMadam(params=pars,
detweights=detweights,
common_flag_mask=args.common_flag_mask)
madam.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking")
comm.comm_world.barrier()
stop = MPI.Wtime()
dur = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(dur), flush=True)
return
