def create_observations(args, comm, fp, all_ces, site):
""" Simulate constant elevation scans.
Simulate constant elevation scans at "site" matching entries in
"all_ces". Each operational day is assigned to a different
process group to allow making day maps.
"""
start = MPI.Wtime()
autotimer = timing.auto_timer()
data = toast.Data(comm)
site_name, site_lat, site_lon, site_alt = site
detectors = sorted(fp.keys())
detquats = {}
for d in detectors:
detquats[d] = fp[d]['quat']
nces = len(all_ces)
breaks = []
do_break = False
for i in range(nces - 1):
# If current and next CES are on different days, insert a break
tz = args.timezone / 24.
start1 = all_ces[i][3] # MJD start
start2 = all_ces[i + 1][3] # MJD start
scan1 = all_ces[i][4]
scan2 = all_ces[i + 1][4]
if scan1 != scan2 and do_break:
breaks.append(i + 1)
do_break = False
continue
day1 = int(start1 + tz)
day2 = int(start2 + tz)
if day1 != day2:
if scan1 == scan2:
# We want an entire CES, even if it crosses the day bound.
# Wait until the scan number changes.
do_break = True
else:
breaks.append(i + 1)
nbreak = len(breaks)
if nbreak != comm.ngroups - 1:
raise RuntimeError(
'Number of observing days ({}) does not match number of process '
'groups ({}).'.format(nbreak + 1, comm.ngroups))
groupdist = toast.distribute_uniform(nces, comm.ngroups, breaks=breaks)
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
# Create the noise model used by all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]['fmin']
fknee[d] = fp[d]['fknee']
alpha[d] = fp[d]['alpha']
NET[d] = fp[d]['NET']
noise = tt.AnalyticNoise(rate=rates,
fmin=fmin,
detectors=detectors,
fknee=fknee,
alpha=alpha,
NET=NET)
for ices in range(group_firstobs, group_firstobs + group_numobs):
ces = all_ces[ices]
CES_start, CES_stop, name, mjdstart, scan, subscan, azmin, azmax, \
el = ces
totsamples = int((CES_stop - CES_start) * args.samplerate)
# create the single TOD for this observation
try:
tod = tt.TODGround(comm.comm_group,
detquats,
totsamples,
detranks=comm.comm_group.size,
firsttime=CES_start,
rate=args.samplerate,
site_lon=site_lon,
site_lat=site_lat,
site_alt=site_alt,
azmin=azmin,
azmax=azmax,
el=el,
scanrate=args.scanrate,
scan_accel=args.scan_accel,
CES_start=None,
CES_stop=None,
sun_angle_min=args.sun_angle_min,
coord=args.coord,
sampsizes=None)
except RuntimeError as e:
print('Failed to create the CES scan: {}'.format(e),
flush=args.flush)
return
# Create the (single) observation
ob = {}
ob['name'] = 'CES-{}-{}-{}'.format(name, scan, subscan)
ob['tod'] = tod
if len(tod.subscans) > 0:
ob['intervals'] = tod.subscans
else:
raise RuntimeError('{} has no valid intervals'.format(ob['name']))
ob['baselines'] = None
ob['noise'] = noise
ob['id'] = int(mjdstart * 10000)
data.obs.append(ob)
for ob in data.obs:
tod = ob['tod']
tod.free_azel_quats()
if comm.comm_group.rank == 0:
print('Group # {:4} has {} observations.'.format(
comm.group, len(data.obs)),
flush=args.flush)
if len(data.obs) == 0:
raise RuntimeError('Too many tasks. Every MPI task must '
'be assigned to at least one observation.')
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Simulated scans in {:.2f} seconds'
''.format(stop - start),
flush=args.flush)
return data
def create_observations(args, comm, fp, all_ces, site):
""" Simulate constant elevation scans.
Simulate constant elevation scans at "site" matching entries in
"all_ces". Each operational day is assigned to a different
process group to allow making day maps.
"""
start = MPI.Wtime()
autotimer = timing.auto_timer()
data = toast.Data(comm)
site_name, site_lat, site_lon, site_alt = site
detectors = sorted(fp.keys())
detquats = {}
for d in detectors:
detquats[d] = fp[d]['quat']
nces = len(all_ces)
breaks = []
do_break = False
for i in range(nces-1):
# If current and next CES are on different days, insert a break
tz = args.timezone / 24.
start1 = all_ces[i][3] # MJD start
start2 = all_ces[i+1][3] # MJD start
scan1 = all_ces[i][4]
scan2 = all_ces[i+1][4]
if scan1 != scan2 and do_break:
breaks.append(i + 1)
do_break = False
continue
day1 = int(start1 + tz)
day2 = int(start2 + tz)
if day1 != day2:
if scan1 == scan2:
# We want an entire CES, even if it crosses the day bound.
# Wait until the scan number changes.
do_break = True
else:
breaks.append(i + 1)
nbreak = len(breaks)
if nbreak != comm.ngroups-1:
raise RuntimeError(
'Number of observing days ({}) does not match number of process '
'groups ({}).'.format(nbreak+1, comm.ngroups))
groupdist = toast.distribute_uniform(nces, comm.ngroups, breaks=breaks)
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
# Create the noise model used by all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]['fmin']
fknee[d] = fp[d]['fknee']
alpha[d] = fp[d]['alpha']
NET[d] = fp[d]['NET']
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
for ices in range(group_firstobs, group_firstobs + group_numobs):
ces = all_ces[ices]
CES_start, CES_stop, name, mjdstart, scan, subscan, azmin, azmax, \
el = ces
totsamples = int((CES_stop - CES_start) * args.samplerate)
# create the single TOD for this observation
try:
tod = tt.TODGround(
comm.comm_group,
detquats,
totsamples,
detranks=comm.comm_group.size,
firsttime=CES_start,
rate=args.samplerate,
site_lon=site_lon,
site_lat=site_lat,
site_alt=site_alt,
azmin=azmin,
azmax=azmax,
el=el,
scanrate=args.scanrate,
scan_accel=args.scan_accel,
CES_start=None,
CES_stop=None,
sun_angle_min=args.sun_angle_min,
coord=args.coord,
sampsizes=None)
except RuntimeError as e:
print('Failed to create the CES scan: {}'.format(e),
flush=args.flush)
return
# Create the (single) observation
ob = {}
ob['name'] = 'CES-{}-{}-{}'.format(name, scan, subscan)
ob['tod'] = tod
if len(tod.subscans) > 0:
ob['intervals'] = tod.subscans
else:
raise RuntimeError('{} has no valid intervals'.format(ob['name']))
ob['baselines'] = None
ob['noise'] = noise
ob['id'] = int(mjdstart * 10000)
data.obs.append(ob)
for ob in data.obs:
tod = ob['tod']
tod.free_azel_quats()
if comm.comm_group.rank == 0:
print('Group # {:4} has {} observations.'.format(
comm.group, len(data.obs)), flush=args.flush)
if len(data.obs) == 0:
raise RuntimeError('Too many tasks. Every MPI task must '
'be assigned to at least one observation.')
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Simulated scans in {:.2f} seconds'
''.format(stop-start), flush=args.flush)
return data
def main():
comm = MPI.COMM_WORLD
if comm.rank == 0:
print("Running with {} processes".format(comm.size))
parser = argparse.ArgumentParser( description='Read a toast covariance matrix and invert it.' )
parser.add_argument( '--input', required=True, default=None, help='The input covariance FITS file' )
parser.add_argument( '--output', required=False, default=None, help='The output inverse covariance FITS file.' )
parser.add_argument( '--rcond', required=False, default=None, help='Optionally write the inverse condition number map to this file.' )
parser.add_argument( '--single', required=False, default=False, action='store_true', help='Write the output in single precision.' )
parser.add_argument( '--threshold', required=False, default=1e-3, type=np.float, help='Reciprocal condition number threshold' )
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer(timing.FILE())
# get options
infile = args.input
outfile = None
if args.output is not None:
outfile = args.output
else:
inmat = re.match(r'(.*)\.fits', infile)
if inmat is None:
print("input file should have .fits extension")
sys.exit(0)
inroot = inmat.group(1)
outfile = "{}_inv.fits".format(inroot)
# We need to read the header to get the size of the matrix.
# This would be a trivial function call in astropy.fits or
# fitsio, but we don't want to bring in a whole new dependency
# just for that. Instead, we open the file with healpy in memmap
# mode so that nothing is actually read except the header.
nside = 0
ncovnz = 0
if comm.rank == 0:
fake, head = hp.read_map(infile, h=True, memmap=True)
for key, val in head:
if key == 'NSIDE':
nside = int(val)
if key == 'TFIELDS':
ncovnz = int(val)
nside = comm.bcast(nside, root=0)
ncovnz = comm.bcast(nnz, root=0)
nnz = int( ( (np.sqrt(8.0*ncovnz) - 1.0) / 2.0 ) + 0.5 )
npix = 12 * nside**2
subnside = int(nside / 16)
if subnside == 0:
subnside = 1
subnpix = 12 * subnside**2
nsubmap = int( npix / subnpix )
# divide the submaps as evenly as possible among processes
dist = toast.distribute_uniform(nsubmap, comm.size)
local = np.arange(dist[comm.rank][0], dist[comm.rank][0] + dist[comm.rank][1])
if comm.rank == 0:
if os.path.isfile(outfile):
os.remove(outfile)
comm.barrier()
# create the covariance and inverse condition number map
cov = None
invcov = None
rcond = None
cov = tm.DistPixels(comm=comm, dtype=np.float64, size=npix, nnz=ncovnz, submap=subnpix, local=local)
if args.single:
invcov = tm.DistPixels(comm=comm, dtype=np.float32, size=npix, nnz=ncovnz, submap=subnpix, local=local)
else:
invcov = cov
if args.rcond is not None:
rcond = tm.DistPixels(comm=comm, dtype=np.float64, size=npix, nnz=nnz, submap=subnpix, local=local)
# read the covariance
cov.read_healpix_fits(infile)
# every process computes its local piece
tm.covariance_invert(cov, args.threshold, rcond=rcond)
if args.single:
invcov.data[:] = cov.data.astype(np.float32)
# write the inverted covariance
invcov.write_healpix_fits(outfile)
# write the condition number
if args.rcond is not None:
rcond.write_healpix_fits(args.rcond)
return
def main():
comm = MPI.COMM_WORLD
if comm.rank == 0:
print("Running with {} processes".format(comm.size))
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description='Read a toast covariance matrix and write the inverse condition number map' )
parser.add_argument( '--input', required=True, default=None, help='The input covariance FITS file' )
parser.add_argument( '--output', required=False, default=None, help='The output inverse condition map FITS file.' )
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer(timing.FILE())
# get options
infile = args.input
outfile = None
if args.output is not None:
outfile = args.output
else:
inmat = re.match(r'(.*)\.fits', infile)
if inmat is None:
print("input file should have .fits extension")
sys.exit(0)
inroot = inmat.group(1)
outfile = "{}_rcond.fits".format(inroot)
# We need to read the header to get the size of the matrix.
# This would be a trivial function call in astropy.fits or
# fitsio, but we don't want to bring in a whole new dependency
# just for that. Instead, we open the file with healpy in memmap
# mode so that nothing is actually read except the header.
nside = 0
nnz = 0
if comm.rank == 0:
fake, head = hp.read_map(infile, h=True, memmap=True)
for key, val in head:
if key == 'NSIDE':
nside = int(val)
if key == 'TFIELDS':
nnz = int(val)
nside = comm.bcast(nside, root=0)
nnz = comm.bcast(nnz, root=0)
npix = 12 * nside**2
subnside = int(nside / 16)
if subnside == 0:
subnside = 1
subnpix = 12 * subnside**2
nsubmap = int( npix / subnpix )
# divide the submaps as evenly as possible among processes
dist = toast.distribute_uniform(nsubmap, comm.size)
local = np.arange(dist[comm.rank][0], dist[comm.rank][0] + dist[comm.rank][1])
if comm.rank == 0:
if os.path.isfile(outfile):
os.remove(outfile)
comm.barrier()
# create the covariance and inverse condition number map
cov = tm.DistPixels(comm=comm, dtype=np.float64, size=npix, nnz=nnz, submap=subnpix, local=local)
# read the covariance
cov.read_healpix_fits(infile)
# every process computes its local piece
rcond = tm.covariance_rcond(cov)
# write the map
rcond.write_healpix_fits(outfile)
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description="Simulate satellite "
"boresight pointing and make a noise map.", fromfile_prefix_chars="@" )
parser.add_argument( "--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute observations" )
parser.add_argument( "--samplerate", required=False, type=float,
default=40.0, help="Detector sample rate (Hz)" )
parser.add_argument( "--starttime", required=False, type=float,
default=0.0, help="The overall start time of the simulation" )
parser.add_argument( "--spinperiod", required=False, type=float,
default=10.0, help="The period (in minutes) of the rotation about the "
"spin axis" )
parser.add_argument( "--spinangle", required=False, type=float,
default=30.0, help="The opening angle (in degrees) of the boresight "
"from the spin axis" )
parser.add_argument( "--precperiod", required=False, type=float,
default=50.0, help="The period (in minutes) of the rotation about the "
"precession axis" )
parser.add_argument( "--precangle", required=False, type=float,
default=65.0, help="The opening angle (in degrees) of the spin axis "
"from the precession axis" )
parser.add_argument( "--hwprpm", required=False, type=float,
default=0.0, help="The rate (in RPM) of the HWP rotation" )
parser.add_argument( "--hwpstep", required=False, default=None,
help="For stepped HWP, the angle in degrees of each step" )
parser.add_argument( "--hwpsteptime", required=False, type=float,
default=0.0, help="For stepped HWP, the the time in seconds between "
"steps" )
parser.add_argument( "--obs", required=False, type=float, default=1.0,
help="Number of hours in one science observation" )
parser.add_argument( "--gap", required=False, type=float, default=0.0,
help="Cooler cycle time in hours between science obs" )
parser.add_argument( "--numobs", required=False, type=int, default=1,
help="Number of complete observations" )
parser.add_argument( "--outdir", required=False, default="out",
help="Output directory" )
parser.add_argument( "--debug", required=False, default=False,
action="store_true", help="Write diagnostics" )
parser.add_argument( "--nside", required=False, type=int, default=64,
help="Healpix NSIDE" )
parser.add_argument( "--subnside", required=False, type=int, default=4,
help="Distributed pixel sub-map NSIDE" )
parser.add_argument( "--baseline", required=False, type=float,
default=60.0, help="Destriping baseline length (seconds)" )
parser.add_argument( "--noisefilter", required=False, default=False,
action="store_true", help="Destripe with the noise filter enabled" )
parser.add_argument( "--madam", required=False, default=False,
action="store_true", help="If specified, use libmadam for map-making" )
parser.add_argument( "--madampar", required=False, default=None,
help="Madam parameter file" )
parser.add_argument('--flush',
required=False, default=False, action='store_true',
help='Flush every print statement.')
parser.add_argument( "--MC_start", required=False, type=int, default=0,
help="First Monte Carlo noise realization" )
parser.add_argument( "--MC_count", required=False, type=int, default=1,
help="Number of Monte Carlo noise realizations" )
parser.add_argument( "--fp", required=False, default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
"a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" "
"(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" "
"(float). For optional plotting, the key \"color\" can specify a "
"valid matplotlib color string." )
parser.add_argument('--tidas',
required=False, default=None,
help='Output TIDAS export path')
parser.add_argument('--input_map', required=False,
help='Input map for signal')
parser.add_argument('--input_pysm_model', required=False,
help='Comma separated models for on-the-fly PySM '
'simulation, e.g. s3,d6,f1,a2"')
parser.add_argument('--apply_beam', required=False, action='store_true',
help='Apply beam convolution to input map with gaussian '
'beam parameters defined in focalplane')
parser.add_argument('--input_dipole', required=False,
help='Simulate dipole, possible values are '
'total, orbital, solar')
parser.add_argument('--input_dipole_solar_speed_kms', required=False,
help='Solar system speed [km/s]', type=float,
default=369.0)
parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False,
help='Solar system speed galactic latitude [degrees]',
type=float, default=48.26)
parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False,
help='Solar system speed galactic longitude[degrees]',
type=float, default=263.99)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# get options
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > args.nside:
subnside = args.nside
subnpix = 12 * subnside * subnside
start = MPI.Wtime()
fp = None
# Load focalplane information
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.fp, "rb") as p:
fp = pickle.load(p)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Create focalplane: {:.2f} seconds".format(stop-start),
flush=True)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
tt.plot_focalplane(fp, 10.0, 10.0, outfile)
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
if groupsize > len(fp.keys()):
if comm.comm_world.rank == 0:
print("process group is too large for the number of detectors",
flush=True)
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(fp.keys())
detquats = {}
detindx = None
if "index" in fp[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = fp[d]["quat"]
if detindx is not None:
detindx[d] = fp[d]["index"]
# Distribute the observations uniformly
groupdist = toast.distribute_uniform(args.numobs, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = tt.regular_intervals(args.numobs, args.starttime, 0,
args.samplerate, 3600*args.obs, 3600*args.gap)
# Create the noise model used for all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]["fmin"]
fknee[d] = fp[d]["fknee"]
alpha[d] = fp[d]["alpha"]
NET[d] = fp[d]["NET"]
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
mem_counter = tt.OpMemoryCounter()
# The distributed timestream data
data = toast.Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = tt.TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
firsttime=obsrange[ob].start,
rate=args.samplerate,
spinperiod=args.spinperiod,
spinangle=args.spinangle,
precperiod=args.precperiod,
precangle=args.precangle,
detindx=detindx,
detranks=comm.group_size
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Read parameters, compute data distribution: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = tt.slew_precession_axis(nsim=tod.local_samples[1],
firstsamp=obsoffset, samplerate=args.samplerate,
degday=degday)
tod.set_prec_axis(qprec=precquat)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Construct boresight pointing: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# make a Healpix pointing matrix.
pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime)
pointing.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Pointing generation took {:.3f} s".format(elapsed), flush=True)
start = stop
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = "signal"
if args.input_pysm_model:
simulate_sky_signal(args, comm, data, mem_counter,
[fp], subnpix, localsm, signalname=signalname)
if args.input_dipole:
print("Simulating dipole")
op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole,
solar_speed=args.input_dipole_solar_speed_kms,
solar_gal_lat=args.input_dipole_solar_gal_lat_deg,
solar_gal_lon=args.input_dipole_solar_gal_lon_deg,
out=signalname,
keep_quats=True,
keep_vel=False,
subtract=False,
coord="G",
freq=0, # we could use frequency for quadrupole correction
flag_mask=255, common_flag_mask=255)
op_sim_dipole.exec(data)
# Mapmaking. For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in detectors:
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.samplerate * net * net)
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing N_pp took {:.3f} s".format(elapsed),
flush=True)
start = stop
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Dumping debug data distribution took "
"{:.3f} s".format(elapsed), flush=True)
start = stop
mcstart = start
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# create output directory for this realization
outpath = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(outpath):
os.makedirs(outpath)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Creating output dir {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# clear all signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
if mc == firstmc:
# For the first realization, optionally export the
# timestream data to a TIDAS volume.
if args.tidas is not None:
from toast.tod.tidas import OpTidasExport
tidas_path = os.path.abspath(args.tidas)
export = OpTidasExport(tidas_path, name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Noise simulation {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal",
detweights=detweights)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Building noise weighted map {:04d} took {:.3f} s".format(
mc, elapsed), flush=True)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Computing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.write_healpix_fits(os.path.join(outpath, "binned.fits"))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Writing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
elapsed = stop - mcstart
if comm.comm_world.rank == 0:
print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed),
flush=True)
start = stop
else:
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = cross
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = args.nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# clear all total signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Noise simulation took {:.3f} s".format(elapsed),
flush=True)
start = stop
# create output directory for this realization
pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(pars["path_output"]):
os.makedirs(pars["path_output"])
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open(os.path.join(pars["path_output"],
"distdata.txt"), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
madam = tm.OpMadam(params=pars, detweights=detweights,
name="tot_signal")
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Mapmaking took {:.3f} s".format(elapsed), flush=True)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description="Simulate satellite "
"boresight pointing and make a noise map.", fromfile_prefix_chars="@" )
parser.add_argument( "--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute observations" )
parser.add_argument( "--samplerate", required=False, type=float,
default=40.0, help="Detector sample rate (Hz)" )
parser.add_argument( "--starttime", required=False, type=float,
default=0.0, help="The overall start time of the simulation" )
parser.add_argument( "--spinperiod", required=False, type=float,
default=10.0, help="The period (in minutes) of the rotation about the "
"spin axis" )
parser.add_argument( "--spinangle", required=False, type=float,
default=30.0, help="The opening angle (in degrees) of the boresight "
"from the spin axis" )
parser.add_argument( "--precperiod", required=False, type=float,
default=50.0, help="The period (in minutes) of the rotation about the "
"precession axis" )
parser.add_argument( "--precangle", required=False, type=float,
default=65.0, help="The opening angle (in degrees) of the spin axis "
"from the precession axis" )
parser.add_argument( "--hwprpm", required=False, type=float,
default=0.0, help="The rate (in RPM) of the HWP rotation" )
parser.add_argument( "--hwpstep", required=False, default=None,
help="For stepped HWP, the angle in degrees of each step" )
parser.add_argument( "--hwpsteptime", required=False, type=float,
default=0.0, help="For stepped HWP, the the time in seconds between "
"steps" )
parser.add_argument( "--obs", required=False, type=float, default=1.0,
help="Number of hours in one science observation" )
parser.add_argument( "--gap", required=False, type=float, default=0.0,
help="Cooler cycle time in hours between science obs" )
parser.add_argument( "--numobs", required=False, type=int, default=1,
help="Number of complete observations" )
parser.add_argument( "--outdir", required=False, default="out",
help="Output directory" )
parser.add_argument( "--debug", required=False, default=False,
action="store_true", help="Write diagnostics" )
parser.add_argument( "--nside", required=False, type=int, default=64,
help="Healpix NSIDE" )
parser.add_argument( "--subnside", required=False, type=int, default=4,
help="Distributed pixel sub-map NSIDE" )
parser.add_argument('--coord', required=False, default='E',
help='Sky coordinate system [C,E,G]')
parser.add_argument( "--baseline", required=False, type=float,
default=60.0, help="Destriping baseline length (seconds)" )
parser.add_argument( "--noisefilter", required=False, default=False,
action="store_true", help="Destripe with the noise filter enabled" )
parser.add_argument( "--madam", required=False, default=False,
action="store_true", help="If specified, use libmadam for map-making" )
parser.add_argument( "--madampar", required=False, default=None,
help="Madam parameter file" )
parser.add_argument('--flush',
required=False, default=False, action='store_true',
help='Flush every print statement.')
parser.add_argument( "--MC_start", required=False, type=int, default=0,
help="First Monte Carlo noise realization" )
parser.add_argument( "--MC_count", required=False, type=int, default=1,
help="Number of Monte Carlo noise realizations" )
parser.add_argument( "--fp", required=False, default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
"a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" "
"(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" "
"(float). For optional plotting, the key \"color\" can specify a "
"valid matplotlib color string." )
parser.add_argument( "--gain", required=False, default=None,
help= "Calibrate the input timelines with a set of gains from a"
"FITS file containing 3 extensions:"
"HDU named DETECTORS : table with list of detector names in a column named DETECTORS"
"HDU named TIME: table with common timestamps column named TIME"
"HDU named GAINS: 2D image of floats with one row per detector and one column per value.")
parser.add_argument('--tidas',
required=False, default=None,
help='Output TIDAS export path')
parser.add_argument('--spt3g',
required=False, default=None,
help='Output SPT3G export path')
parser.add_argument('--input_map', required=False,
help='Input map for signal')
parser.add_argument('--input_pysm_model', required=False,
help='Comma separated models for on-the-fly PySM '
'simulation, e.g. s3,d6,f1,a2"')
parser.add_argument('--input_pysm_precomputed_cmb_K_CMB', required=False,
help='Precomputed CMB map for PySM in K_CMB'
'it overrides any model defined in input_pysm_model"')
parser.add_argument('--apply_beam', required=False, action='store_true',
help='Apply beam convolution to input map with gaussian '
'beam parameters defined in focalplane')
parser.add_argument('--input_dipole', required=False,
help='Simulate dipole, possible values are '
'total, orbital, solar')
parser.add_argument('--input_dipole_solar_speed_kms', required=False,
help='Solar system speed [km/s]', type=float,
default=369.0)
parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False,
help='Solar system speed galactic latitude [degrees]',
type=float, default=48.26)
parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False,
help='Solar system speed galactic longitude[degrees]',
type=float, default=263.99)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot export")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# get options
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > args.nside:
subnside = args.nside
subnpix = 12 * subnside * subnside
start = MPI.Wtime()
fp = None
gain = None
# Load focalplane information
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.fp, "rb") as p:
fp = pickle.load(p)
if args.gain is not None:
gain = {}
with fits.open(args.gain) as f:
gain["TIME"] = np.array(f["TIME"].data["TIME"])
for i_det, det_name in f["DETECTORS"].data["DETECTORS"]:
gain[det_name] = np.array(f["GAINS"].data[i_det, :])
if args.gain is not None:
gain = comm.comm_world.bcast(gain, root=0)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Create focalplane ({} dets): {:.2f} seconds"\
.format(len(fp.keys()), stop-start), flush=True)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
dquats = { x : fp[x]["quat"] for x in fp.keys() }
dfwhm = { x : fp[x]["fwhm"] for x in fp.keys() }
tt.plot_focalplane(dquats, 10.0, 10.0, outfile, fwhm=dfwhm)
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
if groupsize > len(fp.keys()):
if comm.comm_world.rank == 0:
print("process group is too large for the number of detectors",
flush=True)
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(fp.keys())
detquats = {}
detindx = None
if "index" in fp[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = fp[d]["quat"]
if detindx is not None:
detindx[d] = fp[d]["index"]
# Distribute the observations uniformly
groupdist = toast.distribute_uniform(args.numobs, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = tt.regular_intervals(args.numobs, args.starttime, 0,
args.samplerate, 3600*args.obs, 3600*args.gap)
# Create the noise model used for all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]["fmin"]
fknee[d] = fp[d]["fknee"]
alpha[d] = fp[d]["alpha"]
NET[d] = fp[d]["NET"]
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
mem_counter = tt.OpMemoryCounter()
# The distributed timestream data
data = toast.Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = tt.TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
coord=args.coord,
firstsamp=obsrange[ob].first,
firsttime=obsrange[ob].start,
rate=args.samplerate,
spinperiod=args.spinperiod,
spinangle=args.spinangle,
precperiod=args.precperiod,
precangle=args.precangle,
detindx=detindx,
detranks=comm.group_size
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Read parameters, compute data distribution: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = np.empty(4*tod.local_samples[1],
dtype=np.float64).reshape((-1,4))
tt.slew_precession_axis(precquat,
firstsamp=(obsoffset + tod.local_samples[0]),
samplerate=args.samplerate, degday=degday)
tod.set_prec_axis(qprec=precquat)
del precquat
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Construct boresight pointing: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# make a Healpix pointing matrix.
pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime)
pointing.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Pointing generation took {:.3f} s".format(elapsed), flush=True)
start = stop
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = "signal"
has_signal = False
if args.input_pysm_model:
has_signal = True
simulate_sky_signal(args, comm, data, mem_counter,
[fp], subnpix, localsm, signalname=signalname)
if args.input_dipole:
print("Simulating dipole")
has_signal = True
op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole,
solar_speed=args.input_dipole_solar_speed_kms,
solar_gal_lat=args.input_dipole_solar_gal_lat_deg,
solar_gal_lon=args.input_dipole_solar_gal_lon_deg,
out=signalname,
keep_quats=False,
keep_vel=False,
subtract=False,
coord=args.coord,
freq=0, # we could use frequency for quadrupole correction
flag_mask=255, common_flag_mask=255)
op_sim_dipole.exec(data)
del op_sim_dipole
# Mapmaking. For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in detectors:
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.samplerate * net * net)
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing N_pp took {:.3f} s".format(elapsed),
flush=True)
start = stop
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Dumping debug data distribution took "
"{:.3f} s".format(elapsed), flush=True)
start = stop
mcstart = start
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# create output directory for this realization
outpath = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(outpath):
os.makedirs(outpath)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Creating output dir {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# clear all signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Noise simulation {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# add sky signal
if has_signal:
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Add sky signal {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
if gain is not None:
op_apply_gain = tt.OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
if mc == firstmc:
# For the first realization, optionally export the
# timestream data. If we had observation intervals defined,
# we could pass "use_interval=True" to the export operators,
# which would ensure breaks in the exported data at
# acceptable places.
if args.tidas is not None:
tidas_path = os.path.abspath(args.tidas)
export = OpTidasExport(tidas_path, TODTidas, backend="hdf5",
use_todchunks=True,
create_opts={"group_dets":"sim"},
ctor_opts={"group_dets":"sim"},
cache_name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Tidas export took {:.3f} s"\
.format(elapsed), flush=True)
start = stop
if args.spt3g is not None:
spt3g_path = os.path.abspath(args.spt3g)
export = Op3GExport(spt3g_path, TOD3G, use_todchunks=True,
export_opts={"prefix" : "sim"},
cache_name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" SPT3G export took {:.3f} s"\
.format(elapsed), flush=True)
start = stop
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal",
detweights=detweights)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Building noise weighted map {:04d} took {:.3f} s".format(
mc, elapsed), flush=True)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Computing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.write_healpix_fits(os.path.join(outpath, "binned.fits"))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Writing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
elapsed = stop - mcstart
if comm.comm_world.rank == 0:
print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed),
flush=True)
start = stop
else:
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = cross
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = args.nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# clear all total signal data from the cache, so that we can generate
# new noise timestreams.
for obs in data.obs:
tod = obs['tod']
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
if has_signal:
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
if gain is not None:
op_apply_gain = tt.OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Noise simulation took {:.3f} s".format(elapsed),
flush=True)
start = stop
# create output directory for this realization
pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(pars["path_output"]):
os.makedirs(pars["path_output"])
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open(os.path.join(pars["path_output"],
"distdata.txt"), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
madam = tm.OpMadam(params=pars, detweights=detweights,
name="tot_signal")
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Mapmaking took {:.3f} s".format(elapsed), flush=True)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(elapsed), flush=True)