def expand_pointing(args, comm, data):
""" Expand the bore sight pointing to every detector.
"""
start = MPI.Wtime()
autotimer = timing.auto_timer()
hwprpm = args.hwprpm
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
hwpsteptime = args.hwpsteptime
npix = 12 * args.nside**2
if comm.comm_world.rank == 0:
print('Expanding pointing', flush=args.flush)
pointing = tt.OpPointingHpix(nside=args.nside,
nest=True,
mode='IQU',
hwprpm=hwprpm,
hwpstep=hwpstep,
hwpsteptime=hwpsteptime)
pointing.exec(data)
# Only purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if args.tidas is None:
for ob in data.obs:
tod = ob['tod']
tod.free_radec_quats()
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Pointing generation took {:.3f} s'.format(stop - start),
flush=args.flush)
return
# Plot some local data from the first observation
import matplotlib.pyplot as plt
ob = data.obs[0]
tod = ob["tod"]
boloname = tod.local_dets[0]
bolodata = tod.cache.reference("signal_{}".format(boloname))
fig = plt.figure()
plt.plot(np.arange(len(bolodata)), bolodata)
plt.savefig("bolo_{}.png".format(boloname))
del bolodata
# Construct a pointing matrix
nside = 2048
pointing = tt.OpPointingHpix(nside=nside,
nest=True,
mode="IQU",
pixels="pixels",
weights="weights")
pointing.exec(data)
# Apply a polynomial filter per subscan
polyfilter = tt.OpPolyFilter(order=3,
name="signal",
common_flag_name="flags_common",
common_flag_mask=1)
polyfilter.exec(data)
# Make a binned map
npix = 12 * nside**2
subnpix = 12 * 16**2
binned_map(data, npix, subnpix)
def main():
# We are going to group our processes in a single group. This is fine
# if we have fewer processes than detectors. Otherwise we should group
# them in a reasonable size that is smaller than the number of detectors
# and which divides evenly into the total number of processes.
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=MPI.COMM_WORLD.size)
# Make a fake focalplane. Plot it just for fun (don't waste time on this
# for very large runs though).
fp = fake_focalplane()
if comm.comm_world.rank == 0:
outfile = "custom_example_focalplane.png"
set_backend()
tt.plot_focalplane(fp, 6.0, 6.0, outfile)
# Read in 2 boresight files
borefiles = [
"../data/custom_example_boresight_1.txt",
"../data/custom_example_boresight_2.txt"
]
# Set up the distributed data
rate = 100.0
data = create_observations(comm, rate, fp, borefiles)
# Configure the healpix pixelization we will use for map-making and
# also the "submap" resolution, which sets granularity of the locally
# stored pieces of the sky.
map_nside = 512
map_npix = 12 * map_nside**2
sub_nside = 4
sub_npix = 12 * sub_nside**2
# Compute a pointing matrix with healpix pixels and weights.
pointing = tt.OpPointingHpix(nside=map_nside, nest=True, mode="IQU",
pixels="pixels", weights="weights")
pointing.exec(data)
# Compute the locally hit submaps
local_submaps = pixel_dist(data, sub_npix)
# Sources of simulated data: scan from a symmetric beam convolved sky
# and then add some simulated noise.
signalmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3,
dtype=np.float64, submap=sub_npix, local=local_submaps)
signalmap.read_healpix_fits("../data/custom_example_sky.fits")
scanmap = tt.OpSimScan(distmap=signalmap, pixels='pixels',
weights='weights', out="sim")
scanmap.exec(data)
nse = tt.OpSimNoise(out="sim", realization=0)
nse.exec(data)
# Accumulate the hits and inverse diagonal pixel covariance, as well as the
# noise weighted map. Here we simply use inverse noise weighting.
detweights = {}
for d in fp.keys():
net = fp[d]["NET"]
detweights[d] = 1.0 / (rate * net * net)
invnpp = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=6,
dtype=np.float64, submap=sub_npix, local=local_submaps)
hits = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=1,
dtype=np.int64, submap=sub_npix, local=local_submaps)
zmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3,
dtype=np.float64, submap=sub_npix, local=local_submaps)
invnpp.data.fill(0.0)
hits.data.fill(0)
zmap.data.fill(0.0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits, zmap=zmap, name="sim")
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
zmap.allreduce()
# Write these products out
hits.write_healpix_fits("custom_example_hits.fits")
invnpp.write_healpix_fits("custom_example_invnpp.fits")
zmap.write_healpix_fits("custom_example_zmap.fits")
# Invert the covariance and write
tm.covariance_invert(invnpp, 1.0e-3)
invnpp.write_healpix_fits("custom_example_npp.fits")
# Apply covariance to make the binned map
tm.covariance_apply(invnpp, zmap)
zmap.write_healpix_fits("custom_example_binned.fits")
MPI.Finalize()
return
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser(
description="Read existing data and make a simple map.",
fromfile_prefix_chars="@",
)
parser.add_argument(
"--groupsize",
required=False,
type=int,
default=0,
help="size of processor groups used to distribute "
"observations",
)
parser.add_argument(
"--hwprpm",
required=False,
type=float,
default=0.0,
help="The rate (in RPM) of the HWP rotation",
)
parser.add_argument(
"--samplerate",
required=False,
default=100.0,
type=np.float,
help="Detector sample rate (Hz)",
)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument("--nside",
required=False,
type=int,
default=64,
help="Healpix NSIDE")
parser.add_argument(
"--subnside",
required=False,
type=int,
default=8,
help="Distributed pixel sub-map NSIDE",
)
parser.add_argument("--coord",
required=False,
default="E",
help="Sky coordinate system [C,E,G]")
parser.add_argument(
"--baseline",
required=False,
type=float,
default=60.0,
help="Destriping baseline length (seconds)",
)
parser.add_argument(
"--noisefilter",
required=False,
default=False,
action="store_true",
help="Destripe with the noise filter enabled",
)
parser.add_argument(
"--madam",
required=False,
default=False,
action="store_true",
help="If specified, use libmadam for map-making",
)
parser.add_argument("--madampar",
required=False,
default=None,
help="Madam parameter file")
parser.add_argument(
"--polyorder",
required=False,
type=int,
help="Polynomial order for the polyfilter",
)
parser.add_argument(
"--wbin_ground",
required=False,
type=float,
help="Ground template bin width [degrees]",
)
parser.add_argument(
"--flush",
required=False,
default=False,
action="store_true",
help="Flush every print statement.",
)
parser.add_argument("--tidas",
required=False,
default=None,
help="Input TIDAS volume")
parser.add_argument("--tidas_detgroup",
required=False,
default=None,
help="TIDAS detector group")
parser.add_argument("--spt3g",
required=False,
default=None,
help="Input SPT3G data directory")
parser.add_argument(
"--spt3g_prefix",
required=False,
default=None,
help="SPT3G data frame file prefix",
)
parser.add_argument(
"--common_flag_mask",
required=False,
default=0,
type=np.uint8,
help="Common flag mask",
)
parser.add_argument(
"--debug",
required=False,
default=False,
action="store_true",
help="Write data distribution info and focalplane plot",
)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
# args = parser.parse_args(sys.argv)
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if (args.tidas is not None) and (args.spt3g is not None):
raise RuntimeError("Cannot read two datasets!")
if (args.tidas is None) and (args.spt3g is None):
raise RuntimeError("No dataset specified!")
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot load")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot load")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# Pixelization
nside = args.nside
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > nside:
subnside = nside
subnpix = 12 * subnside * subnside
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print(
"WARNING: process groupsize does not evenly divide into "
"total number of processes",
flush=True,
)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# Create output directory
mtime = MPI.Wtime()
if comm.comm_world.rank == 0:
if not os.path.isdir(args.outdir):
os.makedirs(args.outdir)
mtime = elapsed(comm.comm_world, mtime, "Creating output directory")
# The distributed timestream data
data = None
if args.tidas is not None:
if args.tidas_detgroup is None:
raise RuntimeError("you must specify the detector group")
data = tds.load_tidas(
comm,
comm.group_size,
args.tidas,
"r",
args.tidas_detgroup,
tds.TODTidas,
group_dets=args.tidas_detgroup,
distintervals="chunks",
)
if args.spt3g is not None:
if args.spt3g_prefix is None:
raise RuntimeError("you must specify the frame file prefix")
data = s3g.load_spt3g(
comm,
comm.group_size,
args.spt3g,
args.spt3g_prefix,
s3g.obsweight_spt3g,
s3g.TOD3G,
)
mtime = elapsed(comm.comm_world, mtime, "Distribute data")
# In debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
mtime = elapsed(comm.comm_world, mtime,
"Dumping debug data distribution")
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
# Just plot the dets from the first TOD
temptod = data.obs[0]["tod"]
# FIXME: change this once we store det info in the metadata.
dfwhm = {x: 10.0 for x in temptod.detectors}
tt.plot_focalplane(temptod.detoffset(),
10.0,
10.0,
outfile,
fwhm=dfwhm)
comm.comm_world.barrier()
mtime = elapsed(comm.comm_world, mtime, "Plotting debug focalplane")
# Compute pointing matrix
pointing = tt.OpPointingHpix(nside=args.nside,
nest=True,
mode="IQU",
hwprpm=args.hwprpm)
pointing.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Expand pointing")
# Mapmaking.
# FIXME: We potentially have a different noise model for every
# observation. We need to have both spt3g and tidas format Noise
# classes which read the information from disk. Then the mapmaking
# operators need to get these noise weights from each observation.
detweights = {d: 1.0 for d in data.obs[0]["tod"].detectors}
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# Filter data if desired
if args.polyorder:
polyfilter = tt.OpPolyFilter(
order=args.polyorder, common_flag_mask=args.common_flag_mask)
polyfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering")
if args.wbin_ground:
groundfilter = tt.OpGroundFilter(
wbin=args.wbin_ground, common_flag_mask=args.common_flag_mask)
groundfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime,
"Ground template filtering")
# Compute pixel space distribution
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
if localpix is None:
raise RuntimeError(
"Process {} has no hit pixels. Perhaps there are fewer "
"detectors than processes in the group?".format(
comm.comm_world.rank))
localsm = np.unique(np.floor_divide(localpix, subnpix))
mtime = elapsed(comm.comm_world, mtime, "Compute local submaps")
# construct distributed maps to store the covariance,
# noise weighted map, and hits
mtime = MPI.Wtime()
invnpp = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=6,
dtype=np.float64,
submap=subnpix,
local=localsm,
)
hits = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=1,
dtype=np.int64,
submap=subnpix,
local=localsm,
)
zmap = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=3,
dtype=np.float64,
submap=subnpix,
local=localsm,
)
# compute the hits and covariance.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(
detweights=detweights,
invnpp=invnpp,
hits=hits,
common_flag_mask=args.common_flag_mask,
)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1")
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1")
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1")
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing N_pp")
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap,
detweights=detweights,
common_flag_mask=args.common_flag_mask)
build_zmap.exec(data)
zmap.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map")
tm.covariance_apply(invnpp, zmap)
mtime = elapsed(comm.comm_world, mtime, "Computing binned map")
zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits"))
mtime = elapsed(comm.comm_world, mtime, "Writing binned map")
else:
# Set up MADAM map making.
pars = {}
pars["temperature_only"] = "F"
pars["force_pol"] = "T"
pars["kfirst"] = "T"
pars["concatenate_messages"] = "T"
pars["write_map"] = "T"
pars["write_binmap"] = "T"
pars["write_matrix"] = "T"
pars["write_wcov"] = "T"
pars["write_hits"] = "T"
pars["nside_cross"] = nside // 2
pars["nside_submap"] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars["base_first"] = args.baseline
pars["nside_map"] = nside
if args.noisefilter:
pars["kfilter"] = "T"
else:
pars["kfilter"] = "F"
pars["fsample"] = args.samplerate
madam = tm.OpMadam(params=pars,
detweights=detweights,
common_flag_mask=args.common_flag_mask)
madam.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking")
comm.comm_world.barrier()
stop = MPI.Wtime()
dur = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(dur), flush=True)
return
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description="Simulate satellite "
"boresight pointing and make a noise map.", fromfile_prefix_chars="@" )
parser.add_argument( "--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute observations" )
parser.add_argument( "--samplerate", required=False, type=float,
default=40.0, help="Detector sample rate (Hz)" )
parser.add_argument( "--starttime", required=False, type=float,
default=0.0, help="The overall start time of the simulation" )
parser.add_argument( "--spinperiod", required=False, type=float,
default=10.0, help="The period (in minutes) of the rotation about the "
"spin axis" )
parser.add_argument( "--spinangle", required=False, type=float,
default=30.0, help="The opening angle (in degrees) of the boresight "
"from the spin axis" )
parser.add_argument( "--precperiod", required=False, type=float,
default=50.0, help="The period (in minutes) of the rotation about the "
"precession axis" )
parser.add_argument( "--precangle", required=False, type=float,
default=65.0, help="The opening angle (in degrees) of the spin axis "
"from the precession axis" )
parser.add_argument( "--hwprpm", required=False, type=float,
default=0.0, help="The rate (in RPM) of the HWP rotation" )
parser.add_argument( "--hwpstep", required=False, default=None,
help="For stepped HWP, the angle in degrees of each step" )
parser.add_argument( "--hwpsteptime", required=False, type=float,
default=0.0, help="For stepped HWP, the the time in seconds between "
"steps" )
parser.add_argument( "--obs", required=False, type=float, default=1.0,
help="Number of hours in one science observation" )
parser.add_argument( "--gap", required=False, type=float, default=0.0,
help="Cooler cycle time in hours between science obs" )
parser.add_argument( "--numobs", required=False, type=int, default=1,
help="Number of complete observations" )
parser.add_argument( "--outdir", required=False, default="out",
help="Output directory" )
parser.add_argument( "--debug", required=False, default=False,
action="store_true", help="Write diagnostics" )
parser.add_argument( "--nside", required=False, type=int, default=64,
help="Healpix NSIDE" )
parser.add_argument( "--subnside", required=False, type=int, default=4,
help="Distributed pixel sub-map NSIDE" )
parser.add_argument( "--baseline", required=False, type=float,
default=60.0, help="Destriping baseline length (seconds)" )
parser.add_argument( "--noisefilter", required=False, default=False,
action="store_true", help="Destripe with the noise filter enabled" )
parser.add_argument( "--madam", required=False, default=False,
action="store_true", help="If specified, use libmadam for map-making" )
parser.add_argument( "--madampar", required=False, default=None,
help="Madam parameter file" )
parser.add_argument('--flush',
required=False, default=False, action='store_true',
help='Flush every print statement.')
parser.add_argument( "--MC_start", required=False, type=int, default=0,
help="First Monte Carlo noise realization" )
parser.add_argument( "--MC_count", required=False, type=int, default=1,
help="Number of Monte Carlo noise realizations" )
parser.add_argument( "--fp", required=False, default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
"a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" "
"(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" "
"(float). For optional plotting, the key \"color\" can specify a "
"valid matplotlib color string." )
parser.add_argument('--tidas',
required=False, default=None,
help='Output TIDAS export path')
parser.add_argument('--input_map', required=False,
help='Input map for signal')
parser.add_argument('--input_pysm_model', required=False,
help='Comma separated models for on-the-fly PySM '
'simulation, e.g. s3,d6,f1,a2"')
parser.add_argument('--apply_beam', required=False, action='store_true',
help='Apply beam convolution to input map with gaussian '
'beam parameters defined in focalplane')
parser.add_argument('--input_dipole', required=False,
help='Simulate dipole, possible values are '
'total, orbital, solar')
parser.add_argument('--input_dipole_solar_speed_kms', required=False,
help='Solar system speed [km/s]', type=float,
default=369.0)
parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False,
help='Solar system speed galactic latitude [degrees]',
type=float, default=48.26)
parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False,
help='Solar system speed galactic longitude[degrees]',
type=float, default=263.99)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# get options
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > args.nside:
subnside = args.nside
subnpix = 12 * subnside * subnside
start = MPI.Wtime()
fp = None
# Load focalplane information
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.fp, "rb") as p:
fp = pickle.load(p)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Create focalplane: {:.2f} seconds".format(stop-start),
flush=True)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
tt.plot_focalplane(fp, 10.0, 10.0, outfile)
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
if groupsize > len(fp.keys()):
if comm.comm_world.rank == 0:
print("process group is too large for the number of detectors",
flush=True)
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(fp.keys())
detquats = {}
detindx = None
if "index" in fp[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = fp[d]["quat"]
if detindx is not None:
detindx[d] = fp[d]["index"]
# Distribute the observations uniformly
groupdist = toast.distribute_uniform(args.numobs, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = tt.regular_intervals(args.numobs, args.starttime, 0,
args.samplerate, 3600*args.obs, 3600*args.gap)
# Create the noise model used for all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]["fmin"]
fknee[d] = fp[d]["fknee"]
alpha[d] = fp[d]["alpha"]
NET[d] = fp[d]["NET"]
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
mem_counter = tt.OpMemoryCounter()
# The distributed timestream data
data = toast.Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = tt.TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
firsttime=obsrange[ob].start,
rate=args.samplerate,
spinperiod=args.spinperiod,
spinangle=args.spinangle,
precperiod=args.precperiod,
precangle=args.precangle,
detindx=detindx,
detranks=comm.group_size
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Read parameters, compute data distribution: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = tt.slew_precession_axis(nsim=tod.local_samples[1],
firstsamp=obsoffset, samplerate=args.samplerate,
degday=degday)
tod.set_prec_axis(qprec=precquat)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Construct boresight pointing: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# make a Healpix pointing matrix.
pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime)
pointing.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Pointing generation took {:.3f} s".format(elapsed), flush=True)
start = stop
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = "signal"
if args.input_pysm_model:
simulate_sky_signal(args, comm, data, mem_counter,
[fp], subnpix, localsm, signalname=signalname)
if args.input_dipole:
print("Simulating dipole")
op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole,
solar_speed=args.input_dipole_solar_speed_kms,
solar_gal_lat=args.input_dipole_solar_gal_lat_deg,
solar_gal_lon=args.input_dipole_solar_gal_lon_deg,
out=signalname,
keep_quats=True,
keep_vel=False,
subtract=False,
coord="G",
freq=0, # we could use frequency for quadrupole correction
flag_mask=255, common_flag_mask=255)
op_sim_dipole.exec(data)
# Mapmaking. For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in detectors:
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.samplerate * net * net)
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing N_pp took {:.3f} s".format(elapsed),
flush=True)
start = stop
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Dumping debug data distribution took "
"{:.3f} s".format(elapsed), flush=True)
start = stop
mcstart = start
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# create output directory for this realization
outpath = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(outpath):
os.makedirs(outpath)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Creating output dir {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# clear all signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
if mc == firstmc:
# For the first realization, optionally export the
# timestream data to a TIDAS volume.
if args.tidas is not None:
from toast.tod.tidas import OpTidasExport
tidas_path = os.path.abspath(args.tidas)
export = OpTidasExport(tidas_path, name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Noise simulation {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal",
detweights=detweights)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Building noise weighted map {:04d} took {:.3f} s".format(
mc, elapsed), flush=True)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Computing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.write_healpix_fits(os.path.join(outpath, "binned.fits"))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Writing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
elapsed = stop - mcstart
if comm.comm_world.rank == 0:
print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed),
flush=True)
start = stop
else:
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = cross
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = args.nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# clear all total signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Noise simulation took {:.3f} s".format(elapsed),
flush=True)
start = stop
# create output directory for this realization
pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(pars["path_output"]):
os.makedirs(pars["path_output"])
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open(os.path.join(pars["path_output"],
"distdata.txt"), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
madam = tm.OpMadam(params=pars, detweights=detweights,
name="tot_signal")
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Mapmaking took {:.3f} s".format(elapsed), flush=True)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main(parameter_file, outdir):
'The main function'
# Set up the logging system
logformat = '[%(asctime)s %(levelname)s] %(message)s'
log.basicConfig(level=log.INFO,
format=logformat)
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
log.info("running with %d processes", comm.comm_world.size)
# Read the parameter file
conf = None
if comm.comm_world.rank == 0:
configuration_parser = ConfigParser()
configuration_parser.read(parameter_file)
conf = build_configuration_obj(configuration_parser)
log.info('configuration read from file "%s"', parameter_file)
conf = comm.comm_world.bcast(conf, root=0)
if outdir != '.':
if not os.path.isdir(outdir):
os.mkdir(outdir)
# construct the list of intervals which split the whole observation time
intervals = tt.regular_intervals(n=int(conf.num_obs),
start=0.0,
first=0,
rate=conf.sample_rate_Hz,
duration=3600*float(conf.hours_per_obs),
gap=3600*float(conf.gap_width_hours))
detquats = {} # type: Dict[str, Any]
for det in conf.detectors:
detquats[det.name] = det.quat
# Create the noise model for this observation
rate = {} # type: Dict[str, float]
fmin = {} # type: Dict[str, float]
fknee = {} # type: Dict[str, float]
alpha = {} # type: Dict[str, float]
net = {} # type: Dict[str, float]
for det in conf.detectors:
rate[det.name] = conf.sample_rate_Hz
fmin[det.name] = 1.0 / (3600.0 * conf.hours_per_obs * conf.num_obs)
fknee[det.name] = det.fknee_Hz
alpha[det.name] = det.alpha
net[det.name] = det.net
noise = tt.AnalyticNoise(rate=rate,
fmin=fmin,
detectors=[x.name for x in conf.detectors],
fknee=fknee,
alpha=alpha,
NET=net)
# The distributed time stream data
data = toast.Data(comm)
intsamp = None
if len(intervals) == 1:
intsamp = intervals[0].last - intervals[0].first + 1
else:
# include also the samples in the gap
intsamp = intervals[1].first - intervals[0].first
intchunk = int(intsamp / conf.chunks_per_obs)
distint = [] # type: List[int]
for _ in range(conf.chunks_per_obs - 1):
distint.append(intchunk)
distint.append(intsamp - (intchunk * (conf.chunks_per_obs - 1)))
distsizes = []
for _ in intervals:
distsizes.extend(distint)
totsamples = np.sum(distsizes)
# create the single TOD for this observation
tod = tt.TODSatellite(
mpicomm=comm.comm_group,
detectors=detquats,
samples=totsamples,
firsttime=0.0,
rate=conf.sample_rate_Hz,
spinperiod=conf.spin_period_min,
spinangle=conf.spin_angle_deg,
precperiod=conf.prec_period_min,
precangle=conf.prec_angle_deg,
sizes=distsizes)
# Create the (single) observation
ob = {} # type: Dict[str, Any]
ob['id'] = 0
ob['name'] = 'mission'
ob['tod'] = tod
ob['intervals'] = intervals
ob['noise'] = noise
data.obs.append(ob)
precquat = tt.slew_precession_axis(nsim=tod.local_samples[1],
firstsamp=tod.local_samples[0],
samplerate=conf.sample_rate_Hz,
degday=DEG_ANGLE_PER_DAY)
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
tod.set_prec_axis(qprec=precquat)
# simulate noise
nse = tt.OpSimNoise(out='noise', stream=0, realization=0)
nse.exec(data)
comm.comm_world.barrier()
# make a Healpix pointing matrix. By setting purge_pntg=True,
# we purge the detector quaternion pointing to save memory.
# If we ever change this pipeline in a way that needs this
# pointing at a later stage, we need to set this to False
# and run at higher concurrency.
pointing = tt.OpPointingHpix(nside=conf.nside,
nest=False,
mode='IQU',
hwprpm=conf.hwp_rpm,
hwpstep=None,
hwpsteptime=None)
pointing.exec(data)
comm.comm_world.barrier()
if comm.comm_world.rank == 0:
log.info('reading map "%s"', conf.sky_map)
foreground_map = healpy.read_map(conf.sky_map,
field=(conf.i_column, conf.q_column, conf.u_column),
verbose=False)
for det in conf.detectors:
if comm.comm_world.rank == 0:
log.info('processing detector "%s"', det.name)
times = tod.read_times()
flags, glflags = tod.read_flags(detector=det.name)
pixels = tod.cache.reference('pixels_{0}'.format(det.name))
theta, phi = healpy.pix2ang(conf.nside, pixels)
vect = healpy.pix2vec(conf.nside, pixels)
weights = tod.cache.reference('weights_{0}'.format(det.name))
psi = 0.5 * np.arctan2(weights[:, 2], weights[:, 1])
psi += np.deg2rad(det.polarisation_angle_deg)
noise = tod.cache.reference('noise_{0}'.format(det.name))
# Estimate the total TOD using the formula
# P = I + Q cos 2psi + U sin 2psi
# where Q and U are expressed in the reference frame of the detector
sky_i, sky_q, sky_u = [healpy.get_interp_val(foreground_map[x], theta, phi)
for x in (0, 1, 2)]
det_q = sky_q * np.cos(2. * psi) + sky_u * np.sin(2. * psi)
det_u = -sky_q * np.sin(2. * psi) + sky_u * np.cos(2. * psi)
foreground_tod = sky_i + det_q * np.cos(2. * psi) + det_u * np.sin(2. * psi)
dipole_tod = dip.get_dipole_temperature(np.column_stack(vect).T)
total_tod = det.gain * (foreground_tod + dipole_tod + noise)
hdu = fits.BinTableHDU.from_columns([fits.Column(name='TIME',
format='D',
unit='s',
array=times),
fits.Column(name='THETA',
format='E',
unit='rad',
array=theta),
fits.Column(name='PHI',
format='E',
unit='rad',
array=phi),
fits.Column(name='PSI',
format='E',
unit='rad',
array=psi),
fits.Column(name='FGTOD',
format='E',
unit='K',
array=foreground_tod),
fits.Column(name='DIPTOD',
format='E',
unit='K',
array=dipole_tod),
fits.Column(name='NOISETOD',
format='E',
unit='K',
array=noise),
fits.Column(name='TOTALTOD',
format='E',
unit='V',
array=total_tod),
fits.Column(name='FLAGS',
format='I',
unit='',
array=flags),
fits.Column(name='GLFLAGS',
format='I',
unit='',
array=glflags)])
hdu.header['COMMENT'] = 'Angles are expressed in the Ecliptic system'
hdu.header['FIRSTT'] = (times[0], 'Time of the first sample [s]')
hdu.header['LASTT'] = (times[-1], 'Time of the last sample [s]')
hdu.header['GAIN'] = (det.gain, 'Detector gain [V/K]')
hdu.header['NAME'] = (det.name, 'Name of the detector')
hdu.header['VERSION'] = (__version__,
'Version of the code used to create '
'this file')
hdu.header['net'] = (det.net, 'net')
hdu.header['ALPHA'] = (det.alpha, 'Slope of the 1/f noise')
hdu.header['FKNEE'] = (det.fknee_Hz,
'Knee frequency of the 1/f noise [Hz]')
output_file = os.path.join(outdir,
('tod_{0}_mpi{1:04d}.fits'
.format(det.name, comm.comm_world.rank)))
# Save a copy of the parameter file into the primary HDU of the file,
# so that it will be always possible to know the value of the input parameters
# used to create it
prim_hdu = fits.PrimaryHDU(data=conf.param_file_contents)
hdu_list = fits.HDUList([prim_hdu, hdu])
hdu_list.writeto(output_file, clobber=True)
if comm.comm_world.rank == 0:
log.info('pointings have been created successfully')
