def scan_signal(args, comm, data, localsm, subnpix):
""" Scan time-ordered signal from a map.
"""
signalname = None
if args.input_map:
if comm.comm_world.rank == 0:
print('Scanning input map', flush=args.flush)
start = MPI.Wtime()
autotimer = timing.auto_timer()
npix = 12*args.nside**2
# Scan the sky signal
if comm.comm_world.rank == 0 and not os.path.isfile(args.input_map):
raise RuntimeError(
'Input map does not exist: {}'.format(args.input_map))
distmap = tm.DistPixels(
comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float32, submap=subnpix, local=localsm)
distmap.read_healpix_fits(args.input_map)
scansim = tt.OpSimScan(distmap=distmap, out='signal')
scansim.exec(data)
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Read and sampled input map: {:.2f} seconds'
''.format(stop-start), flush=args.flush)
signalname = 'signal'
return signalname
def binned_map(data, npix, subnpix, out="."):
"""Make a binned map
This function should exist in toast, but all the pieces do. If we are
doing MCs we break these operations into two pieces and only generate
the noise weighted map each realization.
"""
# The global MPI communicator
cworld = data.comm.comm_world
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(data, nnz=6, dtype=np.float64)
hits = tm.DistPixels(data, nnz=1, dtype=np.int64)
zmap = tm.DistPixels(data, nnz=3, dtype=np.float64)
invnpp.data.fill(0.0)
hits.data.fill(0)
zmap.data.fill(0.0)
start = MPI.Wtime()
if cworld.rank == 0:
print("Accumulating hits and N_pp'^-1 ...", flush=True)
# Setting detweights to None gives uniform weighting.
build_invnpp = tm.OpAccumDiag(detweights=None,
invnpp=invnpp,
hits=hits,
zmap=zmap,
name="signal",
pixels="pixels",
weights="weights",
common_flag_name="flags_common",
common_flag_mask=1)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
zmap.allreduce()
cworld.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if cworld.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
if cworld.rank == 0:
print("Writing hits and N_pp'^-1 ...", flush=True)
hits.write_healpix_fits(os.path.join(out, "hits.fits"))
invnpp.write_healpix_fits(os.path.join(out, "invnpp.fits"))
start = stop
if cworld.rank == 0:
print("Inverting N_pp'^-1 ...", flush=True)
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
cworld.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if cworld.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed), flush=True)
if cworld.rank == 0:
print("Writing N_pp' ...", flush=True)
invnpp.write_healpix_fits(os.path.join(out, "npp.fits"))
start = stop
if cworld.rank == 0:
print("Computing binned map ...", flush=True)
tm.covariance_apply(invnpp, zmap)
cworld.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if cworld.rank == 0:
print("Computing binned map took {:.3f} s".format(elapsed), flush=True)
start = stop
if cworld.rank == 0:
print("Writing binned map ...", flush=True)
zmap.write_healpix_fits(os.path.join(out, "binned.fits"))
if cworld.rank == 0:
print("Binned map done", flush=True)
return
def build_npp(args, comm, data, localsm, subnpix, detweights, flag_name,
common_flag_name):
""" Build pixel-pixel noise covariance matrices.
"""
if not args.skip_bin:
if comm.comm_world.rank == 0:
print('Preparing distributed map', flush=args.flush)
start0 = MPI.Wtime()
start = start0
autotimer = timing.auto_timer()
npix = 12 * args.nside**2
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world,
size=npix,
nnz=6,
dtype=np.float64,
submap=subnpix,
local=localsm)
invnpp.data.fill(0.0)
hits = tm.DistPixels(comm=comm.comm_world,
size=npix,
nnz=1,
dtype=np.int64,
submap=subnpix,
local=localsm)
hits.data.fill(0)
zmap = tm.DistPixels(comm=comm.comm_world,
size=npix,
nnz=3,
dtype=np.float64,
submap=subnpix,
local=localsm)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects initialized in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
invnpp_group = None
hits_group = None
zmap_group = None
if comm.comm_group.size < comm.comm_world.size:
invnpp_group = tm.DistPixels(comm=comm.comm_group,
size=npix,
nnz=6,
dtype=np.float64,
submap=subnpix,
local=localsm)
invnpp_group.data.fill(0.0)
hits_group = tm.DistPixels(comm=comm.comm_group,
size=npix,
nnz=1,
dtype=np.int64,
submap=subnpix,
local=localsm)
hits_group.data.fill(0)
zmap_group = tm.DistPixels(comm=comm.comm_group,
size=npix,
nnz=3,
dtype=np.float64,
submap=subnpix,
local=localsm)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects initialized in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
build_invnpp = tm.OpAccumDiag(detweights=detweights,
invnpp=invnpp,
hits=hits,
flag_name=flag_name,
common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask)
build_invnpp.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects accumulated in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
invnpp.allreduce()
if not args.skip_hits:
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects reduced in {:.3f} s'.format(stop - start),
flush=args.flush)
start = stop
if invnpp_group is not None:
build_invnpp_group = tm.OpAccumDiag(
detweights=detweights,
invnpp=invnpp_group,
hits=hits_group,
flag_name=flag_name,
common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask)
build_invnpp_group.exec(data)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects accumulated in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
invnpp_group.allreduce()
if not args.skip_hits:
hits_group.allreduce()
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects reduced in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/hits.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
hits.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing hit map to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
del hits
if hits_group is not None:
if not args.skip_hits:
fn = '{}/hits_group_{:04}.fits'.format(args.outdir, comm.group)
if args.zip:
fn += '.gz'
hits_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group hit map to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
del hits_group
if not args.skip_hits:
fn = '{}/invnpp.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
invnpp.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing N_pp^-1 to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
if not args.skip_hits:
if invnpp_group is not None:
fn = '{}/invnpp_group_{:04}.fits'.format(
args.outdir, comm.group)
if args.zip:
fn += '.gz'
invnpp_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group N_pp^-1 to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Inverting N_pp^-1 took {:.3f} s'.format(stop - start),
flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/npp.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
invnpp.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing N_pp to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
if invnpp_group is not None:
tm.covariance_invert(invnpp_group, 1.0e-3)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Inverting group N_pp^-1 took {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/npp_group_{:04}.fits'.format(args.outdir, comm.group)
if args.zip:
fn += '.gz'
invnpp_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group N_pp to {} took {:.3f} s'.format(
fn, stop - start),
flush=args.flush)
start = stop
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print('Building Npp took {:.3f} s'.format(stop - start0),
flush=args.flush)
return invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name
def main():
comm = MPI.COMM_WORLD
if comm.rank == 0:
print("Running with {} processes".format(comm.size))
parser = argparse.ArgumentParser( description='Read a toast covariance matrix and invert it.' )
parser.add_argument( '--input', required=True, default=None, help='The input covariance FITS file' )
parser.add_argument( '--output', required=False, default=None, help='The output inverse covariance FITS file.' )
parser.add_argument( '--rcond', required=False, default=None, help='Optionally write the inverse condition number map to this file.' )
parser.add_argument( '--single', required=False, default=False, action='store_true', help='Write the output in single precision.' )
parser.add_argument( '--threshold', required=False, default=1e-3, type=np.float, help='Reciprocal condition number threshold' )
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer(timing.FILE())
# get options
infile = args.input
outfile = None
if args.output is not None:
outfile = args.output
else:
inmat = re.match(r'(.*)\.fits', infile)
if inmat is None:
print("input file should have .fits extension")
sys.exit(0)
inroot = inmat.group(1)
outfile = "{}_inv.fits".format(inroot)
# We need to read the header to get the size of the matrix.
# This would be a trivial function call in astropy.fits or
# fitsio, but we don't want to bring in a whole new dependency
# just for that. Instead, we open the file with healpy in memmap
# mode so that nothing is actually read except the header.
nside = 0
ncovnz = 0
if comm.rank == 0:
fake, head = hp.read_map(infile, h=True, memmap=True)
for key, val in head:
if key == 'NSIDE':
nside = int(val)
if key == 'TFIELDS':
ncovnz = int(val)
nside = comm.bcast(nside, root=0)
ncovnz = comm.bcast(nnz, root=0)
nnz = int( ( (np.sqrt(8.0*ncovnz) - 1.0) / 2.0 ) + 0.5 )
npix = 12 * nside**2
subnside = int(nside / 16)
if subnside == 0:
subnside = 1
subnpix = 12 * subnside**2
nsubmap = int( npix / subnpix )
# divide the submaps as evenly as possible among processes
dist = toast.distribute_uniform(nsubmap, comm.size)
local = np.arange(dist[comm.rank][0], dist[comm.rank][0] + dist[comm.rank][1])
if comm.rank == 0:
if os.path.isfile(outfile):
os.remove(outfile)
comm.barrier()
# create the covariance and inverse condition number map
cov = None
invcov = None
rcond = None
cov = tm.DistPixels(comm=comm, dtype=np.float64, size=npix, nnz=ncovnz, submap=subnpix, local=local)
if args.single:
invcov = tm.DistPixels(comm=comm, dtype=np.float32, size=npix, nnz=ncovnz, submap=subnpix, local=local)
else:
invcov = cov
if args.rcond is not None:
rcond = tm.DistPixels(comm=comm, dtype=np.float64, size=npix, nnz=nnz, submap=subnpix, local=local)
# read the covariance
cov.read_healpix_fits(infile)
# every process computes its local piece
tm.covariance_invert(cov, args.threshold, rcond=rcond)
if args.single:
invcov.data[:] = cov.data.astype(np.float32)
# write the inverted covariance
invcov.write_healpix_fits(outfile)
# write the condition number
if args.rcond is not None:
rcond.write_healpix_fits(args.rcond)
return
def main():
# We are going to group our processes in a single group. This is fine
# if we have fewer processes than detectors. Otherwise we should group
# them in a reasonable size that is smaller than the number of detectors
# and which divides evenly into the total number of processes.
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=MPI.COMM_WORLD.size)
# Make a fake focalplane. Plot it just for fun (don't waste time on this
# for very large runs though).
fp = fake_focalplane()
if comm.comm_world.rank == 0:
outfile = "custom_example_focalplane.png"
set_backend()
tt.plot_focalplane(fp, 6.0, 6.0, outfile)
# Read in 2 boresight files
borefiles = [
"../data/custom_example_boresight_1.txt",
"../data/custom_example_boresight_2.txt"
]
# Set up the distributed data
rate = 100.0
data = create_observations(comm, rate, fp, borefiles)
# Configure the healpix pixelization we will use for map-making and
# also the "submap" resolution, which sets granularity of the locally
# stored pieces of the sky.
map_nside = 512
map_npix = 12 * map_nside**2
sub_nside = 4
sub_npix = 12 * sub_nside**2
# Compute a pointing matrix with healpix pixels and weights.
pointing = tt.OpPointingHpix(nside=map_nside, nest=True, mode="IQU",
pixels="pixels", weights="weights")
pointing.exec(data)
# Compute the locally hit submaps
local_submaps = pixel_dist(data, sub_npix)
# Sources of simulated data: scan from a symmetric beam convolved sky
# and then add some simulated noise.
signalmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3,
dtype=np.float64, submap=sub_npix, local=local_submaps)
signalmap.read_healpix_fits("../data/custom_example_sky.fits")
scanmap = tt.OpSimScan(distmap=signalmap, pixels='pixels',
weights='weights', out="sim")
scanmap.exec(data)
nse = tt.OpSimNoise(out="sim", realization=0)
nse.exec(data)
# Accumulate the hits and inverse diagonal pixel covariance, as well as the
# noise weighted map. Here we simply use inverse noise weighting.
detweights = {}
for d in fp.keys():
net = fp[d]["NET"]
detweights[d] = 1.0 / (rate * net * net)
invnpp = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=6,
dtype=np.float64, submap=sub_npix, local=local_submaps)
hits = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=1,
dtype=np.int64, submap=sub_npix, local=local_submaps)
zmap = tm.DistPixels(comm=comm.comm_world, size=map_npix, nnz=3,
dtype=np.float64, submap=sub_npix, local=local_submaps)
invnpp.data.fill(0.0)
hits.data.fill(0)
zmap.data.fill(0.0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits, zmap=zmap, name="sim")
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
zmap.allreduce()
# Write these products out
hits.write_healpix_fits("custom_example_hits.fits")
invnpp.write_healpix_fits("custom_example_invnpp.fits")
zmap.write_healpix_fits("custom_example_zmap.fits")
# Invert the covariance and write
tm.covariance_invert(invnpp, 1.0e-3)
invnpp.write_healpix_fits("custom_example_npp.fits")
# Apply covariance to make the binned map
tm.covariance_apply(invnpp, zmap)
zmap.write_healpix_fits("custom_example_binned.fits")
MPI.Finalize()
return
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser(
description="Read existing data and make a simple map.",
fromfile_prefix_chars="@",
)
parser.add_argument(
"--groupsize",
required=False,
type=int,
default=0,
help="size of processor groups used to distribute "
"observations",
)
parser.add_argument(
"--hwprpm",
required=False,
type=float,
default=0.0,
help="The rate (in RPM) of the HWP rotation",
)
parser.add_argument(
"--samplerate",
required=False,
default=100.0,
type=np.float,
help="Detector sample rate (Hz)",
)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument("--nside",
required=False,
type=int,
default=64,
help="Healpix NSIDE")
parser.add_argument(
"--subnside",
required=False,
type=int,
default=8,
help="Distributed pixel sub-map NSIDE",
)
parser.add_argument("--coord",
required=False,
default="E",
help="Sky coordinate system [C,E,G]")
parser.add_argument(
"--baseline",
required=False,
type=float,
default=60.0,
help="Destriping baseline length (seconds)",
)
parser.add_argument(
"--noisefilter",
required=False,
default=False,
action="store_true",
help="Destripe with the noise filter enabled",
)
parser.add_argument(
"--madam",
required=False,
default=False,
action="store_true",
help="If specified, use libmadam for map-making",
)
parser.add_argument("--madampar",
required=False,
default=None,
help="Madam parameter file")
parser.add_argument(
"--polyorder",
required=False,
type=int,
help="Polynomial order for the polyfilter",
)
parser.add_argument(
"--wbin_ground",
required=False,
type=float,
help="Ground template bin width [degrees]",
)
parser.add_argument(
"--flush",
required=False,
default=False,
action="store_true",
help="Flush every print statement.",
)
parser.add_argument("--tidas",
required=False,
default=None,
help="Input TIDAS volume")
parser.add_argument("--tidas_detgroup",
required=False,
default=None,
help="TIDAS detector group")
parser.add_argument("--spt3g",
required=False,
default=None,
help="Input SPT3G data directory")
parser.add_argument(
"--spt3g_prefix",
required=False,
default=None,
help="SPT3G data frame file prefix",
)
parser.add_argument(
"--common_flag_mask",
required=False,
default=0,
type=np.uint8,
help="Common flag mask",
)
parser.add_argument(
"--debug",
required=False,
default=False,
action="store_true",
help="Write data distribution info and focalplane plot",
)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
# args = parser.parse_args(sys.argv)
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if (args.tidas is not None) and (args.spt3g is not None):
raise RuntimeError("Cannot read two datasets!")
if (args.tidas is None) and (args.spt3g is None):
raise RuntimeError("No dataset specified!")
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot load")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot load")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# Pixelization
nside = args.nside
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > nside:
subnside = nside
subnpix = 12 * subnside * subnside
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print(
"WARNING: process groupsize does not evenly divide into "
"total number of processes",
flush=True,
)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# Create output directory
mtime = MPI.Wtime()
if comm.comm_world.rank == 0:
if not os.path.isdir(args.outdir):
os.makedirs(args.outdir)
mtime = elapsed(comm.comm_world, mtime, "Creating output directory")
# The distributed timestream data
data = None
if args.tidas is not None:
if args.tidas_detgroup is None:
raise RuntimeError("you must specify the detector group")
data = tds.load_tidas(
comm,
comm.group_size,
args.tidas,
"r",
args.tidas_detgroup,
tds.TODTidas,
group_dets=args.tidas_detgroup,
distintervals="chunks",
)
if args.spt3g is not None:
if args.spt3g_prefix is None:
raise RuntimeError("you must specify the frame file prefix")
data = s3g.load_spt3g(
comm,
comm.group_size,
args.spt3g,
args.spt3g_prefix,
s3g.obsweight_spt3g,
s3g.TOD3G,
)
mtime = elapsed(comm.comm_world, mtime, "Distribute data")
# In debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
mtime = elapsed(comm.comm_world, mtime,
"Dumping debug data distribution")
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
# Just plot the dets from the first TOD
temptod = data.obs[0]["tod"]
# FIXME: change this once we store det info in the metadata.
dfwhm = {x: 10.0 for x in temptod.detectors}
tt.plot_focalplane(temptod.detoffset(),
10.0,
10.0,
outfile,
fwhm=dfwhm)
comm.comm_world.barrier()
mtime = elapsed(comm.comm_world, mtime, "Plotting debug focalplane")
# Compute pointing matrix
pointing = tt.OpPointingHpix(nside=args.nside,
nest=True,
mode="IQU",
hwprpm=args.hwprpm)
pointing.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Expand pointing")
# Mapmaking.
# FIXME: We potentially have a different noise model for every
# observation. We need to have both spt3g and tidas format Noise
# classes which read the information from disk. Then the mapmaking
# operators need to get these noise weights from each observation.
detweights = {d: 1.0 for d in data.obs[0]["tod"].detectors}
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# Filter data if desired
if args.polyorder:
polyfilter = tt.OpPolyFilter(
order=args.polyorder, common_flag_mask=args.common_flag_mask)
polyfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering")
if args.wbin_ground:
groundfilter = tt.OpGroundFilter(
wbin=args.wbin_ground, common_flag_mask=args.common_flag_mask)
groundfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime,
"Ground template filtering")
# Compute pixel space distribution
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
if localpix is None:
raise RuntimeError(
"Process {} has no hit pixels. Perhaps there are fewer "
"detectors than processes in the group?".format(
comm.comm_world.rank))
localsm = np.unique(np.floor_divide(localpix, subnpix))
mtime = elapsed(comm.comm_world, mtime, "Compute local submaps")
# construct distributed maps to store the covariance,
# noise weighted map, and hits
mtime = MPI.Wtime()
invnpp = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=6,
dtype=np.float64,
submap=subnpix,
local=localsm,
)
hits = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=1,
dtype=np.int64,
submap=subnpix,
local=localsm,
)
zmap = tm.DistPixels(
comm=comm.comm_world,
size=npix,
nnz=3,
dtype=np.float64,
submap=subnpix,
local=localsm,
)
# compute the hits and covariance.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(
detweights=detweights,
invnpp=invnpp,
hits=hits,
common_flag_mask=args.common_flag_mask,
)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1")
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1")
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1")
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing N_pp")
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap,
detweights=detweights,
common_flag_mask=args.common_flag_mask)
build_zmap.exec(data)
zmap.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map")
tm.covariance_apply(invnpp, zmap)
mtime = elapsed(comm.comm_world, mtime, "Computing binned map")
zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits"))
mtime = elapsed(comm.comm_world, mtime, "Writing binned map")
else:
# Set up MADAM map making.
pars = {}
pars["temperature_only"] = "F"
pars["force_pol"] = "T"
pars["kfirst"] = "T"
pars["concatenate_messages"] = "T"
pars["write_map"] = "T"
pars["write_binmap"] = "T"
pars["write_matrix"] = "T"
pars["write_wcov"] = "T"
pars["write_hits"] = "T"
pars["nside_cross"] = nside // 2
pars["nside_submap"] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars["base_first"] = args.baseline
pars["nside_map"] = nside
if args.noisefilter:
pars["kfilter"] = "T"
else:
pars["kfilter"] = "F"
pars["fsample"] = args.samplerate
madam = tm.OpMadam(params=pars,
detweights=detweights,
common_flag_mask=args.common_flag_mask)
madam.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking")
comm.comm_world.barrier()
stop = MPI.Wtime()
dur = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(dur), flush=True)
return
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description="Simulate satellite "
"boresight pointing and make a noise map.", fromfile_prefix_chars="@" )
parser.add_argument( "--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute observations" )
parser.add_argument( "--samplerate", required=False, type=float,
default=40.0, help="Detector sample rate (Hz)" )
parser.add_argument( "--starttime", required=False, type=float,
default=0.0, help="The overall start time of the simulation" )
parser.add_argument( "--spinperiod", required=False, type=float,
default=10.0, help="The period (in minutes) of the rotation about the "
"spin axis" )
parser.add_argument( "--spinangle", required=False, type=float,
default=30.0, help="The opening angle (in degrees) of the boresight "
"from the spin axis" )
parser.add_argument( "--precperiod", required=False, type=float,
default=50.0, help="The period (in minutes) of the rotation about the "
"precession axis" )
parser.add_argument( "--precangle", required=False, type=float,
default=65.0, help="The opening angle (in degrees) of the spin axis "
"from the precession axis" )
parser.add_argument( "--hwprpm", required=False, type=float,
default=0.0, help="The rate (in RPM) of the HWP rotation" )
parser.add_argument( "--hwpstep", required=False, default=None,
help="For stepped HWP, the angle in degrees of each step" )
parser.add_argument( "--hwpsteptime", required=False, type=float,
default=0.0, help="For stepped HWP, the the time in seconds between "
"steps" )
parser.add_argument( "--obs", required=False, type=float, default=1.0,
help="Number of hours in one science observation" )
parser.add_argument( "--gap", required=False, type=float, default=0.0,
help="Cooler cycle time in hours between science obs" )
parser.add_argument( "--numobs", required=False, type=int, default=1,
help="Number of complete observations" )
parser.add_argument( "--outdir", required=False, default="out",
help="Output directory" )
parser.add_argument( "--debug", required=False, default=False,
action="store_true", help="Write diagnostics" )
parser.add_argument( "--nside", required=False, type=int, default=64,
help="Healpix NSIDE" )
parser.add_argument( "--subnside", required=False, type=int, default=4,
help="Distributed pixel sub-map NSIDE" )
parser.add_argument( "--baseline", required=False, type=float,
default=60.0, help="Destriping baseline length (seconds)" )
parser.add_argument( "--noisefilter", required=False, default=False,
action="store_true", help="Destripe with the noise filter enabled" )
parser.add_argument( "--madam", required=False, default=False,
action="store_true", help="If specified, use libmadam for map-making" )
parser.add_argument( "--madampar", required=False, default=None,
help="Madam parameter file" )
parser.add_argument('--flush',
required=False, default=False, action='store_true',
help='Flush every print statement.')
parser.add_argument( "--MC_start", required=False, type=int, default=0,
help="First Monte Carlo noise realization" )
parser.add_argument( "--MC_count", required=False, type=int, default=1,
help="Number of Monte Carlo noise realizations" )
parser.add_argument( "--fp", required=False, default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
"a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" "
"(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" "
"(float). For optional plotting, the key \"color\" can specify a "
"valid matplotlib color string." )
parser.add_argument('--tidas',
required=False, default=None,
help='Output TIDAS export path')
parser.add_argument('--input_map', required=False,
help='Input map for signal')
parser.add_argument('--input_pysm_model', required=False,
help='Comma separated models for on-the-fly PySM '
'simulation, e.g. s3,d6,f1,a2"')
parser.add_argument('--apply_beam', required=False, action='store_true',
help='Apply beam convolution to input map with gaussian '
'beam parameters defined in focalplane')
parser.add_argument('--input_dipole', required=False,
help='Simulate dipole, possible values are '
'total, orbital, solar')
parser.add_argument('--input_dipole_solar_speed_kms', required=False,
help='Solar system speed [km/s]', type=float,
default=369.0)
parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False,
help='Solar system speed galactic latitude [degrees]',
type=float, default=48.26)
parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False,
help='Solar system speed galactic longitude[degrees]',
type=float, default=263.99)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# get options
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > args.nside:
subnside = args.nside
subnpix = 12 * subnside * subnside
start = MPI.Wtime()
fp = None
# Load focalplane information
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.fp, "rb") as p:
fp = pickle.load(p)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Create focalplane: {:.2f} seconds".format(stop-start),
flush=True)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
tt.plot_focalplane(fp, 10.0, 10.0, outfile)
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
if groupsize > len(fp.keys()):
if comm.comm_world.rank == 0:
print("process group is too large for the number of detectors",
flush=True)
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(fp.keys())
detquats = {}
detindx = None
if "index" in fp[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = fp[d]["quat"]
if detindx is not None:
detindx[d] = fp[d]["index"]
# Distribute the observations uniformly
groupdist = toast.distribute_uniform(args.numobs, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = tt.regular_intervals(args.numobs, args.starttime, 0,
args.samplerate, 3600*args.obs, 3600*args.gap)
# Create the noise model used for all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]["fmin"]
fknee[d] = fp[d]["fknee"]
alpha[d] = fp[d]["alpha"]
NET[d] = fp[d]["NET"]
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
mem_counter = tt.OpMemoryCounter()
# The distributed timestream data
data = toast.Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = tt.TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
firsttime=obsrange[ob].start,
rate=args.samplerate,
spinperiod=args.spinperiod,
spinangle=args.spinangle,
precperiod=args.precperiod,
precangle=args.precangle,
detindx=detindx,
detranks=comm.group_size
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Read parameters, compute data distribution: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = tt.slew_precession_axis(nsim=tod.local_samples[1],
firstsamp=obsoffset, samplerate=args.samplerate,
degday=degday)
tod.set_prec_axis(qprec=precquat)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Construct boresight pointing: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# make a Healpix pointing matrix.
pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime)
pointing.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Pointing generation took {:.3f} s".format(elapsed), flush=True)
start = stop
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = "signal"
if args.input_pysm_model:
simulate_sky_signal(args, comm, data, mem_counter,
[fp], subnpix, localsm, signalname=signalname)
if args.input_dipole:
print("Simulating dipole")
op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole,
solar_speed=args.input_dipole_solar_speed_kms,
solar_gal_lat=args.input_dipole_solar_gal_lat_deg,
solar_gal_lon=args.input_dipole_solar_gal_lon_deg,
out=signalname,
keep_quats=True,
keep_vel=False,
subtract=False,
coord="G",
freq=0, # we could use frequency for quadrupole correction
flag_mask=255, common_flag_mask=255)
op_sim_dipole.exec(data)
# Mapmaking. For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in detectors:
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.samplerate * net * net)
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing N_pp took {:.3f} s".format(elapsed),
flush=True)
start = stop
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Dumping debug data distribution took "
"{:.3f} s".format(elapsed), flush=True)
start = stop
mcstart = start
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# create output directory for this realization
outpath = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(outpath):
os.makedirs(outpath)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Creating output dir {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# clear all signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
if mc == firstmc:
# For the first realization, optionally export the
# timestream data to a TIDAS volume.
if args.tidas is not None:
from toast.tod.tidas import OpTidasExport
tidas_path = os.path.abspath(args.tidas)
export = OpTidasExport(tidas_path, name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Noise simulation {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal",
detweights=detweights)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Building noise weighted map {:04d} took {:.3f} s".format(
mc, elapsed), flush=True)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Computing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.write_healpix_fits(os.path.join(outpath, "binned.fits"))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Writing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
elapsed = stop - mcstart
if comm.comm_world.rank == 0:
print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed),
flush=True)
start = stop
else:
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = cross
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = args.nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# clear all total signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Noise simulation took {:.3f} s".format(elapsed),
flush=True)
start = stop
# create output directory for this realization
pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(pars["path_output"]):
os.makedirs(pars["path_output"])
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open(os.path.join(pars["path_output"],
"distdata.txt"), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
madam = tm.OpMadam(params=pars, detweights=detweights,
name="tot_signal")
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Mapmaking took {:.3f} s".format(elapsed), flush=True)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(elapsed), flush=True)