def _destripe(self, comm, parstring, ndet, detstring, nsamp, nnz, periods,
psdinfo):
""" Destripe the buffered data
"""
auto_timer = timing.auto_timer(type(self).__name__)
fcomm = comm.py2f()
if self._cached:
# destripe
outpath = ''
if 'path_output' in self.params:
outpath = self.params['path_output']
outpath = outpath.encode('ascii')
libmadam.destripe_with_cache(fcomm, ndet, nsamp, nnz,
self._madam_timestamps,
self._madam_pixels,
self._madam_pixweights,
self._madam_signal, outpath)
else:
(detweights, npsd, npsdtot, psdstarts, npsdbin, psdfreqs, npsdval,
psdvals) = psdinfo
# destripe
libmadam.destripe(fcomm, parstring.encode(), ndet,
detstring.encode(), detweights, nsamp, nnz,
self._madam_timestamps, self._madam_pixels,
self._madam_pixweights, self._madam_signal,
len(periods), periods, npsd, npsdtot, psdstarts,
npsdbin, psdfreqs, npsdval, psdvals)
if self._mcmode:
self._cached = True
return
def _stage_signal(self, data, detectors, nsamp, ndet, obs_period_ranges):
""" Stage signal
"""
auto_timer = timing.auto_timer(type(self).__name__)
self._madam_signal = self._cache.create('signal', np.float64,
(nsamp * ndet, ))
global_offset = 0
for iobs, obs in enumerate(data.obs):
tod = obs['tod']
period_ranges = obs_period_ranges[iobs]
for idet, det in enumerate(detectors):
# Get the signal.
signal = tod.local_signal(det, self._name)
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dslice = slice(idet * nsamp + offset,
idet * nsamp + offset + nn)
self._madam_signal[dslice] = signal[istart:istop]
offset += nn
del signal
for idet, det in enumerate(detectors):
if self._name is not None and (self._purge_tod or self._name
== self._name_out):
cachename = "{}_{}".format(self._name, det)
tod.cache.clear(pattern=cachename)
global_offset = offset
return
def __init__(self, comm=None, signal_map="signal_map",
lmax=None, grid=None, fwhm_deg=None, beam=None,
out="smoothed_signal_map"):
autotimer = timing.auto_timer(type(self).__name__)
# We call the parent class constructor, which currently does nothing
super().__init__()
self.comm = comm
self.signal_map = signal_map
self.lmax = lmax
self.out = out
self.grid = grid
# distribute alms
local_m_indices = np.arange(self.comm.rank, lmax + 1, self.comm.size,
dtype=np.int32)
self.order = libsharp.packed_real_order(lmax, ms=local_m_indices)
if (fwhm_deg is not None) and (beam is not None):
raise Exception("OpSmooth error, specify either fwhm_deg or beam, "
"not both")
if (fwhm_deg is None) and (beam is None):
raise Exception("OpSmooth error, specify fwhm_deg or beam")
if fwhm_deg is not None:
self.beam = hp.gauss_beam(fwhm=np.radians(fwhm_deg), lmax=lmax,
pol=True)
else:
self.beam = beam
def add_sky_signal(args, comm, data, totalname, signalname):
""" Add signalname to totalname in the obs tod
"""
if signalname is not None:
autotimer = timing.auto_timer()
for obs in data.obs:
tod = obs['tod']
for det in tod.local_dets:
cachename_in = '{}_{}'.format(signalname, det)
cachename_out = '{}_{}'.format(totalname, det)
ref_in = tod.cache.reference(cachename_in)
if comm.comm_world.rank == 0 and args.debug:
print('add_sky_signal', signalname, 'to', totalname, flush=args.flush)
print(signalname, 'min max', ref_in.min(), ref_in.max())
if tod.cache.exists(cachename_out):
ref_out = tod.cache.reference(cachename_out)
if comm.comm_world.rank == 0 and args.debug:
print(totalname, 'min max', ref_out.min(), ref_out.max())
ref_out += ref_in
else:
ref_out = tod.cache.put(cachename_out, ref_in)
if comm.comm_world.rank == 0 and args.debug:
print('final', 'min max', ref_out.min(), ref_out.max())
del ref_in, ref_out
return
def get_submaps(args, comm, data):
""" Get a list of locally hit pixels and submaps on every process.
"""
autotimer = timing.auto_timer()
if comm.comm_world.rank == 0:
print('Scanning local pixels', flush=args.flush)
start = MPI.Wtime()
# Prepare for using distpixels objects
nside = args.nside
subnside = 16
if subnside > nside:
subnside = nside
subnpix = 12 * subnside * subnside
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
if localpix is None:
raise RuntimeError(
'Process {} has no hit pixels. Perhaps there are fewer '
'detectors than processes in the group?'.format(
comm.comm_world.rank))
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print('Local submaps identified in {:.3f} s'.format(elapsed),
flush=args.flush)
return localpix, localsm, subnpix
def apply_madam(args, comm, data, madampars, outpath, detweights,
totalname_madam, flag_name, common_flag_name):
if args.madam:
if comm.comm_world.rank == 0:
print('Destriping signal', flush=args.flush)
start = MPI.Wtime()
autotimer = timing.auto_timer()
# create output directory for this realization
madampars['path_output'] = outpath
madam = tm.OpMadam(params=madampars,
detweights=detweights,
name=totalname_madam,
common_flag_name=common_flag_name,
flag_name=flag_name,
common_flag_mask=args.common_flag_mask,
purge_tod=True)
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Madam took {:.3f} s'.format(stop - start), flush=args.flush)
return
def output_tidas(args, comm, data, totalname, common_flag_name, flag_name):
if args.tidas is None:
return
autotimer = timing.auto_timer()
from toast.tod.tidas import OpTidasExport
tidas_path = os.path.abspath(args.tidas)
comm.comm_world.Barrier()
if comm.comm_world.rank == 0:
print('Exporting TOD to a TIDAS volume at {}'.format(tidas_path),
flush=args.flush)
start = MPI.Wtime()
export = OpTidasExport(tidas_path,
name=totalname,
common_flag_name=common_flag_name,
flag_name=flag_name,
usedist=True)
export.exec(data)
comm.comm_world.Barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Wrote simulated TOD to {}:{} in {:.2f} s'
''.format(tidas_path, totalname, stop - start),
flush=args.flush)
return
def apply_madam(args, comm, data, madampars, outpath,
detweights, totalname_madam, flag_name, common_flag_name):
if args.madam:
if comm.comm_world.rank == 0:
print('Destriping signal', flush=args.flush)
start = MPI.Wtime()
autotimer = timing.auto_timer()
# create output directory for this realization
madampars['path_output'] = outpath
madam = tm.OpMadam(
params=madampars, detweights=detweights,
name=totalname_madam,
common_flag_name=common_flag_name, flag_name=flag_name,
common_flag_mask=args.common_flag_mask,
purge_tod=True)
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Madam took {:.3f} s'.format(stop-start), flush=args.flush)
return
def exec(self, data):
"""
Apply the gains.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
for obs in data.obs:
tod = obs['tod']
for det in tod.local_dets:
# Cache the output signal
ref = tod.local_signal(det, self._name)
obs_times = tod.read_times()
calibrate(obs_times, ref, self._gain["TIME"], self._gain[det], order=0, inplace=True)
assert np.isnan(ref).sum() == 0, "The signal timestream includes NaN"
del ref
return
def get_submaps(args, comm, data):
""" Get a list of locally hit pixels and submaps on every process.
"""
autotimer = timing.auto_timer()
if comm.comm_world.rank == 0:
print('Scanning local pixels', flush=args.flush)
start = MPI.Wtime()
# Prepare for using distpixels objects
nside = args.nside
subnside = 16
if subnside > nside:
subnside = nside
subnpix = 12 * subnside * subnside
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
if localpix is None:
raise RuntimeError(
'Process {} has no hit pixels. Perhaps there are fewer '
'detectors than processes in the group?'.format(
comm.comm_world.rank))
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print('Local submaps identified in {:.3f} s'.format(elapsed),
flush=args.flush)
return localpix, localsm, subnpix
def output_tidas(args, comm, data, totalname, common_flag_name, flag_name):
if args.tidas is None:
return
autotimer = timing.auto_timer()
from toast.tod.tidas import OpTidasExport, TODTidas
tidas_path = os.path.abspath(args.tidas)
comm.comm_world.Barrier()
if comm.comm_world.rank == 0:
print('Exporting TOD to a TIDAS volume at {}'.format(tidas_path),
flush=args.flush)
start = MPI.Wtime()
export = OpTidasExport(tidas_path, TODTidas, backend="hdf5",
use_todchunks=True,
ctor_opts={"group_dets":"sim"},
cache_name=totalname)
export.exec(data)
comm.comm_world.Barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Wrote simulated TOD to {}:{} in {:.2f} s'
''.format(tidas_path, totalname,
stop-start), flush=args.flush)
return
def add_sky_signal(args, comm, data, totalname, signalname):
""" Add signalname to totalname in the obs tod
"""
if signalname is not None:
autotimer = timing.auto_timer()
for obs in data.obs:
tod = obs['tod']
for det in tod.local_dets:
cachename_in = '{}_{}'.format(signalname, det)
cachename_out = '{}_{}'.format(totalname, det)
ref_in = tod.cache.reference(cachename_in)
if comm.comm_world.rank == 0 and args.debug:
print('add_sky_signal',
signalname,
'to',
totalname,
flush=args.flush)
print(signalname, 'min max', ref_in.min(), ref_in.max())
if tod.cache.exists(cachename_out):
ref_out = tod.cache.reference(cachename_out)
if comm.comm_world.rank == 0 and args.debug:
print(totalname, 'min max', ref_out.min(),
ref_out.max())
ref_out += ref_in
else:
ref_out = tod.cache.put(cachename_out, ref_in)
if comm.comm_world.rank == 0 and args.debug:
print('final', 'min max', ref_out.min(), ref_out.max())
del ref_in, ref_out
return
def scan_signal(args, comm, data, localsm, subnpix):
""" Scan time-ordered signal from a map.
"""
signalname = None
if args.input_map:
if comm.comm_world.rank == 0:
print('Scanning input map', flush=args.flush)
start = MPI.Wtime()
autotimer = timing.auto_timer()
npix = 12*args.nside**2
# Scan the sky signal
if comm.comm_world.rank == 0 and not os.path.isfile(args.input_map):
raise RuntimeError(
'Input map does not exist: {}'.format(args.input_map))
distmap = tm.DistPixels(
comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float32, submap=subnpix, local=localsm)
distmap.read_healpix_fits(args.input_map)
scansim = tt.OpSimScan(distmap=distmap, out='signal')
scansim.exec(data)
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Read and sampled input map: {:.2f} seconds'
''.format(stop-start), flush=args.flush)
signalname = 'signal'
return signalname
def load_schedule(args, comm):
start = MPI.Wtime()
autotimer = timing.auto_timer()
if comm.comm_world.rank == 0:
fn = args.schedule
if not os.path.isfile(fn):
raise RuntimeError('No such schedule file: {}'.format(fn))
start = MPI.Wtime()
f = open(fn, 'r')
while True:
line = f.readline()
if line.startswith('#'):
continue
(site_name, telescope, site_lat, site_lon, site_alt) = line.split()
site_alt = float(site_alt)
site = [site_name, site_lat, site_lon, site_alt]
break
all_ces = []
for line in f:
if line.startswith('#'):
continue
start_date, start_time, stop_date, stop_time, mjdstart, mjdstop, \
name, azmin, azmax, el, rs, \
sun_el1, sun_az1, sun_el2, sun_az2, \
moon_el1, moon_az1, moon_el2, moon_az2, moon_phase, \
scan, subscan = line.split()
start_time = start_date + ' ' + start_time
stop_time = stop_date + ' ' + stop_time
try:
start_time = dateutil.parser.parse(start_time + ' +0000')
stop_time = dateutil.parser.parse(stop_time + ' +0000')
except:
start_time = dateutil.parser.parse(start_time)
stop_time = dateutil.parser.parse(stop_time)
start_timestamp = start_time.timestamp()
stop_timestamp = stop_time.timestamp()
all_ces.append([
start_timestamp, stop_timestamp, name, float(mjdstart),
int(scan), int(subscan), float(azmin), float(azmax), float(el)])
f.close()
stop = MPI.Wtime()
elapsed = stop - start
print('Load schedule: {:.2f} seconds'.format(stop-start),
flush=args.flush)
else:
site = None
all_ces = None
site = comm.comm_world.bcast(site)
all_ces = comm.comm_world.bcast(all_ces)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Loading schedule {:.3f} s'.format(stop-start), flush=args.flush)
return site, all_ces
def exec(self, data):
"""
Scramble the gains.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
for obs in data.obs:
obsindx = 0
if 'id' in obs:
obsindx = obs['id']
else:
print("Warning: observation ID is not set, using zero!")
telescope = 0
if 'telescope' in obs:
telescope = obs['telescope_id']
tod = obs['tod']
pat = re.compile(self._pattern)
for det in tod.local_dets:
# Test the detector pattern
if not pat.match(det):
continue
detindx = tod.detindx[det]
# Cache the output signal
ref = tod.local_signal(det, self._name)
"""
key1 = realization * 2^32 + telescope * 2^16 + component
key2 = obsindx * 2^32 + detindx
counter1 = currently unused (0)
counter2 = currently unused (0)
"""
key1 = (self._realization * 4294967296 + telescope * 65536 +
self._component)
key2 = obsindx * 4294967296 + detindx
counter1 = 0
counter2 = 0
rngdata = rng.random(1,
sampler="gaussian",
key=(key1, key2),
counter=(counter1, counter2))
gain = self._center + rngdata[0] * self._sigma
ref *= gain
del ref
return
def _prepare(self, data, comm):
""" Examine the data object.
"""
auto_timer = timing.auto_timer(type(self).__name__)
nsamp = self._count_samples(data)
# Determine the detectors and the pointing matrix non-zeros
# from the first observation. Madam will expect these to remain
# unchanged across observations.
tod = data.obs[0]['tod']
if self._dets is None:
detectors = tod.local_dets
else:
detectors = [det for det in tod.local_dets if det in self._dets]
ndet = len(detectors)
detstring = self._dets2detstring(detectors)
# to get the number of Non-zero pointing weights per pixel,
# we use the fact that for Madam, all processes have all detectors
# for some slice of time. So we can get this information from the
# shape of the data from the first detector
nnzname = "{}_{}".format(self._weights, detectors[0])
nnz_full = tod.cache.reference(nnzname).shape[1]
if 'temperature_only' in self.params \
and self.params['temperature_only'] in [
'T', 'True', 'TRUE', 'true', True]:
if nnz_full not in [1, 3]:
raise RuntimeError(
'OpMadam: Don\'t know how to make a temperature map '
'with nnz={}'.format(nnz_full))
nnz = 1
nnz_stride = nnz_full
else:
nnz = nnz_full
nnz_stride = 1
if 'nside_map' not in self.params:
raise RuntimeError(
'OpMadam: "nside_map" must be set in the parameter dictionary')
nside = int(self.params['nside_map'])
parstring = self._dict2parstring(self.params)
# Inspect the valid intervals across all observations to
# determine the number of samples per detector
obs_period_ranges, psdfreqs, periods = self._get_period_ranges(
comm, data, detectors, nsamp)
return (parstring, detstring, nsamp, ndet, nnz, nnz_full, nnz_stride,
periods, obs_period_ranges, psdfreqs, detectors, nside)
def exec(self, data):
"""
Scramble the gains.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
for obs in data.obs:
obsindx = 0
if 'id' in obs:
obsindx = obs['id']
else:
print("Warning: observation ID is not set, using zero!")
telescope = 0
if 'telescope' in obs:
telescope = obs['telescope_id']
tod = obs['tod']
pat = re.compile(self._pattern)
for det in tod.local_dets:
# Test the detector pattern
if not pat.match(det):
continue
detindx = tod.detindx[det]
# Cache the output signal
ref = tod.local_signal(det, self._name)
"""
key1 = realization * 2^32 + telescope * 2^16 + component
key2 = obsindx * 2^32 + detindx
counter1 = currently unused (0)
counter2 = currently unused (0)
"""
key1 = (self._realization * 4294967296 + telescope * 65536
+ self._component)
key2 = obsindx * 4294967296 + detindx
counter1 = 0
counter2 = 0
rngdata = rng.random(1, sampler="gaussian", key=(key1, key2),
counter=(counter1, counter2))
gain = self._center + rngdata[0] * self._sigma
ref *= gain
del ref
return
def exec(self, data):
"""
Apply the polynomial filter to the signal.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
for obs in data.obs:
tod = obs['tod']
if self._intervals in obs:
intervals = obs[self._intervals]
else:
intervals = None
local_intervals = tod.local_intervals(intervals)
common_ref = tod.local_common_flags(self._common_flag_name)
pat = re.compile(self._pattern)
for det in tod.local_dets:
# Test the detector pattern
if not pat.match(det):
continue
ref = tod.local_signal(det, self._name)
flag_ref = tod.local_flags(det, self._flag_name)
# Iterate over each interval
local_starts = []
local_stops = []
for ival in local_intervals:
local_starts.append(ival.first)
local_stops.append(ival.last)
local_starts = np.array(local_starts)
local_stops = np.array(local_stops)
flg = common_ref & self._common_flag_mask
flg |= flag_ref & self._flag_mask
filter_polyfilter(
self._order, [ref], flg, local_starts, local_stops)
flag_ref[flg != 0] |= self._poly_flag_mask
del ref
del flag_ref
del common_ref
return
def exec(self, data):
"""
Apply the polynomial filter to the signal.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
for obs in data.obs:
tod = obs['tod']
if self._intervals in obs:
intervals = obs[self._intervals]
else:
intervals = None
local_intervals = tod.local_intervals(intervals)
common_ref = tod.local_common_flags(self._common_flag_name)
pat = re.compile(self._pattern)
for det in tod.local_dets:
# Test the detector pattern
if not pat.match(det):
continue
ref = tod.local_signal(det, self._name)
flag_ref = tod.local_flags(det, self._flag_name)
# Iterate over each interval
local_starts = []
local_stops = []
for ival in local_intervals:
local_starts.append(ival.first)
local_stops.append(ival.last)
local_starts = np.array(local_starts)
local_stops = np.array(local_stops)
flg = common_ref & self._common_flag_mask
flg |= flag_ref & self._flag_mask
filter_polyfilter(self._order, [ref], flg, local_starts,
local_stops)
flag_ref[flg != 0] |= self._poly_flag_mask
del ref
del flag_ref
del common_ref
return
def setup_madam(args, comm):
""" Prepare to run Madam on the stored TOD.
"""
pars = None
if args.madam:
autotimer = timing.auto_timer()
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
submap = 16
if submap > args.nside:
submap = args.nside
pars['temperature_only'] = False
pars['force_pol'] = True
pars['kfirst'] = True
pars['write_map'] = True
pars['write_binmap'] = True
pars['write_matrix'] = True
pars['write_wcov'] = True
pars['write_hits'] = True
pars['nside_cross'] = cross
pars['nside_submap'] = submap
pars['allreduce'] = args.madam_allreduce
if args.madampar is not None:
pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*')
comment = re.compile(r'^#.*')
with open(args.madampar, 'r') as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars['base_first'] = args.madam_baseline_length
pars['basis_order'] = args.madam_baseline_order
pars['nside_map'] = args.nside
if args.madam_noisefilter:
pars['kfilter'] = True
else:
pars['kfilter'] = False
pars['precond_width'] = 1
pars['fsample'] = args.samplerate
pars['iter_max'] = args.madam_iter_max
return pars
def setup_madam(args, comm):
""" Prepare to run Madam on the stored TOD.
"""
pars = None
if args.madam:
autotimer = timing.auto_timer()
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
submap = 16
if submap > args.nside:
submap = args.nside
pars['temperature_only'] = False
pars['force_pol'] = True
pars['kfirst'] = True
pars['write_map'] = True
pars['write_binmap'] = True
pars['write_matrix'] = True
pars['write_wcov'] = True
pars['write_hits'] = True
pars['nside_cross'] = cross
pars['nside_submap'] = submap
pars['allreduce'] = args.madam_allreduce
if args.madampar is not None:
pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*')
comment = re.compile(r'^#.*')
with open(args.madampar, 'r') as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars['base_first'] = args.madam_baseline_length
pars['basis_order'] = args.madam_baseline_order
pars['nside_map'] = args.nside
if args.madam_noisefilter:
pars['kfilter'] = True
else:
pars['kfilter'] = False
pars['precond_width'] = 1
pars['fsample'] = args.samplerate
pars['iter_max'] = args.madam_iter_max
return pars
def main():
args, start_timestamp, stop_timestamp = parse_args()
autotimer = timing.auto_timer(timing.FILE())
patches = parse_patches(args)
build_schedule(args, start_timestamp, stop_timestamp,
args.sun_el_max * degree, args.sun_avoidance_angle * degree,
args.sun_angle_min * degree, args.moon_angle_min * degree,
args.el_min * degree, args.el_max * degree,
args.fp_radius * degree, patches)
def _stage_pixweights(
self, data, detectors, nsamp, ndet, nnz, nnz_full, nnz_stride, obs_period_ranges
):
"""Now collect the pixel weights
"""
auto_timer = timing.auto_timer(type(self).__name__)
self._madam_pixweights = self._cache.create(
"pixweights", madam.WEIGHT_TYPE, (nsamp * ndet * nnz,)
)
self._madam_pixweights[:] = 0
global_offset = 0
for iobs, obs in enumerate(data.obs):
tod = obs["tod"]
period_ranges = obs_period_ranges[iobs]
for idet, det in enumerate(detectors):
# get the pixels and weights for the valid intervals
# from the cache
weightsname = "{}_{}".format(self._weights, det)
weights = tod.cache.reference(weightsname)
weight_dtype = weights.dtype
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dwslice = slice(
(idet * nsamp + offset) * nnz,
(idet * nsamp + offset + nn) * nnz,
)
self._madam_pixweights[dwslice] = weights[istart:istop].flatten()[
::nnz_stride
]
offset += nn
del weights
# Purge the weights but restore them from the Madam
# buffers when purge_weights=False.
# Handle special case when Madam only stores a subset of
# the weights.
if not self._purge_weights and (nnz != nnz_full):
pass
else:
for idet, det in enumerate(detectors):
# get the pixels and weights for the valid intervals
# from the cache
weightsname = "{}_{}".format(self._weights, det)
tod.cache.clear(pattern=weightsname)
global_offset = offset
return weight_dtype
def exec(self, data):
"""
Create the timestreams...
This loops over all observations and detectors and uses the pointing
matrix to ...
Args:
data (toast.Data): The distributed data.
"""
if libsharp is None:
raise RuntimeError('libsharp not available')
autotimer = timing.auto_timer(type(self).__name__)
has_pol = len(data[self.signal_map]) > 1
alm_sharp_I = libsharp.analysis(
self.grid,
self.order,
np.ascontiguousarray(data[self.signal_map][0].reshape((1, 1, -1))),
spin=0,
comm=self.comm)
self.order.almxfl(alm_sharp_I, np.ascontiguousarray(self.beam[:, 0:1]))
out = libsharp.synthesis(self.grid,
self.order,
alm_sharp_I,
spin=0,
comm=self.comm)[0]
if has_pol:
alm_sharp_P = libsharp.analysis(
self.grid,
self.order,
np.ascontiguousarray(data[self.signal_map][1:3, :].reshape(
(1, 2, -1))),
spin=2,
comm=self.comm)
self.order.almxfl(alm_sharp_P,
np.ascontiguousarray(self.beam[:, (1, 2)]))
signal_map_P = libsharp.synthesis(self.grid,
self.order,
alm_sharp_P,
spin=2,
comm=self.comm)[0]
out = np.vstack((out, signal_map_P))
data[self.out] = out
assert data[self.signal_map].shape == data[self.out].shape
def _stage_pixweights(self, data, detectors, nsamp, ndet, nnz, nnz_full,
nnz_stride, obs_period_ranges):
"""Now collect the pixel weights
"""
auto_timer = timing.auto_timer(type(self).__name__)
self._madam_pixweights = self._cache.create("pixweights",
madam.WEIGHT_TYPE,
(nsamp * ndet * nnz, ))
self._madam_pixweights[:] = 0
global_offset = 0
for iobs, obs in enumerate(data.obs):
tod = obs["tod"]
period_ranges = obs_period_ranges[iobs]
for idet, det in enumerate(detectors):
# get the pixels and weights for the valid intervals
# from the cache
weightsname = "{}_{}".format(self._weights, det)
weights = tod.cache.reference(weightsname)
weight_dtype = weights.dtype
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dwslice = slice(
(idet * nsamp + offset) * nnz,
(idet * nsamp + offset + nn) * nnz,
)
self._madam_pixweights[dwslice] = weights[
istart:istop].flatten()[::nnz_stride]
offset += nn
del weights
# Purge the weights but restore them from the Madam
# buffers when purge_weights=False.
# Handle special case when Madam only stores a subset of
# the weights.
if not self._purge_weights and (nnz != nnz_full):
pass
else:
for idet, det in enumerate(detectors):
# get the pixels and weights for the valid intervals
# from the cache
weightsname = "{}_{}".format(self._weights, det)
tod.cache.clear(pattern=weightsname)
global_offset = offset
return weight_dtype
def _stage_time(self, data, detectors, nsamp, obs_period_ranges):
""" Stage the timestamps and use them to build PSD inputs.
"""
auto_timer = timing.auto_timer(type(self).__name__)
self._madam_timestamps = self._cache.create('timestamps', np.float64,
(nsamp, ))
global_offset = 0
time_offset = 0
psds = {}
for iobs, obs in enumerate(data.obs):
tod = obs['tod']
period_ranges = obs_period_ranges[iobs]
# Collect the timestamps for the valid intervals
timestamps = tod.local_times()
# Translate the time stamps to be monotonous
timestamps -= timestamps[0] - time_offset
time_offset = timestamps[-1] + 1
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
ind = slice(offset, offset + nn)
self._madam_timestamps[ind] = timestamps[istart:istop]
offset += nn
# get the noise object for this observation and create new
# entries in the dictionary when the PSD actually changes
if self._noisekey in obs.keys():
nse = obs[self._noisekey]
if 'noise_scale' in obs:
noise_scale = obs['noise_scale']
else:
noise_scale = 1
if nse is not None:
for det in detectors:
psd = nse.psd(det) * noise_scale**2
if det not in psds:
psds[det] = [(0, psd)]
else:
if not np.allclose(psds[det][-1][1], psd):
psds[det] += [(timestamps[0], psd)]
global_offset = offset
return psds
def _stage_time(self, data, detectors, nsamp, obs_period_ranges):
""" Stage the timestamps and use them to build PSD inputs.
"""
auto_timer = timing.auto_timer(type(self).__name__)
self._madam_timestamps = self._cache.create(
"timestamps", madam.TIMESTAMP_TYPE, (nsamp,)
)
offset = 0
time_offset = 0
psds = {}
for iobs, obs in enumerate(data.obs):
tod = obs["tod"]
period_ranges = obs_period_ranges[iobs]
# Collect the timestamps for the valid intervals
timestamps = tod.local_times().copy()
if self._translate_timestamps:
# Translate the time stamps to be monotonous
timestamps -= timestamps[0] - time_offset
time_offset = timestamps[-1] + 1
for istart, istop in period_ranges:
nn = istop - istart
ind = slice(offset, offset + nn)
self._madam_timestamps[ind] = timestamps[istart:istop]
offset += nn
# get the noise object for this observation and create new
# entries in the dictionary when the PSD actually changes
if self._noisekey in obs.keys():
nse = obs[self._noisekey]
if "noise_scale" in obs:
noise_scale = obs["noise_scale"]
else:
noise_scale = 1
if nse is not None:
for det in detectors:
psd = nse.psd(det) * noise_scale ** 2
if det not in psds:
psds[det] = [(0, psd)]
else:
if not np.allclose(psds[det][-1][1], psd):
psds[det] += [(timestamps[0], psd)]
return psds
def main():
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
# Create an argparse and add custom arguments
parser = argparse.ArgumentParser(description=“...")
parser.add_argument('--groupsize',
required=False, type=np.int,
help='Size of a process group assigned to a CES')
# pass the argparse object to timing module which will add timing
# arguments and return "parse.parse_args() result after handling
# the timing specific options
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
# create the primary auto timer for the entire script
autotimer = timing.auto_timer(timing.FILE())
def _destripe(self, comm, pars, dets, periods, psdinfo):
""" Destripe the buffered data
"""
auto_timer = timing.auto_timer(type(self).__name__)
if self._verbose:
memreport(comm, "just before calling libmadam.destripe")
if self._cached:
# destripe
outpath = ""
if "path_output" in self.params:
outpath = self.params["path_output"]
outpath = outpath.encode("ascii")
madam.destripe_with_cache(
comm,
self._madam_timestamps,
self._madam_pixels,
self._madam_pixweights,
self._madam_signal,
outpath,
)
else:
(detweights, npsd, psdstarts, psdfreqs, psdvals) = psdinfo
# destripe
madam.destripe(
comm,
pars,
dets,
detweights,
self._madam_timestamps,
self._madam_pixels,
self._madam_pixweights,
self._madam_signal,
periods,
npsd,
psdstarts,
psdfreqs,
psdvals,
)
if self._mcmode:
self._cached = True
return
def _destripe(self, comm, pars, dets, periods, psdinfo):
""" Destripe the buffered data
"""
auto_timer = timing.auto_timer(type(self).__name__)
if self._verbose:
memreport(comm, "just before calling libmadam.destripe")
if self._cached:
# destripe
outpath = ""
if "path_output" in self.params:
outpath = self.params["path_output"]
outpath = outpath.encode("ascii")
madam.destripe_with_cache(
comm,
self._madam_timestamps,
self._madam_pixels,
self._madam_pixweights,
self._madam_signal,
outpath,
)
else:
(detweights, npsd, psdstarts, psdfreqs, psdvals) = psdinfo
# destripe
madam.destripe(
comm,
pars,
dets,
detweights,
self._madam_timestamps,
self._madam_pixels,
self._madam_pixweights,
self._madam_signal,
periods,
npsd,
psdstarts,
psdfreqs,
psdvals,
)
if self._mcmode:
self._cached = True
return
def add_sky_signal(args, comm, data, totalname, signalname):
""" Add signalname to totalname in the obs tod
"""
if signalname is not None:
autotimer = timing.auto_timer()
for obs in data.obs:
tod = obs['tod']
for det in tod.local_dets:
cachename_in = '{}_{}'.format(signalname, det)
cachename_out = '{}_{}'.format(totalname, det)
ref_in = tod.cache.reference(cachename_in)
if tod.cache.exists(cachename_out):
ref_out = tod.cache.reference(cachename_out)
ref_out += ref_in
else:
ref_out = tod.cache.put(cachename_out, ref_in)
del ref_in, ref_out
return
def expand_pointing(args, comm, data):
""" Expand the bore sight pointing to every detector.
"""
start = MPI.Wtime()
autotimer = timing.auto_timer()
hwprpm = args.hwprpm
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
hwpsteptime = args.hwpsteptime
npix = 12 * args.nside**2
if comm.comm_world.rank == 0:
print('Expanding pointing', flush=args.flush)
pointing = tt.OpPointingHpix(nside=args.nside,
nest=True,
mode='IQU',
hwprpm=hwprpm,
hwpstep=hwpstep,
hwpsteptime=hwpsteptime)
pointing.exec(data)
# Only purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if args.tidas is None:
for ob in data.obs:
tod = ob['tod']
tod.free_radec_quats()
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Pointing generation took {:.3f} s'.format(stop - start),
flush=args.flush)
return
def exec(self, data):
"""
Create the timestreams...
This loops over all observations and detectors and uses the pointing
matrix to ...
Args:
data (toast.Data): The distributed data.
"""
if libsharp is None:
raise RuntimeError('libsharp not available')
autotimer = timing.auto_timer(type(self).__name__)
has_pol = len(data[self.signal_map]) > 1
alm_sharp_I = libsharp.analysis(
self.grid, self.order,
np.ascontiguousarray(data[self.signal_map][0].reshape((1, 1, -1))),
spin=0, comm=self.comm)
self.order.almxfl(alm_sharp_I, np.ascontiguousarray(self.beam[:, 0:1]))
out = libsharp.synthesis(self.grid, self.order, alm_sharp_I, spin=0,
comm=self.comm)[0]
if has_pol:
alm_sharp_P = libsharp.analysis(
self.grid, self.order,
np.ascontiguousarray(
data[self.signal_map][1:3, :].reshape((1, 2, -1))),
spin=2, comm=self.comm)
self.order.almxfl(alm_sharp_P,
np.ascontiguousarray(self.beam[:, (1, 2)]))
signal_map_P = libsharp.synthesis(
self.grid, self.order, alm_sharp_P, spin=2, comm=self.comm)[0]
out = np.vstack((
out,
signal_map_P))
data[self.out] = out
assert data[self.signal_map].shape == data[self.out].shape
def apply_polyfilter(args, comm, data, totalname_freq):
if args.polyorder:
if comm.comm_world.rank == 0:
print('Polyfiltering signal', flush=args.flush)
start = MPI.Wtime()
autotimer = timing.auto_timer()
common_flag_name = 'common_flags'
flag_name = 'flags'
polyfilter = tt.OpPolyFilter(
order=args.polyorder, name=totalname_freq,
common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask,
flag_name=flag_name)
polyfilter.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Polynomial filtering took {:.3f} s'.format(stop-start),
flush=args.flush)
return
def apply_polyfilter(args, comm, data, totalname_freq):
if args.polyorder:
if comm.comm_world.rank == 0:
print('Polyfiltering signal', flush=args.flush)
start = MPI.Wtime()
autotimer = timing.auto_timer()
common_flag_name = 'common_flags'
flag_name = 'flags'
polyfilter = tt.OpPolyFilter(order=args.polyorder,
name=totalname_freq,
common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask,
flag_name=flag_name)
polyfilter.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Polynomial filtering took {:.3f} s'.format(stop - start),
flush=args.flush)
return
def apply_groundfilter(args, comm, data, totalname_freq):
if args.wbin_ground:
if comm.comm_world.rank == 0:
print('Ground filtering signal', flush=args.flush)
start = MPI.Wtime()
autotimer = timing.auto_timer()
common_flag_name = 'common_flags'
flag_name = 'flags'
groundfilter = tt.OpGroundFilter(
wbin=args.wbin_ground, name=totalname_freq,
common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask,
flag_name=flag_name)
groundfilter.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Ground filtering took {:.3f} s'.format(stop-start),
flush=args.flush)
return
def expand_pointing(args, comm, data):
""" Expand the bore sight pointing to every detector.
"""
start = MPI.Wtime()
autotimer = timing.auto_timer()
hwprpm = args.hwprpm
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
hwpsteptime = args.hwpsteptime
npix = 12 * args.nside**2
if comm.comm_world.rank == 0:
print('Expanding pointing', flush=args.flush)
pointing = tt.OpPointingHpix(
nside=args.nside, nest=True, mode='IQU',
hwprpm=hwprpm, hwpstep=hwpstep, hwpsteptime=hwpsteptime)
pointing.exec(data)
# Only purge the pointing if we are NOT going to export the
# data to a TIDAS volume
if args.tidas is None:
for ob in data.obs:
tod = ob['tod']
tod.free_radec_quats()
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Pointing generation took {:.3f} s'.format(stop-start),
flush=args.flush)
return
def __init__(self,
comm=None,
signal_map="signal_map",
lmax=None,
grid=None,
fwhm_deg=None,
beam=None,
out="smoothed_signal_map"):
autotimer = timing.auto_timer(type(self).__name__)
# We call the parent class constructor, which currently does nothing
super().__init__()
self.comm = comm
self.signal_map = signal_map
self.lmax = lmax
self.out = out
self.grid = grid
# distribute alms
local_m_indices = np.arange(self.comm.rank,
lmax + 1,
self.comm.size,
dtype=np.int32)
self.order = libsharp.packed_real_order(lmax, ms=local_m_indices)
if (fwhm_deg is not None) and (beam is not None):
raise Exception("OpSmooth error, specify either fwhm_deg or beam, "
"not both")
if (fwhm_deg is None) and (beam is None):
raise Exception("OpSmooth error, specify fwhm_deg or beam")
if fwhm_deg is not None:
self.beam = hp.gauss_beam(fwhm=np.radians(fwhm_deg),
lmax=lmax,
pol=True)
else:
self.beam = beam
def _stage_signal(self, data, detectors, nsamp, ndet, obs_period_ranges):
""" Stage signal
"""
auto_timer = timing.auto_timer(type(self).__name__)
self._madam_signal = self._cache.create(
"signal", madam.SIGNAL_TYPE, (nsamp * ndet,)
)
self._madam_signal[:] = np.nan
global_offset = 0
for iobs, obs in enumerate(data.obs):
tod = obs["tod"]
period_ranges = obs_period_ranges[iobs]
for idet, det in enumerate(detectors):
# Get the signal.
signal = tod.local_signal(det, self._name)
signal_dtype = signal.dtype
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dslice = slice(idet * nsamp + offset, idet * nsamp + offset + nn)
self._madam_signal[dslice] = signal[istart:istop]
offset += nn
del signal
for idet, det in enumerate(detectors):
if self._name is not None and (
self._purge_tod or self._name == self._name_out
):
cachename = "{}_{}".format(self._name, det)
tod.cache.clear(pattern=cachename)
global_offset = offset
return signal_dtype
def exec(self, data, comm=None):
"""
Copy data to Madam-compatible buffers and make a map.
Args:
data (toast.Data): The distributed data.
"""
if libmadam is None:
raise RuntimeError("Cannot find libmadam")
if len(data.obs) == 0:
raise RuntimeError(
'OpMadam requires every supplied data object to '
'contain at least one observation')
auto_timer = timing.auto_timer(type(self).__name__)
if comm is None:
# Just use COMM_WORLD
comm = data.comm.comm_world
(parstring, detstring, nsamp, ndet, nnz, nnz_full, nnz_stride, periods,
obs_period_ranges, psdfreqs, detectors,
nside) = self._prepare(data, comm)
psdinfo, pixels_dtype, weight_dtype = self._stage_data(
data, comm, nsamp, ndet, nnz, nnz_full, nnz_stride,
obs_period_ranges, psdfreqs, detectors, nside)
self._destripe(comm, parstring, ndet, detstring, nsamp, nnz, periods,
psdinfo)
self._unstage_data(comm, data, nsamp, nnz, nnz_full, obs_period_ranges,
detectors, pixels_dtype, nside, weight_dtype)
return
def build_npp(args, comm, data, localsm, subnpix, detweights,
flag_name, common_flag_name):
""" Build pixel-pixel noise covariance matrices.
"""
if not args.skip_bin:
if comm.comm_world.rank == 0:
print('Preparing distributed map', flush=args.flush)
start0 = MPI.Wtime()
start = start0
autotimer = timing.auto_timer()
npix = 12*args.nside**2
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
invnpp.data.fill(0.0)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
hits.data.fill(0)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects initialized in {:.3f} s'
''.format(stop-start), flush=args.flush)
start = stop
invnpp_group = None
hits_group = None
zmap_group = None
if comm.comm_group.size < comm.comm_world.size:
invnpp_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=6,
dtype=np.float64, submap=subnpix,
local=localsm)
invnpp_group.data.fill(0.0)
hits_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=1,
dtype=np.int64, submap=subnpix,
local=localsm)
hits_group.data.fill(0)
zmap_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=3,
dtype=np.float64, submap=subnpix,
local=localsm)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects initialized in {:.3f} s'
''.format(stop-start), flush=args.flush)
start = stop
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
build_invnpp = tm.OpAccumDiag(
detweights=detweights, invnpp=invnpp, hits=hits,
flag_name=flag_name, common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask)
build_invnpp.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects accumulated in {:.3f} s'
''.format(stop-start), flush=args.flush)
start = stop
invnpp.allreduce()
if not args.skip_hits:
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects reduced in {:.3f} s'.format(stop-start),
flush=args.flush)
start = stop
if invnpp_group is not None:
build_invnpp_group = tm.OpAccumDiag(
detweights=detweights, invnpp=invnpp_group, hits=hits_group,
flag_name=flag_name, common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask)
build_invnpp_group.exec(data)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects accumulated in {:.3f} s'
''.format(stop-start), flush=args.flush)
start = stop
invnpp_group.allreduce()
if not args.skip_hits:
hits_group.allreduce()
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects reduced in {:.3f} s'
''.format(stop-start), flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/hits.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
hits.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing hit map to {} took {:.3f} s'
''.format(fn, stop-start), flush=args.flush)
start = stop
del hits
if hits_group is not None:
if not args.skip_hits:
fn = '{}/hits_group_{:04}.fits'.format(args.outdir, comm.group)
if args.zip:
fn += '.gz'
hits_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group hit map to {} took {:.3f} s'
''.format(fn, stop-start), flush=args.flush)
start = stop
del hits_group
if not args.skip_hits:
fn = '{}/invnpp.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
invnpp.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing N_pp^-1 to {} took {:.3f} s'
''.format(fn, stop-start), flush=args.flush)
start = stop
if not args.skip_hits:
if invnpp_group is not None:
fn = '{}/invnpp_group_{:04}.fits'.format(args.outdir,
comm.group)
if args.zip:
fn += '.gz'
invnpp_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group N_pp^-1 to {} took {:.3f} s'
''.format(fn, stop-start), flush=args.flush)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Inverting N_pp^-1 took {:.3f} s'.format(stop-start),
flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/npp.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
invnpp.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing N_pp to {} took {:.3f} s'
''.format(fn, stop-start), flush=args.flush)
start = stop
if invnpp_group is not None:
tm.covariance_invert(invnpp_group, 1.0e-3)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Inverting group N_pp^-1 took {:.3f} s'
''.format(stop-start), flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/npp_group_{:04}.fits'.format(args.outdir, comm.group)
if args.zip:
fn += '.gz'
invnpp_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group N_pp to {} took {:.3f} s'.format(
fn, stop-start), flush=args.flush)
start = stop
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print('Building Npp took {:.3f} s'.format(
stop-start0), flush=args.flush)
return invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name
def build_npp(args, comm, data, localsm, subnpix, detweights, flag_name,
common_flag_name):
""" Build pixel-pixel noise covariance matrices.
"""
if not args.skip_bin:
if comm.comm_world.rank == 0:
print('Preparing distributed map', flush=args.flush)
start0 = MPI.Wtime()
start = start0
autotimer = timing.auto_timer()
npix = 12 * args.nside**2
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world,
size=npix,
nnz=6,
dtype=np.float64,
submap=subnpix,
local=localsm)
invnpp.data.fill(0.0)
hits = tm.DistPixels(comm=comm.comm_world,
size=npix,
nnz=1,
dtype=np.int64,
submap=subnpix,
local=localsm)
hits.data.fill(0)
zmap = tm.DistPixels(comm=comm.comm_world,
size=npix,
nnz=3,
dtype=np.float64,
submap=subnpix,
local=localsm)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects initialized in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
invnpp_group = None
hits_group = None
zmap_group = None
if comm.comm_group.size < comm.comm_world.size:
invnpp_group = tm.DistPixels(comm=comm.comm_group,
size=npix,
nnz=6,
dtype=np.float64,
submap=subnpix,
local=localsm)
invnpp_group.data.fill(0.0)
hits_group = tm.DistPixels(comm=comm.comm_group,
size=npix,
nnz=1,
dtype=np.int64,
submap=subnpix,
local=localsm)
hits_group.data.fill(0)
zmap_group = tm.DistPixels(comm=comm.comm_group,
size=npix,
nnz=3,
dtype=np.float64,
submap=subnpix,
local=localsm)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects initialized in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
build_invnpp = tm.OpAccumDiag(detweights=detweights,
invnpp=invnpp,
hits=hits,
flag_name=flag_name,
common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask)
build_invnpp.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects accumulated in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
invnpp.allreduce()
if not args.skip_hits:
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - distobjects reduced in {:.3f} s'.format(stop - start),
flush=args.flush)
start = stop
if invnpp_group is not None:
build_invnpp_group = tm.OpAccumDiag(
detweights=detweights,
invnpp=invnpp_group,
hits=hits_group,
flag_name=flag_name,
common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask)
build_invnpp_group.exec(data)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects accumulated in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
invnpp_group.allreduce()
if not args.skip_hits:
hits_group.allreduce()
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - group distobjects reduced in {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/hits.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
hits.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing hit map to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
del hits
if hits_group is not None:
if not args.skip_hits:
fn = '{}/hits_group_{:04}.fits'.format(args.outdir, comm.group)
if args.zip:
fn += '.gz'
hits_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group hit map to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
del hits_group
if not args.skip_hits:
fn = '{}/invnpp.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
invnpp.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing N_pp^-1 to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
if not args.skip_hits:
if invnpp_group is not None:
fn = '{}/invnpp_group_{:04}.fits'.format(
args.outdir, comm.group)
if args.zip:
fn += '.gz'
invnpp_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group N_pp^-1 to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Inverting N_pp^-1 took {:.3f} s'.format(stop - start),
flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/npp.fits'.format(args.outdir)
if args.zip:
fn += '.gz'
invnpp.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing N_pp to {} took {:.3f} s'
''.format(fn, stop - start),
flush=args.flush)
start = stop
if invnpp_group is not None:
tm.covariance_invert(invnpp_group, 1.0e-3)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Inverting group N_pp^-1 took {:.3f} s'
''.format(stop - start),
flush=args.flush)
start = stop
if not args.skip_hits:
fn = '{}/npp_group_{:04}.fits'.format(args.outdir, comm.group)
if args.zip:
fn += '.gz'
invnpp_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group N_pp to {} took {:.3f} s'.format(
fn, stop - start),
flush=args.flush)
start = stop
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print('Building Npp took {:.3f} s'.format(stop - start0),
flush=args.flush)
return invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name
def exec(self, data):
"""
Apply the ground filter to the signal.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
# the two-level pytoast communicator
comm = data.comm
# the communicator within the group
cgroup = comm.comm_group
# Each group loops over its own CES:es
for obs in data.obs:
tod = obs["tod"]
nsamp_tot = tod.total_samples
my_offset, my_nsamp = tod.local_samples
if self._intervals in obs:
intervals = obs[self._intervals]
else:
intervals = None
local_intervals = tod.local_intervals(intervals)
# Construct trend templates. Full domain for x is [-1, 1]
x = np.arange(my_offset, my_offset + my_nsamp) / nsamp_tot * 2 - 1
ntrend = self._trend_order
# Do not include the offset in the trend. It will be part of
# of the ground template
cheby_trend = chebval(x, np.eye(ntrend + 1), tensor=True)[1:]
try:
(azmin, azmax, _, _) = tod.scan_range
az = tod.read_boresight_az()
except Exception as e:
raise RuntimeError(
"Failed to get boresight azimuth from TOD. Perhaps it is "
'not ground TOD? "{}"'.format(e)
)
# Cache the output common flags
common_ref = tod.local_common_flags(self._common_flag_name)
# The azimuth vector is assumed to be arranged so that the
# azimuth increases monotonously even across the zero meridian.
phase = (az - azmin) / (azmax - azmin) * 2 - 1
nfilter = self._filter_order + 1
cheby_templates = chebval(phase, np.eye(nfilter), tensor=True)
if not self._split_template:
cheby_filter = cheby_templates
else:
# Create separate templates for alternating scans
cheby_filter = []
mask1 = common_ref != 0
mask2 = mask1.copy()
for i, ival in enumerate(local_intervals):
mask = [mask1, mask2][i % 2]
mask[ival.first : ival.last + 1] = True
for template in cheby_templates:
for mask in mask1, mask2:
temp = template.copy()
temp[mask] = 0
cheby_filter.append(temp)
templates = []
for temp in cheby_trend, cheby_filter:
for template in temp:
templates.append(template)
for det in tod.detectors:
if det in tod.local_dets:
ref = tod.local_signal(det, self._name)
flag_ref = tod.local_flags(det, self._flag_name)
good = np.logical_and(
common_ref & self._common_flag_mask == 0,
flag_ref & self._flag_mask == 0,
)
del flag_ref
else:
ref = None
good = None
coeff = self.fit_templates(tod, det, templates, ref, good)
if det in tod.local_dets:
# Trend
trend = np.zeros_like(ref)
for cc, template in zip(coeff[:ntrend], cheby_trend):
trend += cc * template
if self._detrend:
ref[good] -= trend[good]
# Ground template
grtemplate = np.zeros_like(ref)
for cc, template in zip(coeff[ntrend:], cheby_filter):
grtemplate += cc * template
ref[good] -= grtemplate[good]
ref[np.logical_not(good)] = 0
del ref
del common_ref
return
def create_observations(args, comm, fp, all_ces, site):
""" Simulate constant elevation scans.
Simulate constant elevation scans at "site" matching entries in
"all_ces". Each operational day is assigned to a different
process group to allow making day maps.
"""
start = MPI.Wtime()
autotimer = timing.auto_timer()
data = toast.Data(comm)
site_name, site_lat, site_lon, site_alt = site
detectors = sorted(fp.keys())
detquats = {}
for d in detectors:
detquats[d] = fp[d]['quat']
nces = len(all_ces)
breaks = []
do_break = False
for i in range(nces - 1):
# If current and next CES are on different days, insert a break
tz = args.timezone / 24.
start1 = all_ces[i][3] # MJD start
start2 = all_ces[i + 1][3] # MJD start
scan1 = all_ces[i][4]
scan2 = all_ces[i + 1][4]
if scan1 != scan2 and do_break:
breaks.append(i + 1)
do_break = False
continue
day1 = int(start1 + tz)
day2 = int(start2 + tz)
if day1 != day2:
if scan1 == scan2:
# We want an entire CES, even if it crosses the day bound.
# Wait until the scan number changes.
do_break = True
else:
breaks.append(i + 1)
nbreak = len(breaks)
if nbreak != comm.ngroups - 1:
raise RuntimeError(
'Number of observing days ({}) does not match number of process '
'groups ({}).'.format(nbreak + 1, comm.ngroups))
groupdist = toast.distribute_uniform(nces, comm.ngroups, breaks=breaks)
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
# Create the noise model used by all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]['fmin']
fknee[d] = fp[d]['fknee']
alpha[d] = fp[d]['alpha']
NET[d] = fp[d]['NET']
noise = tt.AnalyticNoise(rate=rates,
fmin=fmin,
detectors=detectors,
fknee=fknee,
alpha=alpha,
NET=NET)
for ices in range(group_firstobs, group_firstobs + group_numobs):
ces = all_ces[ices]
CES_start, CES_stop, name, mjdstart, scan, subscan, azmin, azmax, \
el = ces
totsamples = int((CES_stop - CES_start) * args.samplerate)
# create the single TOD for this observation
try:
tod = tt.TODGround(comm.comm_group,
detquats,
totsamples,
detranks=comm.comm_group.size,
firsttime=CES_start,
rate=args.samplerate,
site_lon=site_lon,
site_lat=site_lat,
site_alt=site_alt,
azmin=azmin,
azmax=azmax,
el=el,
scanrate=args.scanrate,
scan_accel=args.scan_accel,
CES_start=None,
CES_stop=None,
sun_angle_min=args.sun_angle_min,
coord=args.coord,
sampsizes=None)
except RuntimeError as e:
print('Failed to create the CES scan: {}'.format(e),
flush=args.flush)
return
# Create the (single) observation
ob = {}
ob['name'] = 'CES-{}-{}-{}'.format(name, scan, subscan)
ob['tod'] = tod
if len(tod.subscans) > 0:
ob['intervals'] = tod.subscans
else:
raise RuntimeError('{} has no valid intervals'.format(ob['name']))
ob['baselines'] = None
ob['noise'] = noise
ob['id'] = int(mjdstart * 10000)
data.obs.append(ob)
for ob in data.obs:
tod = ob['tod']
tod.free_azel_quats()
if comm.comm_group.rank == 0:
print('Group # {:4} has {} observations.'.format(
comm.group, len(data.obs)),
flush=args.flush)
if len(data.obs) == 0:
raise RuntimeError('Too many tasks. Every MPI task must '
'be assigned to at least one observation.')
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Simulated scans in {:.2f} seconds'
''.format(stop - start),
flush=args.flush)
return data
def main():
comm = MPI.COMM_WORLD
if comm.rank == 0:
print("Running with {} processes".format(comm.size))
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description='Read a toast covariance matrix and write the inverse condition number map' )
parser.add_argument( '--input', required=True, default=None, help='The input covariance FITS file' )
parser.add_argument( '--output', required=False, default=None, help='The output inverse condition map FITS file.' )
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer(timing.FILE())
# get options
infile = args.input
outfile = None
if args.output is not None:
outfile = args.output
else:
inmat = re.match(r'(.*)\.fits', infile)
if inmat is None:
print("input file should have .fits extension")
sys.exit(0)
inroot = inmat.group(1)
outfile = "{}_rcond.fits".format(inroot)
# We need to read the header to get the size of the matrix.
# This would be a trivial function call in astropy.fits or
# fitsio, but we don't want to bring in a whole new dependency
# just for that. Instead, we open the file with healpy in memmap
# mode so that nothing is actually read except the header.
nside = 0
nnz = 0
if comm.rank == 0:
fake, head = hp.read_map(infile, h=True, memmap=True)
for key, val in head:
if key == 'NSIDE':
nside = int(val)
if key == 'TFIELDS':
nnz = int(val)
nside = comm.bcast(nside, root=0)
nnz = comm.bcast(nnz, root=0)
npix = 12 * nside**2
subnside = int(nside / 16)
if subnside == 0:
subnside = 1
subnpix = 12 * subnside**2
nsubmap = int( npix / subnpix )
# divide the submaps as evenly as possible among processes
dist = toast.distribute_uniform(nsubmap, comm.size)
local = np.arange(dist[comm.rank][0], dist[comm.rank][0] + dist[comm.rank][1])
if comm.rank == 0:
if os.path.isfile(outfile):
os.remove(outfile)
comm.barrier()
# create the covariance and inverse condition number map
cov = tm.DistPixels(comm=comm, dtype=np.float64, size=npix, nnz=nnz, submap=subnpix, local=local)
# read the covariance
cov.read_healpix_fits(infile)
# every process computes its local piece
rcond = tm.covariance_rcond(cov)
# write the map
rcond.write_healpix_fits(outfile)
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser( description="Simulate satellite "
"boresight pointing and make a noise map.", fromfile_prefix_chars="@" )
parser.add_argument( "--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute observations" )
parser.add_argument( "--samplerate", required=False, type=float,
default=40.0, help="Detector sample rate (Hz)" )
parser.add_argument( "--starttime", required=False, type=float,
default=0.0, help="The overall start time of the simulation" )
parser.add_argument( "--spinperiod", required=False, type=float,
default=10.0, help="The period (in minutes) of the rotation about the "
"spin axis" )
parser.add_argument( "--spinangle", required=False, type=float,
default=30.0, help="The opening angle (in degrees) of the boresight "
"from the spin axis" )
parser.add_argument( "--precperiod", required=False, type=float,
default=50.0, help="The period (in minutes) of the rotation about the "
"precession axis" )
parser.add_argument( "--precangle", required=False, type=float,
default=65.0, help="The opening angle (in degrees) of the spin axis "
"from the precession axis" )
parser.add_argument( "--hwprpm", required=False, type=float,
default=0.0, help="The rate (in RPM) of the HWP rotation" )
parser.add_argument( "--hwpstep", required=False, default=None,
help="For stepped HWP, the angle in degrees of each step" )
parser.add_argument( "--hwpsteptime", required=False, type=float,
default=0.0, help="For stepped HWP, the the time in seconds between "
"steps" )
parser.add_argument( "--obs", required=False, type=float, default=1.0,
help="Number of hours in one science observation" )
parser.add_argument( "--gap", required=False, type=float, default=0.0,
help="Cooler cycle time in hours between science obs" )
parser.add_argument( "--numobs", required=False, type=int, default=1,
help="Number of complete observations" )
parser.add_argument( "--outdir", required=False, default="out",
help="Output directory" )
parser.add_argument( "--debug", required=False, default=False,
action="store_true", help="Write diagnostics" )
parser.add_argument( "--nside", required=False, type=int, default=64,
help="Healpix NSIDE" )
parser.add_argument( "--subnside", required=False, type=int, default=4,
help="Distributed pixel sub-map NSIDE" )
parser.add_argument('--coord', required=False, default='E',
help='Sky coordinate system [C,E,G]')
parser.add_argument( "--baseline", required=False, type=float,
default=60.0, help="Destriping baseline length (seconds)" )
parser.add_argument( "--noisefilter", required=False, default=False,
action="store_true", help="Destripe with the noise filter enabled" )
parser.add_argument( "--madam", required=False, default=False,
action="store_true", help="If specified, use libmadam for map-making" )
parser.add_argument( "--madampar", required=False, default=None,
help="Madam parameter file" )
parser.add_argument('--flush',
required=False, default=False, action='store_true',
help='Flush every print statement.')
parser.add_argument( "--MC_start", required=False, type=int, default=0,
help="First Monte Carlo noise realization" )
parser.add_argument( "--MC_count", required=False, type=int, default=1,
help="Number of Monte Carlo noise realizations" )
parser.add_argument( "--fp", required=False, default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
"a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" "
"(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" "
"(float). For optional plotting, the key \"color\" can specify a "
"valid matplotlib color string." )
parser.add_argument( "--gain", required=False, default=None,
help= "Calibrate the input timelines with a set of gains from a"
"FITS file containing 3 extensions:"
"HDU named DETECTORS : table with list of detector names in a column named DETECTORS"
"HDU named TIME: table with common timestamps column named TIME"
"HDU named GAINS: 2D image of floats with one row per detector and one column per value.")
parser.add_argument('--tidas',
required=False, default=None,
help='Output TIDAS export path')
parser.add_argument('--spt3g',
required=False, default=None,
help='Output SPT3G export path')
parser.add_argument('--input_map', required=False,
help='Input map for signal')
parser.add_argument('--input_pysm_model', required=False,
help='Comma separated models for on-the-fly PySM '
'simulation, e.g. s3,d6,f1,a2"')
parser.add_argument('--input_pysm_precomputed_cmb_K_CMB', required=False,
help='Precomputed CMB map for PySM in K_CMB'
'it overrides any model defined in input_pysm_model"')
parser.add_argument('--apply_beam', required=False, action='store_true',
help='Apply beam convolution to input map with gaussian '
'beam parameters defined in focalplane')
parser.add_argument('--input_dipole', required=False,
help='Simulate dipole, possible values are '
'total, orbital, solar')
parser.add_argument('--input_dipole_solar_speed_kms', required=False,
help='Solar system speed [km/s]', type=float,
default=369.0)
parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False,
help='Solar system speed galactic latitude [degrees]',
type=float, default=48.26)
parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False,
help='Solar system speed galactic longitude[degrees]',
type=float, default=263.99)
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot export")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# get options
hwpstep = None
if args.hwpstep is not None:
hwpstep = float(args.hwpstep)
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > args.nside:
subnside = args.nside
subnpix = 12 * subnside * subnside
start = MPI.Wtime()
fp = None
gain = None
# Load focalplane information
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0])
fake["fwhm"] = 30.0
fake["fknee"] = 0.0
fake["alpha"] = 1.0
fake["NET"] = 1.0
fake["color"] = "r"
fp = {}
fp["bore"] = fake
else:
with open(args.fp, "rb") as p:
fp = pickle.load(p)
if args.gain is not None:
gain = {}
with fits.open(args.gain) as f:
gain["TIME"] = np.array(f["TIME"].data["TIME"])
for i_det, det_name in f["DETECTORS"].data["DETECTORS"]:
gain[det_name] = np.array(f["GAINS"].data[i_det, :])
if args.gain is not None:
gain = comm.comm_world.bcast(gain, root=0)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Create focalplane ({} dets): {:.2f} seconds"\
.format(len(fp.keys()), stop-start), flush=True)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
dquats = { x : fp[x]["quat"] for x in fp.keys() }
dfwhm = { x : fp[x]["fwhm"] for x in fp.keys() }
tt.plot_focalplane(dquats, 10.0, 10.0, outfile, fwhm=dfwhm)
# Since we are simulating noise timestreams, we want
# them to be contiguous and reproducible over the whole
# observation. We distribute data by detector within an
# observation, so ensure that our group size is not larger
# than the number of detectors we have.
if groupsize > len(fp.keys()):
if comm.comm_world.rank == 0:
print("process group is too large for the number of detectors",
flush=True)
comm.comm_world.Abort()
# Detector information from the focalplane
detectors = sorted(fp.keys())
detquats = {}
detindx = None
if "index" in fp[detectors[0]]:
detindx = {}
for d in detectors:
detquats[d] = fp[d]["quat"]
if detindx is not None:
detindx[d] = fp[d]["index"]
# Distribute the observations uniformly
groupdist = toast.distribute_uniform(args.numobs, comm.ngroups)
# Compute global time and sample ranges of all observations
obsrange = tt.regular_intervals(args.numobs, args.starttime, 0,
args.samplerate, 3600*args.obs, 3600*args.gap)
# Create the noise model used for all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]["fmin"]
fknee[d] = fp[d]["fknee"]
alpha[d] = fp[d]["alpha"]
NET[d] = fp[d]["NET"]
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
mem_counter = tt.OpMemoryCounter()
# The distributed timestream data
data = toast.Data(comm)
# Every process group creates its observations
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
for ob in range(group_firstobs, group_firstobs + group_numobs):
tod = tt.TODSatellite(
comm.comm_group,
detquats,
obsrange[ob].samples,
coord=args.coord,
firstsamp=obsrange[ob].first,
firsttime=obsrange[ob].start,
rate=args.samplerate,
spinperiod=args.spinperiod,
spinangle=args.spinangle,
precperiod=args.precperiod,
precangle=args.precangle,
detindx=detindx,
detranks=comm.group_size
)
obs = {}
obs["name"] = "science_{:05d}".format(ob)
obs["tod"] = tod
obs["intervals"] = None
obs["baselines"] = None
obs["noise"] = noise
obs["id"] = ob
data.obs.append(obs)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Read parameters, compute data distribution: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# we set the precession axis now, which will trigger calculation
# of the boresight pointing.
for ob in range(group_numobs):
curobs = data.obs[ob]
tod = curobs["tod"]
# Get the global sample offset from the original distribution of
# intervals
obsoffset = obsrange[group_firstobs + ob].first
# Constantly slewing precession axis
degday = 360.0 / 365.25
precquat = np.empty(4*tod.local_samples[1],
dtype=np.float64).reshape((-1,4))
tt.slew_precession_axis(precquat,
firstsamp=(obsoffset + tod.local_samples[0]),
samplerate=args.samplerate, degday=degday)
tod.set_prec_axis(qprec=precquat)
del precquat
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Construct boresight pointing: "
"{:.2f} seconds".format(stop-start), flush=True)
start = stop
# make a Healpix pointing matrix.
pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime)
pointing.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Pointing generation took {:.3f} s".format(elapsed), flush=True)
start = stop
localpix, localsm, subnpix = get_submaps(args, comm, data)
signalname = "signal"
has_signal = False
if args.input_pysm_model:
has_signal = True
simulate_sky_signal(args, comm, data, mem_counter,
[fp], subnpix, localsm, signalname=signalname)
if args.input_dipole:
print("Simulating dipole")
has_signal = True
op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole,
solar_speed=args.input_dipole_solar_speed_kms,
solar_gal_lat=args.input_dipole_solar_gal_lat_deg,
solar_gal_lon=args.input_dipole_solar_gal_lon_deg,
out=signalname,
keep_quats=False,
keep_vel=False,
subtract=False,
coord=args.coord,
freq=0, # we could use frequency for quadrupole correction
flag_mask=255, common_flag_mask=255)
op_sim_dipole.exec(data)
del op_sim_dipole
# Mapmaking. For purposes of this simulation, we use detector noise
# weights based on the NET (white noise level). If the destriping
# baseline is too long, this will not be the best choice.
detweights = {}
for d in detectors:
net = fp[d]["NET"]
detweights[d] = 1.0 / (args.samplerate * net * net)
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# get locally hit pixels
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance once, since the pointing and noise
# weights are fixed.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Inverting N_pp^-1 took {:.3f} s".format(elapsed),
flush=True)
start = stop
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Writing N_pp took {:.3f} s".format(elapsed),
flush=True)
start = stop
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Dumping debug data distribution took "
"{:.3f} s".format(elapsed), flush=True)
start = stop
mcstart = start
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# create output directory for this realization
outpath = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(outpath):
os.makedirs(outpath)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Creating output dir {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# clear all signal data from the cache, so that we can generate
# new noise timestreams.
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Noise simulation {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
# add sky signal
if has_signal:
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Add sky signal {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
if gain is not None:
op_apply_gain = tt.OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
if mc == firstmc:
# For the first realization, optionally export the
# timestream data. If we had observation intervals defined,
# we could pass "use_interval=True" to the export operators,
# which would ensure breaks in the exported data at
# acceptable places.
if args.tidas is not None:
tidas_path = os.path.abspath(args.tidas)
export = OpTidasExport(tidas_path, TODTidas, backend="hdf5",
use_todchunks=True,
create_opts={"group_dets":"sim"},
ctor_opts={"group_dets":"sim"},
cache_name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Tidas export took {:.3f} s"\
.format(elapsed), flush=True)
start = stop
if args.spt3g is not None:
spt3g_path = os.path.abspath(args.spt3g)
export = Op3GExport(spt3g_path, TOD3G, use_todchunks=True,
export_opts={"prefix" : "sim"},
cache_name="tot_signal")
export.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" SPT3G export took {:.3f} s"\
.format(elapsed), flush=True)
start = stop
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal",
detweights=detweights)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Building noise weighted map {:04d} took {:.3f} s".format(
mc, elapsed), flush=True)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Computing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
start = stop
zmap.write_healpix_fits(os.path.join(outpath, "binned.fits"))
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print(" Writing binned map {:04d} took {:.3f} s".format(mc,
elapsed), flush=True)
elapsed = stop - mcstart
if comm.comm_world.rank == 0:
print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed),
flush=True)
start = stop
else:
# Set up MADAM map making.
pars = {}
cross = args.nside // 2
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = cross
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = args.nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
# Loop over Monte Carlos
firstmc = int(args.MC_start)
nmc = int(args.MC_count)
for mc in range(firstmc, firstmc+nmc):
# clear all total signal data from the cache, so that we can generate
# new noise timestreams.
for obs in data.obs:
tod = obs['tod']
tod.cache.clear("tot_signal_.*")
# simulate noise
nse = tt.OpSimNoise(out="tot_signal", realization=mc)
nse.exec(data)
# add sky signal
if has_signal:
add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname)
if gain is not None:
op_apply_gain = tt.OpApplyGain(gain, name="tot_signal")
op_apply_gain.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Noise simulation took {:.3f} s".format(elapsed),
flush=True)
start = stop
# create output directory for this realization
pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc)
if comm.comm_world.rank == 0:
if not os.path.isdir(pars["path_output"]):
os.makedirs(pars["path_output"])
# in debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open(os.path.join(pars["path_output"],
"distdata.txt"), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
madam = tm.OpMadam(params=pars, detweights=detweights,
name="tot_signal")
madam.exec(data)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print("Mapmaking took {:.3f} s".format(elapsed), flush=True)
comm.comm_world.barrier()
stop = MPI.Wtime()
elapsed = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main():
if MPI.COMM_WORLD.rank == 0:
print("Running with {} processes".format(MPI.COMM_WORLD.size),
flush=True)
global_start = MPI.Wtime()
parser = argparse.ArgumentParser(
description="Read existing data and make a simple map.",
fromfile_prefix_chars="@")
parser.add_argument("--groupsize", required=False, type=int, default=0,
help="size of processor groups used to distribute "\
"observations")
parser.add_argument("--hwprpm", required=False, type=float,
default=0.0,
help="The rate (in RPM) of the HWP rotation")
parser.add_argument('--samplerate',
required=False, default=100.0, type=np.float,
help='Detector sample rate (Hz)')
parser.add_argument("--outdir", required=False, default="out",
help="Output directory")
parser.add_argument("--nside", required=False, type=int, default=64,
help="Healpix NSIDE")
parser.add_argument("--subnside", required=False, type=int, default=8,
help="Distributed pixel sub-map NSIDE")
parser.add_argument("--coord", required=False, default="E",
help="Sky coordinate system [C,E,G]")
parser.add_argument("--baseline", required=False, type=float,
default=60.0,
help="Destriping baseline length (seconds)")
parser.add_argument("--noisefilter", required=False, default=False,
action="store_true",
help="Destripe with the noise filter enabled")
parser.add_argument("--madam", required=False, default=False,
action="store_true",
help="If specified, use libmadam for map-making")
parser.add_argument("--madampar", required=False, default=None,
help="Madam parameter file")
parser.add_argument("--polyorder",
required=False, type=int,
help="Polynomial order for the polyfilter")
parser.add_argument("--wbin_ground",
required=False, type=float,
help="Ground template bin width [degrees]")
parser.add_argument("--flush", required=False, default=False,
action="store_true",
help="Flush every print statement.")
parser.add_argument("--tidas", required=False, default=None,
help="Input TIDAS volume")
parser.add_argument("--tidas_detgroup", required=False, default=None,
help="TIDAS detector group")
parser.add_argument("--spt3g", required=False, default=None,
help="Input SPT3G data directory")
parser.add_argument("--spt3g_prefix", required=False, default=None,
help="SPT3G data frame file prefix")
parser.add_argument("--common_flag_mask",
required=False, default=0, type=np.uint8,
help="Common flag mask")
parser.add_argument("--debug", required=False, default=False,
action="store_true",
help="Write data distribution info and focalplane plot")
args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True))
#args = parser.parse_args(sys.argv)
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
if (args.tidas is not None) and (args.spt3g is not None):
raise RuntimeError("Cannot read two datasets!")
if (args.tidas is None) and (args.spt3g is None):
raise RuntimeError("No dataset specified!")
if args.tidas is not None:
if not tt.tidas_available:
raise RuntimeError("TIDAS not found- cannot load")
if args.spt3g is not None:
if not tt.spt3g_available:
raise RuntimeError("SPT3G not found- cannot load")
groupsize = args.groupsize
if groupsize == 0:
groupsize = MPI.COMM_WORLD.size
# Pixelization
nside = args.nside
npix = 12 * args.nside * args.nside
subnside = args.subnside
if subnside > nside:
subnside = nside
subnpix = 12 * subnside * subnside
# This is the 2-level toast communicator.
if MPI.COMM_WORLD.size % groupsize != 0:
if MPI.COMM_WORLD.rank == 0:
print("WARNING: process groupsize does not evenly divide into "
"total number of processes", flush=True)
comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize)
# Create output directory
mtime = MPI.Wtime()
if comm.comm_world.rank == 0:
if not os.path.isdir(args.outdir):
os.makedirs(args.outdir)
mtime = elapsed(comm.comm_world, mtime, "Creating output directory")
# The distributed timestream data
data = None
if args.tidas is not None:
if args.tidas_detgroup is None:
raise RuntimeError("you must specify the detector group")
data = tds.load_tidas(comm, comm.group_size, args.tidas,
"r", args.tidas_detgroup, tds.TODTidas,
group_dets=args.tidas_detgroup,
distintervals="chunks")
if args.spt3g is not None:
if args.spt3g_prefix is None:
raise RuntimeError("you must specify the frame file prefix")
data = s3g.load_spt3g(comm, comm.group_size, args.spt3g,
args.spt3g_prefix, s3g.obsweight_spt3g,
s3g.TOD3G)
mtime = elapsed(comm.comm_world, mtime, "Distribute data")
# In debug mode, print out data distribution information
if args.debug:
handle = None
if comm.comm_world.rank == 0:
handle = open("{}_distdata.txt".format(args.outdir), "w")
data.info(handle)
if comm.comm_world.rank == 0:
handle.close()
mtime = elapsed(comm.comm_world, mtime,
"Dumping debug data distribution")
if comm.comm_world.rank == 0:
outfile = "{}_focalplane.png".format(args.outdir)
set_backend()
# Just plot the dets from the first TOD
temptod = data.obs[0]["tod"]
# FIXME: change this once we store det info in the metadata.
dfwhm = { x : 10.0 for x in temptod.detectors }
tt.plot_focalplane(temptod.detoffset(), 10.0, 10.0, outfile, fwhm=dfwhm)
comm.comm_world.barrier()
mtime = elapsed(comm.comm_world, mtime,
"Plotting debug focalplane")
# Compute pointing matrix
pointing = tt.OpPointingHpix(
nside=args.nside, nest=True, mode="IQU",
hwprpm=args.hwprpm)
pointing.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Expand pointing")
# Mapmaking.
# FIXME: We potentially have a different noise model for every
# observation. We need to have both spt3g and tidas format Noise
# classes which read the information from disk. Then the mapmaking
# operators need to get these noise weights from each observation.
detweights = { d : 1.0 for d in data.obs[0]["tod"].detectors }
if not args.madam:
if comm.comm_world.rank == 0:
print("Not using Madam, will only make a binned map!", flush=True)
# Filter data if desired
if args.polyorder:
polyfilter = tt.OpPolyFilter(
order=args.polyorder,
common_flag_mask=args.common_flag_mask)
polyfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering")
if args.wbin_ground:
groundfilter = tt.OpGroundFilter(
wbin=args.wbin_ground,
common_flag_mask=args.common_flag_mask)
groundfilter.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Ground template filtering")
# Compute pixel space distribution
lc = tm.OpLocalPixels()
localpix = lc.exec(data)
if localpix is None:
raise RuntimeError(
"Process {} has no hit pixels. Perhaps there are fewer "
"detectors than processes in the group?".format(
comm.comm_world.rank))
localsm = np.unique(np.floor_divide(localpix, subnpix))
mtime = elapsed(comm.comm_world, mtime, "Compute local submaps")
# construct distributed maps to store the covariance,
# noise weighted map, and hits
mtime = MPI.Wtime()
invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6,
dtype=np.float64, submap=subnpix, local=localsm)
hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1,
dtype=np.int64, submap=subnpix, local=localsm)
zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3,
dtype=np.float64, submap=subnpix, local=localsm)
# compute the hits and covariance.
invnpp.data.fill(0.0)
hits.data.fill(0)
build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp,
hits=hits, common_flag_mask=args.common_flag_mask)
build_invnpp.exec(data)
invnpp.allreduce()
hits.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1")
hits.write_healpix_fits("{}_hits.fits".format(args.outdir))
invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1")
# invert it
tm.covariance_invert(invnpp, 1.0e-3)
mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1")
invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir))
mtime = elapsed(comm.comm_world, mtime, "Writing N_pp")
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(zmap=zmap, detweights=detweights,
common_flag_mask=args.common_flag_mask)
build_zmap.exec(data)
zmap.allreduce()
mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map")
tm.covariance_apply(invnpp, zmap)
mtime = elapsed(comm.comm_world, mtime, "Computing binned map")
zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits"))
mtime = elapsed(comm.comm_world, mtime, "Writing binned map")
else:
# Set up MADAM map making.
pars = {}
pars[ "temperature_only" ] = "F"
pars[ "force_pol" ] = "T"
pars[ "kfirst" ] = "T"
pars[ "concatenate_messages" ] = "T"
pars[ "write_map" ] = "T"
pars[ "write_binmap" ] = "T"
pars[ "write_matrix" ] = "T"
pars[ "write_wcov" ] = "T"
pars[ "write_hits" ] = "T"
pars[ "nside_cross" ] = nside // 2
pars[ "nside_submap" ] = subnside
if args.madampar is not None:
pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*")
comment = re.compile(r"^#.*")
with open(args.madampar, "r") as f:
for line in f:
if comment.match(line) is None:
result = pat.match(line)
if result is not None:
key, value = result.group(1), result.group(2)
pars[key] = value
pars[ "base_first" ] = args.baseline
pars[ "nside_map" ] = nside
if args.noisefilter:
pars[ "kfilter" ] = "T"
else:
pars[ "kfilter" ] = "F"
pars[ "fsample" ] = args.samplerate
madam = tm.OpMadam(params=pars, detweights=detweights,
common_flag_mask=args.common_flag_mask)
madam.exec(data)
mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking")
comm.comm_world.barrier()
stop = MPI.Wtime()
dur = stop - global_start
if comm.comm_world.rank == 0:
print("Total Time: {:.2f} seconds".format(dur), flush=True)
return
def main():
# This is the 2-level toast communicator. By default,
# there is just one group which spans MPI_COMM_WORLD.
comm = toast.Comm()
if comm.comm_world.rank == 0:
print('Running with {} processes at {}'.format(
comm.comm_world.size, str(datetime.now())), flush=True)
global_timer = timing.simple_timer('Total time')
global_timer.start()
args, comm = parse_arguments(comm)
autotimer = timing.auto_timer("@{}".format(timing.FILE()))
# Load and broadcast the schedule file
site, all_ces = load_schedule(args, comm)
# load or simulate the focalplane
fp, detweights = load_fp(args, comm)
# Create the TOAST data object to match the schedule. This will
# include simulating the boresight pointing.
data = create_observations(args, comm, fp, all_ces, site)
# Expand boresight quaternions into detector pointing weights and
# pixel numbers
expand_pointing(args, comm, data)
# Prepare auxiliary information for distributed map objects
localpix, localsm, subnpix = get_submaps(args, comm, data)
# Scan input map
signalname = scan_signal(args, comm, data, localsm, subnpix)
# Set up objects to take copies of the TOD at appropriate times
signalname_madam, sigcopy_madam, sigclear \
= setup_sigcopy(args, comm, signalname)
common_flag_name = None
flag_name = None
invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name \
= build_npp(args, comm, data, localsm, subnpix, detweights,
flag_name, common_flag_name)
madampars = setup_madam(args, comm)
output_tidas(args, comm, data, signalname, common_flag_name, flag_name)
outpath = setup_output(args, comm)
# Make a copy of the signal for Madam
copy_signal_madam(args, comm, data, sigcopy_madam)
# Bin unprocessed signal for reference
bin_maps(args, comm, data, 'binned', zmap, invnpp, zmap_group, invnpp_group,
detweights, signalname, flag_name, common_flag_name, outpath)
# Filter signal
apply_polyfilter(args, comm, data, signalname)
apply_groundfilter(args, comm, data, signalname)
# Bin the filtered signal
if args.polyorder or args.wbin_ground:
bin_maps(args, comm, data, 'filtered', zmap, invnpp, zmap_group,
invnpp_group, detweights, signalname, flag_name,
common_flag_name, outpath)
clear_signal(args, comm, data, sigclear)
# Now run Madam on the unprocessed copy of the signal
apply_madam(args, comm, data, madampars, outpath, detweights,
signalname_madam, flag_name, common_flag_name)
comm.comm_world.barrier()
global_timer.stop()
if comm.comm_world.rank == 0:
global_timer.report()
def load_fp(args, comm):
start = MPI.Wtime()
autotimer = timing.auto_timer()
fp = None
# Load focalplane information
nullquat = np.array([0,0,0,1], dtype=np.float64)
if comm.comm_world.rank == 0:
if args.fp is None:
# in this case, create a fake detector at the boresight
# with a pure white noise spectrum.
fake = {}
fake['quat'] = nullquat
fake['fwhm'] = 30.0
fake['fknee'] = 0.0
fake['fmin'] = 1e-9
fake['alpha'] = 1.0
fake['NET'] = 1.0
fake['color'] = 'r'
fp = {}
# Second detector at 22.5 degree polarization angle
fp['bore1'] = fake
fake2 = {}
zrot = qa.rotation(ZAXIS, 22.5*degree)
fake2['quat'] = qa.mult(fake['quat'], zrot)
fake2['fwhm'] = 30.0
fake2['fknee'] = 0.0
fake2['fmin'] = 1e-9
fake2['alpha'] = 1.0
fake2['NET'] = 1.0
fake2['color'] = 'r'
fp['bore2'] = fake2
# Third detector at 45 degree polarization angle
fake3 = {}
zrot = qa.rotation(ZAXIS, 45*degree)
fake3['quat'] = qa.mult(fake['quat'], zrot)
fake3['fwhm'] = 30.0
fake3['fknee'] = 0.0
fake3['fmin'] = 1e-9
fake3['alpha'] = 1.0
fake3['NET'] = 1.0
fake3['color'] = 'r'
fp['bore3'] = fake3
# Fourth detector at 67.5 degree polarization angle
fake4 = {}
zrot = qa.rotation(ZAXIS, 67.5*degree)
fake4['quat'] = qa.mult(fake['quat'], zrot)
fake4['fwhm'] = 30.0
fake4['fknee'] = 0.0
fake4['fmin'] = 1e-9
fake4['alpha'] = 1.0
fake4['NET'] = 1.0
fake4['color'] = 'r'
fp['bore4'] = fake4
else:
with open(args.fp, 'rb') as p:
fp = pickle.load(p)
fp = comm.comm_world.bcast(fp, root=0)
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_world.rank == 0:
print('Create focalplane: {:.2f} seconds'.format(stop-start),
flush=args.flush)
start = stop
if args.debug:
if comm.comm_world.rank == 0:
outfile = '{}/focalplane.png'.format(args.outdir)
tt.plot_focalplane(fp, 6, 6, outfile)
detectors = sorted(fp.keys())
detweights = {}
for d in detectors:
net = fp[d]['NET']
detweights[d] = 1.0 / (args.samplerate * net * net)
return fp, detweights
def bin_maps(args, comm, data, rootname,
zmap, invnpp, zmap_group, invnpp_group, detweights, totalname_freq,
flag_name, common_flag_name, outpath):
""" Use TOAST facilities to bin stored signal.
"""
if not args.skip_bin:
if comm.comm_world.rank == 0:
print('Binning unfiltered maps', flush=args.flush)
start0 = MPI.Wtime()
start = start0
autotimer = timing.auto_timer()
# Bin a map using the toast facilities
zmap.data.fill(0.0)
build_zmap = tm.OpAccumDiag(
detweights=detweights, zmap=zmap, name=totalname_freq,
flag_name=flag_name, common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask)
build_zmap.exec(data)
zmap.allreduce()
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Building noise weighted map took {:.3f} s'
''.format(stop-start), flush=args.flush)
start = stop
tm.covariance_apply(invnpp, zmap)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Computing {} map took {:.3f} s'
''.format(rootname, stop-start), flush=args.flush)
start = stop
fn = os.path.join(outpath, rootname+'.fits')
if args.zip:
fn += '.gz'
zmap.write_healpix_fits(fn)
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print(' - Writing {} map to {} took {:.3f} s'
''.format(rootname, fn, stop-start), flush=args.flush)
if zmap_group is not None:
zmap_group.data.fill(0.0)
build_zmap_group = tm.OpAccumDiag(
detweights=detweights, zmap=zmap_group,
name=totalname_freq,
flag_name=flag_name, common_flag_name=common_flag_name,
common_flag_mask=args.common_flag_mask)
build_zmap_group.exec(data)
zmap_group.allreduce()
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Building group noise weighted map took '
'{:.3f} s'.format(stop-start), flush=args.flush)
start = stop
tm.covariance_apply(invnpp_group, zmap_group)
comm.comm_group.barrier()
stop = MPI.Wtime()
elapsed = stop - start
if comm.comm_group.rank == 0:
print(' - Computing {} map took {:.3f} s'
''.format(rootname, stop-start), flush=args.flush)
start = stop
fn = os.path.join(outpath, '{}_group_{:04}.fits'
''.format(rootname, comm.group))
if args.zip:
fn += '.gz'
zmap_group.write_healpix_fits(fn)
comm.comm_group.barrier()
stop = MPI.Wtime()
if comm.comm_group.rank == 0:
print(' - Writing group {} map to {} took '
'{:.3f} s'.format(rootname, fn, stop-start),
flush=args.flush)
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Mapmaking took {:.3f} s'
''.format(stop-start0), flush=args.flush)
return
def create_observations(args, comm, fp, all_ces, site):
""" Simulate constant elevation scans.
Simulate constant elevation scans at "site" matching entries in
"all_ces". Each operational day is assigned to a different
process group to allow making day maps.
"""
start = MPI.Wtime()
autotimer = timing.auto_timer()
data = toast.Data(comm)
site_name, site_lat, site_lon, site_alt = site
detectors = sorted(fp.keys())
detquats = {}
for d in detectors:
detquats[d] = fp[d]['quat']
nces = len(all_ces)
breaks = []
do_break = False
for i in range(nces-1):
# If current and next CES are on different days, insert a break
tz = args.timezone / 24.
start1 = all_ces[i][3] # MJD start
start2 = all_ces[i+1][3] # MJD start
scan1 = all_ces[i][4]
scan2 = all_ces[i+1][4]
if scan1 != scan2 and do_break:
breaks.append(i + 1)
do_break = False
continue
day1 = int(start1 + tz)
day2 = int(start2 + tz)
if day1 != day2:
if scan1 == scan2:
# We want an entire CES, even if it crosses the day bound.
# Wait until the scan number changes.
do_break = True
else:
breaks.append(i + 1)
nbreak = len(breaks)
if nbreak != comm.ngroups-1:
raise RuntimeError(
'Number of observing days ({}) does not match number of process '
'groups ({}).'.format(nbreak+1, comm.ngroups))
groupdist = toast.distribute_uniform(nces, comm.ngroups, breaks=breaks)
group_firstobs = groupdist[comm.group][0]
group_numobs = groupdist[comm.group][1]
# Create the noise model used by all observations
fmin = {}
fknee = {}
alpha = {}
NET = {}
rates = {}
for d in detectors:
rates[d] = args.samplerate
fmin[d] = fp[d]['fmin']
fknee[d] = fp[d]['fknee']
alpha[d] = fp[d]['alpha']
NET[d] = fp[d]['NET']
noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors,
fknee=fknee, alpha=alpha, NET=NET)
for ices in range(group_firstobs, group_firstobs + group_numobs):
ces = all_ces[ices]
CES_start, CES_stop, name, mjdstart, scan, subscan, azmin, azmax, \
el = ces
totsamples = int((CES_stop - CES_start) * args.samplerate)
# create the single TOD for this observation
try:
tod = tt.TODGround(
comm.comm_group,
detquats,
totsamples,
detranks=comm.comm_group.size,
firsttime=CES_start,
rate=args.samplerate,
site_lon=site_lon,
site_lat=site_lat,
site_alt=site_alt,
azmin=azmin,
azmax=azmax,
el=el,
scanrate=args.scanrate,
scan_accel=args.scan_accel,
CES_start=None,
CES_stop=None,
sun_angle_min=args.sun_angle_min,
coord=args.coord,
sampsizes=None)
except RuntimeError as e:
print('Failed to create the CES scan: {}'.format(e),
flush=args.flush)
return
# Create the (single) observation
ob = {}
ob['name'] = 'CES-{}-{}-{}'.format(name, scan, subscan)
ob['tod'] = tod
if len(tod.subscans) > 0:
ob['intervals'] = tod.subscans
else:
raise RuntimeError('{} has no valid intervals'.format(ob['name']))
ob['baselines'] = None
ob['noise'] = noise
ob['id'] = int(mjdstart * 10000)
data.obs.append(ob)
for ob in data.obs:
tod = ob['tod']
tod.free_azel_quats()
if comm.comm_group.rank == 0:
print('Group # {:4} has {} observations.'.format(
comm.group, len(data.obs)), flush=args.flush)
if len(data.obs) == 0:
raise RuntimeError('Too many tasks. Every MPI task must '
'be assigned to at least one observation.')
comm.comm_world.barrier()
stop = MPI.Wtime()
if comm.comm_world.rank == 0:
print('Simulated scans in {:.2f} seconds'
''.format(stop-start), flush=args.flush)
return data
def exec(self, data):
"""
Apply the ground filter to the signal.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
# the two-level pytoast communicator
comm = data.comm
# the communicator within the group
cgroup = comm.comm_group
# Each group loops over its own CES:es
for obs in data.obs:
tod = obs['tod']
try:
(azmin, azmax, _, _) = tod.scan_range
az = tod.read_boresight_az()
except Exception as e:
raise RuntimeError(
'Failed to get boresight azimuth from TOD. Perhaps it is '
'not ground TOD? "{}"'.format(e))
# Cache the output common flags
common_ref = tod.local_common_flags(self._common_flag_name)
# The azimuth vector is assumed to be arranged so that the
# azimuth increases monotonously even across the zero meridian.
wbin = np.radians(self._wbin)
nbin = int((azmax - azmin) // wbin + 1)
ibin = ((az - azmin) // wbin).astype(np.int)
for det in tod.detectors:
hits = np.zeros(nbin)
binned = np.zeros(nbin)
# Bin the local data
if det in tod.local_dets:
ref = tod.local_signal(det, self._name)
flag_ref = tod.local_flags(det, self._flag_name)
good = np.logical_and(
common_ref & self._common_flag_mask == 0,
flag_ref & self._flag_mask == 0)
# If binning ever becomes a bottleneck, it must be
# implemented in Cython or compiled code. The range
# checks are very expensive.
for i, s in zip(ibin[good], ref[good]):
hits[i] += 1
binned[i] += s
del flag_ref
# Reduce the binned data. The detector signal may be
# distributed across the group communicator.
cgroup.Allreduce(MPI.IN_PLACE, hits, op=MPI.SUM)
cgroup.Allreduce(MPI.IN_PLACE, binned, op=MPI.SUM)
good = hits != 0
binned[good] /= hits[good]
# Subtract the ground template
if det in tod.local_dets:
ref -= binned[ibin]
del ref
del common_ref
return
def exec(self, data):
"""
Generate atmosphere timestreams.
This iterates over all observations and detectors and generates
the atmosphere timestreams.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
group = data.comm.group
for obs in data.obs:
try:
obsname = obs['name']
except Exception:
obsname = 'observation'
prefix = '{} : {} : '.format(group, obsname)
tod = self._get_from_obs('tod', obs)
comm = tod.mpicomm
obsindx = self._get_from_obs('id', obs)
telescope = self._get_from_obs('telescope_id', obs)
site = self._get_from_obs('site_id', obs)
altitude = self._get_from_obs('altitude', obs)
weather = self._get_from_obs('weather', obs)
fp_radius = np.radians(self._get_from_obs('fpradius', obs))
# Get the observation time span and initialize the weather
# object if one is provided.
times = tod.local_times()
tmin = times[0]
tmax = times[-1]
tmin_tot = comm.allreduce(tmin, op=MPI.MIN)
tmax_tot = comm.allreduce(tmax, op=MPI.MAX)
weather.set(site, self._realization, tmin_tot)
"""
The random number generator accepts a key and a counter,
each made of two 64bit integers.
Following tod_math.py we set
key1 = realization * 2^32 + telescope * 2^16 + component
key2 = obsindx * 2^32
counter1 = currently unused (0)
counter2 = sample in stream (incremented internally in the atm code)
"""
key1 = self._realization * 2 ** 32 + telescope * 2 ** 16 \
+ self._component
key2 = site * 2**16 + obsindx
counter1 = 0
counter2 = 0
if self._freq is not None:
absorption = atm_get_absorption_coefficient(
altitude, weather.air_temperature,
weather.surface_pressure, weather.pwv, self._freq)
loading = atm_get_atmospheric_loading(altitude,
weather.air_temperature,
weather.surface_pressure,
weather.pwv, self._freq)
tod.meta['loading'] = loading
else:
absorption = None
if self._cachedir is None:
cachedir = None
else:
# The number of atmospheric realizations can be large. Use
# sub-directories under cachedir.
subdir = str(int((obsindx % 1000) // 100))
subsubdir = str(int((obsindx % 100) // 10))
subsubsubdir = str(obsindx % 10)
cachedir = os.path.join(self._cachedir, subdir, subsubdir,
subsubsubdir)
if comm.rank == 0:
try:
os.makedirs(cachedir)
except FileExistsError:
pass
comm.Barrier()
if comm.rank == 0:
print(prefix + 'Setting up atmosphere simulation',
flush=self._flush)
comm.Barrier()
# Cache the output common flags
common_ref = tod.local_common_flags(self._common_flag_name)
# Read the extent of the AZ/EL boresight pointing, and use that
# to compute the range of angles needed for simulating the slab.
(min_az_bore, max_az_bore, min_el_bore,
max_el_bore) = tod.scan_range
# print("boresight scan range = {}, {}, {}, {}".format(
# min_az_bore, max_az_bore, min_el_bore, max_el_bore))
# Use a fixed focal plane radius so that changing the actual
# set of detectors will not affect the simulated atmosphere.
elfac = 1 / np.cos(max_el_bore + fp_radius)
azmin = min_az_bore - fp_radius * elfac
azmax = max_az_bore + fp_radius * elfac
if azmin < -2 * np.pi:
azmin += 2 * np.pi
azmax += 2 * np.pi
elif azmax > 2 * np.pi:
azmin -= 2 * np.pi
azmax -= 2 * np.pi
elmin = min_el_bore - fp_radius
elmax = max_el_bore + fp_radius
azmin = comm.allreduce(azmin, op=MPI.MIN)
azmax = comm.allreduce(azmax, op=MPI.MAX)
elmin = comm.allreduce(elmin, op=MPI.MIN)
elmax = comm.allreduce(elmax, op=MPI.MAX)
if elmin < 0 or elmax > np.pi / 2:
raise RuntimeError(
'Error in CES elevation: elmin = {:.2f}, elmax = {:.2f}'
''.format(elmin, elmax))
comm.Barrier()
# Loop over the time span in "wind_time"-sized chunks.
# wind_time is intended to reflect the correlation length
# in the atmospheric noise.
tmin = tmin_tot
istart = 0
while tmin < tmax_tot:
while times[istart] < tmin:
istart += 1
tmax = tmin + self._wind_time
if tmax < tmax_tot:
# Extend the scan to the next turnaround
istop = istart
while istop < times.size and times[istop] < tmax:
istop += 1
while istop < times.size and (common_ref[istop]
| tod.TURNAROUND == 0):
istop += 1
if istop < times.size:
tmax = times[istop]
else:
tmax = tmax_tot
else:
tmax = tmax_tot
istop = times.size
ind = slice(istart, istop)
nind = istop - istart
if self._report_timing:
comm.Barrier()
tstart = MPI.Wtime()
comm.Barrier()
if comm.rank == 0:
print(prefix + 'Instantiating the atmosphere for t = {}'
''.format(tmin - tmin_tot),
flush=self._flush)
comm.Barrier()
T0_center = weather.air_temperature
wx = weather.west_wind
wy = weather.south_wind
w_center = np.sqrt(wx**2 + wy**2)
wdir_center = np.arctan2(wy, wx)
sim = atm_sim_alloc(
azmin, azmax, elmin, elmax, tmin, tmax, self._lmin_center,
self._lmin_sigma, self._lmax_center, self._lmax_sigma,
w_center, 0, wdir_center, 0, self._z0_center,
self._z0_sigma, T0_center, 0, self._zatm, self._zmax,
self._xstep, self._ystep, self._zstep, self._nelem_sim_max,
self._verbosity, comm, self._gangsize, key1, key2,
counter1, counter2, cachedir)
if sim == 0:
raise RuntimeError(prefix +
'Failed to allocate simulation')
if self._report_timing:
comm.Barrier()
tstop = MPI.Wtime()
if comm.rank == 0 and tstop - tstart > 1:
print(prefix + 'OpSimAtmosphere: Initialized '
'atmosphere in {:.2f} s'.format(tstop - tstart),
flush=self._flush)
tstart = tstop
comm.Barrier()
use_cache = cachedir is not None
if comm.rank == 0:
fname = os.path.join(
cachedir, '{}_{}_{}_{}_metadata.txt'.format(
key1, key2, counter1, counter2))
if use_cache and os.path.isfile(fname):
print(prefix + 'Loading the atmosphere for t = {} '
'from {}'.format(tmin - tmin_tot, fname),
flush=self._flush)
cached = True
else:
print(prefix + 'Simulating the atmosphere for t = {}'
''.format(tmin - tmin_tot),
flush=self._flush)
cached = False
err = atm_sim_simulate(sim, use_cache)
if err != 0:
raise RuntimeError(prefix + 'Simulation failed.')
# Advance the sample counter in case wind_time broke the
# observation in parts
counter2 += 100000000
if self._report_timing:
comm.Barrier()
tstop = MPI.Wtime()
if comm.rank == 0 and tstop - tstart > 1:
if cached:
op = 'Loaded'
else:
op = 'Simulated'
print(prefix + 'OpSimAtmosphere: {} atmosphere in '
'{:.2f} s'.format(op, tstop - tstart),
flush=self._flush)
tstart = tstop
if self._verbosity > 0:
self._plot_snapshots(sim, prefix, obsname, azmin, azmax,
elmin, elmax, tmin, tmax, comm)
nsamp = tod.local_samples[1]
if self._report_timing:
comm.Barrier()
tstart = MPI.Wtime()
if comm.rank == 0:
print(prefix + 'Observing the atmosphere',
flush=self._flush)
for det in tod.local_dets:
# Cache the output signal
cachename = '{}_{}'.format(self._out, det)
if tod.cache.exists(cachename):
ref = tod.cache.reference(cachename)
else:
ref = tod.cache.create(cachename, np.float64,
(nsamp, ))
# Cache the output flags
flag_ref = tod.local_flags(det, self._flag_name)
if self._apply_flags:
good = np.logical_and(
common_ref[ind] & self._common_flag_mask == 0,
flag_ref[ind] & self._flag_mask == 0)
ngood = np.sum(good)
if ngood == 0:
continue
azelquat = tod.read_pntg(detector=det,
local_start=istart,
n=nind,
azel=True)[good]
atmdata = np.zeros(ngood, dtype=np.float64)
else:
ngood = nind
azelquat = tod.read_pntg(detector=det,
local_start=istart,
n=nind,
azel=True)
atmdata = np.zeros(nind, dtype=np.float64)
# Convert Az/El quaternion of the detector back into
# angles for the simulation.
theta, phi, _ = qa.to_angles(azelquat)
# Azimuth is measured in the opposite direction
# than longitude
az = 2 * np.pi - phi
el = np.pi / 2 - theta
if np.ptp(az) < np.pi:
azmin_det = np.amin(az)
azmax_det = np.amax(az)
else:
# Scanning across the zero azimuth.
azmin_det = np.amin(az[az > np.pi]) - 2 * np.pi
azmax_det = np.amax(az[az < np.pi])
elmin_det = np.amin(el)
elmax_det = np.amax(el)
if ((not (azmin <= azmin_det and azmax_det <= azmax)
and not (azmin <= azmin_det - 2 * np.pi
and azmax_det - 2 * np.pi <= azmax)) or
not (elmin <= elmin_det and elmin_det <= elmax)):
raise RuntimeError(
prefix + 'Detector Az/El: [{:.5f}, {:.5f}], '
'[{:.5f}, {:.5f}] is not contained in '
'[{:.5f}, {:.5f}], [{:.5f} {:.5f}]'
''.format(azmin_det, azmax_det, elmin_det,
elmax_det, azmin, azmax, elmin, elmax))
# Integrate detector signal
err = atm_sim_observe(sim, times[ind], az, el, atmdata,
ngood, 0)
if err != 0:
# Observing failed
print(prefix + 'OpSimAtmosphere: Observing FAILED. '
'det = {}, rank = {}'.format(det, comm.rank),
flush=self._flush)
atmdata[:] = 0
flag_ref[ind] = 255
if self._gain:
atmdata *= self._gain
if absorption is not None:
# Apply the frequency-dependent absorption-coefficient
atmdata *= absorption
if self._apply_flags:
ref[ind][good] += atmdata
else:
ref[ind] += atmdata
del ref
err = atm_sim_free(sim)
if err != 0:
raise RuntimeError(prefix + 'Failed to free simulation.')
if self._report_timing:
comm.Barrier()
tstop = MPI.Wtime()
if comm.rank == 0 and tstop - tstart > 1:
print(prefix + 'OpSimAtmosphere: Observed atmosphere '
'in {:.2f} s'.format(tstop - tstart),
flush=self._flush)
tmin = tmax
return
def exec(self, data):
"""
Generate atmosphere timestreams.
This iterates over all observations and detectors and generates
the atmosphere timestreams.
Args:
data (toast.Data): The distributed data.
"""
autotimer = timing.auto_timer(type(self).__name__)
group = data.comm.group
for obs in data.obs:
try:
obsname = obs["name"]
except Exception:
obsname = "observation"
prefix = "{} : {} : ".format(group, obsname)
tod = self._get_from_obs("tod", obs)
comm = tod.mpicomm
site = self._get_from_obs("site_id", obs)
weather = self._get_from_obs("weather", obs)
# Get the observation time span and initialize the weather
# object if one is provided.
times = tod.local_times()
tmin = times[0]
tmax = times[-1]
tmin_tot = comm.allreduce(tmin, op=MPI.MIN)
tmax_tot = comm.allreduce(tmax, op=MPI.MAX)
weather.set(site, self._realization, tmin_tot)
key1, key2, counter1, counter2 = self._get_rng_keys(obs)
absorption = self._get_absorption_and_loading(obs)
cachedir = self._get_cache_dir(obs, comm)
comm.Barrier()
if comm.rank == 0:
print(prefix + "Setting up atmosphere simulation", flush=self._flush)
comm.Barrier()
# Cache the output common flags
common_ref = tod.local_common_flags(self._common_flag_name)
scan_range = self._get_scan_range(obs, comm)
# Loop over the time span in "wind_time"-sized chunks.
# wind_time is intended to reflect the correlation length
# in the atmospheric noise.
if self._report_timing:
comm.Barrier()
tstart = MPI.Wtime()
tmin = tmin_tot
istart = 0
counter1start = counter1
while tmin < tmax_tot:
if self._report_timing:
comm.Barrier()
tstart = MPI.Wtime()
comm.Barrier()
if comm.rank == 0:
print(
prefix + "Instantiating the atmosphere for t = {}"
"".format(tmin - tmin_tot),
flush=self._flush,
)
istart, istop, tmax = self._get_time_range(
tmin, istart, times, tmax_tot, common_ref, tod, weather
)
ind = slice(istart, istop)
nind = istop - istart
comm.Barrier()
rmin = 0
rmax = 100
scale = 10
counter2start = counter2
counter1 = counter1start
xstart, ystart, zstart = self._xstep, self._ystep, self._zstep
while rmax < 100000:
sim, counter2 = self._simulate_atmosphere(
weather,
scan_range,
tmin,
tmax,
comm,
key1,
key2,
counter1,
counter2start,
cachedir,
prefix,
tmin_tot,
tmax_tot,
rmin,
rmax,
)
if self._verbosity > 15:
self._plot_snapshots(
sim,
prefix,
obsname,
scan_range,
tmin,
tmax,
comm,
rmin,
rmax,
)
self._observe_atmosphere(
sim,
tod,
comm,
prefix,
common_ref,
istart,
nind,
ind,
scan_range,
times,
absorption,
)
rmin = rmax
rmax *= scale
self._xstep *= np.sqrt(scale)
self._ystep *= np.sqrt(scale)
self._zstep *= np.sqrt(scale)
counter1 += 1
if self._verbosity > 5:
self._save_tod(
obsname, tod, times, istart, nind, ind, comm, common_ref
)
self._xstep, self._ystep, self._zstep = xstart, ystart, zstart
tmin = tmax
if self._report_timing:
comm.Barrier()
tstop = MPI.Wtime()
if comm.rank == 0:
print(
prefix
+ "Simulated and observed atmosphere in {:.2f} s".format(
tstop - tstart
),
flush=self._flush,
)
return
