def _destripe(self, comm, parstring, ndet, detstring, nsamp, nnz, periods, psdinfo): """ Destripe the buffered data """ auto_timer = timing.auto_timer(type(self).__name__) fcomm = comm.py2f() if self._cached: # destripe outpath = '' if 'path_output' in self.params: outpath = self.params['path_output'] outpath = outpath.encode('ascii') libmadam.destripe_with_cache(fcomm, ndet, nsamp, nnz, self._madam_timestamps, self._madam_pixels, self._madam_pixweights, self._madam_signal, outpath) else: (detweights, npsd, npsdtot, psdstarts, npsdbin, psdfreqs, npsdval, psdvals) = psdinfo # destripe libmadam.destripe(fcomm, parstring.encode(), ndet, detstring.encode(), detweights, nsamp, nnz, self._madam_timestamps, self._madam_pixels, self._madam_pixweights, self._madam_signal, len(periods), periods, npsd, npsdtot, psdstarts, npsdbin, psdfreqs, npsdval, psdvals) if self._mcmode: self._cached = True return
def _stage_signal(self, data, detectors, nsamp, ndet, obs_period_ranges): """ Stage signal """ auto_timer = timing.auto_timer(type(self).__name__) self._madam_signal = self._cache.create('signal', np.float64, (nsamp * ndet, )) global_offset = 0 for iobs, obs in enumerate(data.obs): tod = obs['tod'] period_ranges = obs_period_ranges[iobs] for idet, det in enumerate(detectors): # Get the signal. signal = tod.local_signal(det, self._name) offset = global_offset for istart, istop in period_ranges: nn = istop - istart dslice = slice(idet * nsamp + offset, idet * nsamp + offset + nn) self._madam_signal[dslice] = signal[istart:istop] offset += nn del signal for idet, det in enumerate(detectors): if self._name is not None and (self._purge_tod or self._name == self._name_out): cachename = "{}_{}".format(self._name, det) tod.cache.clear(pattern=cachename) global_offset = offset return
def __init__(self, comm=None, signal_map="signal_map", lmax=None, grid=None, fwhm_deg=None, beam=None, out="smoothed_signal_map"): autotimer = timing.auto_timer(type(self).__name__) # We call the parent class constructor, which currently does nothing super().__init__() self.comm = comm self.signal_map = signal_map self.lmax = lmax self.out = out self.grid = grid # distribute alms local_m_indices = np.arange(self.comm.rank, lmax + 1, self.comm.size, dtype=np.int32) self.order = libsharp.packed_real_order(lmax, ms=local_m_indices) if (fwhm_deg is not None) and (beam is not None): raise Exception("OpSmooth error, specify either fwhm_deg or beam, " "not both") if (fwhm_deg is None) and (beam is None): raise Exception("OpSmooth error, specify fwhm_deg or beam") if fwhm_deg is not None: self.beam = hp.gauss_beam(fwhm=np.radians(fwhm_deg), lmax=lmax, pol=True) else: self.beam = beam
def add_sky_signal(args, comm, data, totalname, signalname): """ Add signalname to totalname in the obs tod """ if signalname is not None: autotimer = timing.auto_timer() for obs in data.obs: tod = obs['tod'] for det in tod.local_dets: cachename_in = '{}_{}'.format(signalname, det) cachename_out = '{}_{}'.format(totalname, det) ref_in = tod.cache.reference(cachename_in) if comm.comm_world.rank == 0 and args.debug: print('add_sky_signal', signalname, 'to', totalname, flush=args.flush) print(signalname, 'min max', ref_in.min(), ref_in.max()) if tod.cache.exists(cachename_out): ref_out = tod.cache.reference(cachename_out) if comm.comm_world.rank == 0 and args.debug: print(totalname, 'min max', ref_out.min(), ref_out.max()) ref_out += ref_in else: ref_out = tod.cache.put(cachename_out, ref_in) if comm.comm_world.rank == 0 and args.debug: print('final', 'min max', ref_out.min(), ref_out.max()) del ref_in, ref_out return
def get_submaps(args, comm, data): """ Get a list of locally hit pixels and submaps on every process. """ autotimer = timing.auto_timer() if comm.comm_world.rank == 0: print('Scanning local pixels', flush=args.flush) start = MPI.Wtime() # Prepare for using distpixels objects nside = args.nside subnside = 16 if subnside > nside: subnside = nside subnpix = 12 * subnside * subnside # get locally hit pixels lc = tm.OpLocalPixels() localpix = lc.exec(data) if localpix is None: raise RuntimeError( 'Process {} has no hit pixels. Perhaps there are fewer ' 'detectors than processes in the group?'.format( comm.comm_world.rank)) # find the locally hit submaps. localsm = np.unique(np.floor_divide(localpix, subnpix)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print('Local submaps identified in {:.3f} s'.format(elapsed), flush=args.flush) return localpix, localsm, subnpix
def apply_madam(args, comm, data, madampars, outpath, detweights, totalname_madam, flag_name, common_flag_name): if args.madam: if comm.comm_world.rank == 0: print('Destriping signal', flush=args.flush) start = MPI.Wtime() autotimer = timing.auto_timer() # create output directory for this realization madampars['path_output'] = outpath madam = tm.OpMadam(params=madampars, detweights=detweights, name=totalname_madam, common_flag_name=common_flag_name, flag_name=flag_name, common_flag_mask=args.common_flag_mask, purge_tod=True) madam.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Madam took {:.3f} s'.format(stop - start), flush=args.flush) return
def output_tidas(args, comm, data, totalname, common_flag_name, flag_name): if args.tidas is None: return autotimer = timing.auto_timer() from toast.tod.tidas import OpTidasExport tidas_path = os.path.abspath(args.tidas) comm.comm_world.Barrier() if comm.comm_world.rank == 0: print('Exporting TOD to a TIDAS volume at {}'.format(tidas_path), flush=args.flush) start = MPI.Wtime() export = OpTidasExport(tidas_path, name=totalname, common_flag_name=common_flag_name, flag_name=flag_name, usedist=True) export.exec(data) comm.comm_world.Barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Wrote simulated TOD to {}:{} in {:.2f} s' ''.format(tidas_path, totalname, stop - start), flush=args.flush) return
def apply_madam(args, comm, data, madampars, outpath, detweights, totalname_madam, flag_name, common_flag_name): if args.madam: if comm.comm_world.rank == 0: print('Destriping signal', flush=args.flush) start = MPI.Wtime() autotimer = timing.auto_timer() # create output directory for this realization madampars['path_output'] = outpath madam = tm.OpMadam( params=madampars, detweights=detweights, name=totalname_madam, common_flag_name=common_flag_name, flag_name=flag_name, common_flag_mask=args.common_flag_mask, purge_tod=True) madam.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Madam took {:.3f} s'.format(stop-start), flush=args.flush) return
def exec(self, data): """ Apply the gains. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) for obs in data.obs: tod = obs['tod'] for det in tod.local_dets: # Cache the output signal ref = tod.local_signal(det, self._name) obs_times = tod.read_times() calibrate(obs_times, ref, self._gain["TIME"], self._gain[det], order=0, inplace=True) assert np.isnan(ref).sum() == 0, "The signal timestream includes NaN" del ref return
def get_submaps(args, comm, data): """ Get a list of locally hit pixels and submaps on every process. """ autotimer = timing.auto_timer() if comm.comm_world.rank == 0: print('Scanning local pixels', flush=args.flush) start = MPI.Wtime() # Prepare for using distpixels objects nside = args.nside subnside = 16 if subnside > nside: subnside = nside subnpix = 12 * subnside * subnside # get locally hit pixels lc = tm.OpLocalPixels() localpix = lc.exec(data) if localpix is None: raise RuntimeError( 'Process {} has no hit pixels. Perhaps there are fewer ' 'detectors than processes in the group?'.format( comm.comm_world.rank)) # find the locally hit submaps. localsm = np.unique(np.floor_divide(localpix, subnpix)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print('Local submaps identified in {:.3f} s'.format(elapsed), flush=args.flush) return localpix, localsm, subnpix
def output_tidas(args, comm, data, totalname, common_flag_name, flag_name): if args.tidas is None: return autotimer = timing.auto_timer() from toast.tod.tidas import OpTidasExport, TODTidas tidas_path = os.path.abspath(args.tidas) comm.comm_world.Barrier() if comm.comm_world.rank == 0: print('Exporting TOD to a TIDAS volume at {}'.format(tidas_path), flush=args.flush) start = MPI.Wtime() export = OpTidasExport(tidas_path, TODTidas, backend="hdf5", use_todchunks=True, ctor_opts={"group_dets":"sim"}, cache_name=totalname) export.exec(data) comm.comm_world.Barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Wrote simulated TOD to {}:{} in {:.2f} s' ''.format(tidas_path, totalname, stop-start), flush=args.flush) return
def add_sky_signal(args, comm, data, totalname, signalname): """ Add signalname to totalname in the obs tod """ if signalname is not None: autotimer = timing.auto_timer() for obs in data.obs: tod = obs['tod'] for det in tod.local_dets: cachename_in = '{}_{}'.format(signalname, det) cachename_out = '{}_{}'.format(totalname, det) ref_in = tod.cache.reference(cachename_in) if comm.comm_world.rank == 0 and args.debug: print('add_sky_signal', signalname, 'to', totalname, flush=args.flush) print(signalname, 'min max', ref_in.min(), ref_in.max()) if tod.cache.exists(cachename_out): ref_out = tod.cache.reference(cachename_out) if comm.comm_world.rank == 0 and args.debug: print(totalname, 'min max', ref_out.min(), ref_out.max()) ref_out += ref_in else: ref_out = tod.cache.put(cachename_out, ref_in) if comm.comm_world.rank == 0 and args.debug: print('final', 'min max', ref_out.min(), ref_out.max()) del ref_in, ref_out return
def scan_signal(args, comm, data, localsm, subnpix): """ Scan time-ordered signal from a map. """ signalname = None if args.input_map: if comm.comm_world.rank == 0: print('Scanning input map', flush=args.flush) start = MPI.Wtime() autotimer = timing.auto_timer() npix = 12*args.nside**2 # Scan the sky signal if comm.comm_world.rank == 0 and not os.path.isfile(args.input_map): raise RuntimeError( 'Input map does not exist: {}'.format(args.input_map)) distmap = tm.DistPixels( comm=comm.comm_world, size=npix, nnz=3, dtype=np.float32, submap=subnpix, local=localsm) distmap.read_healpix_fits(args.input_map) scansim = tt.OpSimScan(distmap=distmap, out='signal') scansim.exec(data) stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Read and sampled input map: {:.2f} seconds' ''.format(stop-start), flush=args.flush) signalname = 'signal' return signalname
def load_schedule(args, comm): start = MPI.Wtime() autotimer = timing.auto_timer() if comm.comm_world.rank == 0: fn = args.schedule if not os.path.isfile(fn): raise RuntimeError('No such schedule file: {}'.format(fn)) start = MPI.Wtime() f = open(fn, 'r') while True: line = f.readline() if line.startswith('#'): continue (site_name, telescope, site_lat, site_lon, site_alt) = line.split() site_alt = float(site_alt) site = [site_name, site_lat, site_lon, site_alt] break all_ces = [] for line in f: if line.startswith('#'): continue start_date, start_time, stop_date, stop_time, mjdstart, mjdstop, \ name, azmin, azmax, el, rs, \ sun_el1, sun_az1, sun_el2, sun_az2, \ moon_el1, moon_az1, moon_el2, moon_az2, moon_phase, \ scan, subscan = line.split() start_time = start_date + ' ' + start_time stop_time = stop_date + ' ' + stop_time try: start_time = dateutil.parser.parse(start_time + ' +0000') stop_time = dateutil.parser.parse(stop_time + ' +0000') except: start_time = dateutil.parser.parse(start_time) stop_time = dateutil.parser.parse(stop_time) start_timestamp = start_time.timestamp() stop_timestamp = stop_time.timestamp() all_ces.append([ start_timestamp, stop_timestamp, name, float(mjdstart), int(scan), int(subscan), float(azmin), float(azmax), float(el)]) f.close() stop = MPI.Wtime() elapsed = stop - start print('Load schedule: {:.2f} seconds'.format(stop-start), flush=args.flush) else: site = None all_ces = None site = comm.comm_world.bcast(site) all_ces = comm.comm_world.bcast(all_ces) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Loading schedule {:.3f} s'.format(stop-start), flush=args.flush) return site, all_ces
def exec(self, data): """ Scramble the gains. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) for obs in data.obs: obsindx = 0 if 'id' in obs: obsindx = obs['id'] else: print("Warning: observation ID is not set, using zero!") telescope = 0 if 'telescope' in obs: telescope = obs['telescope_id'] tod = obs['tod'] pat = re.compile(self._pattern) for det in tod.local_dets: # Test the detector pattern if not pat.match(det): continue detindx = tod.detindx[det] # Cache the output signal ref = tod.local_signal(det, self._name) """ key1 = realization * 2^32 + telescope * 2^16 + component key2 = obsindx * 2^32 + detindx counter1 = currently unused (0) counter2 = currently unused (0) """ key1 = (self._realization * 4294967296 + telescope * 65536 + self._component) key2 = obsindx * 4294967296 + detindx counter1 = 0 counter2 = 0 rngdata = rng.random(1, sampler="gaussian", key=(key1, key2), counter=(counter1, counter2)) gain = self._center + rngdata[0] * self._sigma ref *= gain del ref return
def _prepare(self, data, comm): """ Examine the data object. """ auto_timer = timing.auto_timer(type(self).__name__) nsamp = self._count_samples(data) # Determine the detectors and the pointing matrix non-zeros # from the first observation. Madam will expect these to remain # unchanged across observations. tod = data.obs[0]['tod'] if self._dets is None: detectors = tod.local_dets else: detectors = [det for det in tod.local_dets if det in self._dets] ndet = len(detectors) detstring = self._dets2detstring(detectors) # to get the number of Non-zero pointing weights per pixel, # we use the fact that for Madam, all processes have all detectors # for some slice of time. So we can get this information from the # shape of the data from the first detector nnzname = "{}_{}".format(self._weights, detectors[0]) nnz_full = tod.cache.reference(nnzname).shape[1] if 'temperature_only' in self.params \ and self.params['temperature_only'] in [ 'T', 'True', 'TRUE', 'true', True]: if nnz_full not in [1, 3]: raise RuntimeError( 'OpMadam: Don\'t know how to make a temperature map ' 'with nnz={}'.format(nnz_full)) nnz = 1 nnz_stride = nnz_full else: nnz = nnz_full nnz_stride = 1 if 'nside_map' not in self.params: raise RuntimeError( 'OpMadam: "nside_map" must be set in the parameter dictionary') nside = int(self.params['nside_map']) parstring = self._dict2parstring(self.params) # Inspect the valid intervals across all observations to # determine the number of samples per detector obs_period_ranges, psdfreqs, periods = self._get_period_ranges( comm, data, detectors, nsamp) return (parstring, detstring, nsamp, ndet, nnz, nnz_full, nnz_stride, periods, obs_period_ranges, psdfreqs, detectors, nside)
def exec(self, data): """ Scramble the gains. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) for obs in data.obs: obsindx = 0 if 'id' in obs: obsindx = obs['id'] else: print("Warning: observation ID is not set, using zero!") telescope = 0 if 'telescope' in obs: telescope = obs['telescope_id'] tod = obs['tod'] pat = re.compile(self._pattern) for det in tod.local_dets: # Test the detector pattern if not pat.match(det): continue detindx = tod.detindx[det] # Cache the output signal ref = tod.local_signal(det, self._name) """ key1 = realization * 2^32 + telescope * 2^16 + component key2 = obsindx * 2^32 + detindx counter1 = currently unused (0) counter2 = currently unused (0) """ key1 = (self._realization * 4294967296 + telescope * 65536 + self._component) key2 = obsindx * 4294967296 + detindx counter1 = 0 counter2 = 0 rngdata = rng.random(1, sampler="gaussian", key=(key1, key2), counter=(counter1, counter2)) gain = self._center + rngdata[0] * self._sigma ref *= gain del ref return
def exec(self, data): """ Apply the polynomial filter to the signal. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) for obs in data.obs: tod = obs['tod'] if self._intervals in obs: intervals = obs[self._intervals] else: intervals = None local_intervals = tod.local_intervals(intervals) common_ref = tod.local_common_flags(self._common_flag_name) pat = re.compile(self._pattern) for det in tod.local_dets: # Test the detector pattern if not pat.match(det): continue ref = tod.local_signal(det, self._name) flag_ref = tod.local_flags(det, self._flag_name) # Iterate over each interval local_starts = [] local_stops = [] for ival in local_intervals: local_starts.append(ival.first) local_stops.append(ival.last) local_starts = np.array(local_starts) local_stops = np.array(local_stops) flg = common_ref & self._common_flag_mask flg |= flag_ref & self._flag_mask filter_polyfilter( self._order, [ref], flg, local_starts, local_stops) flag_ref[flg != 0] |= self._poly_flag_mask del ref del flag_ref del common_ref return
def exec(self, data): """ Apply the polynomial filter to the signal. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) for obs in data.obs: tod = obs['tod'] if self._intervals in obs: intervals = obs[self._intervals] else: intervals = None local_intervals = tod.local_intervals(intervals) common_ref = tod.local_common_flags(self._common_flag_name) pat = re.compile(self._pattern) for det in tod.local_dets: # Test the detector pattern if not pat.match(det): continue ref = tod.local_signal(det, self._name) flag_ref = tod.local_flags(det, self._flag_name) # Iterate over each interval local_starts = [] local_stops = [] for ival in local_intervals: local_starts.append(ival.first) local_stops.append(ival.last) local_starts = np.array(local_starts) local_stops = np.array(local_stops) flg = common_ref & self._common_flag_mask flg |= flag_ref & self._flag_mask filter_polyfilter(self._order, [ref], flg, local_starts, local_stops) flag_ref[flg != 0] |= self._poly_flag_mask del ref del flag_ref del common_ref return
def setup_madam(args, comm): """ Prepare to run Madam on the stored TOD. """ pars = None if args.madam: autotimer = timing.auto_timer() # Set up MADAM map making. pars = {} cross = args.nside // 2 submap = 16 if submap > args.nside: submap = args.nside pars['temperature_only'] = False pars['force_pol'] = True pars['kfirst'] = True pars['write_map'] = True pars['write_binmap'] = True pars['write_matrix'] = True pars['write_wcov'] = True pars['write_hits'] = True pars['nside_cross'] = cross pars['nside_submap'] = submap pars['allreduce'] = args.madam_allreduce if args.madampar is not None: pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*') comment = re.compile(r'^#.*') with open(args.madampar, 'r') as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars['base_first'] = args.madam_baseline_length pars['basis_order'] = args.madam_baseline_order pars['nside_map'] = args.nside if args.madam_noisefilter: pars['kfilter'] = True else: pars['kfilter'] = False pars['precond_width'] = 1 pars['fsample'] = args.samplerate pars['iter_max'] = args.madam_iter_max return pars
def setup_madam(args, comm): """ Prepare to run Madam on the stored TOD. """ pars = None if args.madam: autotimer = timing.auto_timer() # Set up MADAM map making. pars = {} cross = args.nside // 2 submap = 16 if submap > args.nside: submap = args.nside pars['temperature_only'] = False pars['force_pol'] = True pars['kfirst'] = True pars['write_map'] = True pars['write_binmap'] = True pars['write_matrix'] = True pars['write_wcov'] = True pars['write_hits'] = True pars['nside_cross'] = cross pars['nside_submap'] = submap pars['allreduce'] = args.madam_allreduce if args.madampar is not None: pat = re.compile(r'\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*') comment = re.compile(r'^#.*') with open(args.madampar, 'r') as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars['base_first'] = args.madam_baseline_length pars['basis_order'] = args.madam_baseline_order pars['nside_map'] = args.nside if args.madam_noisefilter: pars['kfilter'] = True else: pars['kfilter'] = False pars['precond_width'] = 1 pars['fsample'] = args.samplerate pars['iter_max'] = args.madam_iter_max return pars
def main(): args, start_timestamp, stop_timestamp = parse_args() autotimer = timing.auto_timer(timing.FILE()) patches = parse_patches(args) build_schedule(args, start_timestamp, stop_timestamp, args.sun_el_max * degree, args.sun_avoidance_angle * degree, args.sun_angle_min * degree, args.moon_angle_min * degree, args.el_min * degree, args.el_max * degree, args.fp_radius * degree, patches)
def _stage_pixweights( self, data, detectors, nsamp, ndet, nnz, nnz_full, nnz_stride, obs_period_ranges ): """Now collect the pixel weights """ auto_timer = timing.auto_timer(type(self).__name__) self._madam_pixweights = self._cache.create( "pixweights", madam.WEIGHT_TYPE, (nsamp * ndet * nnz,) ) self._madam_pixweights[:] = 0 global_offset = 0 for iobs, obs in enumerate(data.obs): tod = obs["tod"] period_ranges = obs_period_ranges[iobs] for idet, det in enumerate(detectors): # get the pixels and weights for the valid intervals # from the cache weightsname = "{}_{}".format(self._weights, det) weights = tod.cache.reference(weightsname) weight_dtype = weights.dtype offset = global_offset for istart, istop in period_ranges: nn = istop - istart dwslice = slice( (idet * nsamp + offset) * nnz, (idet * nsamp + offset + nn) * nnz, ) self._madam_pixweights[dwslice] = weights[istart:istop].flatten()[ ::nnz_stride ] offset += nn del weights # Purge the weights but restore them from the Madam # buffers when purge_weights=False. # Handle special case when Madam only stores a subset of # the weights. if not self._purge_weights and (nnz != nnz_full): pass else: for idet, det in enumerate(detectors): # get the pixels and weights for the valid intervals # from the cache weightsname = "{}_{}".format(self._weights, det) tod.cache.clear(pattern=weightsname) global_offset = offset return weight_dtype
def exec(self, data): """ Create the timestreams... This loops over all observations and detectors and uses the pointing matrix to ... Args: data (toast.Data): The distributed data. """ if libsharp is None: raise RuntimeError('libsharp not available') autotimer = timing.auto_timer(type(self).__name__) has_pol = len(data[self.signal_map]) > 1 alm_sharp_I = libsharp.analysis( self.grid, self.order, np.ascontiguousarray(data[self.signal_map][0].reshape((1, 1, -1))), spin=0, comm=self.comm) self.order.almxfl(alm_sharp_I, np.ascontiguousarray(self.beam[:, 0:1])) out = libsharp.synthesis(self.grid, self.order, alm_sharp_I, spin=0, comm=self.comm)[0] if has_pol: alm_sharp_P = libsharp.analysis( self.grid, self.order, np.ascontiguousarray(data[self.signal_map][1:3, :].reshape( (1, 2, -1))), spin=2, comm=self.comm) self.order.almxfl(alm_sharp_P, np.ascontiguousarray(self.beam[:, (1, 2)])) signal_map_P = libsharp.synthesis(self.grid, self.order, alm_sharp_P, spin=2, comm=self.comm)[0] out = np.vstack((out, signal_map_P)) data[self.out] = out assert data[self.signal_map].shape == data[self.out].shape
def _stage_pixweights(self, data, detectors, nsamp, ndet, nnz, nnz_full, nnz_stride, obs_period_ranges): """Now collect the pixel weights """ auto_timer = timing.auto_timer(type(self).__name__) self._madam_pixweights = self._cache.create("pixweights", madam.WEIGHT_TYPE, (nsamp * ndet * nnz, )) self._madam_pixweights[:] = 0 global_offset = 0 for iobs, obs in enumerate(data.obs): tod = obs["tod"] period_ranges = obs_period_ranges[iobs] for idet, det in enumerate(detectors): # get the pixels and weights for the valid intervals # from the cache weightsname = "{}_{}".format(self._weights, det) weights = tod.cache.reference(weightsname) weight_dtype = weights.dtype offset = global_offset for istart, istop in period_ranges: nn = istop - istart dwslice = slice( (idet * nsamp + offset) * nnz, (idet * nsamp + offset + nn) * nnz, ) self._madam_pixweights[dwslice] = weights[ istart:istop].flatten()[::nnz_stride] offset += nn del weights # Purge the weights but restore them from the Madam # buffers when purge_weights=False. # Handle special case when Madam only stores a subset of # the weights. if not self._purge_weights and (nnz != nnz_full): pass else: for idet, det in enumerate(detectors): # get the pixels and weights for the valid intervals # from the cache weightsname = "{}_{}".format(self._weights, det) tod.cache.clear(pattern=weightsname) global_offset = offset return weight_dtype
def _stage_time(self, data, detectors, nsamp, obs_period_ranges): """ Stage the timestamps and use them to build PSD inputs. """ auto_timer = timing.auto_timer(type(self).__name__) self._madam_timestamps = self._cache.create('timestamps', np.float64, (nsamp, )) global_offset = 0 time_offset = 0 psds = {} for iobs, obs in enumerate(data.obs): tod = obs['tod'] period_ranges = obs_period_ranges[iobs] # Collect the timestamps for the valid intervals timestamps = tod.local_times() # Translate the time stamps to be monotonous timestamps -= timestamps[0] - time_offset time_offset = timestamps[-1] + 1 offset = global_offset for istart, istop in period_ranges: nn = istop - istart ind = slice(offset, offset + nn) self._madam_timestamps[ind] = timestamps[istart:istop] offset += nn # get the noise object for this observation and create new # entries in the dictionary when the PSD actually changes if self._noisekey in obs.keys(): nse = obs[self._noisekey] if 'noise_scale' in obs: noise_scale = obs['noise_scale'] else: noise_scale = 1 if nse is not None: for det in detectors: psd = nse.psd(det) * noise_scale**2 if det not in psds: psds[det] = [(0, psd)] else: if not np.allclose(psds[det][-1][1], psd): psds[det] += [(timestamps[0], psd)] global_offset = offset return psds
def _stage_time(self, data, detectors, nsamp, obs_period_ranges): """ Stage the timestamps and use them to build PSD inputs. """ auto_timer = timing.auto_timer(type(self).__name__) self._madam_timestamps = self._cache.create( "timestamps", madam.TIMESTAMP_TYPE, (nsamp,) ) offset = 0 time_offset = 0 psds = {} for iobs, obs in enumerate(data.obs): tod = obs["tod"] period_ranges = obs_period_ranges[iobs] # Collect the timestamps for the valid intervals timestamps = tod.local_times().copy() if self._translate_timestamps: # Translate the time stamps to be monotonous timestamps -= timestamps[0] - time_offset time_offset = timestamps[-1] + 1 for istart, istop in period_ranges: nn = istop - istart ind = slice(offset, offset + nn) self._madam_timestamps[ind] = timestamps[istart:istop] offset += nn # get the noise object for this observation and create new # entries in the dictionary when the PSD actually changes if self._noisekey in obs.keys(): nse = obs[self._noisekey] if "noise_scale" in obs: noise_scale = obs["noise_scale"] else: noise_scale = 1 if nse is not None: for det in detectors: psd = nse.psd(det) * noise_scale ** 2 if det not in psds: psds[det] = [(0, psd)] else: if not np.allclose(psds[det][-1][1], psd): psds[det] += [(timestamps[0], psd)] return psds
def main(): # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() # Create an argparse and add custom arguments parser = argparse.ArgumentParser(description=“...") parser.add_argument('--groupsize', required=False, type=np.int, help='Size of a process group assigned to a CES') # pass the argparse object to timing module which will add timing # arguments and return "parse.parse_args() result after handling # the timing specific options args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) # create the primary auto timer for the entire script autotimer = timing.auto_timer(timing.FILE())
def _destripe(self, comm, pars, dets, periods, psdinfo): """ Destripe the buffered data """ auto_timer = timing.auto_timer(type(self).__name__) if self._verbose: memreport(comm, "just before calling libmadam.destripe") if self._cached: # destripe outpath = "" if "path_output" in self.params: outpath = self.params["path_output"] outpath = outpath.encode("ascii") madam.destripe_with_cache( comm, self._madam_timestamps, self._madam_pixels, self._madam_pixweights, self._madam_signal, outpath, ) else: (detweights, npsd, psdstarts, psdfreqs, psdvals) = psdinfo # destripe madam.destripe( comm, pars, dets, detweights, self._madam_timestamps, self._madam_pixels, self._madam_pixweights, self._madam_signal, periods, npsd, psdstarts, psdfreqs, psdvals, ) if self._mcmode: self._cached = True return
def _destripe(self, comm, pars, dets, periods, psdinfo): """ Destripe the buffered data """ auto_timer = timing.auto_timer(type(self).__name__) if self._verbose: memreport(comm, "just before calling libmadam.destripe") if self._cached: # destripe outpath = "" if "path_output" in self.params: outpath = self.params["path_output"] outpath = outpath.encode("ascii") madam.destripe_with_cache( comm, self._madam_timestamps, self._madam_pixels, self._madam_pixweights, self._madam_signal, outpath, ) else: (detweights, npsd, psdstarts, psdfreqs, psdvals) = psdinfo # destripe madam.destripe( comm, pars, dets, detweights, self._madam_timestamps, self._madam_pixels, self._madam_pixweights, self._madam_signal, periods, npsd, psdstarts, psdfreqs, psdvals, ) if self._mcmode: self._cached = True return
def add_sky_signal(args, comm, data, totalname, signalname): """ Add signalname to totalname in the obs tod """ if signalname is not None: autotimer = timing.auto_timer() for obs in data.obs: tod = obs['tod'] for det in tod.local_dets: cachename_in = '{}_{}'.format(signalname, det) cachename_out = '{}_{}'.format(totalname, det) ref_in = tod.cache.reference(cachename_in) if tod.cache.exists(cachename_out): ref_out = tod.cache.reference(cachename_out) ref_out += ref_in else: ref_out = tod.cache.put(cachename_out, ref_in) del ref_in, ref_out return
def expand_pointing(args, comm, data): """ Expand the bore sight pointing to every detector. """ start = MPI.Wtime() autotimer = timing.auto_timer() hwprpm = args.hwprpm hwpstep = None if args.hwpstep is not None: hwpstep = float(args.hwpstep) hwpsteptime = args.hwpsteptime npix = 12 * args.nside**2 if comm.comm_world.rank == 0: print('Expanding pointing', flush=args.flush) pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode='IQU', hwprpm=hwprpm, hwpstep=hwpstep, hwpsteptime=hwpsteptime) pointing.exec(data) # Only purge the pointing if we are NOT going to export the # data to a TIDAS volume if args.tidas is None: for ob in data.obs: tod = ob['tod'] tod.free_radec_quats() comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Pointing generation took {:.3f} s'.format(stop - start), flush=args.flush) return
def exec(self, data): """ Create the timestreams... This loops over all observations and detectors and uses the pointing matrix to ... Args: data (toast.Data): The distributed data. """ if libsharp is None: raise RuntimeError('libsharp not available') autotimer = timing.auto_timer(type(self).__name__) has_pol = len(data[self.signal_map]) > 1 alm_sharp_I = libsharp.analysis( self.grid, self.order, np.ascontiguousarray(data[self.signal_map][0].reshape((1, 1, -1))), spin=0, comm=self.comm) self.order.almxfl(alm_sharp_I, np.ascontiguousarray(self.beam[:, 0:1])) out = libsharp.synthesis(self.grid, self.order, alm_sharp_I, spin=0, comm=self.comm)[0] if has_pol: alm_sharp_P = libsharp.analysis( self.grid, self.order, np.ascontiguousarray( data[self.signal_map][1:3, :].reshape((1, 2, -1))), spin=2, comm=self.comm) self.order.almxfl(alm_sharp_P, np.ascontiguousarray(self.beam[:, (1, 2)])) signal_map_P = libsharp.synthesis( self.grid, self.order, alm_sharp_P, spin=2, comm=self.comm)[0] out = np.vstack(( out, signal_map_P)) data[self.out] = out assert data[self.signal_map].shape == data[self.out].shape
def apply_polyfilter(args, comm, data, totalname_freq): if args.polyorder: if comm.comm_world.rank == 0: print('Polyfiltering signal', flush=args.flush) start = MPI.Wtime() autotimer = timing.auto_timer() common_flag_name = 'common_flags' flag_name = 'flags' polyfilter = tt.OpPolyFilter( order=args.polyorder, name=totalname_freq, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask, flag_name=flag_name) polyfilter.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Polynomial filtering took {:.3f} s'.format(stop-start), flush=args.flush) return
def apply_polyfilter(args, comm, data, totalname_freq): if args.polyorder: if comm.comm_world.rank == 0: print('Polyfiltering signal', flush=args.flush) start = MPI.Wtime() autotimer = timing.auto_timer() common_flag_name = 'common_flags' flag_name = 'flags' polyfilter = tt.OpPolyFilter(order=args.polyorder, name=totalname_freq, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask, flag_name=flag_name) polyfilter.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Polynomial filtering took {:.3f} s'.format(stop - start), flush=args.flush) return
def apply_groundfilter(args, comm, data, totalname_freq): if args.wbin_ground: if comm.comm_world.rank == 0: print('Ground filtering signal', flush=args.flush) start = MPI.Wtime() autotimer = timing.auto_timer() common_flag_name = 'common_flags' flag_name = 'flags' groundfilter = tt.OpGroundFilter( wbin=args.wbin_ground, name=totalname_freq, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask, flag_name=flag_name) groundfilter.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Ground filtering took {:.3f} s'.format(stop-start), flush=args.flush) return
def expand_pointing(args, comm, data): """ Expand the bore sight pointing to every detector. """ start = MPI.Wtime() autotimer = timing.auto_timer() hwprpm = args.hwprpm hwpstep = None if args.hwpstep is not None: hwpstep = float(args.hwpstep) hwpsteptime = args.hwpsteptime npix = 12 * args.nside**2 if comm.comm_world.rank == 0: print('Expanding pointing', flush=args.flush) pointing = tt.OpPointingHpix( nside=args.nside, nest=True, mode='IQU', hwprpm=hwprpm, hwpstep=hwpstep, hwpsteptime=hwpsteptime) pointing.exec(data) # Only purge the pointing if we are NOT going to export the # data to a TIDAS volume if args.tidas is None: for ob in data.obs: tod = ob['tod'] tod.free_radec_quats() comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Pointing generation took {:.3f} s'.format(stop-start), flush=args.flush) return
def __init__(self, comm=None, signal_map="signal_map", lmax=None, grid=None, fwhm_deg=None, beam=None, out="smoothed_signal_map"): autotimer = timing.auto_timer(type(self).__name__) # We call the parent class constructor, which currently does nothing super().__init__() self.comm = comm self.signal_map = signal_map self.lmax = lmax self.out = out self.grid = grid # distribute alms local_m_indices = np.arange(self.comm.rank, lmax + 1, self.comm.size, dtype=np.int32) self.order = libsharp.packed_real_order(lmax, ms=local_m_indices) if (fwhm_deg is not None) and (beam is not None): raise Exception("OpSmooth error, specify either fwhm_deg or beam, " "not both") if (fwhm_deg is None) and (beam is None): raise Exception("OpSmooth error, specify fwhm_deg or beam") if fwhm_deg is not None: self.beam = hp.gauss_beam(fwhm=np.radians(fwhm_deg), lmax=lmax, pol=True) else: self.beam = beam
def _stage_signal(self, data, detectors, nsamp, ndet, obs_period_ranges): """ Stage signal """ auto_timer = timing.auto_timer(type(self).__name__) self._madam_signal = self._cache.create( "signal", madam.SIGNAL_TYPE, (nsamp * ndet,) ) self._madam_signal[:] = np.nan global_offset = 0 for iobs, obs in enumerate(data.obs): tod = obs["tod"] period_ranges = obs_period_ranges[iobs] for idet, det in enumerate(detectors): # Get the signal. signal = tod.local_signal(det, self._name) signal_dtype = signal.dtype offset = global_offset for istart, istop in period_ranges: nn = istop - istart dslice = slice(idet * nsamp + offset, idet * nsamp + offset + nn) self._madam_signal[dslice] = signal[istart:istop] offset += nn del signal for idet, det in enumerate(detectors): if self._name is not None and ( self._purge_tod or self._name == self._name_out ): cachename = "{}_{}".format(self._name, det) tod.cache.clear(pattern=cachename) global_offset = offset return signal_dtype
def exec(self, data, comm=None): """ Copy data to Madam-compatible buffers and make a map. Args: data (toast.Data): The distributed data. """ if libmadam is None: raise RuntimeError("Cannot find libmadam") if len(data.obs) == 0: raise RuntimeError( 'OpMadam requires every supplied data object to ' 'contain at least one observation') auto_timer = timing.auto_timer(type(self).__name__) if comm is None: # Just use COMM_WORLD comm = data.comm.comm_world (parstring, detstring, nsamp, ndet, nnz, nnz_full, nnz_stride, periods, obs_period_ranges, psdfreqs, detectors, nside) = self._prepare(data, comm) psdinfo, pixels_dtype, weight_dtype = self._stage_data( data, comm, nsamp, ndet, nnz, nnz_full, nnz_stride, obs_period_ranges, psdfreqs, detectors, nside) self._destripe(comm, parstring, ndet, detstring, nsamp, nnz, periods, psdinfo) self._unstage_data(comm, data, nsamp, nnz, nnz_full, obs_period_ranges, detectors, pixels_dtype, nside, weight_dtype) return
def build_npp(args, comm, data, localsm, subnpix, detweights, flag_name, common_flag_name): """ Build pixel-pixel noise covariance matrices. """ if not args.skip_bin: if comm.comm_world.rank == 0: print('Preparing distributed map', flush=args.flush) start0 = MPI.Wtime() start = start0 autotimer = timing.auto_timer() npix = 12*args.nside**2 # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) invnpp.data.fill(0.0) hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) hits.data.fill(0) zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - distobjects initialized in {:.3f} s' ''.format(stop-start), flush=args.flush) start = stop invnpp_group = None hits_group = None zmap_group = None if comm.comm_group.size < comm.comm_world.size: invnpp_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) invnpp_group.data.fill(0.0) hits_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) hits_group.data.fill(0) zmap_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - group distobjects initialized in {:.3f} s' ''.format(stop-start), flush=args.flush) start = stop # compute the hits and covariance once, since the pointing and noise # weights are fixed. build_invnpp = tm.OpAccumDiag( detweights=detweights, invnpp=invnpp, hits=hits, flag_name=flag_name, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask) build_invnpp.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - distobjects accumulated in {:.3f} s' ''.format(stop-start), flush=args.flush) start = stop invnpp.allreduce() if not args.skip_hits: hits.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - distobjects reduced in {:.3f} s'.format(stop-start), flush=args.flush) start = stop if invnpp_group is not None: build_invnpp_group = tm.OpAccumDiag( detweights=detweights, invnpp=invnpp_group, hits=hits_group, flag_name=flag_name, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask) build_invnpp_group.exec(data) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - group distobjects accumulated in {:.3f} s' ''.format(stop-start), flush=args.flush) start = stop invnpp_group.allreduce() if not args.skip_hits: hits_group.allreduce() comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - group distobjects reduced in {:.3f} s' ''.format(stop-start), flush=args.flush) start = stop if not args.skip_hits: fn = '{}/hits.fits'.format(args.outdir) if args.zip: fn += '.gz' hits.write_healpix_fits(fn) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Writing hit map to {} took {:.3f} s' ''.format(fn, stop-start), flush=args.flush) start = stop del hits if hits_group is not None: if not args.skip_hits: fn = '{}/hits_group_{:04}.fits'.format(args.outdir, comm.group) if args.zip: fn += '.gz' hits_group.write_healpix_fits(fn) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Writing group hit map to {} took {:.3f} s' ''.format(fn, stop-start), flush=args.flush) start = stop del hits_group if not args.skip_hits: fn = '{}/invnpp.fits'.format(args.outdir) if args.zip: fn += '.gz' invnpp.write_healpix_fits(fn) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Writing N_pp^-1 to {} took {:.3f} s' ''.format(fn, stop-start), flush=args.flush) start = stop if not args.skip_hits: if invnpp_group is not None: fn = '{}/invnpp_group_{:04}.fits'.format(args.outdir, comm.group) if args.zip: fn += '.gz' invnpp_group.write_healpix_fits(fn) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Writing group N_pp^-1 to {} took {:.3f} s' ''.format(fn, stop-start), flush=args.flush) start = stop # invert it tm.covariance_invert(invnpp, 1.0e-3) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Inverting N_pp^-1 took {:.3f} s'.format(stop-start), flush=args.flush) start = stop if not args.skip_hits: fn = '{}/npp.fits'.format(args.outdir) if args.zip: fn += '.gz' invnpp.write_healpix_fits(fn) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Writing N_pp to {} took {:.3f} s' ''.format(fn, stop-start), flush=args.flush) start = stop if invnpp_group is not None: tm.covariance_invert(invnpp_group, 1.0e-3) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Inverting group N_pp^-1 took {:.3f} s' ''.format(stop-start), flush=args.flush) start = stop if not args.skip_hits: fn = '{}/npp_group_{:04}.fits'.format(args.outdir, comm.group) if args.zip: fn += '.gz' invnpp_group.write_healpix_fits(fn) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Writing group N_pp to {} took {:.3f} s'.format( fn, stop-start), flush=args.flush) start = stop stop = MPI.Wtime() if comm.comm_group.rank == 0: print('Building Npp took {:.3f} s'.format( stop-start0), flush=args.flush) return invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name
def build_npp(args, comm, data, localsm, subnpix, detweights, flag_name, common_flag_name): """ Build pixel-pixel noise covariance matrices. """ if not args.skip_bin: if comm.comm_world.rank == 0: print('Preparing distributed map', flush=args.flush) start0 = MPI.Wtime() start = start0 autotimer = timing.auto_timer() npix = 12 * args.nside**2 # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) invnpp.data.fill(0.0) hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) hits.data.fill(0) zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - distobjects initialized in {:.3f} s' ''.format(stop - start), flush=args.flush) start = stop invnpp_group = None hits_group = None zmap_group = None if comm.comm_group.size < comm.comm_world.size: invnpp_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) invnpp_group.data.fill(0.0) hits_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) hits_group.data.fill(0) zmap_group = tm.DistPixels(comm=comm.comm_group, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - group distobjects initialized in {:.3f} s' ''.format(stop - start), flush=args.flush) start = stop # compute the hits and covariance once, since the pointing and noise # weights are fixed. build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits, flag_name=flag_name, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask) build_invnpp.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - distobjects accumulated in {:.3f} s' ''.format(stop - start), flush=args.flush) start = stop invnpp.allreduce() if not args.skip_hits: hits.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - distobjects reduced in {:.3f} s'.format(stop - start), flush=args.flush) start = stop if invnpp_group is not None: build_invnpp_group = tm.OpAccumDiag( detweights=detweights, invnpp=invnpp_group, hits=hits_group, flag_name=flag_name, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask) build_invnpp_group.exec(data) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - group distobjects accumulated in {:.3f} s' ''.format(stop - start), flush=args.flush) start = stop invnpp_group.allreduce() if not args.skip_hits: hits_group.allreduce() comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - group distobjects reduced in {:.3f} s' ''.format(stop - start), flush=args.flush) start = stop if not args.skip_hits: fn = '{}/hits.fits'.format(args.outdir) if args.zip: fn += '.gz' hits.write_healpix_fits(fn) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Writing hit map to {} took {:.3f} s' ''.format(fn, stop - start), flush=args.flush) start = stop del hits if hits_group is not None: if not args.skip_hits: fn = '{}/hits_group_{:04}.fits'.format(args.outdir, comm.group) if args.zip: fn += '.gz' hits_group.write_healpix_fits(fn) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Writing group hit map to {} took {:.3f} s' ''.format(fn, stop - start), flush=args.flush) start = stop del hits_group if not args.skip_hits: fn = '{}/invnpp.fits'.format(args.outdir) if args.zip: fn += '.gz' invnpp.write_healpix_fits(fn) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Writing N_pp^-1 to {} took {:.3f} s' ''.format(fn, stop - start), flush=args.flush) start = stop if not args.skip_hits: if invnpp_group is not None: fn = '{}/invnpp_group_{:04}.fits'.format( args.outdir, comm.group) if args.zip: fn += '.gz' invnpp_group.write_healpix_fits(fn) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Writing group N_pp^-1 to {} took {:.3f} s' ''.format(fn, stop - start), flush=args.flush) start = stop # invert it tm.covariance_invert(invnpp, 1.0e-3) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Inverting N_pp^-1 took {:.3f} s'.format(stop - start), flush=args.flush) start = stop if not args.skip_hits: fn = '{}/npp.fits'.format(args.outdir) if args.zip: fn += '.gz' invnpp.write_healpix_fits(fn) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Writing N_pp to {} took {:.3f} s' ''.format(fn, stop - start), flush=args.flush) start = stop if invnpp_group is not None: tm.covariance_invert(invnpp_group, 1.0e-3) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Inverting group N_pp^-1 took {:.3f} s' ''.format(stop - start), flush=args.flush) start = stop if not args.skip_hits: fn = '{}/npp_group_{:04}.fits'.format(args.outdir, comm.group) if args.zip: fn += '.gz' invnpp_group.write_healpix_fits(fn) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Writing group N_pp to {} took {:.3f} s'.format( fn, stop - start), flush=args.flush) start = stop stop = MPI.Wtime() if comm.comm_group.rank == 0: print('Building Npp took {:.3f} s'.format(stop - start0), flush=args.flush) return invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name
def exec(self, data): """ Apply the ground filter to the signal. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) # the two-level pytoast communicator comm = data.comm # the communicator within the group cgroup = comm.comm_group # Each group loops over its own CES:es for obs in data.obs: tod = obs["tod"] nsamp_tot = tod.total_samples my_offset, my_nsamp = tod.local_samples if self._intervals in obs: intervals = obs[self._intervals] else: intervals = None local_intervals = tod.local_intervals(intervals) # Construct trend templates. Full domain for x is [-1, 1] x = np.arange(my_offset, my_offset + my_nsamp) / nsamp_tot * 2 - 1 ntrend = self._trend_order # Do not include the offset in the trend. It will be part of # of the ground template cheby_trend = chebval(x, np.eye(ntrend + 1), tensor=True)[1:] try: (azmin, azmax, _, _) = tod.scan_range az = tod.read_boresight_az() except Exception as e: raise RuntimeError( "Failed to get boresight azimuth from TOD. Perhaps it is " 'not ground TOD? "{}"'.format(e) ) # Cache the output common flags common_ref = tod.local_common_flags(self._common_flag_name) # The azimuth vector is assumed to be arranged so that the # azimuth increases monotonously even across the zero meridian. phase = (az - azmin) / (azmax - azmin) * 2 - 1 nfilter = self._filter_order + 1 cheby_templates = chebval(phase, np.eye(nfilter), tensor=True) if not self._split_template: cheby_filter = cheby_templates else: # Create separate templates for alternating scans cheby_filter = [] mask1 = common_ref != 0 mask2 = mask1.copy() for i, ival in enumerate(local_intervals): mask = [mask1, mask2][i % 2] mask[ival.first : ival.last + 1] = True for template in cheby_templates: for mask in mask1, mask2: temp = template.copy() temp[mask] = 0 cheby_filter.append(temp) templates = [] for temp in cheby_trend, cheby_filter: for template in temp: templates.append(template) for det in tod.detectors: if det in tod.local_dets: ref = tod.local_signal(det, self._name) flag_ref = tod.local_flags(det, self._flag_name) good = np.logical_and( common_ref & self._common_flag_mask == 0, flag_ref & self._flag_mask == 0, ) del flag_ref else: ref = None good = None coeff = self.fit_templates(tod, det, templates, ref, good) if det in tod.local_dets: # Trend trend = np.zeros_like(ref) for cc, template in zip(coeff[:ntrend], cheby_trend): trend += cc * template if self._detrend: ref[good] -= trend[good] # Ground template grtemplate = np.zeros_like(ref) for cc, template in zip(coeff[ntrend:], cheby_filter): grtemplate += cc * template ref[good] -= grtemplate[good] ref[np.logical_not(good)] = 0 del ref del common_ref return
def create_observations(args, comm, fp, all_ces, site): """ Simulate constant elevation scans. Simulate constant elevation scans at "site" matching entries in "all_ces". Each operational day is assigned to a different process group to allow making day maps. """ start = MPI.Wtime() autotimer = timing.auto_timer() data = toast.Data(comm) site_name, site_lat, site_lon, site_alt = site detectors = sorted(fp.keys()) detquats = {} for d in detectors: detquats[d] = fp[d]['quat'] nces = len(all_ces) breaks = [] do_break = False for i in range(nces - 1): # If current and next CES are on different days, insert a break tz = args.timezone / 24. start1 = all_ces[i][3] # MJD start start2 = all_ces[i + 1][3] # MJD start scan1 = all_ces[i][4] scan2 = all_ces[i + 1][4] if scan1 != scan2 and do_break: breaks.append(i + 1) do_break = False continue day1 = int(start1 + tz) day2 = int(start2 + tz) if day1 != day2: if scan1 == scan2: # We want an entire CES, even if it crosses the day bound. # Wait until the scan number changes. do_break = True else: breaks.append(i + 1) nbreak = len(breaks) if nbreak != comm.ngroups - 1: raise RuntimeError( 'Number of observing days ({}) does not match number of process ' 'groups ({}).'.format(nbreak + 1, comm.ngroups)) groupdist = toast.distribute_uniform(nces, comm.ngroups, breaks=breaks) group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] # Create the noise model used by all observations fmin = {} fknee = {} alpha = {} NET = {} rates = {} for d in detectors: rates[d] = args.samplerate fmin[d] = fp[d]['fmin'] fknee[d] = fp[d]['fknee'] alpha[d] = fp[d]['alpha'] NET[d] = fp[d]['NET'] noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors, fknee=fknee, alpha=alpha, NET=NET) for ices in range(group_firstobs, group_firstobs + group_numobs): ces = all_ces[ices] CES_start, CES_stop, name, mjdstart, scan, subscan, azmin, azmax, \ el = ces totsamples = int((CES_stop - CES_start) * args.samplerate) # create the single TOD for this observation try: tod = tt.TODGround(comm.comm_group, detquats, totsamples, detranks=comm.comm_group.size, firsttime=CES_start, rate=args.samplerate, site_lon=site_lon, site_lat=site_lat, site_alt=site_alt, azmin=azmin, azmax=azmax, el=el, scanrate=args.scanrate, scan_accel=args.scan_accel, CES_start=None, CES_stop=None, sun_angle_min=args.sun_angle_min, coord=args.coord, sampsizes=None) except RuntimeError as e: print('Failed to create the CES scan: {}'.format(e), flush=args.flush) return # Create the (single) observation ob = {} ob['name'] = 'CES-{}-{}-{}'.format(name, scan, subscan) ob['tod'] = tod if len(tod.subscans) > 0: ob['intervals'] = tod.subscans else: raise RuntimeError('{} has no valid intervals'.format(ob['name'])) ob['baselines'] = None ob['noise'] = noise ob['id'] = int(mjdstart * 10000) data.obs.append(ob) for ob in data.obs: tod = ob['tod'] tod.free_azel_quats() if comm.comm_group.rank == 0: print('Group # {:4} has {} observations.'.format( comm.group, len(data.obs)), flush=args.flush) if len(data.obs) == 0: raise RuntimeError('Too many tasks. Every MPI task must ' 'be assigned to at least one observation.') comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Simulated scans in {:.2f} seconds' ''.format(stop - start), flush=args.flush) return data
def main(): comm = MPI.COMM_WORLD if comm.rank == 0: print("Running with {} processes".format(comm.size)) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description='Read a toast covariance matrix and write the inverse condition number map' ) parser.add_argument( '--input', required=True, default=None, help='The input covariance FITS file' ) parser.add_argument( '--output', required=False, default=None, help='The output inverse condition map FITS file.' ) args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) autotimer = timing.auto_timer(timing.FILE()) # get options infile = args.input outfile = None if args.output is not None: outfile = args.output else: inmat = re.match(r'(.*)\.fits', infile) if inmat is None: print("input file should have .fits extension") sys.exit(0) inroot = inmat.group(1) outfile = "{}_rcond.fits".format(inroot) # We need to read the header to get the size of the matrix. # This would be a trivial function call in astropy.fits or # fitsio, but we don't want to bring in a whole new dependency # just for that. Instead, we open the file with healpy in memmap # mode so that nothing is actually read except the header. nside = 0 nnz = 0 if comm.rank == 0: fake, head = hp.read_map(infile, h=True, memmap=True) for key, val in head: if key == 'NSIDE': nside = int(val) if key == 'TFIELDS': nnz = int(val) nside = comm.bcast(nside, root=0) nnz = comm.bcast(nnz, root=0) npix = 12 * nside**2 subnside = int(nside / 16) if subnside == 0: subnside = 1 subnpix = 12 * subnside**2 nsubmap = int( npix / subnpix ) # divide the submaps as evenly as possible among processes dist = toast.distribute_uniform(nsubmap, comm.size) local = np.arange(dist[comm.rank][0], dist[comm.rank][0] + dist[comm.rank][1]) if comm.rank == 0: if os.path.isfile(outfile): os.remove(outfile) comm.barrier() # create the covariance and inverse condition number map cov = tm.DistPixels(comm=comm, dtype=np.float64, size=npix, nnz=nnz, submap=subnpix, local=local) # read the covariance cov.read_healpix_fits(infile) # every process computes its local piece rcond = tm.covariance_rcond(cov) # write the map rcond.write_healpix_fits(outfile)
def main(): if MPI.COMM_WORLD.rank == 0: print("Running with {} processes".format(MPI.COMM_WORLD.size), flush=True) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description="Simulate satellite " "boresight pointing and make a noise map.", fromfile_prefix_chars="@" ) parser.add_argument( "--groupsize", required=False, type=int, default=0, help="size of processor groups used to distribute observations" ) parser.add_argument( "--samplerate", required=False, type=float, default=40.0, help="Detector sample rate (Hz)" ) parser.add_argument( "--starttime", required=False, type=float, default=0.0, help="The overall start time of the simulation" ) parser.add_argument( "--spinperiod", required=False, type=float, default=10.0, help="The period (in minutes) of the rotation about the " "spin axis" ) parser.add_argument( "--spinangle", required=False, type=float, default=30.0, help="The opening angle (in degrees) of the boresight " "from the spin axis" ) parser.add_argument( "--precperiod", required=False, type=float, default=50.0, help="The period (in minutes) of the rotation about the " "precession axis" ) parser.add_argument( "--precangle", required=False, type=float, default=65.0, help="The opening angle (in degrees) of the spin axis " "from the precession axis" ) parser.add_argument( "--hwprpm", required=False, type=float, default=0.0, help="The rate (in RPM) of the HWP rotation" ) parser.add_argument( "--hwpstep", required=False, default=None, help="For stepped HWP, the angle in degrees of each step" ) parser.add_argument( "--hwpsteptime", required=False, type=float, default=0.0, help="For stepped HWP, the the time in seconds between " "steps" ) parser.add_argument( "--obs", required=False, type=float, default=1.0, help="Number of hours in one science observation" ) parser.add_argument( "--gap", required=False, type=float, default=0.0, help="Cooler cycle time in hours between science obs" ) parser.add_argument( "--numobs", required=False, type=int, default=1, help="Number of complete observations" ) parser.add_argument( "--outdir", required=False, default="out", help="Output directory" ) parser.add_argument( "--debug", required=False, default=False, action="store_true", help="Write diagnostics" ) parser.add_argument( "--nside", required=False, type=int, default=64, help="Healpix NSIDE" ) parser.add_argument( "--subnside", required=False, type=int, default=4, help="Distributed pixel sub-map NSIDE" ) parser.add_argument('--coord', required=False, default='E', help='Sky coordinate system [C,E,G]') parser.add_argument( "--baseline", required=False, type=float, default=60.0, help="Destriping baseline length (seconds)" ) parser.add_argument( "--noisefilter", required=False, default=False, action="store_true", help="Destripe with the noise filter enabled" ) parser.add_argument( "--madam", required=False, default=False, action="store_true", help="If specified, use libmadam for map-making" ) parser.add_argument( "--madampar", required=False, default=None, help="Madam parameter file" ) parser.add_argument('--flush', required=False, default=False, action='store_true', help='Flush every print statement.') parser.add_argument( "--MC_start", required=False, type=int, default=0, help="First Monte Carlo noise realization" ) parser.add_argument( "--MC_count", required=False, type=int, default=1, help="Number of Monte Carlo noise realizations" ) parser.add_argument( "--fp", required=False, default=None, help="Pickle file containing a dictionary of detector properties. " "The keys of this dict are the detector names, and each value is also " "a dictionary with keys \"quat\" (4 element ndarray), \"fwhm\" " "(float, arcmin), \"fknee\" (float, Hz), \"alpha\" (float), and \"NET\" " "(float). For optional plotting, the key \"color\" can specify a " "valid matplotlib color string." ) parser.add_argument( "--gain", required=False, default=None, help= "Calibrate the input timelines with a set of gains from a" "FITS file containing 3 extensions:" "HDU named DETECTORS : table with list of detector names in a column named DETECTORS" "HDU named TIME: table with common timestamps column named TIME" "HDU named GAINS: 2D image of floats with one row per detector and one column per value.") parser.add_argument('--tidas', required=False, default=None, help='Output TIDAS export path') parser.add_argument('--spt3g', required=False, default=None, help='Output SPT3G export path') parser.add_argument('--input_map', required=False, help='Input map for signal') parser.add_argument('--input_pysm_model', required=False, help='Comma separated models for on-the-fly PySM ' 'simulation, e.g. s3,d6,f1,a2"') parser.add_argument('--input_pysm_precomputed_cmb_K_CMB', required=False, help='Precomputed CMB map for PySM in K_CMB' 'it overrides any model defined in input_pysm_model"') parser.add_argument('--apply_beam', required=False, action='store_true', help='Apply beam convolution to input map with gaussian ' 'beam parameters defined in focalplane') parser.add_argument('--input_dipole', required=False, help='Simulate dipole, possible values are ' 'total, orbital, solar') parser.add_argument('--input_dipole_solar_speed_kms', required=False, help='Solar system speed [km/s]', type=float, default=369.0) parser.add_argument('--input_dipole_solar_gal_lat_deg', required=False, help='Solar system speed galactic latitude [degrees]', type=float, default=48.26) parser.add_argument('--input_dipole_solar_gal_lon_deg', required=False, help='Solar system speed galactic longitude[degrees]', type=float, default=263.99) args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) autotimer = timing.auto_timer("@{}".format(timing.FILE())) if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot export") if args.spt3g is not None: if not tt.spt3g_available: raise RuntimeError("SPT3G not found- cannot export") groupsize = args.groupsize if groupsize == 0: groupsize = MPI.COMM_WORLD.size # This is the 2-level toast communicator. if MPI.COMM_WORLD.size % groupsize != 0: if MPI.COMM_WORLD.rank == 0: print("WARNING: process groupsize does not evenly divide into " "total number of processes", flush=True) comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize) # get options hwpstep = None if args.hwpstep is not None: hwpstep = float(args.hwpstep) npix = 12 * args.nside * args.nside subnside = args.subnside if subnside > args.nside: subnside = args.nside subnpix = 12 * subnside * subnside start = MPI.Wtime() fp = None gain = None # Load focalplane information if comm.comm_world.rank == 0: if args.fp is None: # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake["quat"] = np.array([0.0, 0.0, 1.0, 0.0]) fake["fwhm"] = 30.0 fake["fknee"] = 0.0 fake["alpha"] = 1.0 fake["NET"] = 1.0 fake["color"] = "r" fp = {} fp["bore"] = fake else: with open(args.fp, "rb") as p: fp = pickle.load(p) if args.gain is not None: gain = {} with fits.open(args.gain) as f: gain["TIME"] = np.array(f["TIME"].data["TIME"]) for i_det, det_name in f["DETECTORS"].data["DETECTORS"]: gain[det_name] = np.array(f["GAINS"].data[i_det, :]) if args.gain is not None: gain = comm.comm_world.bcast(gain, root=0) fp = comm.comm_world.bcast(fp, root=0) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Create focalplane ({} dets): {:.2f} seconds"\ .format(len(fp.keys()), stop-start), flush=True) start = stop if args.debug: if comm.comm_world.rank == 0: outfile = "{}_focalplane.png".format(args.outdir) set_backend() dquats = { x : fp[x]["quat"] for x in fp.keys() } dfwhm = { x : fp[x]["fwhm"] for x in fp.keys() } tt.plot_focalplane(dquats, 10.0, 10.0, outfile, fwhm=dfwhm) # Since we are simulating noise timestreams, we want # them to be contiguous and reproducible over the whole # observation. We distribute data by detector within an # observation, so ensure that our group size is not larger # than the number of detectors we have. if groupsize > len(fp.keys()): if comm.comm_world.rank == 0: print("process group is too large for the number of detectors", flush=True) comm.comm_world.Abort() # Detector information from the focalplane detectors = sorted(fp.keys()) detquats = {} detindx = None if "index" in fp[detectors[0]]: detindx = {} for d in detectors: detquats[d] = fp[d]["quat"] if detindx is not None: detindx[d] = fp[d]["index"] # Distribute the observations uniformly groupdist = toast.distribute_uniform(args.numobs, comm.ngroups) # Compute global time and sample ranges of all observations obsrange = tt.regular_intervals(args.numobs, args.starttime, 0, args.samplerate, 3600*args.obs, 3600*args.gap) # Create the noise model used for all observations fmin = {} fknee = {} alpha = {} NET = {} rates = {} for d in detectors: rates[d] = args.samplerate fmin[d] = fp[d]["fmin"] fknee[d] = fp[d]["fknee"] alpha[d] = fp[d]["alpha"] NET[d] = fp[d]["NET"] noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors, fknee=fknee, alpha=alpha, NET=NET) mem_counter = tt.OpMemoryCounter() # The distributed timestream data data = toast.Data(comm) # Every process group creates its observations group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] for ob in range(group_firstobs, group_firstobs + group_numobs): tod = tt.TODSatellite( comm.comm_group, detquats, obsrange[ob].samples, coord=args.coord, firstsamp=obsrange[ob].first, firsttime=obsrange[ob].start, rate=args.samplerate, spinperiod=args.spinperiod, spinangle=args.spinangle, precperiod=args.precperiod, precangle=args.precangle, detindx=detindx, detranks=comm.group_size ) obs = {} obs["name"] = "science_{:05d}".format(ob) obs["tod"] = tod obs["intervals"] = None obs["baselines"] = None obs["noise"] = noise obs["id"] = ob data.obs.append(obs) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Read parameters, compute data distribution: " "{:.2f} seconds".format(stop-start), flush=True) start = stop # we set the precession axis now, which will trigger calculation # of the boresight pointing. for ob in range(group_numobs): curobs = data.obs[ob] tod = curobs["tod"] # Get the global sample offset from the original distribution of # intervals obsoffset = obsrange[group_firstobs + ob].first # Constantly slewing precession axis degday = 360.0 / 365.25 precquat = np.empty(4*tod.local_samples[1], dtype=np.float64).reshape((-1,4)) tt.slew_precession_axis(precquat, firstsamp=(obsoffset + tod.local_samples[0]), samplerate=args.samplerate, degday=degday) tod.set_prec_axis(qprec=precquat) del precquat stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Construct boresight pointing: " "{:.2f} seconds".format(stop-start), flush=True) start = stop # make a Healpix pointing matrix. pointing = tt.OpPointingHpix(nside=args.nside, nest=True, mode="IQU", hwprpm=args.hwprpm, hwpstep=hwpstep, hwpsteptime=args.hwpsteptime) pointing.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Pointing generation took {:.3f} s".format(elapsed), flush=True) start = stop localpix, localsm, subnpix = get_submaps(args, comm, data) signalname = "signal" has_signal = False if args.input_pysm_model: has_signal = True simulate_sky_signal(args, comm, data, mem_counter, [fp], subnpix, localsm, signalname=signalname) if args.input_dipole: print("Simulating dipole") has_signal = True op_sim_dipole = tt.OpSimDipole(mode=args.input_dipole, solar_speed=args.input_dipole_solar_speed_kms, solar_gal_lat=args.input_dipole_solar_gal_lat_deg, solar_gal_lon=args.input_dipole_solar_gal_lon_deg, out=signalname, keep_quats=False, keep_vel=False, subtract=False, coord=args.coord, freq=0, # we could use frequency for quadrupole correction flag_mask=255, common_flag_mask=255) op_sim_dipole.exec(data) del op_sim_dipole # Mapmaking. For purposes of this simulation, we use detector noise # weights based on the NET (white noise level). If the destriping # baseline is too long, this will not be the best choice. detweights = {} for d in detectors: net = fp[d]["NET"] detweights[d] = 1.0 / (args.samplerate * net * net) if not args.madam: if comm.comm_world.rank == 0: print("Not using Madam, will only make a binned map!", flush=True) # get locally hit pixels lc = tm.OpLocalPixels() localpix = lc.exec(data) # find the locally hit submaps. localsm = np.unique(np.floor_divide(localpix, subnpix)) # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) # compute the hits and covariance once, since the pointing and noise # weights are fixed. invnpp.data.fill(0.0) hits.data.fill(0) build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits) build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Building hits and N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop hits.write_healpix_fits("{}_hits.fits".format(args.outdir)) invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Writing hits and N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop # invert it tm.covariance_invert(invnpp, 1.0e-3) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Inverting N_pp^-1 took {:.3f} s".format(elapsed), flush=True) start = stop invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir)) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Writing N_pp took {:.3f} s".format(elapsed), flush=True) start = stop # in debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open("{}_distdata.txt".format(args.outdir), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Dumping debug data distribution took " "{:.3f} s".format(elapsed), flush=True) start = stop mcstart = start # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc+nmc): # create output directory for this realization outpath = "{}_{:03d}".format(args.outdir, mc) if comm.comm_world.rank == 0: if not os.path.isdir(outpath): os.makedirs(outpath) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Creating output dir {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop # clear all signal data from the cache, so that we can generate # new noise timestreams. tod.cache.clear("tot_signal_.*") # simulate noise nse = tt.OpSimNoise(out="tot_signal", realization=mc) nse.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Noise simulation {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop # add sky signal if has_signal: add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Add sky signal {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop if gain is not None: op_apply_gain = tt.OpApplyGain(gain, name="tot_signal") op_apply_gain.exec(data) if mc == firstmc: # For the first realization, optionally export the # timestream data. If we had observation intervals defined, # we could pass "use_interval=True" to the export operators, # which would ensure breaks in the exported data at # acceptable places. if args.tidas is not None: tidas_path = os.path.abspath(args.tidas) export = OpTidasExport(tidas_path, TODTidas, backend="hdf5", use_todchunks=True, create_opts={"group_dets":"sim"}, ctor_opts={"group_dets":"sim"}, cache_name="tot_signal") export.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Tidas export took {:.3f} s"\ .format(elapsed), flush=True) start = stop if args.spt3g is not None: spt3g_path = os.path.abspath(args.spt3g) export = Op3GExport(spt3g_path, TOD3G, use_todchunks=True, export_opts={"prefix" : "sim"}, cache_name="tot_signal") export.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" SPT3G export took {:.3f} s"\ .format(elapsed), flush=True) start = stop zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag(zmap=zmap, name="tot_signal", detweights=detweights) build_zmap.exec(data) zmap.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Building noise weighted map {:04d} took {:.3f} s".format( mc, elapsed), flush=True) start = stop tm.covariance_apply(invnpp, zmap) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Computing binned map {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop zmap.write_healpix_fits(os.path.join(outpath, "binned.fits")) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print(" Writing binned map {:04d} took {:.3f} s".format(mc, elapsed), flush=True) elapsed = stop - mcstart if comm.comm_world.rank == 0: print(" Mapmaking {:04d} took {:.3f} s".format(mc, elapsed), flush=True) start = stop else: # Set up MADAM map making. pars = {} cross = args.nside // 2 pars[ "temperature_only" ] = "F" pars[ "force_pol" ] = "T" pars[ "kfirst" ] = "T" pars[ "concatenate_messages" ] = "T" pars[ "write_map" ] = "T" pars[ "write_binmap" ] = "T" pars[ "write_matrix" ] = "T" pars[ "write_wcov" ] = "T" pars[ "write_hits" ] = "T" pars[ "nside_cross" ] = cross pars[ "nside_submap" ] = subnside if args.madampar is not None: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars[ "base_first" ] = args.baseline pars[ "nside_map" ] = args.nside if args.noisefilter: pars[ "kfilter" ] = "T" else: pars[ "kfilter" ] = "F" pars[ "fsample" ] = args.samplerate # Loop over Monte Carlos firstmc = int(args.MC_start) nmc = int(args.MC_count) for mc in range(firstmc, firstmc+nmc): # clear all total signal data from the cache, so that we can generate # new noise timestreams. for obs in data.obs: tod = obs['tod'] tod.cache.clear("tot_signal_.*") # simulate noise nse = tt.OpSimNoise(out="tot_signal", realization=mc) nse.exec(data) # add sky signal if has_signal: add_sky_signal(args, comm, data, totalname="tot_signal", signalname=signalname) if gain is not None: op_apply_gain = tt.OpApplyGain(gain, name="tot_signal") op_apply_gain.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Noise simulation took {:.3f} s".format(elapsed), flush=True) start = stop # create output directory for this realization pars[ "path_output" ] = "{}_{:03d}".format(args.outdir, mc) if comm.comm_world.rank == 0: if not os.path.isdir(pars["path_output"]): os.makedirs(pars["path_output"]) # in debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open(os.path.join(pars["path_output"], "distdata.txt"), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() madam = tm.OpMadam(params=pars, detweights=detweights, name="tot_signal") madam.exec(data) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print("Mapmaking took {:.3f} s".format(elapsed), flush=True) comm.comm_world.barrier() stop = MPI.Wtime() elapsed = stop - global_start if comm.comm_world.rank == 0: print("Total Time: {:.2f} seconds".format(elapsed), flush=True)
def main(): if MPI.COMM_WORLD.rank == 0: print("Running with {} processes".format(MPI.COMM_WORLD.size), flush=True) global_start = MPI.Wtime() parser = argparse.ArgumentParser( description="Read existing data and make a simple map.", fromfile_prefix_chars="@") parser.add_argument("--groupsize", required=False, type=int, default=0, help="size of processor groups used to distribute "\ "observations") parser.add_argument("--hwprpm", required=False, type=float, default=0.0, help="The rate (in RPM) of the HWP rotation") parser.add_argument('--samplerate', required=False, default=100.0, type=np.float, help='Detector sample rate (Hz)') parser.add_argument("--outdir", required=False, default="out", help="Output directory") parser.add_argument("--nside", required=False, type=int, default=64, help="Healpix NSIDE") parser.add_argument("--subnside", required=False, type=int, default=8, help="Distributed pixel sub-map NSIDE") parser.add_argument("--coord", required=False, default="E", help="Sky coordinate system [C,E,G]") parser.add_argument("--baseline", required=False, type=float, default=60.0, help="Destriping baseline length (seconds)") parser.add_argument("--noisefilter", required=False, default=False, action="store_true", help="Destripe with the noise filter enabled") parser.add_argument("--madam", required=False, default=False, action="store_true", help="If specified, use libmadam for map-making") parser.add_argument("--madampar", required=False, default=None, help="Madam parameter file") parser.add_argument("--polyorder", required=False, type=int, help="Polynomial order for the polyfilter") parser.add_argument("--wbin_ground", required=False, type=float, help="Ground template bin width [degrees]") parser.add_argument("--flush", required=False, default=False, action="store_true", help="Flush every print statement.") parser.add_argument("--tidas", required=False, default=None, help="Input TIDAS volume") parser.add_argument("--tidas_detgroup", required=False, default=None, help="TIDAS detector group") parser.add_argument("--spt3g", required=False, default=None, help="Input SPT3G data directory") parser.add_argument("--spt3g_prefix", required=False, default=None, help="SPT3G data frame file prefix") parser.add_argument("--common_flag_mask", required=False, default=0, type=np.uint8, help="Common flag mask") parser.add_argument("--debug", required=False, default=False, action="store_true", help="Write data distribution info and focalplane plot") args = timing.add_arguments_and_parse(parser, timing.FILE(noquotes=True)) #args = parser.parse_args(sys.argv) autotimer = timing.auto_timer("@{}".format(timing.FILE())) if (args.tidas is not None) and (args.spt3g is not None): raise RuntimeError("Cannot read two datasets!") if (args.tidas is None) and (args.spt3g is None): raise RuntimeError("No dataset specified!") if args.tidas is not None: if not tt.tidas_available: raise RuntimeError("TIDAS not found- cannot load") if args.spt3g is not None: if not tt.spt3g_available: raise RuntimeError("SPT3G not found- cannot load") groupsize = args.groupsize if groupsize == 0: groupsize = MPI.COMM_WORLD.size # Pixelization nside = args.nside npix = 12 * args.nside * args.nside subnside = args.subnside if subnside > nside: subnside = nside subnpix = 12 * subnside * subnside # This is the 2-level toast communicator. if MPI.COMM_WORLD.size % groupsize != 0: if MPI.COMM_WORLD.rank == 0: print("WARNING: process groupsize does not evenly divide into " "total number of processes", flush=True) comm = toast.Comm(world=MPI.COMM_WORLD, groupsize=groupsize) # Create output directory mtime = MPI.Wtime() if comm.comm_world.rank == 0: if not os.path.isdir(args.outdir): os.makedirs(args.outdir) mtime = elapsed(comm.comm_world, mtime, "Creating output directory") # The distributed timestream data data = None if args.tidas is not None: if args.tidas_detgroup is None: raise RuntimeError("you must specify the detector group") data = tds.load_tidas(comm, comm.group_size, args.tidas, "r", args.tidas_detgroup, tds.TODTidas, group_dets=args.tidas_detgroup, distintervals="chunks") if args.spt3g is not None: if args.spt3g_prefix is None: raise RuntimeError("you must specify the frame file prefix") data = s3g.load_spt3g(comm, comm.group_size, args.spt3g, args.spt3g_prefix, s3g.obsweight_spt3g, s3g.TOD3G) mtime = elapsed(comm.comm_world, mtime, "Distribute data") # In debug mode, print out data distribution information if args.debug: handle = None if comm.comm_world.rank == 0: handle = open("{}_distdata.txt".format(args.outdir), "w") data.info(handle) if comm.comm_world.rank == 0: handle.close() mtime = elapsed(comm.comm_world, mtime, "Dumping debug data distribution") if comm.comm_world.rank == 0: outfile = "{}_focalplane.png".format(args.outdir) set_backend() # Just plot the dets from the first TOD temptod = data.obs[0]["tod"] # FIXME: change this once we store det info in the metadata. dfwhm = { x : 10.0 for x in temptod.detectors } tt.plot_focalplane(temptod.detoffset(), 10.0, 10.0, outfile, fwhm=dfwhm) comm.comm_world.barrier() mtime = elapsed(comm.comm_world, mtime, "Plotting debug focalplane") # Compute pointing matrix pointing = tt.OpPointingHpix( nside=args.nside, nest=True, mode="IQU", hwprpm=args.hwprpm) pointing.exec(data) mtime = elapsed(comm.comm_world, mtime, "Expand pointing") # Mapmaking. # FIXME: We potentially have a different noise model for every # observation. We need to have both spt3g and tidas format Noise # classes which read the information from disk. Then the mapmaking # operators need to get these noise weights from each observation. detweights = { d : 1.0 for d in data.obs[0]["tod"].detectors } if not args.madam: if comm.comm_world.rank == 0: print("Not using Madam, will only make a binned map!", flush=True) # Filter data if desired if args.polyorder: polyfilter = tt.OpPolyFilter( order=args.polyorder, common_flag_mask=args.common_flag_mask) polyfilter.exec(data) mtime = elapsed(comm.comm_world, mtime, "Polynomial filtering") if args.wbin_ground: groundfilter = tt.OpGroundFilter( wbin=args.wbin_ground, common_flag_mask=args.common_flag_mask) groundfilter.exec(data) mtime = elapsed(comm.comm_world, mtime, "Ground template filtering") # Compute pixel space distribution lc = tm.OpLocalPixels() localpix = lc.exec(data) if localpix is None: raise RuntimeError( "Process {} has no hit pixels. Perhaps there are fewer " "detectors than processes in the group?".format( comm.comm_world.rank)) localsm = np.unique(np.floor_divide(localpix, subnpix)) mtime = elapsed(comm.comm_world, mtime, "Compute local submaps") # construct distributed maps to store the covariance, # noise weighted map, and hits mtime = MPI.Wtime() invnpp = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=6, dtype=np.float64, submap=subnpix, local=localsm) hits = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=1, dtype=np.int64, submap=subnpix, local=localsm) zmap = tm.DistPixels(comm=comm.comm_world, size=npix, nnz=3, dtype=np.float64, submap=subnpix, local=localsm) # compute the hits and covariance. invnpp.data.fill(0.0) hits.data.fill(0) build_invnpp = tm.OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits, common_flag_mask=args.common_flag_mask) build_invnpp.exec(data) invnpp.allreduce() hits.allreduce() mtime = elapsed(comm.comm_world, mtime, "Building hits and N_pp^-1") hits.write_healpix_fits("{}_hits.fits".format(args.outdir)) invnpp.write_healpix_fits("{}_invnpp.fits".format(args.outdir)) mtime = elapsed(comm.comm_world, mtime, "Writing hits and N_pp^-1") # invert it tm.covariance_invert(invnpp, 1.0e-3) mtime = elapsed(comm.comm_world, mtime, "Inverting N_pp^-1") invnpp.write_healpix_fits("{}_npp.fits".format(args.outdir)) mtime = elapsed(comm.comm_world, mtime, "Writing N_pp") zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag(zmap=zmap, detweights=detweights, common_flag_mask=args.common_flag_mask) build_zmap.exec(data) zmap.allreduce() mtime = elapsed(comm.comm_world, mtime, "Building noise weighted map") tm.covariance_apply(invnpp, zmap) mtime = elapsed(comm.comm_world, mtime, "Computing binned map") zmap.write_healpix_fits(os.path.join(args.outdir, "binned.fits")) mtime = elapsed(comm.comm_world, mtime, "Writing binned map") else: # Set up MADAM map making. pars = {} pars[ "temperature_only" ] = "F" pars[ "force_pol" ] = "T" pars[ "kfirst" ] = "T" pars[ "concatenate_messages" ] = "T" pars[ "write_map" ] = "T" pars[ "write_binmap" ] = "T" pars[ "write_matrix" ] = "T" pars[ "write_wcov" ] = "T" pars[ "write_hits" ] = "T" pars[ "nside_cross" ] = nside // 2 pars[ "nside_submap" ] = subnside if args.madampar is not None: pat = re.compile(r"\s*(\S+)\s*=\s*(\S+(\s+\S+)*)\s*") comment = re.compile(r"^#.*") with open(args.madampar, "r") as f: for line in f: if comment.match(line) is None: result = pat.match(line) if result is not None: key, value = result.group(1), result.group(2) pars[key] = value pars[ "base_first" ] = args.baseline pars[ "nside_map" ] = nside if args.noisefilter: pars[ "kfilter" ] = "T" else: pars[ "kfilter" ] = "F" pars[ "fsample" ] = args.samplerate madam = tm.OpMadam(params=pars, detweights=detweights, common_flag_mask=args.common_flag_mask) madam.exec(data) mtime = elapsed(comm.comm_world, mtime, "Madam mapmaking") comm.comm_world.barrier() stop = MPI.Wtime() dur = stop - global_start if comm.comm_world.rank == 0: print("Total Time: {:.2f} seconds".format(dur), flush=True) return
def main(): # This is the 2-level toast communicator. By default, # there is just one group which spans MPI_COMM_WORLD. comm = toast.Comm() if comm.comm_world.rank == 0: print('Running with {} processes at {}'.format( comm.comm_world.size, str(datetime.now())), flush=True) global_timer = timing.simple_timer('Total time') global_timer.start() args, comm = parse_arguments(comm) autotimer = timing.auto_timer("@{}".format(timing.FILE())) # Load and broadcast the schedule file site, all_ces = load_schedule(args, comm) # load or simulate the focalplane fp, detweights = load_fp(args, comm) # Create the TOAST data object to match the schedule. This will # include simulating the boresight pointing. data = create_observations(args, comm, fp, all_ces, site) # Expand boresight quaternions into detector pointing weights and # pixel numbers expand_pointing(args, comm, data) # Prepare auxiliary information for distributed map objects localpix, localsm, subnpix = get_submaps(args, comm, data) # Scan input map signalname = scan_signal(args, comm, data, localsm, subnpix) # Set up objects to take copies of the TOD at appropriate times signalname_madam, sigcopy_madam, sigclear \ = setup_sigcopy(args, comm, signalname) common_flag_name = None flag_name = None invnpp, zmap, invnpp_group, zmap_group, flag_name, common_flag_name \ = build_npp(args, comm, data, localsm, subnpix, detweights, flag_name, common_flag_name) madampars = setup_madam(args, comm) output_tidas(args, comm, data, signalname, common_flag_name, flag_name) outpath = setup_output(args, comm) # Make a copy of the signal for Madam copy_signal_madam(args, comm, data, sigcopy_madam) # Bin unprocessed signal for reference bin_maps(args, comm, data, 'binned', zmap, invnpp, zmap_group, invnpp_group, detweights, signalname, flag_name, common_flag_name, outpath) # Filter signal apply_polyfilter(args, comm, data, signalname) apply_groundfilter(args, comm, data, signalname) # Bin the filtered signal if args.polyorder or args.wbin_ground: bin_maps(args, comm, data, 'filtered', zmap, invnpp, zmap_group, invnpp_group, detweights, signalname, flag_name, common_flag_name, outpath) clear_signal(args, comm, data, sigclear) # Now run Madam on the unprocessed copy of the signal apply_madam(args, comm, data, madampars, outpath, detweights, signalname_madam, flag_name, common_flag_name) comm.comm_world.barrier() global_timer.stop() if comm.comm_world.rank == 0: global_timer.report()
def load_fp(args, comm): start = MPI.Wtime() autotimer = timing.auto_timer() fp = None # Load focalplane information nullquat = np.array([0,0,0,1], dtype=np.float64) if comm.comm_world.rank == 0: if args.fp is None: # in this case, create a fake detector at the boresight # with a pure white noise spectrum. fake = {} fake['quat'] = nullquat fake['fwhm'] = 30.0 fake['fknee'] = 0.0 fake['fmin'] = 1e-9 fake['alpha'] = 1.0 fake['NET'] = 1.0 fake['color'] = 'r' fp = {} # Second detector at 22.5 degree polarization angle fp['bore1'] = fake fake2 = {} zrot = qa.rotation(ZAXIS, 22.5*degree) fake2['quat'] = qa.mult(fake['quat'], zrot) fake2['fwhm'] = 30.0 fake2['fknee'] = 0.0 fake2['fmin'] = 1e-9 fake2['alpha'] = 1.0 fake2['NET'] = 1.0 fake2['color'] = 'r' fp['bore2'] = fake2 # Third detector at 45 degree polarization angle fake3 = {} zrot = qa.rotation(ZAXIS, 45*degree) fake3['quat'] = qa.mult(fake['quat'], zrot) fake3['fwhm'] = 30.0 fake3['fknee'] = 0.0 fake3['fmin'] = 1e-9 fake3['alpha'] = 1.0 fake3['NET'] = 1.0 fake3['color'] = 'r' fp['bore3'] = fake3 # Fourth detector at 67.5 degree polarization angle fake4 = {} zrot = qa.rotation(ZAXIS, 67.5*degree) fake4['quat'] = qa.mult(fake['quat'], zrot) fake4['fwhm'] = 30.0 fake4['fknee'] = 0.0 fake4['fmin'] = 1e-9 fake4['alpha'] = 1.0 fake4['NET'] = 1.0 fake4['color'] = 'r' fp['bore4'] = fake4 else: with open(args.fp, 'rb') as p: fp = pickle.load(p) fp = comm.comm_world.bcast(fp, root=0) stop = MPI.Wtime() elapsed = stop - start if comm.comm_world.rank == 0: print('Create focalplane: {:.2f} seconds'.format(stop-start), flush=args.flush) start = stop if args.debug: if comm.comm_world.rank == 0: outfile = '{}/focalplane.png'.format(args.outdir) tt.plot_focalplane(fp, 6, 6, outfile) detectors = sorted(fp.keys()) detweights = {} for d in detectors: net = fp[d]['NET'] detweights[d] = 1.0 / (args.samplerate * net * net) return fp, detweights
def bin_maps(args, comm, data, rootname, zmap, invnpp, zmap_group, invnpp_group, detweights, totalname_freq, flag_name, common_flag_name, outpath): """ Use TOAST facilities to bin stored signal. """ if not args.skip_bin: if comm.comm_world.rank == 0: print('Binning unfiltered maps', flush=args.flush) start0 = MPI.Wtime() start = start0 autotimer = timing.auto_timer() # Bin a map using the toast facilities zmap.data.fill(0.0) build_zmap = tm.OpAccumDiag( detweights=detweights, zmap=zmap, name=totalname_freq, flag_name=flag_name, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask) build_zmap.exec(data) zmap.allreduce() comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Building noise weighted map took {:.3f} s' ''.format(stop-start), flush=args.flush) start = stop tm.covariance_apply(invnpp, zmap) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Computing {} map took {:.3f} s' ''.format(rootname, stop-start), flush=args.flush) start = stop fn = os.path.join(outpath, rootname+'.fits') if args.zip: fn += '.gz' zmap.write_healpix_fits(fn) comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print(' - Writing {} map to {} took {:.3f} s' ''.format(rootname, fn, stop-start), flush=args.flush) if zmap_group is not None: zmap_group.data.fill(0.0) build_zmap_group = tm.OpAccumDiag( detweights=detweights, zmap=zmap_group, name=totalname_freq, flag_name=flag_name, common_flag_name=common_flag_name, common_flag_mask=args.common_flag_mask) build_zmap_group.exec(data) zmap_group.allreduce() comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Building group noise weighted map took ' '{:.3f} s'.format(stop-start), flush=args.flush) start = stop tm.covariance_apply(invnpp_group, zmap_group) comm.comm_group.barrier() stop = MPI.Wtime() elapsed = stop - start if comm.comm_group.rank == 0: print(' - Computing {} map took {:.3f} s' ''.format(rootname, stop-start), flush=args.flush) start = stop fn = os.path.join(outpath, '{}_group_{:04}.fits' ''.format(rootname, comm.group)) if args.zip: fn += '.gz' zmap_group.write_healpix_fits(fn) comm.comm_group.barrier() stop = MPI.Wtime() if comm.comm_group.rank == 0: print(' - Writing group {} map to {} took ' '{:.3f} s'.format(rootname, fn, stop-start), flush=args.flush) stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Mapmaking took {:.3f} s' ''.format(stop-start0), flush=args.flush) return
def create_observations(args, comm, fp, all_ces, site): """ Simulate constant elevation scans. Simulate constant elevation scans at "site" matching entries in "all_ces". Each operational day is assigned to a different process group to allow making day maps. """ start = MPI.Wtime() autotimer = timing.auto_timer() data = toast.Data(comm) site_name, site_lat, site_lon, site_alt = site detectors = sorted(fp.keys()) detquats = {} for d in detectors: detquats[d] = fp[d]['quat'] nces = len(all_ces) breaks = [] do_break = False for i in range(nces-1): # If current and next CES are on different days, insert a break tz = args.timezone / 24. start1 = all_ces[i][3] # MJD start start2 = all_ces[i+1][3] # MJD start scan1 = all_ces[i][4] scan2 = all_ces[i+1][4] if scan1 != scan2 and do_break: breaks.append(i + 1) do_break = False continue day1 = int(start1 + tz) day2 = int(start2 + tz) if day1 != day2: if scan1 == scan2: # We want an entire CES, even if it crosses the day bound. # Wait until the scan number changes. do_break = True else: breaks.append(i + 1) nbreak = len(breaks) if nbreak != comm.ngroups-1: raise RuntimeError( 'Number of observing days ({}) does not match number of process ' 'groups ({}).'.format(nbreak+1, comm.ngroups)) groupdist = toast.distribute_uniform(nces, comm.ngroups, breaks=breaks) group_firstobs = groupdist[comm.group][0] group_numobs = groupdist[comm.group][1] # Create the noise model used by all observations fmin = {} fknee = {} alpha = {} NET = {} rates = {} for d in detectors: rates[d] = args.samplerate fmin[d] = fp[d]['fmin'] fknee[d] = fp[d]['fknee'] alpha[d] = fp[d]['alpha'] NET[d] = fp[d]['NET'] noise = tt.AnalyticNoise(rate=rates, fmin=fmin, detectors=detectors, fknee=fknee, alpha=alpha, NET=NET) for ices in range(group_firstobs, group_firstobs + group_numobs): ces = all_ces[ices] CES_start, CES_stop, name, mjdstart, scan, subscan, azmin, azmax, \ el = ces totsamples = int((CES_stop - CES_start) * args.samplerate) # create the single TOD for this observation try: tod = tt.TODGround( comm.comm_group, detquats, totsamples, detranks=comm.comm_group.size, firsttime=CES_start, rate=args.samplerate, site_lon=site_lon, site_lat=site_lat, site_alt=site_alt, azmin=azmin, azmax=azmax, el=el, scanrate=args.scanrate, scan_accel=args.scan_accel, CES_start=None, CES_stop=None, sun_angle_min=args.sun_angle_min, coord=args.coord, sampsizes=None) except RuntimeError as e: print('Failed to create the CES scan: {}'.format(e), flush=args.flush) return # Create the (single) observation ob = {} ob['name'] = 'CES-{}-{}-{}'.format(name, scan, subscan) ob['tod'] = tod if len(tod.subscans) > 0: ob['intervals'] = tod.subscans else: raise RuntimeError('{} has no valid intervals'.format(ob['name'])) ob['baselines'] = None ob['noise'] = noise ob['id'] = int(mjdstart * 10000) data.obs.append(ob) for ob in data.obs: tod = ob['tod'] tod.free_azel_quats() if comm.comm_group.rank == 0: print('Group # {:4} has {} observations.'.format( comm.group, len(data.obs)), flush=args.flush) if len(data.obs) == 0: raise RuntimeError('Too many tasks. Every MPI task must ' 'be assigned to at least one observation.') comm.comm_world.barrier() stop = MPI.Wtime() if comm.comm_world.rank == 0: print('Simulated scans in {:.2f} seconds' ''.format(stop-start), flush=args.flush) return data
def exec(self, data): """ Apply the ground filter to the signal. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) # the two-level pytoast communicator comm = data.comm # the communicator within the group cgroup = comm.comm_group # Each group loops over its own CES:es for obs in data.obs: tod = obs['tod'] try: (azmin, azmax, _, _) = tod.scan_range az = tod.read_boresight_az() except Exception as e: raise RuntimeError( 'Failed to get boresight azimuth from TOD. Perhaps it is ' 'not ground TOD? "{}"'.format(e)) # Cache the output common flags common_ref = tod.local_common_flags(self._common_flag_name) # The azimuth vector is assumed to be arranged so that the # azimuth increases monotonously even across the zero meridian. wbin = np.radians(self._wbin) nbin = int((azmax - azmin) // wbin + 1) ibin = ((az - azmin) // wbin).astype(np.int) for det in tod.detectors: hits = np.zeros(nbin) binned = np.zeros(nbin) # Bin the local data if det in tod.local_dets: ref = tod.local_signal(det, self._name) flag_ref = tod.local_flags(det, self._flag_name) good = np.logical_and( common_ref & self._common_flag_mask == 0, flag_ref & self._flag_mask == 0) # If binning ever becomes a bottleneck, it must be # implemented in Cython or compiled code. The range # checks are very expensive. for i, s in zip(ibin[good], ref[good]): hits[i] += 1 binned[i] += s del flag_ref # Reduce the binned data. The detector signal may be # distributed across the group communicator. cgroup.Allreduce(MPI.IN_PLACE, hits, op=MPI.SUM) cgroup.Allreduce(MPI.IN_PLACE, binned, op=MPI.SUM) good = hits != 0 binned[good] /= hits[good] # Subtract the ground template if det in tod.local_dets: ref -= binned[ibin] del ref del common_ref return
def exec(self, data): """ Generate atmosphere timestreams. This iterates over all observations and detectors and generates the atmosphere timestreams. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) group = data.comm.group for obs in data.obs: try: obsname = obs['name'] except Exception: obsname = 'observation' prefix = '{} : {} : '.format(group, obsname) tod = self._get_from_obs('tod', obs) comm = tod.mpicomm obsindx = self._get_from_obs('id', obs) telescope = self._get_from_obs('telescope_id', obs) site = self._get_from_obs('site_id', obs) altitude = self._get_from_obs('altitude', obs) weather = self._get_from_obs('weather', obs) fp_radius = np.radians(self._get_from_obs('fpradius', obs)) # Get the observation time span and initialize the weather # object if one is provided. times = tod.local_times() tmin = times[0] tmax = times[-1] tmin_tot = comm.allreduce(tmin, op=MPI.MIN) tmax_tot = comm.allreduce(tmax, op=MPI.MAX) weather.set(site, self._realization, tmin_tot) """ The random number generator accepts a key and a counter, each made of two 64bit integers. Following tod_math.py we set key1 = realization * 2^32 + telescope * 2^16 + component key2 = obsindx * 2^32 counter1 = currently unused (0) counter2 = sample in stream (incremented internally in the atm code) """ key1 = self._realization * 2 ** 32 + telescope * 2 ** 16 \ + self._component key2 = site * 2**16 + obsindx counter1 = 0 counter2 = 0 if self._freq is not None: absorption = atm_get_absorption_coefficient( altitude, weather.air_temperature, weather.surface_pressure, weather.pwv, self._freq) loading = atm_get_atmospheric_loading(altitude, weather.air_temperature, weather.surface_pressure, weather.pwv, self._freq) tod.meta['loading'] = loading else: absorption = None if self._cachedir is None: cachedir = None else: # The number of atmospheric realizations can be large. Use # sub-directories under cachedir. subdir = str(int((obsindx % 1000) // 100)) subsubdir = str(int((obsindx % 100) // 10)) subsubsubdir = str(obsindx % 10) cachedir = os.path.join(self._cachedir, subdir, subsubdir, subsubsubdir) if comm.rank == 0: try: os.makedirs(cachedir) except FileExistsError: pass comm.Barrier() if comm.rank == 0: print(prefix + 'Setting up atmosphere simulation', flush=self._flush) comm.Barrier() # Cache the output common flags common_ref = tod.local_common_flags(self._common_flag_name) # Read the extent of the AZ/EL boresight pointing, and use that # to compute the range of angles needed for simulating the slab. (min_az_bore, max_az_bore, min_el_bore, max_el_bore) = tod.scan_range # print("boresight scan range = {}, {}, {}, {}".format( # min_az_bore, max_az_bore, min_el_bore, max_el_bore)) # Use a fixed focal plane radius so that changing the actual # set of detectors will not affect the simulated atmosphere. elfac = 1 / np.cos(max_el_bore + fp_radius) azmin = min_az_bore - fp_radius * elfac azmax = max_az_bore + fp_radius * elfac if azmin < -2 * np.pi: azmin += 2 * np.pi azmax += 2 * np.pi elif azmax > 2 * np.pi: azmin -= 2 * np.pi azmax -= 2 * np.pi elmin = min_el_bore - fp_radius elmax = max_el_bore + fp_radius azmin = comm.allreduce(azmin, op=MPI.MIN) azmax = comm.allreduce(azmax, op=MPI.MAX) elmin = comm.allreduce(elmin, op=MPI.MIN) elmax = comm.allreduce(elmax, op=MPI.MAX) if elmin < 0 or elmax > np.pi / 2: raise RuntimeError( 'Error in CES elevation: elmin = {:.2f}, elmax = {:.2f}' ''.format(elmin, elmax)) comm.Barrier() # Loop over the time span in "wind_time"-sized chunks. # wind_time is intended to reflect the correlation length # in the atmospheric noise. tmin = tmin_tot istart = 0 while tmin < tmax_tot: while times[istart] < tmin: istart += 1 tmax = tmin + self._wind_time if tmax < tmax_tot: # Extend the scan to the next turnaround istop = istart while istop < times.size and times[istop] < tmax: istop += 1 while istop < times.size and (common_ref[istop] | tod.TURNAROUND == 0): istop += 1 if istop < times.size: tmax = times[istop] else: tmax = tmax_tot else: tmax = tmax_tot istop = times.size ind = slice(istart, istop) nind = istop - istart if self._report_timing: comm.Barrier() tstart = MPI.Wtime() comm.Barrier() if comm.rank == 0: print(prefix + 'Instantiating the atmosphere for t = {}' ''.format(tmin - tmin_tot), flush=self._flush) comm.Barrier() T0_center = weather.air_temperature wx = weather.west_wind wy = weather.south_wind w_center = np.sqrt(wx**2 + wy**2) wdir_center = np.arctan2(wy, wx) sim = atm_sim_alloc( azmin, azmax, elmin, elmax, tmin, tmax, self._lmin_center, self._lmin_sigma, self._lmax_center, self._lmax_sigma, w_center, 0, wdir_center, 0, self._z0_center, self._z0_sigma, T0_center, 0, self._zatm, self._zmax, self._xstep, self._ystep, self._zstep, self._nelem_sim_max, self._verbosity, comm, self._gangsize, key1, key2, counter1, counter2, cachedir) if sim == 0: raise RuntimeError(prefix + 'Failed to allocate simulation') if self._report_timing: comm.Barrier() tstop = MPI.Wtime() if comm.rank == 0 and tstop - tstart > 1: print(prefix + 'OpSimAtmosphere: Initialized ' 'atmosphere in {:.2f} s'.format(tstop - tstart), flush=self._flush) tstart = tstop comm.Barrier() use_cache = cachedir is not None if comm.rank == 0: fname = os.path.join( cachedir, '{}_{}_{}_{}_metadata.txt'.format( key1, key2, counter1, counter2)) if use_cache and os.path.isfile(fname): print(prefix + 'Loading the atmosphere for t = {} ' 'from {}'.format(tmin - tmin_tot, fname), flush=self._flush) cached = True else: print(prefix + 'Simulating the atmosphere for t = {}' ''.format(tmin - tmin_tot), flush=self._flush) cached = False err = atm_sim_simulate(sim, use_cache) if err != 0: raise RuntimeError(prefix + 'Simulation failed.') # Advance the sample counter in case wind_time broke the # observation in parts counter2 += 100000000 if self._report_timing: comm.Barrier() tstop = MPI.Wtime() if comm.rank == 0 and tstop - tstart > 1: if cached: op = 'Loaded' else: op = 'Simulated' print(prefix + 'OpSimAtmosphere: {} atmosphere in ' '{:.2f} s'.format(op, tstop - tstart), flush=self._flush) tstart = tstop if self._verbosity > 0: self._plot_snapshots(sim, prefix, obsname, azmin, azmax, elmin, elmax, tmin, tmax, comm) nsamp = tod.local_samples[1] if self._report_timing: comm.Barrier() tstart = MPI.Wtime() if comm.rank == 0: print(prefix + 'Observing the atmosphere', flush=self._flush) for det in tod.local_dets: # Cache the output signal cachename = '{}_{}'.format(self._out, det) if tod.cache.exists(cachename): ref = tod.cache.reference(cachename) else: ref = tod.cache.create(cachename, np.float64, (nsamp, )) # Cache the output flags flag_ref = tod.local_flags(det, self._flag_name) if self._apply_flags: good = np.logical_and( common_ref[ind] & self._common_flag_mask == 0, flag_ref[ind] & self._flag_mask == 0) ngood = np.sum(good) if ngood == 0: continue azelquat = tod.read_pntg(detector=det, local_start=istart, n=nind, azel=True)[good] atmdata = np.zeros(ngood, dtype=np.float64) else: ngood = nind azelquat = tod.read_pntg(detector=det, local_start=istart, n=nind, azel=True) atmdata = np.zeros(nind, dtype=np.float64) # Convert Az/El quaternion of the detector back into # angles for the simulation. theta, phi, _ = qa.to_angles(azelquat) # Azimuth is measured in the opposite direction # than longitude az = 2 * np.pi - phi el = np.pi / 2 - theta if np.ptp(az) < np.pi: azmin_det = np.amin(az) azmax_det = np.amax(az) else: # Scanning across the zero azimuth. azmin_det = np.amin(az[az > np.pi]) - 2 * np.pi azmax_det = np.amax(az[az < np.pi]) elmin_det = np.amin(el) elmax_det = np.amax(el) if ((not (azmin <= azmin_det and azmax_det <= azmax) and not (azmin <= azmin_det - 2 * np.pi and azmax_det - 2 * np.pi <= azmax)) or not (elmin <= elmin_det and elmin_det <= elmax)): raise RuntimeError( prefix + 'Detector Az/El: [{:.5f}, {:.5f}], ' '[{:.5f}, {:.5f}] is not contained in ' '[{:.5f}, {:.5f}], [{:.5f} {:.5f}]' ''.format(azmin_det, azmax_det, elmin_det, elmax_det, azmin, azmax, elmin, elmax)) # Integrate detector signal err = atm_sim_observe(sim, times[ind], az, el, atmdata, ngood, 0) if err != 0: # Observing failed print(prefix + 'OpSimAtmosphere: Observing FAILED. ' 'det = {}, rank = {}'.format(det, comm.rank), flush=self._flush) atmdata[:] = 0 flag_ref[ind] = 255 if self._gain: atmdata *= self._gain if absorption is not None: # Apply the frequency-dependent absorption-coefficient atmdata *= absorption if self._apply_flags: ref[ind][good] += atmdata else: ref[ind] += atmdata del ref err = atm_sim_free(sim) if err != 0: raise RuntimeError(prefix + 'Failed to free simulation.') if self._report_timing: comm.Barrier() tstop = MPI.Wtime() if comm.rank == 0 and tstop - tstart > 1: print(prefix + 'OpSimAtmosphere: Observed atmosphere ' 'in {:.2f} s'.format(tstop - tstart), flush=self._flush) tmin = tmax return
def exec(self, data): """ Generate atmosphere timestreams. This iterates over all observations and detectors and generates the atmosphere timestreams. Args: data (toast.Data): The distributed data. """ autotimer = timing.auto_timer(type(self).__name__) group = data.comm.group for obs in data.obs: try: obsname = obs["name"] except Exception: obsname = "observation" prefix = "{} : {} : ".format(group, obsname) tod = self._get_from_obs("tod", obs) comm = tod.mpicomm site = self._get_from_obs("site_id", obs) weather = self._get_from_obs("weather", obs) # Get the observation time span and initialize the weather # object if one is provided. times = tod.local_times() tmin = times[0] tmax = times[-1] tmin_tot = comm.allreduce(tmin, op=MPI.MIN) tmax_tot = comm.allreduce(tmax, op=MPI.MAX) weather.set(site, self._realization, tmin_tot) key1, key2, counter1, counter2 = self._get_rng_keys(obs) absorption = self._get_absorption_and_loading(obs) cachedir = self._get_cache_dir(obs, comm) comm.Barrier() if comm.rank == 0: print(prefix + "Setting up atmosphere simulation", flush=self._flush) comm.Barrier() # Cache the output common flags common_ref = tod.local_common_flags(self._common_flag_name) scan_range = self._get_scan_range(obs, comm) # Loop over the time span in "wind_time"-sized chunks. # wind_time is intended to reflect the correlation length # in the atmospheric noise. if self._report_timing: comm.Barrier() tstart = MPI.Wtime() tmin = tmin_tot istart = 0 counter1start = counter1 while tmin < tmax_tot: if self._report_timing: comm.Barrier() tstart = MPI.Wtime() comm.Barrier() if comm.rank == 0: print( prefix + "Instantiating the atmosphere for t = {}" "".format(tmin - tmin_tot), flush=self._flush, ) istart, istop, tmax = self._get_time_range( tmin, istart, times, tmax_tot, common_ref, tod, weather ) ind = slice(istart, istop) nind = istop - istart comm.Barrier() rmin = 0 rmax = 100 scale = 10 counter2start = counter2 counter1 = counter1start xstart, ystart, zstart = self._xstep, self._ystep, self._zstep while rmax < 100000: sim, counter2 = self._simulate_atmosphere( weather, scan_range, tmin, tmax, comm, key1, key2, counter1, counter2start, cachedir, prefix, tmin_tot, tmax_tot, rmin, rmax, ) if self._verbosity > 15: self._plot_snapshots( sim, prefix, obsname, scan_range, tmin, tmax, comm, rmin, rmax, ) self._observe_atmosphere( sim, tod, comm, prefix, common_ref, istart, nind, ind, scan_range, times, absorption, ) rmin = rmax rmax *= scale self._xstep *= np.sqrt(scale) self._ystep *= np.sqrt(scale) self._zstep *= np.sqrt(scale) counter1 += 1 if self._verbosity > 5: self._save_tod( obsname, tod, times, istart, nind, ind, comm, common_ref ) self._xstep, self._ystep, self._zstep = xstart, ystart, zstart tmin = tmax if self._report_timing: comm.Barrier() tstop = MPI.Wtime() if comm.rank == 0: print( prefix + "Simulated and observed atmosphere in {:.2f} s".format( tstop - tstart ), flush=self._flush, ) return