def tune_dets(self, session, params): """tune_dets() **Task** - Emulates files that might come from a general tune dets function. These are some of the files found on simons1 registered when running the following ops with a single band: 1. Find-freq 2. setup_notches 3. tracking_setup 4. short g3 stream Args ---- test_mode : bool If True, will skip any wait times associated with writing g3 files. """ # Find Freq action_time = time.time() files = [ 'amp_sweep_freq.txt', 'amp_sweep_resonance.txt', 'amp_sweep_resp.txt' ] for f in files: self._write_smurf_file(f, 'find_freq', action_time=action_time) # Setup Notches action_time = time.time() files = ['channel_assignment_b0.txt', 'tune.npy'] for f in files: self._write_smurf_file(f, 'setup_notches', action_time=action_time) # tracking setup action_time = time.time() fname = f"{int(time.time())}.dat" self._write_smurf_file(fname, 'tracking_setup', prepend_ctime=False) # Short g3 stream frame_gen = G3FrameGenerator(self.stream_id, self.sample_rate, self.nchans) fname = self._get_g3_filename(frame_gen.session_id, 0, makedirs=True) os.makedirs(os.path.dirname(fname), exist_ok=True) session.data['noise_file'] = fname writer = core.G3Writer(fname) writer(frame_gen.get_obs_frame()) writer(frame_gen.get_status_frame()) start = time.time() stop = start + 30 frame_starts = np.arange(start, stop, self.frame_len) frame_stops = frame_starts + self.frame_len for t0, t1 in zip(frame_starts, frame_stops): if not params['test_mode']: now = time.time() if now < t1: time.sleep(t1 - now) writer(frame_gen.get_data_frame(t0, t1)) writer(core.G3Frame(core.G3FrameType.EndProcessing)) return True, "Wrote tune files"
def Process(self, frames): """ Writes frame to current file. If file has not been started or time_per_file has elapsed, file is closed and a new file is created by `filename` function passed to constructor """ for frame in frames: ftype = frame['hkagg_type'] if ftype == so3g.HKFrameType.session: self.last_session = frame elif ftype == so3g.HKFrameType.status: self.last_status = frame if self.writer is None: self.current_file = self.filename() self.log.info("Creating file: {}".format(self.current_file)) self.writer = core.G3Writer(self.current_file) self.file_start_time = time.time() if ftype in [so3g.HKFrameType.data, so3g.HKFrameType.status]: if self.last_session is not None: self.writer(self.last_session) if ftype == so3g.HKFrameType.data: if self.last_status is not None: self.writer(self.last_status) self.writer(frame) if (time.time() - self.file_start_time) > self.time_per_file: self.close_file() return frames
def create_new_file(self): """Create a new file if needed. This is only done if an existing file is closed out (i.e. writer is None). Filename and path will be based on the time the file is made. Acquisitions that are grouped together by file duration will share the same basename as the start of the acquisition and have their suffix incremented. Writes the last meta data frame to the start of the file if there is one. Example: # Three 10 minute observations file_1 = 2019-08-07-01-30-00_000.g3 file_2 = 2019-08-07-01-30-00_001.g3 file_3 = 2019-08-07-01-30-00_002.g3 # Acqusition stopped and new one started on the hour new_file = 2019-08-07-02-00-00_000.g3 """ if self.writer is None: # Used for tracking when to split acquisitions self.start_time = time.time() # Avoid duplicate filenames if new file started within 1 sec if self._last_file_timestamp is None: pass elif int(self.start_time) == self._last_file_timestamp: self.log.debug( "New file started within 1 second of previous " + "file, incrementing filename suffix.") self.filename_suffix += 1 # Only create new dir and basename if we've finished an acquisition if self.filename_suffix == 0: stream_id = self.stream_id for f in self.frames: if f.get('sostream_id') is not None: stream_id = f['sostream_id'] break self.dirname = _create_dirname(self.start_time, self.data_dir, stream_id) self.basename = int(self.start_time) suffix = "_{:03d}".format(self.filename_suffix) filepath = _create_file_path(self.dirname, self.basename, suffix) self.log.info("Writing to file {}".format(filepath)) self.writer = core.G3Writer(filename=filepath) self._last_file_timestamp = int(self.start_time) # Write the last metadata frame to the start of the new file if self.last_meta is not None: self.writer(self.last_meta) return filepath
def write_example_file(filename='hk_out.g3'): """Generate an example g3 file with Wiring Frame. This is based on other test file writing functions, but is unique in that it contains a wiring frame for use in testing Seek/Tell. Structure of frames in this file should be: - Housekeeping - Housekeeping - Wiring - Housekeeping - Housekeeping Args: filename (str): filename to write data to """ test_file = filename # Write a stream of HK frames. w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=1234, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider(description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. f = hksess.session_frame() w.Process(f) f = hksess.status_frame() w.Process(f) # Write dummy wiring frame f = core.G3Frame() f.type = core.G3FrameType.Wiring w.Process(f) # Now make a data frame. f = hksess.data_frame(prov_id=prov_id) # Add a data block. hk = so3g.IrregBlockDouble() hk.prefix = 'hwp_' hk.data['position'] = [1, 2, 3, 4, 5] hk.data['speed'] = [1.2, 1.2, 1.2, 1.2, 1.2] hk.t = [0, 1, 2, 3, 4] f['blocks'].append(hk) # Write two more housekeeping frames. w.Process(f) w.Process(f) del w
def test_30_serialization(self): m0 = get_test_block(100) m1 = get_test_block(200, offset=100) m2 = m0.concatenate(m1) m0.check() m1.check() m2.check() f = core.G3Frame() f['irreg0'] = m0 f['irreg1'] = m1 core.G3Writer('test.g3').Process(f) f = core.G3Reader('test.g3').Process(None)[0] f['irreg0'].check() f['irreg1'].check() f['irreg0'].concatenate(f['irreg1'])['b']
def test_50_compression(self): test_file = 'test_g3super.g3' # Entropy? sigma_bits = 8 sigma = 2**sigma_bits _get_ts = lambda dtype: self._get_ts( 100, 10000, sigma=sigma, dtype=dtype, seed=12345) w = core.G3Writer(test_file) sizes = {d: [] for d in ALL_DTYPES} for dtype in ALL_DTYPES: # No compression f = core.G3Frame() ts = _get_ts(dtype) sizes[dtype].append(ts.data.nbytes) ts.options(enable=0) f['ts_%s' % dtype] = ts w.Process(f) # Yes compression f = core.G3Frame() ts = _get_ts(dtype) f['ts_%s' % dtype] = ts w.Process(f) del w # Readback r = so3g.G3IndexedReader(test_file) last = 0 for dtype in ALL_DTYPES: for i in range(2): r.Process(None)[0] here = r.Tell() sizes[dtype].append(here - last) last = here # Process the results... for dtype in ALL_DTYPES: err_msg = f'Failed for dtype={dtype}' n, s_uncomp, s_comp = sizes[dtype] comp_ratio = 1. - (s_uncomp - s_comp) / n # But really what matters is the bits-per-word, compressed. bits_per_word = comp_ratio * 8 * np.dtype(dtype).itemsize #print(dtype, bits_per_word / sigma_bits) # I think the theoretical limit is 1.3 or so... self.assertLess(bits_per_word, sigma_bits * 1.4, err_msg)
def test_compression(self): """Confirm that minimum necessary int size is used for serialization.""" count = 10000 overhead = 200 for isize, val in bit_sizes: w = core.G3Writer(test_filename) f = core.G3Frame() f['v'] = core.G3VectorInt([val] * count) w(f) del w on_disk = os.path.getsize(test_filename) self.assertTrue( abs(on_disk - count * isize / 8) <= overhead, "Storage for val %i took %.2f bytes/item, " "too far from %.2f bytes/item" % (val, on_disk / count, isize / 8))
def test_serialize(self): """Confirm full ranges can be saved and loaded.""" w = core.G3Writer(test_filename) for isize, val in bit_sizes: f = core.G3Frame() f['v'] = core.G3VectorInt([val] * 10) w(f) del w r = core.G3Reader(test_filename) for isize, val in bit_sizes: f = r(None)[0] v_in = list(f['v']) self.assertTrue(all([_v == val for _v in v_in]), "Failed to save/load value %i" % val) del r
def start_stream(self, session, params=None): if params is None: params = {} time_per_file = params.get("time_per_file", 60*60) # [sec] data_dir = params.get("data_dir", "data/") self.log.info("Writing data to {}".format(data_dir)) self.log.info("New file every {} seconds".format(time_per_file)) reader = core.G3Reader("tcp://localhost:{}".format(self.port)) writer = None last_meta = None self.is_streaming = True while self.is_streaming: if writer is None: start_time = datetime.utcnow() ts = start_time.timestamp() subdir = os.path.join(data_dir, "{:.5}".format(str(ts))) if not os.path.exists(subdir): os.makedirs(subdir) filename = start_time.strftime("%Y-%m-%d-%H-%M-%S.g3") filepath = os.path.join(subdir, filename) writer = core.G3Writer(filename=filepath) if last_meta is not None: writer(last_meta) frames = reader.Process(None) for f in frames: if f.type == core.G3FrameType.Housekeeping: last_meta = f writer(f) if (datetime.utcnow().timestamp() - ts) > time_per_file: writer(core.G3Frame(core.G3FrameType.EndProcessing)) writer = None if writer is not None: writer(core.G3Frame(core.G3FrameType.EndProcessing)) return True, "Finished Streaming"
def write_example_file(filename='hk_out.g3'): """Generate some example HK data and write to file. Args: filename (str): filename to write data to """ test_file = filename # Write a stream of HK frames. # (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.) w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=1234, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider(description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. f = hksess.session_frame() w.Process(f) f = hksess.status_frame() w.Process(f) # Now make a data frame. f = hksess.data_frame(prov_id=prov_id) # Add a data block. hk = so3g.IrregBlockDouble() hk.prefix = 'hwp_' hk.data['position'] = [1, 2, 3, 4, 5] hk.data['speed'] = [1.2, 1.2, 1.2, 1.2, 1.2] hk.t = [0, 1, 2, 3, 4] f['blocks'].append(hk) w.Process(f) del w
def _readback_compare(self, ts, filename='readback_test.g3', cleanup=True, err_msg='(no detail)'): """Cache the data from ts, write ts to a file, read it back from file, compare to cached data. """ # Cache fake_ts = (collections.namedtuple( 'pseudo_ts', ['times', 'names', 'data']))(np.array(ts.times), np.array(ts.names), ts.data.copy()) # Write f = core.G3Frame() f['item'] = ts core.G3Writer(filename).Process(f) # Read ts1 = core.G3Reader(filename).Process(None)[0]['item'] self._check_equal(fake_ts, ts1, err_msg=err_msg) if cleanup: os.remove(filename)
def write_example_file(filename='hk_out.g3', hkagg_version=2): """Generate some example HK data and write to file. Args: filename (str): filename to write data to hkagg_version (int): which HK version to write to file """ test_file = filename # Write a stream of HK frames. # (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.) seeder = Seeder() w = core.G3Pipeline() w.Add(seeder) w.Add(HKTranslator(target_version=hkagg_version)) w.Add(core.G3Writer(test_file)) if hkagg_version <= 1: seeder.extend(get_v0_stream()) else: seeder.extend(get_v2_stream()) w.Run() del w
def test_40_encoding_serialized(self): test_file = 'test_g3super.g3' offsets = { 'int32': [0, 2**25, 2**26 / 3., -1.78 * 2**27], 'int64': [0, 2**25, 2**26 / 3., -1.78 * 2**27], 'float32': [0], 'float64': [0, 2**25, 2**26 / 3., -1.78 * 2**27, 1.8 * 2**35], } decimals = 2 precision = 10**-decimals w = core.G3Writer(test_file) records = [] for dtype in ALL_DTYPES: for offset in offsets[dtype]: f = core.G3Frame() ts = self._get_ts(4, 100, sigma=100, dtype=dtype) ts.data += int(offset) if dtype in FLOAT_DTYPES: ts.data[:] = np.round(ts.data) ts.calibrate([.01] * ts.data.shape[0]) records.append(ts.data.copy()) f['a'] = ts w.Process(f) del w # readback r = core.G3Reader(test_file) for dtype in ALL_DTYPES: for offset in offsets[dtype]: err_msg = f'Failed for dtype={dtype}, offset={offset}' ts2 = r.Process(None)[0]['a'] record = records.pop(0) np.testing.assert_allclose(record, ts2.data, atol=precision * 1e-3, err_msg=err_msg)
# data in the "SO HK" format. When expanding it, check the SO HK # format description to make sure your frame stream is compliant. import time import numpy as np from spt3g import core from so3g import hk # Start a "Session" to help generate template frames. session = hk.HKSessionHelper(hkagg_version=2) # Create an output file and write the initial "session" frame. If # you break the data into multiple files, you write the session frame # at the start of each file. writer = core.G3Writer('hk_example.g3') writer.Process(session.session_frame()) # Create a new data "provider". This represents a single data # source, sending data for some fixed list of fields. prov_id = session.add_provider('platform') # Whenever there is a change in the active "providers", write a # "status" frame. writer.Process(session.status_frame()) # Write, like, 10 half-scans. frame_time = time.time() v_az = 1.5 # degrees/second dt = 0.001 # seconds halfscan = 10 # degrees
import so3g as ss from spt3g import core ss.TestClass().runme() ss.greet() w = core.G3Writer('out.g3') f = core.G3Frame() f.type = core.G3FrameType.Scan f['testv'] = core.G3VectorDouble([ 1., 2., 3., ]) w.Process(f)
# Type failing works? Can't create mask from non-integer Intervals. try: mask3 = so3g.IntervalsDouble.mask([], 8) except ValueError: mask3 = 'failed' assert (mask3 == 'failed') print() print('Map test') tmap = so3g.MapIntervalsTime() tmap['a'] = ti print(' ', tmap) print(' ', tmap['a']) # Can we read and write them? print() test_filename = 'test_intervals.g3' print('Writing to %s' % test_filename) w = core.G3Writer(test_filename) f = core.G3Frame() f['map'] = tmap f['iv'] = iv2 w.Process(f) del w print() print('Reading from %s' % test_filename) for f in core.G3File(test_filename): print(' ', f)
def _export_frames(self, path=None, prefix=None, cache_name=None, cache_common=None, cache_flag_name=None): """Export cache data to frames. This will either export the cache fields that correspond to the "real" data (those manipulated by the read / write methods) or will export alternate cache fields specified by the arguments. Args: path (str): Override any path specified at construction time. prefix (str): Override any prefix specified at construction time. cache_name (str): When exporting data, the name of the cache object (<name>_<detector>) to use for the detector timestream. If None, use the TOD read* methods. cache_common (str): When exporting data, the name of the cache object to use for common flags. If None, use the TOD read* methods. cache_flag_name (str): When exporting data, the name of the cache object (<name>_<detector>) to use for the detector flags. If None, use the TOD read* methods. """ self._cache_init() # Create the frame schema we are using when exporting data common_fields = [(STR_TIME, c3g.G3VectorTime, STR_TIME), (STR_BORE, c3g.G3VectorDouble, STR_BORE), (STR_BOREAZEL, c3g.G3VectorDouble, STR_BOREAZEL), (STR_POS, c3g.G3VectorDouble, STR_POS), (STR_VEL, c3g.G3VectorDouble, STR_VEL)] if cache_common is None: cname = "{}_{}".format(STR_FLAG, STR_COMMON) common_fields.append((cname, c3g.G3VectorUnsignedChar, cname)) else: common_fields.append( (cache_common, c3g.G3VectorUnsignedChar, cache_common)) det_fields = None if cache_name is None: det_fields = [ ("{}_{}".format(STR_DET, d), d) \ for d in self.detectors ] else: det_fields = [ ("{}_{}".format(cache_name, d), d) \ for d in self.detectors ] flag_fields = None if cache_flag_name is None: flag_fields = [ ("{}_{}".format(STR_FLAG, d), d) \ for d in self.detectors ] else: flag_fields = [ ("{}_{}".format(cache_flag_name, d), d) \ for d in self.detectors ] ex_path = self._path if path is not None: ex_path = path ex_prefix = self._prefix if prefix is not None: ex_prefix = prefix if (ex_path is None) or (ex_prefix is None): raise RuntimeError("You must specify the TOD path and prefix, " "either at construction or export") ex_files = [ os.path.join(ex_path, "{}_{:08d}.g3".format(ex_prefix, x)) \ for x in self._file_sample_offs ] for ifile, (ffile, foff) in enumerate(zip(ex_files, self._file_frame_offs)): nframes = None #print(" ifile = {}, ffile = {}, foff = {}".format(ifile, ffile, foff), flush=True) if ifile == len(ex_files) - 1: # we are at the last file nframes = len(self._frame_sizes) - foff else: # get number of frames in this file nframes = self._file_frame_offs[ifile + 1] - foff writer = None if self.mpicomm.rank == 0: writer = c3g.G3Writer(ffile) props = self.meta() props["units"] = self._units props["have_azel"] = self._have_azel #print("Writing props:") #print(props, flush=True) write_spt3g_obs(writer, props, self._detquats, self.total_samples) # Collect data for all frames in the file in one go. frm_offsets = [ self._frame_sample_offs[foff+f] \ for f in range(nframes) ] frm_sizes = [ self._frame_sizes[foff+f] \ for f in range(nframes) ] # if self.mpicomm.rank == 0: # print(" {} file {}".format(self._path, ifile), flush=True) # print(" start frame = {}, nframes = {}".format(foff, nframes), flush=True) # print(" frame offs = ",frm_offsets, flush=True) # print(" frame sizes = ",frm_sizes, flush=True) fdata = s3utils.cache_to_frames(self, foff, nframes, frm_offsets, frm_sizes, common=common_fields, detector_fields=det_fields, flag_fields=flag_fields, detector_map=STR_DET, flag_map=STR_FLAG, units=self._units) if self.mpicomm.rank == 0: for fdt in fdata: writer(fdt) del writer del fdata return
def stream(self, session, params): """stream(duration=None) **Process** - Generates example fake-files organized in the same way as they would be a regular smurf-stream. For end-to-end testing, we want an example of a pysmurf-ancilliary file, and then regular g3 that rotate at regular intervals. The content of the files here don't match what actual G3 or pysmurf files look like, however the directory structure is the same. Parameters: duration (float, optional): If set, will stop stream after specified amount of time (sec). """ session.set_status('starting') # Write initial smurf metadata action_time = time.time() files = ['freq.txt', 'mask.txt'] for f in files: self._write_smurf_file(f, 'take_g3_stream', action_time=action_time) end_time = None if params.get('duration') is not None: end_time = time.time() + params['duration'] session.set_status('running') frame_gen = G3FrameGenerator(self.stream_id, self.sample_rate, self.nchans) session.data['session_id'] = frame_gen.session_id session.data['g3_files'] = [] seq = 0 fname = self._get_g3_filename(frame_gen.session_id, seq, makedirs=True) os.makedirs(os.path.dirname(fname), exist_ok=True) session.data['g3_files'].append(fname) writer = core.G3Writer(fname) file_start = time.time() writer(frame_gen.get_obs_frame()) writer(frame_gen.get_status_frame()) self.streaming = True while self.streaming: start = time.time() time.sleep(self.frame_len) stop = time.time() writer(frame_gen.get_data_frame(start, stop)) if end_time is not None: if stop > end_time: break if time.time() - file_start > self.file_duration: writer(core.G3Frame(core.G3FrameType.EndProcessing)) seq += 1 fname = self._get_g3_filename(frame_gen.session_id, seq, makedirs=True) os.makedirs(os.path.dirname(fname), exist_ok=True) session.data['g3_files'].append(fname) writer = core.G3Writer(fname) file_start = time.time() writer(core.G3Frame(core.G3FrameType.EndProcessing)) return True, "Finished stream"
# Test data. Exercise some complicated things (STL bits) that we don't # necessarily have control over, mapping a few primitive types. f = core.G3Frame() f['Five'] = 5 v = core.G3VectorDouble([2.6, 7.2]) f['Vec'] = v v = core.G3VectorInt([17, 42, 87]) f['VecInt'] = v m = core.G3MapDouble() m['Six'] = 6 m['GoingOnSixteen'] = 15.9 f['Map'] = m if len(sys.argv) > 1: core.G3Writer(sys.argv[1])(f) sys.exit(0) # For now, we test files from big-endian (PPC64) and little-endian (amd64) # 64-bit systems. Should include some 32-bit ones. for test in ['test-be.g3', 'test-le.g3']: print(test) testdata = core.G3Reader(os.path.join(testpath, test))(None)[0] assert (testdata['Five'] == f['Five']) assert (len(testdata['Vec']) == len(f['Vec'])) for i in range(len(testdata['Vec'])): assert (testdata['Vec'][i] == f['Vec'][i]) assert (len(testdata['VecInt']) == len(f['VecInt'])) for i in range(len(testdata['VecInt'])):
return [f] def __call__(self, *args, **kwargs): return self.Process(*args, **kwargs) if __name__ == '__main__': import argparse parser = argparse.ArgumentParser( usage='This program can be used to convert SO HK Frames to the ' 'latest schema version.') parser.add_argument('--output-file', '-o', default='out.g3') parser.add_argument('--target-version', type=int) parser.add_argument('files', nargs='+', help= "SO Housekeeping files to convert.") args = parser.parse_args() # Run me on a G3File containing a Housekeeping stream. core.set_log_level(core.G3LogLevel.LOG_INFO) translator_args = {} if args.target_version is not None: translator_args['target_version'] = args.target_version print(f'Streaming to {args.output_file}') p = core.G3Pipeline() p.Add(core.G3Reader(args.files)) p.Add(HKTranslator(**translator_args)) p.Add(core.G3Writer(args.output_file)) p.Run()
from spt3g import core, calibration, dfmux, mapmaker import numpy as np fsingle = core.G3File('/home/adama/SPT/spt_analysis/20190125_singleprecision/' + \ 'singleprecision/64502043_tod_v13.g3') fdouble = core.G3File('/home/adama/SPT/spt_analysis/20190125_singleprecision/' + \ 'doubleprecision/64502043_tod_v13.g3') write_mix = core.G3Writer('mixedprecision/64502043_v13_single_pixelpointing.g3') fr_single = fsingle.next() fr_double = fdouble.next() while True: if fr_single.type == core.G3FrameType.Scan and \ fr_double.type == core.G3FrameType.Scan: new_frame = core.G3Frame(core.G3FrameType.Scan) new_frame['DeflaggedTimestreams'] = fr_double['DeflaggedTimestreams'] new_frame['OnlineRaDecRotation'] = fr_double['OnlineRaDecRotation'] new_frame['PixelPointing'] = fr_single['PixelPointing'] new_frame['TodWeights'] = fr_double['TodWeights'] bolos_double = np.array(list(fr_double['DeflaggedTimestreams'].keys())) bolos_pointing = np.array(list(fr_single['PixelPointing'].keys())) bolos_toremove = np.setdiff1d(bolos_double, bolos_pointing) for bolo in bolos_toremove: if bolo in new_frame['DeflaggedTimestreams']: new_frame['DeflaggedTimestreams'].pop(bolo) print(new_frame) write_mix.Process(new_frame) else:
def test_00_basic(self): """Write a stream of HK frames and scan it for errors.""" # Write a stream of HK frames. # (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.) print('Streaming to %s' % test_file) w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=None, hkagg_version=2, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider( description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. w.Process(hksess.session_frame()) w.Process(hksess.status_frame()) # Add a bunch of data frames t_next = time.time() for i in range(10): f = hksess.data_frame(prov_id=prov_id, timestamp=t_next) hk = core.G3TimesampleMap() speed = [1.2, 1.2, 1.3, 1.2, 1.3] hk.times = [ core.G3Time(_t * core.G3Units.second) for _t in t_next + np.arange(len(speed)) ] hk['position'] = core.G3VectorDouble(np.arange(len(speed))) hk['speed'] = core.G3VectorDouble(speed) hk['error_bits'] = core.G3VectorInt([10] * len(speed)) hk['mode_str'] = core.G3VectorString(['ok'] * len(speed)) t_next += len(hk) f['blocks'].append(hk) f['block_names'].append('main_block') w.Process(f) w.Flush() del w print('Stream closed.\n\n') # Now play them back... print('Reading back:') for f in core.G3File(test_file): ht = f.get('hkagg_type') if ht == so3g.HKFrameType.session: print('Session: %i' % f['session_id']) elif ht == so3g.HKFrameType.status: print(' Status update: %i providers' % (len(f['providers']))) elif ht == so3g.HKFrameType.data: print(' Data: %i blocks' % len(f['blocks'])) for i, block in enumerate(f['blocks']): print(' Block %i' % i) for k, v in block.items(): print(' %s' % k, v) # Scan and validate. print() print('Running HKScanner on the test data...') scanner = so3g.hk.HKScanner() pipe = core.G3Pipeline() pipe.Add(core.G3Reader(test_file)) pipe.Add(scanner) pipe.Run() print('Stats: ', scanner.stats) print('Providers: ', scanner.providers) self.assertEqual(scanner.stats['concerns']['n_error'], 0) self.assertEqual(scanner.stats['concerns']['n_warning'], 0)
def test_00_basic(self): """Write a stream of HK frames and scan it for errors.""" # Write a stream of HK frames. # (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.) print('Streaming to %s' % test_file) w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=None, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider( description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. w.Process(hksess.session_frame()) w.Process(hksess.status_frame()) # Add a bunch of data frames t_next = time.time() for i in range(10): f = hksess.data_frame(prov_id=prov_id, timestamp=t_next) hk = so3g.IrregBlockDouble() hk.prefix = 'hwp_' hk.data['position'] = [1, 2, 3, 4, 5] hk.data['speed'] = [1.2, 1.2, 1.3, 1.2, 1.3] hk.t = t_next + np.arange(len(hk.data['speed'])) t_next += len(hk.data['speed']) f['blocks'].append(hk) w.Process(f) w.Flush() del w print('Stream closed.\n\n') # Now play them back... print('Reading back:') for f in core.G3File(test_file): ht = f.get('hkagg_type') if ht == so3g.HKFrameType.session: print('Session: %i' % f['session_id']) elif ht == so3g.HKFrameType.status: print(' Status update: %i providers' % (len(f['providers']))) elif ht == so3g.HKFrameType.data: print(' Data: %i blocks' % len(f['blocks'])) for block in f['blocks']: for k, v in block.data.items(): print(' %s%s' % (block.prefix, k), v) # Scan and validate. print() print('Running HKScanner on the test data...') scanner = so3g.hk.HKScanner() pipe = core.G3Pipeline() pipe.Add(core.G3Reader(test_file)) pipe.Add(scanner) pipe.Run() print('Stats: ', scanner.stats) print('Providers: ', scanner.providers) self.assertEqual(scanner.stats['concerns']['n_error'], 0) self.assertEqual(scanner.stats['concerns']['n_warning'], 0)
for spectrum in ['TT', 'EE', 'BB']: ell_range_nobias = (Cls[0.0]['cls']['ell'] > 500) & \ (Cls[0.0]['cls']['ell'] < 2000) ell_range_bias = (Cls[bias_mag]['cls']['ell'] > 500) & \ (Cls[bias_mag]['cls']['ell'] < 2000) cal_factor = np.mean(Cls[0.0]['cls'][spectrum][ell_range_nobias]) / \ np.mean(Cls[bias_mag]['cls'][spectrum][ell_range_bias]) Cls[bias_mag]['cal'][spectrum] = cal_factor Cls[bias_mag]['cls_normalized'][ spectrum] = Cls[bias_mag]['cls'][spectrum] * cal_factor # save the maps to a G3 file if args.save_maps: fname_stub = 'linearbias_{:.1f}percentPerDeg'.format(bias_mag) w = core.G3Writer('{}_{}.g3'.format( fname_stub, os.path.splitext(os.path.basename(args.simskies[jsky]))[0])) w.Process(map_fr) w.Process(core.G3Frame(core.G3FrameType.EndProcessing)) if args.fit_cosmology: if args.norm_to_unbiased and bias_mag != 0: res = minimize(neg2LogL, [67.87, 0.022277, 0.11843], args=(Cls[bias_mag]['cls_normalized']), method='powell', options={ 'xtol': 1e-6, 'disp': True }) else: res = minimize(neg2LogL, [67.87, 0.022277, 0.11843],
newframe = core.G3Frame(core.G3FrameType.Calibration) for field in old_boloprops: if field != 'BolometerProperties': newframe[field] = old_boloprops[field] newframe['BolometerProperties'] = calibration.BolometerPropertiesMap() print(len(old_boloprops['BolometerProperties'])) for bolo in old_boloprops['BolometerProperties'].keys(): if '/' in bolo: newbolo = bolo.split('/')[1] else: newbolo = bolo if newbolo in nominal_boloprops['NominalBolometerProperties'].keys(): newframe['BolometerProperties'][bolo] = old_boloprops[ 'BolometerProperties'][bolo] newframe['BolometerProperties'][ bolo].pol_angle = nominal_boloprops[ 'NominalBolometerProperties'][newbolo].pol_angle #old_boloprops['BolometerProperties'][bolo].pol_angle = nominal_boloprops['NominalBolometerProperties'][newbolo].pol_angle else: old_boloprops['BolometerProperties'].pop(bolo) print('{} not in new nominal bolometer properties!'.format(bolo)) print(len(old_boloprops['BolometerProperties'])) writer = core.G3Writer(fnames_dict['new']) writer.Process(newframe) writer.Process(core.G3Frame(core.G3FrameType.EndProcessing))
def _export_observation(self, obs, cgroup, detgroup=None, detectors=None, keep_offsets=False): """ Export observation in one or more frame files """ grouprank = 0 if cgroup is not None: grouprank = cgroup.rank # Observation information. Anything here that is a simple data # type will get written to the observation frame. props = dict() for k, v in obs.items(): if isinstance(v, (int, str, bool, float)): props[k] = v # Every observation must have a name... obsname = obs["name"] # The TOD tod = obs["tod"] nsamp = tod.total_samples if detectors is None: detquat = tod.detoffset() detindx = tod.detindx detnames = tod.detectors else: detquat_temp = tod.detoffset() detindx_temp = tod.detindx detquat = {} detindx = {} detnames = [] toddets = set(tod.detectors) for det in detectors: if det not in toddets: continue detnames.append(det) detquat[det] = detquat_temp[det] detindx[det] = detindx_temp[det] ndets = len(detquat) # Get any other metadata from the TOD props.update(tod.meta) # First process in the group makes the output directory obsdir = os.path.join(self._outdir, obsname) if cgroup.rank == 0: if not os.path.isdir(obsdir): os.makedirs(obsdir) cgroup.barrier() detranks, sampranks = tod.grid_size framesizes = self._get_framesizes(tod, obs, nsamp, keep_offsets) (flavors, flavor_type, flavor_maptype, copy_flavors ) = self._get_flavors(tod, detnames, grouprank) (frame_sample_offs, file_sample_offs, file_frame_offs ) = self._get_offsets(cgroup, grouprank, keep_offsets, len(detquat), len(copy_flavors), framesizes) if detgroup is None: prefix = self._prefix else: prefix = "{}_{}".format(self._prefix, detgroup) ex_files = [os.path.join(obsdir, "{}_{:08d}.g3".format(prefix, x)) for x in file_sample_offs] # Loop over each frame file. Write the header frames and then # gather the data from all processes before writing the scan # frames. for ifile, (ffile, foff) in enumerate(zip(ex_files, file_frame_offs)): nframes = None # print(" ifile = {}, ffile = {}, foff = {}" # .format(ifile, ffile, foff), flush=True) if ifile == len(ex_files) - 1: # we are at the last file nframes = len(framesizes) - foff else: # get number of frames in this file nframes = file_frame_offs[ifile + 1] - foff writer = None if grouprank == 0: writer = core3g.G3Writer(ffile) self._write_obs(writer, props, detindx) if "noise" in obs: self._write_precal(writer, detquat, obs["noise"]) else: self._write_precal(writer, detquat, None) # Collect data for all frames in the file in one go. frm_offsets = [frame_sample_offs[foff + f] for f in range(nframes)] frm_sizes = [framesizes[foff + f] for f in range(nframes)] if grouprank == 0 and self._verbose: print(" {} file {} detector group {}".format( obsdir, ifile, detgroup), flush=True) print(" start frame = {}, nframes = {}" .format(foff, nframes), flush=True) print(" frame offs = ", frm_offsets, flush=True) print(" frame sizes = ", frm_sizes, flush=True) fdata = tod_to_frames( tod, foff, nframes, frm_offsets, frm_sizes, cache_signal=self._cache_name, cache_flags=self._cache_flag_name, cache_common_flags=self._cache_common, copy_common=None, copy_detector=copy_flavors, units=self._units, dets=detnames, mask_flag_common=self._mask_flag_common, mask_flag=self._mask_flag, compress=self._compress, ) if grouprank == 0: for fdt in fdata: writer(fdt) del writer del fdata return
def exec(self, data): """Export data to a directory tree of so3g frames. For errors that prevent the export, this function will directly call MPI Abort() rather than raise exceptions. This could be changed in the future if additional logic is implemented to ensure that all processes raise an exception when one process encounters an error. Args: data (toast.Data): The distributed data. """ # the two-level toast communicator comm = data.comm # the global communicator cworld = comm.comm_world # the communicator within the group cgroup = comm.comm_group # the communicator with all processes with # the same rank within their group crank = comm.comm_rank # One process checks the path if cworld.rank == 0: if not os.path.isdir(self._outdir): os.makedirs(self._outdir) cworld.barrier() for obs in data.obs: # Observation information. Anything here that is a simple data # type will get written to the observation frame. props = dict() for k, v in obs.items(): if isinstance(v, (int, str, bool, float)): props[k] = v # Every observation must have a name... obsname = obs["name"] # The TOD tod = obs["tod"] nsamp = tod.total_samples detquat = tod.detoffset() detindx = tod.detindx ndets = len(detquat) detnames = tod.detectors # Get any other metadata from the TOD props.update(tod.meta()) # First process in the group makes the output directory obsdir = os.path.join(self._outdir, obsname) if cgroup.rank == 0: if not os.path.isdir(obsdir): os.makedirs(obsdir) cgroup.barrier() detranks, sampranks = tod.grid_size # Determine frame sizes based on the data distribution framesizes = None if self._usechunks: framesizes = tod.total_chunks elif self._useintervals: if "intervals" not in obs: raise RuntimeError( "Observation does not contain intervals, cannot \ distribute using them") framesizes = intervals_to_chunklist(obs["intervals"], nsamp) if framesizes is None: framesizes = [nsamp] # Examine all the cache objects and find the set of prefixes flavors = set() flavor_type = dict() flavor_maptype = dict() pat = re.compile(r"^(.*?)_(.*)") for nm in list(tod.cache.keys()): mat = pat.match(nm) if mat is not None: pref = mat.group(1) md = mat.group(2) if md in detnames: # This cache field has the form <prefix>_<det> if pref not in flavor_type: ref = tod.cache.reference(nm) if ref.dtype == np.dtype(np.float64): flavors.add(pref) flavor_type[pref] = core3g.G3Timestream flavor_maptype[pref] = core3g.G3TimestreamMap elif ref.dtype == np.dtype(np.int32): flavors.add(pref) flavor_type[pref] = core3g.G3VectorInt flavor_maptype[pref] = core3g.G3MapVectorInt elif ref.dtype == np.dtype(np.uint8): flavors.add(pref) flavor_type[pref] = so3g.IntervalsInt flavor_maptype[pref] = so3g.MapIntervalsInt else: msg = "Cache prefix {} has unsupported \ data type. Skipping export" raise RuntimeError(msg) flavors.discard(self._cache_name) flavors.discard(self._cache_flag_name) copy_flavors = [(x, flavor_type[x], flavor_maptype[x], "signal_{}".format(x)) for x in flavors] print("found cache flavors ", flavors, flush=True) # Given the dimensions of this observation, compute the frame # file sizes and all relevant offsets. frame_sample_offs = None file_sample_offs = None file_frame_offs = None if cgroup.rank == 0: # Compute the frame file breaks. We ignore the observation # and calibration frames since they are small. sampbytes = self._bytes_per_sample(len(detquat), len(flavors)) file_sample_offs, file_frame_offs, frame_sample_offs = \ s3utils.compute_file_frames( sampbytes, framesizes, file_size=self._target_framefile) file_sample_offs = cgroup.bcast(file_sample_offs, root=0) file_frame_offs = cgroup.bcast(file_frame_offs, root=0) frame_sample_offs = cgroup.bcast(frame_sample_offs, root=0) ex_files = [ os.path.join(obsdir, "{}_{:08d}.g3".format(self._prefix, x)) for x in file_sample_offs ] # Loop over each frame file. Write the header frames and then # gather the data from all processes before writing the scan # frames. for ifile, (ffile, foff) in enumerate(zip(ex_files, file_frame_offs)): nframes = None print(" ifile = {}, ffile = {}, foff = {}".format( ifile, ffile, foff), flush=True) if ifile == len(ex_files) - 1: # we are at the last file nframes = len(framesizes) - foff else: # get number of frames in this file nframes = file_frame_offs[ifile + 1] - foff writer = None if cgroup.rank == 0: writer = core3g.G3Writer(ffile) self._write_obs(writer, props, detindx) if "noise" in obs: self._write_precal(writer, detquat, obs["noise"]) else: self._write_precal(writer, detquat, None) # Collect data for all frames in the file in one go. frm_offsets = [ frame_sample_offs[foff + f] for f in range(nframes) ] frm_sizes = [framesizes[foff + f] for f in range(nframes)] if cgroup.rank == 0: print(" {} file {}".format(obsdir, ifile), flush=True) print(" start frame = {}, nframes = {}".format( foff, nframes), flush=True) print(" frame offs = ", frm_offsets, flush=True) print(" frame sizes = ", frm_sizes, flush=True) fdata = tod_to_frames(tod, foff, nframes, frm_offsets, frm_sizes, cache_signal=self._cache_name, cache_flags=self._cache_flag_name, cache_common_flags=self._cache_common, copy_common=None, copy_detector=copy_flavors, units=self._units) if cgroup.rank == 0: for fdt in fdata: writer(fdt) del writer del fdata return