def nextreader(self): nextfilelist = self.files.pop(0) self.broadcast = False if isinstance(nextfilelist, str): self.broadcast = True if self.comm.rank == 0: self.cur_reader = core.G3Reader(nextfilelist) else: self.cur_reader = lambda fr: None else: toreadhere = nextfilelist[self.comm.rank::self.comm.size] self.cur_reader = core.G3Reader(toreadhere)
def test_30_serialization(self): m0 = get_test_block(100) m1 = get_test_block(200, offset=100) m2 = m0.concatenate(m1) m0.check() m1.check() m2.check() f = core.G3Frame() f['irreg0'] = m0 f['irreg1'] = m1 core.G3Writer('test.g3').Process(f) f = core.G3Reader('test.g3').Process(None)[0] f['irreg0'].check() f['irreg1'].check() f['irreg0'].concatenate(f['irreg1'])['b']
def test_serialize(self): """Confirm full ranges can be saved and loaded.""" w = core.G3Writer(test_filename) for isize, val in bit_sizes: f = core.G3Frame() f['v'] = core.G3VectorInt([val] * 10) w(f) del w r = core.G3Reader(test_filename) for isize, val in bit_sizes: f = r(None)[0] v_in = list(f['v']) self.assertTrue(all([_v == val for _v in v_in]), "Failed to save/load value %i" % val) del r
def start_stream(self, session, params=None): if params is None: params = {} time_per_file = params.get("time_per_file", 60*60) # [sec] data_dir = params.get("data_dir", "data/") self.log.info("Writing data to {}".format(data_dir)) self.log.info("New file every {} seconds".format(time_per_file)) reader = core.G3Reader("tcp://localhost:{}".format(self.port)) writer = None last_meta = None self.is_streaming = True while self.is_streaming: if writer is None: start_time = datetime.utcnow() ts = start_time.timestamp() subdir = os.path.join(data_dir, "{:.5}".format(str(ts))) if not os.path.exists(subdir): os.makedirs(subdir) filename = start_time.strftime("%Y-%m-%d-%H-%M-%S.g3") filepath = os.path.join(subdir, filename) writer = core.G3Writer(filename=filepath) if last_meta is not None: writer(last_meta) frames = reader.Process(None) for f in frames: if f.type == core.G3FrameType.Housekeeping: last_meta = f writer(f) if (datetime.utcnow().timestamp() - ts) > time_per_file: writer(core.G3Frame(core.G3FrameType.EndProcessing)) writer = None if writer is not None: writer(core.G3Frame(core.G3FrameType.EndProcessing)) return True, "Finished Streaming"
def _establish_reader_connection(self, timeout=5): """Establish the connection to the G3NetworkSender. Attempts to connect once and waits if connection could not be made and last connection attempt was very recent. Parameters ---------- timeout : int Timeout in seconds for the G3Reader, afterwhich the connection will drop and G3Reader will return empty lists on each Process call. Returns ------- reader : spt3g.core.G3Reader The G3Reader object connected to the configured address and port """ reader = None try: reader = core.G3Reader(self.address, timeout=timeout) self.log.debug("G3Reader connection to {addr} established!", addr=self.address) except RuntimeError: self.log.error("G3Reader could not connect.") # Prevent rapid connection attempts if self.last_connection_time is not None: t_diff = time.time() - self.last_connection_time if t_diff < 1: self.log.debug("Last connection was only {d} seconds ago. " + "Sleeping for {t}.", d=t_diff, t=(1 - t_diff)) time.sleep(1 - t_diff) self.last_connection_time = time.time() return reader
def _readback_compare(self, ts, filename='readback_test.g3', cleanup=True, err_msg='(no detail)'): """Cache the data from ts, write ts to a file, read it back from file, compare to cached data. """ # Cache fake_ts = (collections.namedtuple( 'pseudo_ts', ['times', 'names', 'data']))(np.array(ts.times), np.array(ts.names), ts.data.copy()) # Write f = core.G3Frame() f['item'] = ts core.G3Writer(filename).Process(f) # Read ts1 = core.G3Reader(filename).Process(None)[0]['item'] self._check_equal(fake_ts, ts1, err_msg=err_msg) if cleanup: os.remove(filename)
def read_spt3g_obs(file): reader = c3g.G3Reader(file, 1) f = list(reader(None)) obframe = f[0] obspat = re.compile("^obs_(.*)") detpat = re.compile("{}_(.*)".format(STR_QUAT)) obs = dict() props = dict() dets = dict() nsamp = None for k in obframe.keys(): if k == "samples": nsamp = int(obframe[k]) else: obsmat = obspat.match(k) detmat = detpat.match(k) if obsmat is not None: obs[obsmat.group(1)] = s3utils.from_g3_type(obframe[k]) elif detmat is not None: dets[detmat.group(1)] = np.array(obframe[k], dtype=np.float64) else: props[k] = s3utils.from_g3_type(obframe[k]) return obs, props, dets, nsamp
def test_40_encoding_serialized(self): test_file = 'test_g3super.g3' offsets = { 'int32': [0, 2**25, 2**26 / 3., -1.78 * 2**27], 'int64': [0, 2**25, 2**26 / 3., -1.78 * 2**27], 'float32': [0], 'float64': [0, 2**25, 2**26 / 3., -1.78 * 2**27, 1.8 * 2**35], } decimals = 2 precision = 10**-decimals w = core.G3Writer(test_file) records = [] for dtype in ALL_DTYPES: for offset in offsets[dtype]: f = core.G3Frame() ts = self._get_ts(4, 100, sigma=100, dtype=dtype) ts.data += int(offset) if dtype in FLOAT_DTYPES: ts.data[:] = np.round(ts.data) ts.calibrate([.01] * ts.data.shape[0]) records.append(ts.data.copy()) f['a'] = ts w.Process(f) del w # readback r = core.G3Reader(test_file) for dtype in ALL_DTYPES: for offset in offsets[dtype]: err_msg = f'Failed for dtype={dtype}, offset={offset}' ts2 = r.Process(None)[0]['a'] record = records.pop(0) np.testing.assert_allclose(record, ts2.data, atol=precision * 1e-3, err_msg=err_msg)
for k, v in b.data.items(): if len(v) != len(b.t): core.log_error( 'Field "%s" has %i samples but .t has %i samples.' % (k, len(v), len(b.t))) self.stats['concerns']['n_error'] += 1 if len(t_check) and abs(min(t_check) - t_this) > 60: core.log_warn( 'data frame timestamp (%.1f) does not correspond to ' 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') self.stats['concerns']['n_warning'] += 1 return [f] if __name__ == '__main__': # Run me on a G3File containing a Housekeeping stream. core.set_log_level(core.G3LogLevel.LOG_INFO) import sys for f in sys.argv[1:]: p = core.G3Pipeline() p.Add(core.G3Reader(f)) p.Add(HKScanner()) p.Run()
f['Vec'] = v v = core.G3VectorInt([17, 42, 87]) f['VecInt'] = v m = core.G3MapDouble() m['Six'] = 6 m['GoingOnSixteen'] = 15.9 f['Map'] = m if len(sys.argv) > 1: core.G3Writer(sys.argv[1])(f) sys.exit(0) # For now, we test files from big-endian (PPC64) and little-endian (amd64) # 64-bit systems. Should include some 32-bit ones. for test in ['test-be.g3', 'test-le.g3']: print(test) testdata = core.G3Reader(os.path.join(testpath, test))(None)[0] assert (testdata['Five'] == f['Five']) assert (len(testdata['Vec']) == len(f['Vec'])) for i in range(len(testdata['Vec'])): assert (testdata['Vec'][i] == f['Vec'][i]) assert (len(testdata['VecInt']) == len(f['VecInt'])) for i in range(len(testdata['VecInt'])): assert (testdata['VecInt'][i] == f['VecInt'][i]) assert (len(testdata['Map']) == len(f['Map'])) assert (testdata['Map'].keys() == f['Map'].keys()) for i in testdata['Map'].keys(): assert (testdata['Map'][i] == f['Map'][i])
def test_00_basic(self): """Write a stream of HK frames and scan it for errors.""" # Write a stream of HK frames. # (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.) print('Streaming to %s' % test_file) w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=None, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider( description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. w.Process(hksess.session_frame()) w.Process(hksess.status_frame()) # Add a bunch of data frames t_next = time.time() for i in range(10): f = hksess.data_frame(prov_id=prov_id, timestamp=t_next) hk = so3g.IrregBlockDouble() hk.prefix = 'hwp_' hk.data['position'] = [1, 2, 3, 4, 5] hk.data['speed'] = [1.2, 1.2, 1.3, 1.2, 1.3] hk.t = t_next + np.arange(len(hk.data['speed'])) t_next += len(hk.data['speed']) f['blocks'].append(hk) w.Process(f) w.Flush() del w print('Stream closed.\n\n') # Now play them back... print('Reading back:') for f in core.G3File(test_file): ht = f.get('hkagg_type') if ht == so3g.HKFrameType.session: print('Session: %i' % f['session_id']) elif ht == so3g.HKFrameType.status: print(' Status update: %i providers' % (len(f['providers']))) elif ht == so3g.HKFrameType.data: print(' Data: %i blocks' % len(f['blocks'])) for block in f['blocks']: for k, v in block.data.items(): print(' %s%s' % (block.prefix, k), v) # Scan and validate. print() print('Running HKScanner on the test data...') scanner = so3g.hk.HKScanner() pipe = core.G3Pipeline() pipe.Add(core.G3Reader(test_file)) pipe.Add(scanner) pipe.Run() print('Stats: ', scanner.stats) print('Providers: ', scanner.providers) self.assertEqual(scanner.stats['concerns']['n_error'], 0) self.assertEqual(scanner.stats['concerns']['n_warning'], 0)
print('Parent') send = core.G3NetworkSender(hostname='*', port=port) time.sleep(1) # XXX: how to signal that the remote end is ready? print('Sending') for f in frames: send(f) send(core.G3Frame(core.G3FrameType.EndProcessing)) pid, status = os.wait() print('Child Status: ', status) if status == 0: print('OK') sys.exit(status) else: # Child print('Child') recv = core.G3Reader(filename='tcp://localhost:%d' % port) rframes = [] for k in range(len(frames)): chunk = recv(None) print(chunk[0]) rframes += chunk assert (len(rframes)) == len(frames) for i in range(len(rframes)): assert (rframes[i]['Sequence'] == i) sys.exit(0)
return [f] def __call__(self, *args, **kwargs): return self.Process(*args, **kwargs) if __name__ == '__main__': import argparse parser = argparse.ArgumentParser( usage='This program can be used to convert SO HK Frames to the ' 'latest schema version.') parser.add_argument('--output-file', '-o', default='out.g3') parser.add_argument('--target-version', type=int) parser.add_argument('files', nargs='+', help= "SO Housekeeping files to convert.") args = parser.parse_args() # Run me on a G3File containing a Housekeeping stream. core.set_log_level(core.G3LogLevel.LOG_INFO) translator_args = {} if args.target_version is not None: translator_args['target_version'] = args.target_version print(f'Streaming to {args.output_file}') p = core.G3Pipeline() p.Add(core.G3Reader(args.files)) p.Add(HKTranslator(**translator_args)) p.Add(core.G3Writer(args.output_file)) p.Run()
def read(self, session, params=None): """read(src='tcp://localhost:4532') **Process** - Process for reading in G3Frames from a source or list of sources. If this source is an address that begins with ``tcp://``, the agent will attempt to connect to a G3NetworkSender at the specified location. The ``src`` param can also be a filepath or list of filepaths pointing to G3Files to be streamed. If a list of filenames is passed, once the first file is finished streaming, subsequent files will be streamed. """ self._running = True session.set_status('running') src_idx = 0 if isinstance(params['src'], str): sources = [params['src']] else: sources = params['src'] reader = None source = None source_offset = 0 while self._running: if reader is None: try: source = sources[src_idx] source_is_file = not source.startswith('tcp://') reader = core.G3Reader(source, timeout=5) except RuntimeError as e: if source_is_file: # Raise error if file cannot be found raise e else: # If not a file, log error and try again self.log.error( "G3Reader could not connect! Retrying in 10 sec.") time.sleep(10) continue frames = reader.Process(None) if not frames: # If source is a file, start over with next file or break if # finished all sources. If socket, just reset reader and try to # reconnect if source_is_file: src_idx += 1 if src_idx >= len(sources): self.log.info("Finished reading all sources") break reader = None continue frame = frames[0] # If this source is a file, this will shift the timestamps so that # data lines up with the current timestamp instead of using the # timestamps in the file if source_is_file and (not source_offset): source_offset = frame['time'].time / core.G3Units.s \ - time.time() elif not source_is_file: source_offset = 0 if frame.type == core.G3FrameType.Wiring: self._process_status(frame) continue elif frame.type == core.G3FrameType.Scan: out = self._process_data(frame, source_offset=source_offset) else: continue for f in out: # This will block until there's a free spot in the queue. # This is useful if the src is a file and reader.Process does # not block self.out_queue.put(f) return True, "Stopped read process"
if changes: output.append(f) elif f['hkagg_type'] == so3g.HKFrameType.data: fb = self.providers[f['prov_id']] fb.add(f) if fb.ready(): output += fb.rebundle() else: raise ValueError('Invalid hkagg_type') return output if __name__ == '__main__': from so3g.hk import HKScanner, HKReframer import sys core.set_log_level(core.G3LogLevel.LOG_INFO) files = sys.argv[1:] p = core.G3Pipeline() p.Add(core.G3Reader(files)) p.Add(HKScanner()) rf = HKReframer() p.Add(rf) p.Add(HKScanner()) p.Add(core.G3Writer, filename='out.g3') p.Run()
def test_00_basic(self): """Write a stream of HK frames and scan it for errors.""" # Write a stream of HK frames. # (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.) print('Streaming to %s' % test_file) w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=None, hkagg_version=2, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider( description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. w.Process(hksess.session_frame()) w.Process(hksess.status_frame()) # Add a bunch of data frames t_next = time.time() for i in range(10): f = hksess.data_frame(prov_id=prov_id, timestamp=t_next) hk = core.G3TimesampleMap() speed = [1.2, 1.2, 1.3, 1.2, 1.3] hk.times = [ core.G3Time(_t * core.G3Units.second) for _t in t_next + np.arange(len(speed)) ] hk['position'] = core.G3VectorDouble(np.arange(len(speed))) hk['speed'] = core.G3VectorDouble(speed) hk['error_bits'] = core.G3VectorInt([10] * len(speed)) hk['mode_str'] = core.G3VectorString(['ok'] * len(speed)) t_next += len(hk) f['blocks'].append(hk) f['block_names'].append('main_block') w.Process(f) w.Flush() del w print('Stream closed.\n\n') # Now play them back... print('Reading back:') for f in core.G3File(test_file): ht = f.get('hkagg_type') if ht == so3g.HKFrameType.session: print('Session: %i' % f['session_id']) elif ht == so3g.HKFrameType.status: print(' Status update: %i providers' % (len(f['providers']))) elif ht == so3g.HKFrameType.data: print(' Data: %i blocks' % len(f['blocks'])) for i, block in enumerate(f['blocks']): print(' Block %i' % i) for k, v in block.items(): print(' %s' % k, v) # Scan and validate. print() print('Running HKScanner on the test data...') scanner = so3g.hk.HKScanner() pipe = core.G3Pipeline() pipe.Add(core.G3Reader(test_file)) pipe.Add(scanner) pipe.Run() print('Stats: ', scanner.stats) print('Providers: ', scanner.providers) self.assertEqual(scanner.stats['concerns']['n_error'], 0) self.assertEqual(scanner.stats['concerns']['n_warning'], 0)