def create_focal_plane(n_det): # Generate a simple squarish grid. i, p = np.arange(n_det) // 2, np.arange(n_det) % 2 side = max(2, int(i[-1]**.5)) row, col = i // side, i % side pol_fam = (row + col) % 2 pol = (pol_fam * 45 + p * 90) * core.G3Units.deg x = (col / (side - 1) - .5) * 1. * core.G3Units.deg y = (row / (side - 1) - .5) * 1. * core.G3Units.deg # Convert to quternions in the prescribed way. phi = np.arctan2(y, x) theta = np.arcsin((x**2 + y**2)**.5 / core.G3Units.rad) q = (coords.q_euler(2, phi) * coords.q_euler(1, theta) * coords.q_euler(2, -phi) * coords.q_euler(2, pol / core.G3Units.rad)) f = core.G3Frame(FT.Calibration) #booo f['cal_type'] = 'focal_plane' # For now just store a vector of detector names, then a vector of # boresight-relative quaternion rotations for corresponding dets. f['signal_q'] = q f['signal_x'] = core.G3VectorDouble(x) f['signal_y'] = core.G3VectorDouble(y) f['signal_theta'] = core.G3VectorDouble(theta) f['signal_phi'] = core.G3VectorDouble(phi) f['signal_pol'] = core.G3VectorDouble(pol) f['signal_names'] = core.G3VectorString() for j in range(n_det): f['signal_names'].append('det%04i%s' % (i[j], {0: 'A', 1: 'B'}[p[j]])) return f
def stream_fake_data(self, session, params=None): """stream_fake_data() **Process** - Process for streaming fake data. This will queue up G3Frames full of fake data to be sent to lyrebird. """ self._run_fake_stream = True ndets = self.fp.num_dets chans = np.arange(ndets) frame_start = time.time() while self._run_fake_stream: time.sleep(2) frame_stop = time.time() ts = np.arange(frame_start, frame_stop, 1. / self.target_rate) frame_start = frame_stop nframes = len(ts) data_out = np.random.normal(0, 1, (nframes, ndets)) data_out += np.sin(2 * np.pi * ts[:, None] + .2 * chans[None, :]) for t, d in zip(ts, data_out): fr = core.G3Frame(core.G3FrameType.Scan) fr['idx'] = 0 fr['data'] = core.G3VectorDouble(d) fr['timestamp'] = core.G3Time(t * core.G3Units.s) self.out_queue.put(fr) fr = core.G3Frame(core.G3FrameType.Scan) fr['idx'] = 1 fr['data'] = core.G3VectorDouble(np.sin(d)) fr['timestamp'] = core.G3Time(t * core.G3Units.s) self.out_queue.put(fr) return True, "Stopped fake stream process"
def CalibrateValue(data, caldict_entry): '''Apply gain / offset units from G3 cal file to register''' uvalue = UnitValue(caldict_entry) g3type = type(data) # make a copy if np.size(data) == 1: data = data.value data2 = np.array(data, dtype='float64') thisdtype = data2.dtype # calibrate units data2 += np.array(caldict_entry['Offset'], dtype=thisdtype) data2 *= np.array(uvalue / caldict_entry['ReciprocalFactor'], dtype=thisdtype) if not data2.shape: data2 = data2.tolist() # if a register has units, it can't be an int anymore. well, actually, # it can't be an int if we're adding floats to it or multiplying it by # floats either, so convert everything that has an entry in the cal file # to float/double. if g3type == core.G3VectorInt: return core.G3VectorDouble(data2) elif g3type == core.G3MapInt: return core.G3MapDouble(data2) elif g3type == core.G3Int: return core.G3Double(data2) else: return g3type(data2)
def config_frame(self): """ Generates a config frame for lyrebird """ frame = core.G3Frame(core.G3FrameType.Wiring) frame['x'] = core.G3VectorDouble(self.xs) frame['y'] = core.G3VectorDouble(self.ys) frame['cname'] = core.G3VectorString(self.cnames) frame['rotation'] = core.G3VectorDouble(self.rots) frame['templates'] = core.G3VectorString(self.templates) frame['values'] = core.G3VectorString(self.value_names) frame['color_is_dynamic'] = core.G3VectorBool(self.eq_color_is_dynamic) frame['equations'] = core.G3VectorString(self.eqs) frame['eq_labels'] = core.G3VectorString(self.eq_labels) frame['cmaps'] = core.G3VectorString(self.cmaps) return frame
def _write_precal(self, writer, dets, noise): """Write the calibration frame at the start of an observation. This frame nominally contains "preliminary" values for the detectors. For simulations, this contains the true detector offsets and noise properties. """ qname = "detector_offset" f = core3g.G3Frame(core3g.G3FrameType.Calibration) # Add a vector map for quaternions f[qname] = core3g.G3MapVectorDouble() for k, v in dets.items(): f[qname][k] = core3g.G3VectorDouble(v) if noise is not None: kfreq = "noise_stream_freq" kpsd = "noise_stream_psd" kindx = "noise_stream_index" dstr = "noise_detector_streams" dwt = "noise_detector_weights" f[kfreq] = core3g.G3MapVectorDouble() f[kpsd] = core3g.G3MapVectorDouble() f[kindx] = core3g.G3MapInt() f[dstr] = core3g.G3MapVectorInt() f[dwt] = core3g.G3MapVectorDouble() nse_dets = list(noise.detectors) nse_keys = list(noise.keys) st = dict() wts = dict() for d in nse_dets: st[d] = list() wts[d] = list() for k in nse_keys: f[kfreq][k] = core3g.G3VectorDouble(noise.freq(k).tolist()) f[kpsd][k] = core3g.G3VectorDouble(noise.psd(k).tolist()) f[kindx][k] = int(noise.index(k)) for d in nse_dets: wt = noise.weight(d, k) if wt > 0: st[d].append(noise.index(k)) wts[d].append(wt) for d in nse_dets: f[dstr][d] = core3g.G3VectorInt(st[d]) f[dwt][d] = core3g.G3VectorDouble(wts[d]) writer(f) return
def write_spt3g_obs(writer, props, dets, nsamp): f = c3g.G3Frame(c3g.G3FrameType.Observation) for k, v in props.items(): f[k] = s3utils.to_g3_type(v) f["samples"] = c3g.G3Int(nsamp) for k, v in dets.items(): f["{}_{}".format(STR_QUAT, k)] = c3g.G3VectorDouble(v) writer(f) return
def _generate_g3_dicts(self): # Dictionaries of double vectors that will hold CHWP data # Data must be type casted from doubles to uint32_t's later self.encoder_count = core.G3VectorUInt() self.encoder_clock = core.G3VectorUInt() self.encoder_quad = core.G3VectorUInt() # Dictionaries of double vectors that will hold the IRIG data self.irig_time = core.G3VectorDouble() self.irig_clock = core.G3VectorUInt() # self.irig_synch = core.G3VectorDouble() return
def __call__(self, frame): if (frame.type == core.G3FrameType.Timepoint and ('chwp_encoder_clock' not in frame.keys() and 'chwp_irig_clock' not in frame.keys())): return [frame] elif (frame.type == core.G3FrameType.Timepoint and 'chwp_encoder_clock' in frame.keys()): self._encd_quad.append(frame['chwp_encoder_quad']) self._encd_clk.append(frame['chwp_encoder_clock']) self._encd_cnt.append(frame['chwp_encoder_count']) return [frame] elif (frame.type == core.G3FrameType.Timepoint and 'chwp_irig_clock' in frame.keys()): self._irig_clk.append(frame['chwp_irig_clock']) self._irig_tme.append(frame['chwp_irig_time']) return [frame] else: # End of scan -- process the data self._encd_quad = np.array(self._encd_quad).flatten() self._encd_clk = np.array(self._encd_clk).flatten() self._encd_cnt = np.array(self._encd_cnt).flatten() self._irig_clk = np.array(self._irig_clk).flatten() self._irig_tme = np.array(self._irig_tme).flatten() # Calculate angle vs time self._angle_time() # Return a frame with the angle and time out_frame = core.G3Frame(core.G3FrameType.Timepoint) out_frame['chwp_encoder_quad'] = core.G3VectorUInt(self._encd_quad) out_frame['chwp_encoder_clock'] = core.G3VectorUInt(self._encd_clk) out_frame['chwp_encoder_count'] = core.G3VectorUInt(self._encd_cnt) out_frame['chwp_irig_clock'] = core.G3VectorUInt(self._irig_clk) out_frame['chwp_irig_time'] = core.G3VectorUInt(self._irig_tme) out_frame['chwp_time'] = core.G3VectorDouble(self._time) out_frame['chwp_angle'] = core.G3VectorDouble(self._angle) out_frame['chwp_dropped_packets'] = core.G3UInt( int(self._num_dropped_pkts)) return [out_frame, core.G3Frame(core.G3FrameType.EndProcessing)]
def Process(self, frame): if (self.num_frames == 0): return [] self.num_frames -= 1 frame = core.G3Frame(core.G3FrameType.Map) if self.test == 0: frame['ra'] = core.G3VectorDouble([0]) frame['dec'] = core.G3VectorDouble([90 * G3Units.deg]) frame['pol'] = core.G3VectorDouble([0]) else: frame['ra'] = core.G3VectorDouble([15 * G3Units.deg]) frame['dec'] = core.G3VectorDouble([45 * G3Units.deg]) frame['pol'] = core.G3VectorDouble([0]) return frame
def get_v2_stream(): """Generate some example HK data, in schema version 2. Returns a list of frames constituting a valid version 2 HK stream. """ # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=1234, hkagg_version=2, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider( description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. frames = [ hksess.session_frame(), hksess.status_frame(), ] # Now make a data frame. f = hksess.data_frame(prov_id=prov_id) # Add some data blocks. hk = core.G3TimesampleMap() hk.times = core.G3VectorTime([core.G3Time(i*core.G3Units.seconds) for i in [0, 1, 2, 3, 4]]) hk['speed'] = core.G3VectorDouble([1.2, 1.2, 1.2, 1.2, 1.2]) f['blocks'].append(hk) f['block_names'].append('group0') hk = core.G3TimesampleMap() hk.times = core.G3VectorTime([core.G3Time(i*core.G3Units.seconds) for i in [0, 1, 2, 3, 4]]) hk['position'] = core.G3VectorInt([1, 2, 3, 4, 5]) hk['mode'] = core.G3VectorString(['going', 'going', 'going', 'going', 'gone/']) f['blocks'].append(hk) f['block_names'].append('group1') frames.append(f) return frames
def __call__(self, frame): if frame.type == core.G3FrameType.Map and 'Id' in frame and \ fnmatch(frame['Id'], args.mapid) and 'ScanNumber' in frame: if frame['ScanNumber'] % 2 == 0: self.signs.append(rng.choice([-1., 1.])) else: self.signs.append(-1 * self.signs[-1]) flipped_frame = core.G3Frame(core.G3FrameType.Map) for key in frame: if key in ['T', 'Q', 'U']: flipped_frame[key] = float(self.signs[-1] + self.offset) * frame[key] else: flipped_frame[key] = frame[key] del frame return flipped_frame if frame.type == core.G3FrameType.EndProcessing: output_frame = core.G3Frame() output_frame['signs'] = core.G3VectorDouble(self.signs) return [output_frame, frame]
def test_00_basic(self): """Write a stream of HK frames and scan it for errors.""" # Write a stream of HK frames. # (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.) print('Streaming to %s' % test_file) w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=None, hkagg_version=2, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider( description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. w.Process(hksess.session_frame()) w.Process(hksess.status_frame()) # Add a bunch of data frames t_next = time.time() for i in range(10): f = hksess.data_frame(prov_id=prov_id, timestamp=t_next) hk = core.G3TimesampleMap() speed = [1.2, 1.2, 1.3, 1.2, 1.3] hk.times = [ core.G3Time(_t * core.G3Units.second) for _t in t_next + np.arange(len(speed)) ] hk['position'] = core.G3VectorDouble(np.arange(len(speed))) hk['speed'] = core.G3VectorDouble(speed) hk['error_bits'] = core.G3VectorInt([10] * len(speed)) hk['mode_str'] = core.G3VectorString(['ok'] * len(speed)) t_next += len(hk) f['blocks'].append(hk) f['block_names'].append('main_block') w.Process(f) w.Flush() del w print('Stream closed.\n\n') # Now play them back... print('Reading back:') for f in core.G3File(test_file): ht = f.get('hkagg_type') if ht == so3g.HKFrameType.session: print('Session: %i' % f['session_id']) elif ht == so3g.HKFrameType.status: print(' Status update: %i providers' % (len(f['providers']))) elif ht == so3g.HKFrameType.data: print(' Data: %i blocks' % len(f['blocks'])) for i, block in enumerate(f['blocks']): print(' Block %i' % i) for k, v in block.items(): print(' %s' % k, v) # Scan and validate. print() print('Running HKScanner on the test data...') scanner = so3g.hk.HKScanner() pipe = core.G3Pipeline() pipe.Add(core.G3Reader(test_file)) pipe.Add(scanner) pipe.Run() print('Stats: ', scanner.stats) print('Providers: ', scanner.providers) self.assertEqual(scanner.stats['concerns']['n_error'], 0) self.assertEqual(scanner.stats['concerns']['n_warning'], 0)
def CalibrateFrame(f, calibration_file=None): '''Apply gain / offset / units from G3 cal file''' if f.type != core.G3FrameType.GcpSlow: return try: if f['Calibrated'] == True: print('Already calibrated!\n') return except KeyError: f['Calibrated'] = True cf = CalFile.CalFileReader() cd = cf.readCalFile(calibration_file) for board in f.keys(): if board == 'Calibrated': continue cboard = copy.deepcopy(f[board]) for rmap in cboard.keys(): for reg in cboard[rmap].keys(): try: rcd = cd[board][rmap][reg] except KeyError: continue rsize = numpy.size(cboard[rmap][reg]) if rsize > 1: rshape = numpy.shape(cboard[rmap][reg]) if len(rshape) > 1: for i in range(rshape[0]): try: rcdi = rcd[i] except KeyError: rcdi = rcd uvalue = UnitValue(rcdi) datatemp = numpy.asarray(cboard[rmap][reg][i]) datatemp2 = datatemp.copy() # if a register has units, it can't be an # int anymore. # well, actually, it can't be an int if # we're adding floats to it or multiplying # it by floats either, so convert # everything that has an entry in the cal # file to float/double. datatemp2 = numpy.asarray(datatemp2, dtype='float64') thisdtype = datatemp2.dtype datatemp2 += \ numpy.array(rcdi['Offset'],dtype=thisdtype) datatemp2 *= numpy.array(uvalue / rcdi['ReciprocalFactor'], dtype=thisdtype) if type(cboard[rmap][reg][i]) \ is core.G3VectorInt: regitemp = core.G3VectorDouble(datatemp2) elif type(cboard[rmap][reg][i]) \ is core.G3MapInt: regitemp = core.G3MapDouble(datatemp2) elif type(cboard[rmap][reg][i]) \ is core.G3Int: regitemp = core.G3Double(datatemp2) else: regitemp = \ (type(cboard[rmap][reg][i]))(datatemp2) cboard[rmap][reg][i] = regitemp else: try: rcdi = rcd[0] except KeyError: rcdi = rcd uvalue = UnitValue(rcdi) datatemp = numpy.asarray(cboard[rmap][reg]) datatemp2 = datatemp.copy() # if a register has units, it can't be an # int anymore. well, actually (see above)... datatemp2 = numpy.asarray(datatemp2, dtype='float64') thisdtype = datatemp2.dtype datatemp2 += \ numpy.array(rcdi['Offset'],dtype=thisdtype) datatemp2 *= numpy.array(uvalue / rcdi['ReciprocalFactor'], dtype=thisdtype) if type(cboard[rmap][reg]) \ is core.G3VectorInt: regtemp = core.G3VectorDouble(datatemp2) elif type(cboard[rmap][reg]) \ is core.G3MapInt: regtemp = core.G3MapDouble(datatemp2) elif type(cboard[rmap][reg]) \ is core.G3Int: regtemp = core.G3Double(datatemp2) else: regtemp = \ (type(cboard[rmap][reg]))(datatemp2) cboard[rmap][reg] = regtemp else: try: rcdi = rcd[0] except KeyError: rcdi = rcd uvalue = UnitValue(rcdi) datatemp = cboard[rmap][reg].value datatemp2 = datatemp # if a register has units, it can't be an # int anymore. well, actually (see above)... datatemp2 = numpy.float(datatemp2) datatemp2 = datatemp2 + rcdi['Offset'] datatemp2 *= uvalue / rcdi['ReciprocalFactor'] if type(cboard[rmap][reg]) \ is core.G3VectorInt: regtemp = core.G3VectorDouble(datatemp2) elif type(cboard[rmap][reg]) \ is core.G3MapInt: regtemp = core.G3MapDouble(datatemp2) elif type(cboard[rmap][reg]) \ is core.G3Int: regtemp = core.G3Double(datatemp2) else: regtemp = \ (type(cboard[rmap][reg]))(datatemp2) cboard[rmap][reg] = regtemp del f[board] f[board] = cboard
def _g3(arr): return core.G3VectorDouble(arr)
for i in range(10): # Number of samples n = int(halfscan / v_az / dt) # Vector of unix timestamps t = frame_time + dt * np.arange(n) # Vector of az and el az = v_az * dt * np.arange(n) if i % 2: az = -az el = az * 0 + 50. # Construct a "block", which is a named G3TimesampleMap. block = core.G3TimesampleMap() block.times = core.G3VectorTime( [core.G3Time(_t * core.G3Units.s) for _t in t]) block['az'] = core.G3VectorDouble(az) block['el'] = core.G3VectorDouble(el) # Create an output data frame template associated with this # provider. frame = session.data_frame(prov_id) # Add the block and block name to the frame, and write it. frame['block_names'].append('pointing') frame['blocks'].append(block) writer.Process(frame) # For next iteration. frame_time += n * dt
import so3g as ss from spt3g import core ss.TestClass().runme() ss.greet() w = core.G3Writer('out.g3') f = core.G3Frame() f.type = core.G3FrameType.Scan f['testv'] = core.G3VectorDouble([ 1., 2., 3., ]) w.Process(f)
#!/usr/bin/env python from spt3g import core import os, sys # Test that we can read files written on a variety of platforms. Pass a path # to generate test data for whatever this platform is. testpath = os.path.join(os.environ['SPT3G_SOFTWARE_PATH'], 'core/tests/portability') # Test data. Exercise some complicated things (STL bits) that we don't # necessarily have control over, mapping a few primitive types. f = core.G3Frame() f['Five'] = 5 v = core.G3VectorDouble([2.6, 7.2]) f['Vec'] = v v = core.G3VectorInt([17, 42, 87]) f['VecInt'] = v m = core.G3MapDouble() m['Six'] = 6 m['GoingOnSixteen'] = 15.9 f['Map'] = m if len(sys.argv) > 1: core.G3Writer(sys.argv[1])(f) sys.exit(0) # For now, we test files from big-endian (PPC64) and little-endian (amd64) # 64-bit systems. Should include some 32-bit ones.
def _process_data(self, frame, source_offset=0): """ Processes a Scan frame. If lyrebird is enabled, this will return a seq of G3Frames that are formatted for lyrebird to ingest. """ if 'session_id' not in frame: return [] # Calculate downsample factor times_in, data_in = load_frame_data(frame) times_in = times_in - source_offset sample_rate = 1. / np.median(np.diff(times_in)) nsamps = len(times_in) nchans = len(data_in) self._process_monitored_chans(times_in, data_in) if self.avg1 is None: self.avg1 = RollingAvg(200, nchans) self.avg2 = RollingAvg(200, nchans) elif self.avg1.nchans != nchans: self.log.warn( f"Channel count has changed! {self.avg1.nchans}->{nchans}") self.avg1 = RollingAvg(200, nchans) self.avg2 = RollingAvg(200, nchans) # Calc RMS hpf_data = data_in - self.avg1.apply(data_in) # To high-pass filter rms = np.sqrt(self.avg2.apply(hpf_data**2)) # To calc rolling rms ds_factor = sample_rate // self.target_rate if np.isnan(ds_factor): # There is only one element in the timestream ds_factor = 1 ds_factor = max(int(ds_factor), 1) # Prevents downsample factors < 1 # Arrange output data structure sample_idxs = np.arange(0, nsamps, ds_factor, dtype=np.int32) num_frames = len(sample_idxs) times_out = times_in[sample_idxs] abs_chans = self.mask[np.arange(nchans)] raw_out = np.zeros((num_frames, self.fp.num_dets)) rms_out = np.zeros((num_frames, self.fp.num_dets)) for i, c in enumerate(abs_chans): if c >= len(self.fp.chan_mask): continue idx = self.fp.chan_mask[c] if idx >= 0: raw_out[:, idx] = data_in[i, sample_idxs] rms_out[:, idx] = rms[i, sample_idxs] out = [] for i in range(num_frames): fr = core.G3Frame(core.G3FrameType.Scan) fr['idx'] = 0 fr['data'] = core.G3VectorDouble(raw_out[i]) fr['timestamp'] = core.G3Time(times_out[i] * core.G3Units.s) out.append(fr) fr = core.G3Frame(core.G3FrameType.Scan) fr['idx'] = 1 fr['data'] = core.G3VectorDouble(rms_out[i]) fr['timestamp'] = core.G3Time(times_out[i] * core.G3Units.s) out.append(fr) return out
def Process(self, f): """Translates one frame to the target schema. Irrelevant frames are passed through unmodified. Args: f: a G3Frame Returns: A list containing only the translated frame. G3Pipeline compatibility would permit us to return a single frame here, instead of a length-1 list. But we also sometimes call Process outside of a G3Pipeline, where a consistent output type is desirable. Returning lists is most future-compatible; consumers that want to assume length-1 should assert it to be true. """ if f.type == core.G3FrameType.EndProcessing: core.log_info(str(self.stats)) return [f] if f.type != core.G3FrameType.Housekeeping: self.stats['n_other'] += 1 return [f] # It is an HK frame. orig_version = f.get('hkagg_version', 0) self.stats['n_hk'] += 1 self.stats['versions'][orig_version] = self.stats['versions'].get(orig_version, 0) + 1 if orig_version > self.target_version and not self.future_tolerant: raise ValueError( ('Translator to v%i encountered v%i, but future_tolerant=False.') % (self.TARGET_VERSION, orig_version)) if orig_version >= self.target_version: return [f] # Always update the version, even if that's our only change... if 'hkagg_version' in f: if 'hkagg_version_orig' not in f: f['hkagg_version_orig'] = orig_version del f['hkagg_version'] f['hkagg_version'] = self.target_version # No difference in Session/Status for v0, v1, v2. if f.get('hkagg_type') != so3g.HKFrameType.data: return [f] if self.target_version == 0: return [f] if orig_version == 0: # Pop the data blocks out of the frame. orig_blocks = f.pop('blocks') f['blocks'] = core.G3VectorFrameObject() # Now process the data blocks. for block in orig_blocks: new_block = core.G3TimesampleMap() new_block.times = so3g.hk.util.get_g3_time(block.t) for k in block.data.keys(): v = block.data[k] new_block[k] = core.G3VectorDouble(v) f['blocks'].append(new_block) if self.target_version == 1: return [f] if orig_version <= 1: # Add 'block_names'. Since we don't want to start # caching Block Stream information, just compute a good # block name based on the alphabetically first field in # the block. block_names = [] for block in f['blocks']: field_names = list(sorted(block.keys())) block_names.append('block_for_%s' % field_names[0]) assert(len(block_names[-1]) < 256) # What have you done. orig_block_names = [] f['block_names'] = core.G3VectorString(block_names) return [f]