def stream_fake_data(self, session, params=None): """stream_fake_data() **Process** - Process for streaming fake data. This will queue up G3Frames full of fake data to be sent to lyrebird. """ self._run_fake_stream = True ndets = self.fp.num_dets chans = np.arange(ndets) frame_start = time.time() while self._run_fake_stream: time.sleep(2) frame_stop = time.time() ts = np.arange(frame_start, frame_stop, 1. / self.target_rate) frame_start = frame_stop nframes = len(ts) data_out = np.random.normal(0, 1, (nframes, ndets)) data_out += np.sin(2 * np.pi * ts[:, None] + .2 * chans[None, :]) for t, d in zip(ts, data_out): fr = core.G3Frame(core.G3FrameType.Scan) fr['idx'] = 0 fr['data'] = core.G3VectorDouble(d) fr['timestamp'] = core.G3Time(t * core.G3Units.s) self.out_queue.put(fr) fr = core.G3Frame(core.G3FrameType.Scan) fr['idx'] = 1 fr['data'] = core.G3VectorDouble(np.sin(d)) fr['timestamp'] = core.G3Time(t * core.G3Units.s) self.out_queue.put(fr) return True, "Stopped fake stream process"
def _send_cleanse_flowcontrol_frame(self): """Send CLEANSE flowcontrol frames.""" if self.writer is not None: self.log.info("Sending CLEANSE flowcontrol frame") f = core.G3Frame(core.G3FrameType.Observation) f['sostream_flowcontrol'] = FlowControl.CLEANSE.value self.writer.Process(f) f = core.G3Frame(core.G3FrameType.Wiring) f['sostream_flowcontrol'] = FlowControl.CLEANSE.value self.writer.Process(f)
def start_stream(self, session, params=None): if params is None: params = {} delay = params.get('delay', 1) ts_len = params.get('ts_len', 100) # Writes status frame f = core.G3Frame(core.G3FrameType.Housekeeping) f['session_id'] = 0 f['start_time'] = time.time() self.writer.Process(f) self.is_streaming = True frame_num = 0 while self.is_streaming: f = core.G3Frame(core.G3FrameType.Scan) t1 = time.time() t0 = t1 - delay ts = np.arange(t0, t1, 1 / self.freq) f['session_id'] = 0 f['frame_num'] = frame_num f['data'] = core.G3TimestreamMap() for k, c in self.channels.items(): fparams = copy.copy(c) bg = np.random.normal(0, fparams.get('stdev', 0), len(ts)) if fparams['type'] == 'const': xs = bg + fparams['val'] elif fparams['type'] in ['lin', 'linear']: xs = bg + ts * fparams['slope'] + fparams.get('offset', 0) # Wraps from -pi to pi xs = np.mod(xs + np.pi, 2 * np.pi) - np.pi f['data'][k] = core.G3Timestream(xs) f['data'][k].start = core.G3Time(t0 * core.G3Units.sec) f['data'][k].stop = core.G3Time(t1 * core.G3Units.sec) self.log.info("Writing G3 Frame") self.writer.Process(f) frame_num += 1 time.sleep(delay) print("Writing EndProcessingFrame") f = core.G3Frame(core.G3FrameType.EndProcessing) self.writer.Process(f) return True, "Finished streaming"
def _send_start_flowcontrol_frame(self): """Send START flowcontrol frame.""" if self.writer is not None: self.log.info("Sending START flowcontrol frame") f = core.G3Frame(core.G3FrameType.none) f['sostream_flowcontrol'] = FlowControl.START.value self.writer.Process(f) # Send new Observation frame at start of observation f = core.G3Frame(core.G3FrameType.Observation) f['session_id'] = 0 f['start_time'] = time.time() self.writer.Process(f)
def start_stream(self, session, params=None): """ Task to stream fake detector data as G3Frames Args: frame_rate (float, optional): Frequency [Hz] at which G3Frames are sent over the network. Defaults to 1 frame pers sec. sample_rate (float, optional): Sample rate [Hz] for each channel. Defaults to 10 Hz. """ if params is None: params = {} frame_rate = params.get('frame_rate', 1.) sample_rate = params.get('sample_rate', 10.) f = core.G3Frame(core.G3FrameType.Observation) f['session_id'] = 0 f['start_time'] = time.time() self.writer.Process(f) frame_num = 0 self.is_streaming = True while self.is_streaming: frame_start = time.time() time.sleep(1. / frame_rate) frame_stop = time.time() times = np.arange(frame_start, frame_stop, 1. / sample_rate) f = core.G3Frame(core.G3FrameType.Scan) f['session_id'] = 0 f['frame_num'] = frame_num f['data'] = core.G3TimestreamMap() for i, chan in enumerate(self.channels): ts = core.G3Timestream([chan.read(t) for t in times]) ts.start = core.G3Time(frame_start * core.G3Units.sec) ts.stop = core.G3Time(frame_stop * core.G3Units.sec) f['data'][str(i)] = ts self.writer.Process(f) self.log.info("Writing frame...") frame_num += 1 return True, "Finished streaming"
def test_50_compression(self): test_file = 'test_g3super.g3' # Entropy? sigma_bits = 8 sigma = 2**sigma_bits _get_ts = lambda dtype: self._get_ts( 100, 10000, sigma=sigma, dtype=dtype, seed=12345) w = core.G3Writer(test_file) sizes = {d: [] for d in ALL_DTYPES} for dtype in ALL_DTYPES: # No compression f = core.G3Frame() ts = _get_ts(dtype) sizes[dtype].append(ts.data.nbytes) ts.options(enable=0) f['ts_%s' % dtype] = ts w.Process(f) # Yes compression f = core.G3Frame() ts = _get_ts(dtype) f['ts_%s' % dtype] = ts w.Process(f) del w # Readback r = so3g.G3IndexedReader(test_file) last = 0 for dtype in ALL_DTYPES: for i in range(2): r.Process(None)[0] here = r.Tell() sizes[dtype].append(here - last) last = here # Process the results... for dtype in ALL_DTYPES: err_msg = f'Failed for dtype={dtype}' n, s_uncomp, s_comp = sizes[dtype] comp_ratio = 1. - (s_uncomp - s_comp) / n # But really what matters is the bits-per-word, compressed. bits_per_word = comp_ratio * 8 * np.dtype(dtype).itemsize #print(dtype, bits_per_word / sigma_bits) # I think the theoretical limit is 1.3 or so... self.assertLess(bits_per_word, sigma_bits * 1.4, err_msg)
def __call__(self, frame): if self.ran: return frame from pydfmux.core import dfmux as pydfmux hwmf = core.G3Frame(core.G3FrameType.Wiring) hwm = DfMuxWiringMap() for mod in self.hwm.query(pydfmux.ReadoutModule): for bolo in range(self.channels): mapping = DfMuxChannelMapping() mapping.board_ip = struct.unpack( "i", socket.inet_aton( socket.gethostbyname('iceboard' + str(mod.iceboard.serial) + '.local')))[0] mapping.board_serial = int(mod.iceboard.serial) mapping.board_slot = mod.iceboard.slot if mod.iceboard.slot else -1 mapping.crate_serial = int( mod.iceboard.crate.serial) if mod.iceboard.slot else -1 mapping.module = mod.module - 1 # pydfmux HWMs use 1-indexing of modules, while FPGA uses 0-indexing if mod.mezzanine.mezzanine == 2: mapping.module += 4 mapping.channel = bolo hwm[str(mapping)] = mapping hwmf['WiringMap'] = hwm hwmf['ReadoutSystem'] = 'ICE' self.ran = True return [hwmf, frame]
def tune_dets(self, session, params): """tune_dets() **Task** - Emulates files that might come from a general tune dets function. These are some of the files found on simons1 registered when running the following ops with a single band: 1. Find-freq 2. setup_notches 3. tracking_setup 4. short g3 stream Args ---- test_mode : bool If True, will skip any wait times associated with writing g3 files. """ # Find Freq action_time = time.time() files = [ 'amp_sweep_freq.txt', 'amp_sweep_resonance.txt', 'amp_sweep_resp.txt' ] for f in files: self._write_smurf_file(f, 'find_freq', action_time=action_time) # Setup Notches action_time = time.time() files = ['channel_assignment_b0.txt', 'tune.npy'] for f in files: self._write_smurf_file(f, 'setup_notches', action_time=action_time) # tracking setup action_time = time.time() fname = f"{int(time.time())}.dat" self._write_smurf_file(fname, 'tracking_setup', prepend_ctime=False) # Short g3 stream frame_gen = G3FrameGenerator(self.stream_id, self.sample_rate, self.nchans) fname = self._get_g3_filename(frame_gen.session_id, 0, makedirs=True) os.makedirs(os.path.dirname(fname), exist_ok=True) session.data['noise_file'] = fname writer = core.G3Writer(fname) writer(frame_gen.get_obs_frame()) writer(frame_gen.get_status_frame()) start = time.time() stop = start + 30 frame_starts = np.arange(start, stop, self.frame_len) frame_stops = frame_starts + self.frame_len for t0, t1 in zip(frame_starts, frame_stops): if not params['test_mode']: now = time.time() if now < t1: time.sleep(t1 - now) writer(frame_gen.get_data_frame(t0, t1)) writer(core.G3Frame(core.G3FrameType.EndProcessing)) return True, "Wrote tune files"
def create_focal_plane(n_det): # Generate a simple squarish grid. i, p = np.arange(n_det) // 2, np.arange(n_det) % 2 side = max(2, int(i[-1]**.5)) row, col = i // side, i % side pol_fam = (row + col) % 2 pol = (pol_fam * 45 + p * 90) * core.G3Units.deg x = (col / (side - 1) - .5) * 1. * core.G3Units.deg y = (row / (side - 1) - .5) * 1. * core.G3Units.deg # Convert to quternions in the prescribed way. phi = np.arctan2(y, x) theta = np.arcsin((x**2 + y**2)**.5 / core.G3Units.rad) q = (coords.q_euler(2, phi) * coords.q_euler(1, theta) * coords.q_euler(2, -phi) * coords.q_euler(2, pol / core.G3Units.rad)) f = core.G3Frame(FT.Calibration) #booo f['cal_type'] = 'focal_plane' # For now just store a vector of detector names, then a vector of # boresight-relative quaternion rotations for corresponding dets. f['signal_q'] = q f['signal_x'] = core.G3VectorDouble(x) f['signal_y'] = core.G3VectorDouble(y) f['signal_theta'] = core.G3VectorDouble(theta) f['signal_phi'] = core.G3VectorDouble(phi) f['signal_pol'] = core.G3VectorDouble(pol) f['signal_names'] = core.G3VectorString() for j in range(n_det): f['signal_names'].append('det%04i%s' % (i[j], {0: 'A', 1: 'B'}[p[j]])) return f
def apply(self, f): """ Applies the G3Module on an individual G3Frame or G3TimestreamMap. Likely to be useful when debugging Args: f (G3Frame or G3TimestreamMap) Returns: G3Frame: if f is a G3Frame it returns the frame after the module has been applied G3TimestreamMap: if f is a G3TimestreamMap it returns self.output """ if type(f) == core.G3Frame: if f.type != core.G3FrameType.Scan: raise ValueError("""Cannot apply {} on a {} frame""".format(self.__class__, f.type)) self.__call__(f) return f if type(f) == core.G3TimestreamMap: frame = core.G3Frame() frame.type = core.G3FrameType.Scan frame[self.input] = f self.__call__(frame) return frame[self.output] raise ValueError('apply requires a G3Frame or a G3TimestreamMap, you gave me {}'.format(type(f)))
def _find_and_fill_frames(self, fr): # This function does most of the work: # Find frames with separation that is too large, and fill the gaps # with new frames. if self.last_frame is None: self.last_frame = fr return [fr] this_dt = fr['EventHeader'].time - self.last_frame['EventHeader'].time if this_dt > 1.5 * self.dt: # We're missing at least one frame nframes = int(np.round(this_dt / self.dt) - 1) injected_dt = this_dt / (nframes + 1) injected_frs = [core.G3Frame(core.G3FrameType.Timepoint) for i in range(nframes)] for i, new_fr in enumerate(injected_frs): new_fr['EventHeader'] = self.last_frame['EventHeader'] + \ injected_dt * (i + 1) new_fr['DfMux'] = fr['DfMux'].copy() try: new_fr['CalibratorOn'] = fr['CalibratorOn'] except KeyError: pass new_fr['GarbageData'] = True self.last_frame = fr return injected_frs + [fr] self.last_frame = fr return [fr]
def __init__(self, map_ids=None, output_map_id=None, ignore_missing_weights=False): self.coadd_frame = core.G3Frame(core.G3FrameType.Map) self.coadd_frame["Id"] = output_map_id if isinstance(map_ids, str): map_ids = [map_ids] self.map_ids = map_ids self.ignore_missing_weights = ignore_missing_weights
def _write_obs(self, writer, props, detindx): """Write an observation frame. Given a dictionary of scalars, write these to an observation frame. Args: writer (G3Writer): The writer instance. props (dict): Dictionary of properties. detindx (dict): Dictionary of UIDs for each detector. Returns: None """ f = core3g.G3Frame(core3g.G3FrameType.Observation) for k, v in props.items(): if k == "detector_uid": # old indices loaded from so3g file continue f[k] = s3utils.to_g3_type(v) indx = core3g.G3MapInt() for k, v in detindx.items(): indx[k] = int(v) f["detector_uid"] = indx writer(f) return
def _send_end_flowcontrol_frame(self): """Send END flowcontrol frame.""" if self.writer is not None: self.log.info("Sending END flowcontrol frame") f = core.G3Frame(core.G3FrameType.none) f['sostream_flowcontrol'] = FlowControl.END.value self.writer.Process(f)
def __init__( self, map_id, map_stub, polarized=True, weighted=True, pol_conv=maps.MapPolConv.IAU, ): self.map_frame = core.G3Frame(core.G3FrameType.Map) self.map_frame["Id"] = map_id map_stub = map_stub.clone(False) map_stub.weighted = weighted map_stub.pol_conv = pol_conv T = map_stub.clone(False) T.pol_type = maps.MapPolType.T self.map_frame["T"] = T if polarized: Q = map_stub.clone(False) Q.pol_type = maps.MapPolType.Q self.map_frame["Q"] = Q U = map_stub.clone(False) U.pol_type = maps.MapPolType.U self.map_frame["U"] = U if weighted: W = maps.G3SkyMapWeights(map_stub, polarized) self.map_frame["Wpol" if polarized else "Wunpol"] = W
def __init__(self, map_id, maps_in, ignore_missing_weights=False): self.map_frame = core.G3Frame(core.G3FrameType.Map) self.map_frame["Id"] = map_id if isinstance(maps_in, list): for m in maps_in: if isinstance(m, maps.G3SkyMap): k = str(m.pol_type) if k not in "TQU": raise ValueError("Input map has invalid pol_type %s" % k) self.map_frame[k] = m elif isinstance(m, maps.G3SkyMapWeights): self.map_frame["Wpol" if m.polarized else "Wunpol"] = m else: raise TypeError("maps_in must be G3SkyMap or G3SkyMapWeights") elif isinstance(maps_in, dict): for k, m in maps_in.items(): if k not in ["T", "Q", "U", "Wpol", "Wunpol"]: continue self.map_frame[k] = m else: raise TypeError("maps_in must be a list or dict") ValidateMaps(self.map_frame, ignore_missing_weights=ignore_missing_weights)
def noise_scan_frames(n_frames=3, n_dets=20, input='signal', n_samps=200, samp_rate=0.005 * core.G3Units.second, t_start=core.G3Time('2020-1-1T00:00:00')): """ Generate a list of frames filled with noise data and nothing else. Args: n_frames (int): number of frames to make n_dets (int): number of detectors per frame input (str): name of G3TimestreamMap for detectors, should be some form of 'signal' n_samps (int): number of samples per detector timestream samp_rate (G3Unit.second): detector sampling rate t_start (G3Time): start time of the set of frames """ frame_list = [] for n in range(n_frames): f = core.G3Frame() f.type = core.G3FrameType.Scan tsm = core.G3TimestreamMap() z = np.zeros((n_samps, )) for d in enumerate_det_id(n_dets): tsm[d] = core.G3Timestream(z) tsm.start = t_start tsm.stop = t_start + n_samps * samp_rate tsm = MakeNoiseData().apply(tsm) f[input] = tsm t_start += n_samps * samp_rate frame_list.append(f) return frame_list
def __call__(self, frame): if self.trig: self.trig = False f = core.G3Frame(core.G3FrameType.Timepoint) f['Foo'] = 5 return f return []
def test_write_last_meta(self, frame_recorder): """If last_meta exists, write it to file.""" f = core.G3Frame(core.G3FrameType.Observation) f['session_id'] = 0 f['start_time'] = time.time() frame_recorder.last_meta = f frame_recorder.create_new_file()
def __call__(self, frame): if self.ran: return frame hwmf = core.G3Frame(core.G3FrameType.Wiring) hwmf['WiringMap'] = self.hwm hwmf['ReadoutSystem'] = 'DfMux' self.ran = True return [hwmf, frame]
def __init__(self, KeysToIgnore=['PointingOffsetX', 'PointingOffsetY']): ''' Ignores keys in the KeysToIgnore list during merging. By default, set to values written by the flux/pointing calibration that are stored to the BolometerPropertiesMap. ''' self.outframe = core.G3Frame(core.G3FrameType.Calibration) self.ignore_keys = KeysToIgnore
def __call__(self, frame): ret = [] if frame.type == core.G3FrameType.Timepoint: if self.count % self.N == 0: ret.append(core.G3Frame(core.G3FrameType.Housekeeping)) self.count += 1 ret.append(frame) return ret
def start_stream(self, session, params=None): if params is None: params = {} time_per_file = params.get("time_per_file", 60*60) # [sec] data_dir = params.get("data_dir", "data/") self.log.info("Writing data to {}".format(data_dir)) self.log.info("New file every {} seconds".format(time_per_file)) reader = core.G3Reader("tcp://localhost:{}".format(self.port)) writer = None last_meta = None self.is_streaming = True while self.is_streaming: if writer is None: start_time = datetime.utcnow() ts = start_time.timestamp() subdir = os.path.join(data_dir, "{:.5}".format(str(ts))) if not os.path.exists(subdir): os.makedirs(subdir) filename = start_time.strftime("%Y-%m-%d-%H-%M-%S.g3") filepath = os.path.join(subdir, filename) writer = core.G3Writer(filename=filepath) if last_meta is not None: writer(last_meta) frames = reader.Process(None) for f in frames: if f.type == core.G3FrameType.Housekeeping: last_meta = f writer(f) if (datetime.utcnow().timestamp() - ts) > time_per_file: writer(core.G3Frame(core.G3FrameType.EndProcessing)) writer = None if writer is not None: writer(core.G3Frame(core.G3FrameType.EndProcessing)) return True, "Finished Streaming"
def __call__(self, frame): ret = [] if (frame.type == core.G3FrameType.Timepoint and 'DfMux' in frame.keys()): if self._count % self._N == 0: ret.append(core.G3Frame(core.G3FrameType.Scan)) self._count += 1 ret.append(frame) return ret
def write_spt3g_obs(writer, props, dets, nsamp): f = c3g.G3Frame(c3g.G3FrameType.Observation) for k, v in props.items(): f[k] = s3utils.to_g3_type(v) f["samples"] = c3g.G3Int(nsamp) for k, v in dets.items(): f["{}_{}".format(STR_QUAT, k)] = c3g.G3VectorDouble(v) writer(f) return
def test_write_flowcontrol_frame(self, frame_recorder): """Flow control frames should log a warning and continue.""" f = core.G3Frame(core.G3FrameType.none) f['sostream_flowcontrol'] = FlowControl.START.value frame_recorder.frames = [f] frame_recorder.write_frames_to_file() assert frame_recorder.frames == []
def write_example_file(filename='hk_out.g3'): """Generate an example g3 file with Wiring Frame. This is based on other test file writing functions, but is unique in that it contains a wiring frame for use in testing Seek/Tell. Structure of frames in this file should be: - Housekeeping - Housekeeping - Wiring - Housekeeping - Housekeeping Args: filename (str): filename to write data to """ test_file = filename # Write a stream of HK frames. w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=1234, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider(description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. f = hksess.session_frame() w.Process(f) f = hksess.status_frame() w.Process(f) # Write dummy wiring frame f = core.G3Frame() f.type = core.G3FrameType.Wiring w.Process(f) # Now make a data frame. f = hksess.data_frame(prov_id=prov_id) # Add a data block. hk = so3g.IrregBlockDouble() hk.prefix = 'hwp_' hk.data['position'] = [1, 2, 3, 4, 5] hk.data['speed'] = [1.2, 1.2, 1.2, 1.2, 1.2] hk.t = [0, 1, 2, 3, 4] f['blocks'].append(hk) # Write two more housekeeping frames. w.Process(f) w.Process(f) del w
def addinfo(fr): if fr.type == core.G3FrameType.EndProcessing: return global n fr['time'] = core.G3Time(int(time.time() * core.G3Units.s)) fr['count'] = n fr2 = core.G3Frame(core.G3FrameType.Housekeeping) fr2['count'] = n n += 1 return [fr, fr2]
def PyDfMuxBolometerPropertiesInjector(frame, pydfmux_hwm=None, angle_per_mm = 4.186*core.G3Units.deg/1000): ''' Insert a calibration frame following any wiring frame containing a BolometerPropertiesMap named "NominalBolometerProperties" that has the properties of each bolometer as defined by the given pydfmux hardware map. ''' if frame.type != core.G3FrameType.Wiring: return from spt3g import calibration from pydfmux.core import dfmux as pydfmux cal = core.G3Frame(core.G3FrameType.Calibration) bpm = calibration.BolometerPropertiesMap() for bolo in pydfmux_hwm.query(pydfmux.Bolometer): bp = calibration.BolometerProperties() if hasattr(bolo, 'physical_name'): bp.physical_name = str(bolo.wafer.name) + '_' + str(bolo.physical_name) if hasattr(bolo, 'x_mm') and bolo.x_mm is not None: bp.x_offset = bolo.x_mm * angle_per_mm if hasattr(bolo, 'y_mm') and bolo.y_mm is not None: bp.y_offset = bolo.y_mm * angle_per_mm if bolo.wafer is not None: bp.wafer_id = str(bolo.wafer.name) if hasattr(bolo, 'pixel') and bolo.pixel is not None: bp.pixel_id = str(bolo.pixel) if hasattr(bolo, 'pixel_type') and bolo.pixel_type is not None: bp.pixel_type = str(bolo.pixel_type) if hasattr(bolo, 'observing_band') and bolo.observing_band is not None: bp.band = float(bolo.observing_band)*core.G3Units.GHz if hasattr(bolo, 'polarization_angle') and bolo.polarization_angle is not None: bp.pol_angle = float(bolo.polarization_angle)*core.G3Units.deg bp.pol_efficiency = 1.0 if hasattr(bolo, 'coupling') and bolo.coupling is not None: if bolo.coupling == "optical": bp.coupling = calibration.BolometerCouplingType.Optical elif bolo.coupling == "dark_tres": bp.coupling = calibration.BolometerCouplingType.DarkTermination elif bolo.coupling == "dark_xover": bp.coupling = calibration.BolometerCouplingType.DarkCrossover elif bolo.coupling == "resistor": bp.coupling = calibration.BolometerCouplingType.Resistor else: raise ValueError("Unrecognized coupling type %s" % bolo.coupling) bpm[str(bolo.name)] = bp cal['NominalBolometerProperties'] = bpm from pydfmux import current_transferfunction cal['DfMuxTransferFunction'] = current_transferfunction return [frame, cal]
def close_file(self): """Close a file. Clean up with EndProcessing frame, set self.writer to None, rezero suffix. """ if self.writer is not None: # Flush internal cache and clean-up (no frame written) self.writer(core.G3Frame(core.G3FrameType.EndProcessing)) self.log.debug("File closed.") self.writer = None self.filename_suffix = 0