def AddBenchData(f): ''' Add the optical bench positions to the frame. ''' if f.type != core.G3FrameType.GcpSlow: return bench_axes = ['y1', 'y2', 'y3', 'x4', 'x5', 'z6'] benchcom = core.G3TimestreamMap() benchpos = core.G3TimestreamMap() benchzero = core.G3TimestreamMap() for i, key in enumerate(bench_axes): # As of 2017-08-03, SCU time is not trustworthy # start = f['antenna0']['scu']['benchSampleTime'][0][0] # stop = f['antenna0']['scu']['benchSampleTime'][0][-1] # For now, do this bit of evil start = f['antenna0']['tracker']['utc'][0][0] stop = f['antenna0']['tracker']['utc'][0][-1] benchcom[key] = core.G3Timestream(f['antenna0']['scu']['benchExpected'][i]) benchcom[key].start = start benchcom[key].stop = stop benchpos[key] = core.G3Timestream(f['antenna0']['scu']['benchActual'][i]) benchpos[key].start = start benchpos[key].stop = stop benchzero[key] = core.G3Timestream(f['antenna0']['scu']['benchZeros'][i]) benchzero[key].start = start benchzero[key].stop = stop f['BenchPosition'] = benchpos f['BenchCommandedPosition'] = benchcom f['BenchZeros'] = benchzero
def convert_azel_to_radec(az, el, location=spt): ''' When passed G3Timestreams of azimuth and elevation positions and a telescope location (where SPT is, by default), return an (RA, Dec) tuple of timestreams corresponding to the astronomical coordinates at which the telescope was pointing. Example: ra, dec = convert_azel_to_radec(az, el) ''' assert(az.start == el.start) assert(az.stop == el.stop) assert(az.n_samples == el.n_samples) # record locations of bad elevation values to mark them later badel_inds = numpy.where((el < -90. * core.G3Units.deg) | (el > 90. * core.G3Units.deg)) el[badel_inds] = 0. * core.G3Units.deg t = astropy.time.Time(numpy.asarray([i.mjd for i in az.times()]), format='mjd') k = astropy.coordinates.AltAz(az=numpy.asarray(az)/core.G3Units.deg*astropy.units.deg, alt=numpy.asarray(el)/core.G3Units.deg*astropy.units.deg, obstime=t, location=location, pressure=0) kt = k.transform_to(astropy.coordinates.FK5) ra = core.G3Timestream(numpy.asarray(kt.ra/astropy.units.deg)*core.G3Units.deg) dec = core.G3Timestream(numpy.asarray(kt.dec/astropy.units.deg)*core.G3Units.deg) dec[badel_inds] = numpy.nan ra.start = dec.start = az.start ra.stop = dec.stop = az.stop return (ra, dec)
def convert_radec_to_azel(ra, dec, location=spt): ''' When passed G3Timestreams of RA and declination positions and a telescope location (where SPT is, by default), return an (Az, El) tuple of timestreams corresponding to the local coordinates at which the telescope was pointing. Example: az, el = convert_radec_to_azel(az, el) ''' assert(ra.start == dec.start) assert(ra.stop == dec.stop) assert(ra.n_samples == dec.n_samples) t = astropy.time.Time(numpy.asarray([i.mjd for i in ra.times()]), format='mjd') k = astropy.coordinates.FK5(ra=numpy.asarray(ra)/core.G3Units.deg*astropy.units.deg, dec=numpy.asarray(dec)/core.G3Units.deg*astropy.units.deg) kt = k.transform_to(astropy.coordinates.AltAz(obstime=t, location=location, pressure=0)) az = core.G3Timestream(numpy.asarray(kt.az/astropy.units.deg)*core.G3Units.deg) el = core.G3Timestream(numpy.asarray(kt.alt/astropy.units.deg)*core.G3Units.deg) az.start = el.start = ra.start az.stop = el.stop = ra.stop return (az, el)
def process(self, data, det_name): locs = np.random.randint(data.n_samples, size=(np.random.randint(self.max_glitches), )) heights = np.random.randn( len(locs)) * self.height_std_sigma * np.std(data) glitches = np.zeros((data.n_samples, )) glitches[locs] += heights self.glitch_map[det_name] = core.G3Timestream(glitches) return core.G3Timestream(data + glitches)
def convert_radec_to_gal(ra, dec): """ Convert timestreams of right ascension and declination to Galactic longitude and latitude. Arguments --------- ra, dec : np.ndarray or G3Timestream Array of Equatorial sky coordinates. If inputs are G3Timestream objects, G3Timestreams are also returned. Returns ------- glon, glat : np.ndarray or G3Timestream """ singleton = False if isinstance(ra, core.G3Timestream): assert ra.start == dec.start assert ra.stop == dec.stop assert ra.n_samples == dec.n_samples else: try: len(ra) except TypeError: singleton = True if singleton: ra = np.atleast_1d(ra) dec = np.atleast_1d(dec) assert len(ra) == len(dec) k = astropy.coordinates.FK5( ra=np.asarray(ra) / core.G3Units.deg * astropy.units.deg, dec=np.asarray(dec) / core.G3Units.deg * astropy.units.deg, ) kt = k.transform_to(astropy.coordinates.Galactic) glon = np.asarray(kt.l / astropy.units.deg) * core.G3Units.deg glat = np.asarray(kt.b / astropy.units.deg) * core.G3Units.deg if isinstance(ra, core.G3Timestream): glon = core.G3Timestream(glon) glat = core.G3Timestream(glat) glon.start = glat.start = ra.start glon.stop = glat.stop = ra.stop elif singleton: glon = glon[0] glat = glat[0] return (glon, glat)
def convert_gal_to_radec(glon, glat): """ Convert timestreams of Galactic longitude and latitude to right ascension and declination. Arguments --------- glon, glat : np.ndarray or G3Timestream Array of Galactic sky coordinates. If inputs are G3Timestream objects, G3Timestreams are also returned. Returns ------- ra, dec : np.ndarray or G3Timestream """ singleton = False if isinstance(glon, core.G3Timestream): assert glon.start == glat.start assert glon.stop == glat.stop assert glon.n_samples == glat.n_samples else: try: len(glon) except TypeError: singleton = True if singleton: glon = np.atleast_1d(glon) glat = np.atleast_1d(glat) assert len(glon) == len(glat) k = astropy.coordinates.Galactic( l=np.asarray(glon) / core.G3Units.deg * astropy.units.deg, b=np.asarray(glat) / core.G3Units.deg * astropy.units.deg, ) kt = k.transform_to(astropy.coordinates.FK5) ra = np.asarray(kt.ra / astropy.units.deg) * core.G3Units.deg dec = np.asarray(kt.dec / astropy.units.deg) * core.G3Units.deg if isinstance(ra, core.G3Timestream): ra = core.G3Timestream(ra) dec = core.G3Timestream(dec) ra.start = dec.start = glon.start ra.stop = dec.stop = glon.stop elif singleton: ra = ra[0] dec = dec[0] return (ra, dec)
def __call__(self, f): if f.type == FT.Calibration and f['cal_type'] == 'focal_plane': self.focal_plane = f if f.type != FT.Scan: return [f] # As long as we have a focal_plane, we can create signal vectors. if self.focal_plane is None: return [f] f['signal'] = core.G3TimestreamMap() # Determine time samples we will be covering. if self.start_time is None: first = f['vertex_enc_raw'].t[0] * core.G3Units.sec self.start_time = core.G3Time( np.ceil(first / self.tick_step) * self.tick_step) # And we will end before... last = core.G3Time(f['vertex_enc_raw'].t[-1] * core.G3Units.sec) n = int((last.time - self.start_time.time) / self.tick_step) end_time = core.G3Time(self.start_time.time + n * self.tick_step) z = np.zeros(n) for k in self.focal_plane['signal_names']: f['signal'][k] = core.G3Timestream(z) # You can't broadcast-set the start and end time unless the # elements are already populated. f['signal'].start = self.start_time f['signal'].stop = end_time self.start_time = end_time return [f]
def noise_scan_frames(n_frames=3, n_dets=20, input='signal', n_samps=200, samp_rate=0.005 * core.G3Units.second, t_start=core.G3Time('2020-1-1T00:00:00')): """ Generate a list of frames filled with noise data and nothing else. Args: n_frames (int): number of frames to make n_dets (int): number of detectors per frame input (str): name of G3TimestreamMap for detectors, should be some form of 'signal' n_samps (int): number of samples per detector timestream samp_rate (G3Unit.second): detector sampling rate t_start (G3Time): start time of the set of frames """ frame_list = [] for n in range(n_frames): f = core.G3Frame() f.type = core.G3FrameType.Scan tsm = core.G3TimestreamMap() z = np.zeros((n_samps, )) for d in enumerate_det_id(n_dets): tsm[d] = core.G3Timestream(z) tsm.start = t_start tsm.stop = t_start + n_samps * samp_rate tsm = MakeNoiseData().apply(tsm) f[input] = tsm t_start += n_samps * samp_rate frame_list.append(f) return frame_list
def __call__(self, frame): if 'DfMuxTransferFunction' in frame: self.default_tf = frame['DfMuxTransferFunction'] if frame.type == core.G3FrameType.Wiring: self.wiringmap = frame['WiringMap'] self.system = frame['ReadoutSystem'] self.convfactors = {} return if frame.type == core.G3FrameType.Housekeeping: self.hkmap = frame['DfMuxHousekeeping'] self.convfactors = {} return if self.keepconversions and frame.type == core.G3FrameType.Observation: self.convfactors = {} self.hkmap = None return if frame.type != core.G3FrameType.Scan: return # Housekeeping data can also be in Scan frames if 'DfMuxHousekeeping' in frame: if self.hkmap is None or (frame['DfMuxHousekeeping'].values()[0].timestamp != self.hkmap.values()[0].timestamp and not self.keepconversions): # XXX: finer-grained check for same values? self.hkmap = frame['DfMuxHousekeeping'] self.convfactors = {} # Now the meat newts = core.G3TimestreamMap() oldts = frame[self.input] for bolo,ts in oldts.items(): if ts.units not in self.convfactors: self.convfactors[ts.units] = {} if ts.units == self.units: convfactor = 1. elif bolo in self.convfactors[ts.units]: convfactor = self.convfactors[ts.units][bolo] else: tf = self.default_tf # XXX: might get from HK data try: convfactor = get_timestream_unit_conversion(ts.units, self.units, bolo, wiringmap=self.wiringmap, hkmap=self.hkmap, system=self.system, tf=tf) except KeyError: if not self.skiperrors: raise newts[bolo] = ts continue self.convfactors[ts.units][bolo] = convfactor # And cache it # Convert timestream and store results convts = core.G3Timestream(ts) if core.G3TimestreamUnits.Resistance in [self.units, ts.units] and self.units != ts.units: convts = 1. / convts convts.units = self.units convts *= convfactor if convts.units != core.G3TimestreamUnits.Counts: convts.SetFLACCompression(False) newts[bolo] = convts frame[self.output] = newts
def process(self, data, det_name): locs = np.random.randint(data.n_samples, size=(np.random.randint(self.max_jumps), )) heights = np.random.randn( len(locs)) * self.height_std_sigma * np.std(data) jumps = np.zeros((data.n_samples, )) for i in range(len(locs)): jumps[locs[i]:] += heights[i] self.jump_map[det_name] = core.G3Timestream(jumps) return data + jumps
def start_stream(self, session, params=None): if params is None: params = {} delay = params.get('delay', 1) ts_len = params.get('ts_len', 100) # Writes status frame f = core.G3Frame(core.G3FrameType.Housekeeping) f['session_id'] = 0 f['start_time'] = time.time() self.writer.Process(f) self.is_streaming = True frame_num = 0 while self.is_streaming: f = core.G3Frame(core.G3FrameType.Scan) t1 = time.time() t0 = t1 - delay ts = np.arange(t0, t1, 1 / self.freq) f['session_id'] = 0 f['frame_num'] = frame_num f['data'] = core.G3TimestreamMap() for k, c in self.channels.items(): fparams = copy.copy(c) bg = np.random.normal(0, fparams.get('stdev', 0), len(ts)) if fparams['type'] == 'const': xs = bg + fparams['val'] elif fparams['type'] in ['lin', 'linear']: xs = bg + ts * fparams['slope'] + fparams.get('offset', 0) # Wraps from -pi to pi xs = np.mod(xs + np.pi, 2 * np.pi) - np.pi f['data'][k] = core.G3Timestream(xs) f['data'][k].start = core.G3Time(t0 * core.G3Units.sec) f['data'][k].stop = core.G3Time(t1 * core.G3Units.sec) self.log.info("Writing G3 Frame") self.writer.Process(f) frame_num += 1 time.sleep(delay) print("Writing EndProcessingFrame") f = core.G3Frame(core.G3FrameType.EndProcessing) self.writer.Process(f) return True, "Finished streaming"
def start_stream(self, session, params=None): """ Task to stream fake detector data as G3Frames Args: frame_rate (float, optional): Frequency [Hz] at which G3Frames are sent over the network. Defaults to 1 frame pers sec. sample_rate (float, optional): Sample rate [Hz] for each channel. Defaults to 10 Hz. """ if params is None: params = {} frame_rate = params.get('frame_rate', 1.) sample_rate = params.get('sample_rate', 10.) f = core.G3Frame(core.G3FrameType.Observation) f['session_id'] = 0 f['start_time'] = time.time() self.writer.Process(f) frame_num = 0 self.is_streaming = True while self.is_streaming: frame_start = time.time() time.sleep(1. / frame_rate) frame_stop = time.time() times = np.arange(frame_start, frame_stop, 1. / sample_rate) f = core.G3Frame(core.G3FrameType.Scan) f['session_id'] = 0 f['frame_num'] = frame_num f['data'] = core.G3TimestreamMap() for i, chan in enumerate(self.channels): ts = core.G3Timestream([chan.read(t) for t in times]) ts.start = core.G3Time(frame_start * core.G3Units.sec) ts.stop = core.G3Time(frame_stop * core.G3Units.sec) f['data'][str(i)] = ts self.writer.Process(f) self.log.info("Writing frame...") frame_num += 1 return True, "Finished streaming"
def check_for_sim_keys(fr, valid_ids_key = 'valid_ids', out_tsm_key = 'valid_ids_tsm'): ''' CalculatePointing expects a TimestreamMap, so make sure there's one in the frame when doing mock-observations. valid_ids_key should already exist in simstub and point to a list. ''' if fr.type == core.G3FrameType.Scan: if not valid_ids_key in fr: raise KeyError( "%s not found in frame. "%valid_ids_key + "List of valid bolometer ids required for mock-observing.") tsm = core.G3TimestreamMap() for bid in fr[valid_ids_key]: tsm[bid]=core.G3Timestream() fr[out_tsm_key] = tsm
def __call__(self, f): if f.type == core.G3FrameType.Scan: if self.input not in f.keys() or type(f[self.input]) != core.G3TimestreamMap: raise ValueError("""Frame is a Scan but does not have a G3Timestream map named {}""".format(self.input)) processing = core.G3TimestreamMap() for k in f[self.input].keys(): processing[k] = core.G3Timestream( self.process(f[self.input][k], k) ) processing.start = f[self.input].start processing.stop = f[self.input].stop if self.input == self.output: f.pop(self.input) f[self.output] = processing
def convert_azel_to_radec(az, el, location=spt, mjd=None): """ Convert timestreams of local azimuth and elevation to right ascension and declination. Arguments --------- az, el : np.ndarray or G3Timestream Array of local coordinates. If inputs are G3Timestream objects, G3Timestreams are also returned. location : astropy.coordinates.EarthLocation instance The telescope location on Earth. mjd : np.ndarray An array of times for each az/el sample. If input az and el are not G3Timestreams, this argument is required. Returns ------- ra, dec : np.ndarray or G3Timestream """ singleton = False if isinstance(az, core.G3Timestream): assert az.start == el.start assert az.stop == el.stop assert az.n_samples == el.n_samples t = astropy.time.Time(np.asarray([i.mjd for i in az.times()]), format="mjd") else: try: len(az) except TypeError: singleton = True if singleton: az = np.atleast_1d(az) el = np.atleast_1d(el) mjd = np.atleast_1d(mjd) assert len(az) == len(el) t = astropy.time.Time(mjd, format="mjd") check_iers(az.stop) # record locations of bad elevation values to mark them later badel_inds = np.where((el < -90.0 * core.G3Units.deg) | (el > 90.0 * core.G3Units.deg)) el[badel_inds] = 0.0 * core.G3Units.deg k = astropy.coordinates.AltAz( az=np.asarray(az) / core.G3Units.deg * astropy.units.deg, alt=np.asarray(el) / core.G3Units.deg * astropy.units.deg, obstime=t, location=location, pressure=0, ) kt = k.transform_to(astropy.coordinates.FK5) ra = np.asarray(kt.ra / astropy.units.deg) * core.G3Units.deg dec = np.asarray(kt.dec / astropy.units.deg) * core.G3Units.deg dec[badel_inds] = np.nan if isinstance(az, core.G3Timestream): ra = core.G3Timestream(ra) dec = core.G3Timestream(dec) ra.start = dec.start = az.start ra.stop = dec.stop = az.stop elif singleton: ra = ra[0] dec = dec[0] return (ra, dec)
def __call__(self, f): if f.type == FT.Calibration and f['cal_type'] == 'focal_plane': self.focal_plane = f if f.type != FT.Scan: return [f] if f.type == FT.EndProcessing: flush = True else: flush = False self.frame_buffer.append(f) self.raw_buffer.append(f[self.enc_name]) # Figure out what frames we're able to process, given info we have. frames_out = [] # Work in units of seconds. raw_t0, raw_t1 = self.raw_buffer[0].t[0], self.raw_buffer[-1].t[-1] # Process any frame that ends before raw_t1. frame_index = 0 while len(self.frame_buffer) > 0: f = self.frame_buffer[0] if not flush and (f['signal'].stop.time / core.G3Units.sec > raw_t1): break sig = f[self.signal_name] # f['signal'] frame_t0 = sig.start.time / core.G3Units.sec frame_t1 = sig.stop.time / core.G3Units.sec tick_rate = sig.sample_rate / core.G3Units.Hz # Figure out what range of samples we will be able to set. start_index = np.ceil((raw_t0 - frame_t0) * tick_rate) end_index = np.floor((raw_t1 - frame_t0) * tick_rate) + 1 start_index = max(0, int(start_index)) end_index = min(int(end_index), sig.n_samples) if end_index != sig.n_samples and not flush: # Buffer. break # Otherwise, do the interpolations... frames_out.append(self.frame_buffer.pop(0)) t_raw = np.hstack([r.t for r in self.raw_buffer]) t_int = frame_t0 + np.arange(start_index, end_index) / tick_rate boresight = core.G3TimestreamMap() vs = {} for k in ['az', 'el', 'corot']: interp = spline1d( t_raw, np.hstack([r.data[k] for r in self.raw_buffer])) v = np.empty(sig.n_samples) v[:start_index] = np.nan v[start_index:end_index] = interp(t_int) v[end_index:] = np.nan vs[k] = v boresight[k] = core.G3Timestream(vs[k]) boresight.start = sig.start boresight.stop = sig.stop f[self.boresight_name] = boresight # f['boresight'] # Compute quaternion. q = ( # Sky <-- near (el, az=0) coords.q_euler(2, -vs['az'] * np.pi / 180) * # ... sky at az=0 <-- near (el=0,az=0) coords.q_euler(1, -vs['el'] * np.pi / 180) * # ... (1,-xi,eta) <-- (-eta,-xi,1) coords.q_euler(1, np.pi / 2) * # ... (-eta,-xi,1) <-- (eta,xi,1) coords.q_euler(2, np.pi)) # Note that there's no "TimestreamQuat" class. So no timestamps. f[self.boresight_name + '_q'] = q # f['boresight_q'] # Discard raw data we're not using any more. Out of caution, # keep one more frame than we have buffered. while len(self.raw_buffer) - len(self.frame_buffer) > 2: self.raw_buffer.pop(0) return frames_out
def convert_radec_to_azel(ra, dec, location=spt, mjd=None): """ Convert timestreams of right ascension and declination to local azimuth and elevation. Arguments --------- ra, dec : np.ndarray or G3Timestream Array of Equatorial sky coordinates. If inputs are G3Timestream objects, G3Timestreams are also returned. location : astropy.coordinates.EarthLocation instance The telescope location on Earth. mjd : np.ndarray An array of times for each ra/dec sample. If input ra and dec are not G3Timestreams, this argument is required. Returns ------- az, el : np.ndarray or G3Timestream """ singleton = False if isinstance(ra, core.G3Timestream): assert ra.start == dec.start assert ra.stop == dec.stop assert ra.n_samples == dec.n_samples t = astropy.time.Time(np.asarray([i.mjd for i in ra.times()]), format="mjd") else: try: len(ra) except TypeError: singleton = True if singleton: ra = np.atleast_1d(ra) dec = np.atleast_1d(dec) mjd = np.atleast_1d(mjd) assert len(ra) == len(dec) t = astropy.time.Time(mjd, format="mjd") check_iers(ra.stop) k = astropy.coordinates.FK5( ra=np.asarray(ra) / core.G3Units.deg * astropy.units.deg, dec=np.asarray(dec) / core.G3Units.deg * astropy.units.deg, ) kt = k.transform_to( astropy.coordinates.AltAz(obstime=t, location=location, pressure=0)) az = np.asarray(kt.az / astropy.units.deg) * core.G3Units.deg el = np.asarray(kt.alt / astropy.units.deg) * core.G3Units.deg if isinstance(ra, core.G3Timestream): az = core.G3Timestream(az) el = core.G3Timestream(el) az.start = el.start = ra.start az.stop = el.stop = ra.stop elif singleton: az = az[0] el = el[0] return (az, el)
#!/usr/bin/env python import numpy from spt3g import core ts = core.G3Timestream(numpy.zeros(50)) ts.units = core.G3TimestreamUnits.Power ts.start = core.G3Time(0) ts.stop = core.G3Time(10 * core.G3Units.s) assert (ts[5] == 0) # Check scalar getitem ts[12] = 13.4 # Check scalar setitem assert (ts[12] == 13.4) ts[:] = numpy.arange(50) # Test vector setitem assert (ts[12]) == 12.0 # Test units preserved by slicing assert (ts[:5].units == ts.units) # Test length with slicing assert (len(ts[::2]) == len(ts) / 2) assert (len(ts[5:]) == len(ts) - 5) # Test consistency with numpy slicing with steps that do and do not divide evenly into array length assert ((numpy.asarray(ts[::2]) == numpy.asarray(ts)[::2]).all()) assert ((numpy.asarray(ts[::3]) == numpy.asarray(ts)[::3]).all()) assert ((numpy.asarray(ts[5::2]) == numpy.asarray(ts)[5::2]).all()) assert ((numpy.asarray(ts[5::3]) == numpy.asarray(ts)[5::3]).all()) assert ((numpy.asarray(ts[5:-4:2]) == numpy.asarray(ts)[5:-4:2]).all())
def AddBenchData(f): ''' Add the optical bench positions to the frame. ''' if f.type != core.G3FrameType.GcpSlow: return bench_axes = ['y1', 'y2', 'y3', 'x4', 'x5', 'z6'] benchcom = core.G3TimestreamMap() benchpos = core.G3TimestreamMap() benchzero = core.G3TimestreamMap() benchoff = core.G3TimestreamMap() bencherr = core.G3TimestreamMap() bench_info = core.G3TimestreamMap() for i, key in enumerate(bench_axes): # As of 2017-08-03, SCU time is not trustworthy # start = f['antenna0']['scu']['benchSampleTime'][0][0] # stop = f['antenna0']['scu']['benchSampleTime'][0][-1] # For now, do this bit of evil start = f['antenna0']['tracker']['utc'][0][0] stop = f['antenna0']['tracker']['utc'][0][-1] benchcom[key] = core.G3Timestream( f['antenna0']['scu']['benchExpected'][i]) benchcom[key].start = start benchcom[key].stop = stop benchpos[key] = core.G3Timestream( f['antenna0']['scu']['benchActual'][i]) benchpos[key].start = start benchpos[key].stop = stop benchzero[key] = core.G3Timestream( f['antenna0']['scu']['benchZeros'][i]) benchzero[key].start = start benchzero[key].stop = stop benchoff[key] = core.G3Timestream( f['antenna0']['scu']['benchOffsets'][i]) benchoff[key].start = start benchoff[key].stop = stop bencherr[key] = core.G3Timestream( f['antenna0']['scu']['benchErrors'][i]) bencherr[key].start = start bencherr[key].stop = stop info_items = [ 'benchFocus', 'benchDeadBand', 'benchAcquiredThreshold', 'benchPrimaryState', 'benchSecondaryState', 'benchFault', 'timeLocked' ] bench_info = core.G3TimestreamMap() for i, key in enumerate(info_items): start = f['antenna0']['tracker']['utc'][0][0] stop = f['antenna0']['tracker']['utc'][0][-1] bench_info[key] = core.G3Timestream(f['antenna0']['scu'][key][0]) bench_info[key].start = start bench_info[key].stop = stop f['BenchPosition'] = benchpos f['BenchCommandedPosition'] = benchcom f['BenchZeros'] = benchzero f['BenchOffsets'] = benchoff f['BenchErrors'] = bencherr f['BenchInfo'] = bench_info f['BenchSampleTime'] = f['antenna0']['scu']['benchSampleTime'][0]
def start_background_streamer(self, session, params=None): """start_background_streamer(params=None) Process to run streaming process. A data stream is started automatically. It can be stopped and started by the start and stop tasks. Either way keep alive flow control frames are being sent. Parameters ---------- frame_rate : float, optional Frequency [Hz] at which G3Frames are sent over the network. Defaults to 1 frame pers sec. sample_rate : float, optional Sample rate [Hz] for each channel. Defaults to 10 Hz. """ if params is None: params = {} self.writer = core.G3NetworkSender(hostname=self.target_host, port=self.port) frame_rate = params.get('frame_rate', 1.) sample_rate = params.get('sample_rate', 10.) frame_num = 0 self.running_in_background = True # Control flags FIFO stack to keep Writer single threaded self.flags = deque([FlowControl.START]) while self.running_in_background: # Send START frame if next(iter(self.flags), None) is FlowControl.START: self._set_stream_on() # sends start flowcontrol self.is_streaming = True self.flags.popleft() print("stream running in background") self.log.debug("control flags: {f}", f=self.flags) # Send keep alive flow control frame f = core.G3Frame(core.G3FrameType.none) f['sostream_flowcontrol'] = FlowControl.ALIVE.value self.writer.Process(f) if self.is_streaming: frame_start = time.time() time.sleep(1. / frame_rate) frame_stop = time.time() times = np.arange(frame_start, frame_stop, 1. / sample_rate) f = core.G3Frame(core.G3FrameType.Scan) f['session_id'] = 0 f['frame_num'] = frame_num f['sostream_id'] = self.stream_id f['data'] = core.G3TimestreamMap() for i, chan in enumerate(self.channels): ts = core.G3Timestream([chan.read() for t in times]) ts.start = core.G3Time(frame_start * core.G3Units.sec) ts.stop = core.G3Time(frame_stop * core.G3Units.sec) f['data'][f"r{i:04}"] = ts self.writer.Process(f) self.log.info("Writing frame...") frame_num += 1 # Send END frame if next(iter(self.flags), None) is FlowControl.END: self._send_end_flowcontrol_frame() self._send_cleanse_flowcontrol_frame() self.is_streaming = False self.flags.popleft() else: # Don't send keep alive frames too quickly time.sleep(1) # Shutdown streamer if next(iter(self.flags), None) is SHUTDOWN: self.running_in_background = False self.flags.popleft() # Teardown writer self.writer.Close() self.writer = None return True, "Finished streaming"
#!/usr/bin/env python from __future__ import print_function import numpy, sys from spt3g import core then = core.G3Time.Now() now = core.G3Time(then.time + 3 * core.G3Units.second) data = numpy.zeros(200) ts = core.G3Timestream(data) ts.start = then ts.stop = now assert (numpy.abs(ts.sample_rate / core.G3Units.Hz - 66.33333) < 1e-5) times = ts.times() assert (ts.times()[0] == ts.start) print(ts.times()[-1].time, ts.stop.time) assert (numpy.abs(ts.times()[-1].time - ts.stop.time) < 2 ) # Max 1 tick roundoff error tsm = core.G3TimestreamMap() tsm['Test'] = ts assert (numpy.abs(tsm.sample_rate / core.G3Units.Hz - 66.33333) < 1e-5) times = tsm.times()
#!/usr/bin/env python from spt3g import core, coordinateutils import numpy as np np.random.seed(42) n_samps = int(1e3) az_0 = core.G3Timestream(np.random.rand(n_samps) * 2.0 * np.pi - np.pi) pole_avoidance = 0.7 el_0 = core.G3Timestream( np.random.rand(n_samps) * np.pi * pole_avoidance - np.pi / 2.0 * pole_avoidance) az_0.start = core.G3Time('20170329_000001') el_0.start = core.G3Time('20170329_000001') az_0.stop = core.G3Time('20170329_100001') el_0.stop = core.G3Time('20170329_100001') az_1 = az_0 + 10 * core.G3Units.arcmin el_1 = el_0 + 10 * core.G3Units.arcmin ra_0, dec_0 = coordinateutils.azel.convert_azel_to_radec(az_0, el_0) ra_1, dec_1 = coordinateutils.azel.convert_azel_to_radec(az_1, el_1) o_az_0 = az_0 + (np.random.rand() - 0.5) * 2 * core.G3Units.deg o_el_0 = el_0 + (np.random.rand() - 0.5) * 2 * core.G3Units.deg o_ra_0, o_dec_0 = coordinateutils.azel.convert_azel_to_radec(o_az_0, o_el_0) import astropy.coordinates
def cache_to_frames(tod, start_frame, n_frames, frame_offsets, frame_sizes, common=None, detector_fields=None, flag_fields=None, detector_map="detectors", flag_map="flags", units=None): """Gather all data from the distributed cache for a single frame. Args: tod (toast.TOD): instance of a TOD class. start_frame (int): the first frame index. n_frames (int): the number of frames. frame_offsets (list): list of the first samples of all frames. frame_sizes (list): list of the number of samples in each frame. common (tuple): (cache name, G3 type, frame name) of each common field. detector_fields (tuple): (cache name, frame name) of each detector field. flag_fields (tuple): (cache name, frame name) of each flag field. detector_map (str): the name of the frame timestream map. flag_map (str): then name of the frame flag map. units: G3 units of the detector data. """ # Local sample range local_first = tod.local_samples[0] nlocal = tod.local_samples[1] # The process grid detranks, sampranks = tod.grid_size rankdet, ranksamp = tod.grid_ranks # Helper function: # For a given timestream, the gather is done across the # process row which contains the specific detector, or across # the first process row for common telescope data. def gather_field(prow, fld, indx, cacheoff, ncache): gproc = 0 gdata = None # We are going to allreduce this later, so that every process # knows the dimensions of the field. allnnz = 0 if rankdet == prow: #print(" proc {} doing gather of {}".format(tod.mpicomm.rank, fld), flush=True) # This process is in the process row that has this field, # participate in the gather operation. pdata = None # Find the data type and shape from the cache object mtype = None ref = tod.cache.reference(fld) nnz = 1 if (len(ref.shape) > 1) and (ref.shape[1] > 0): nnz = ref.shape[1] if ref.dtype == np.dtype(np.float64): mtype = MPI.DOUBLE elif ref.dtype == np.dtype(np.int64): mtype = MPI.INT64_T elif ref.dtype == np.dtype(np.int32): mtype = MPI.INT32_T elif ref.dtype == np.dtype(np.uint8): mtype = MPI.UINT8_T else: msg = "Cannot gather cache field {} of type {}"\ .format(fld, ref.dtype) raise RuntimeError(msg) #print("field {}: proc {} has nnz = {}".format(fld, tod.mpicomm.rank, nnz), flush=True) pz = 0 if cacheoff is not None: pdata = ref.flatten()[nnz * cacheoff:nnz * (cacheoff + ncache)] pz = nnz * ncache else: pdata = np.zeros(0, dtype=ref.dtype) psizes = tod.grid_comm_row.gather(pz, root=0) disp = None totsize = None if ranksamp == 0: #print("Gathering field {} with type {}".format(fld, mtype), flush=True) # We are the process collecting the gathered data. gproc = tod.mpicomm.rank allnnz = nnz # Compute the displacements into the receive buffer. disp = [0] for ps in psizes[:-1]: last = disp[-1] disp.append(last + ps) totsize = np.sum(psizes) # allocate receive buffer gdata = np.zeros(totsize, dtype=ref.dtype) #print("Gatherv psizes = {}, disp = {}".format(psizes, disp), flush=True) #print("field {}: proc {} start Gatherv".format(fld, tod.mpicomm.rank), flush=True) tod.grid_comm_row.Gatherv(pdata, [gdata, psizes, disp, mtype], root=0) #print("field {}: proc {} finish Gatherv".format(fld, tod.mpicomm.rank), flush=True) del disp del psizes del pdata del ref # Now send this data to the root process of the whole communicator. # Only one process (the first one in process row "prow") has data # to send. # Create a unique message tag mtag = 10 * indx #print(" proc {} hit allreduce of gproc".format(tod.mpicomm.rank), flush=True) # All processes find out which one did the gather gproc = tod.mpicomm.allreduce(gproc, MPI.SUM) # All processes find out the field dimensions allnnz = tod.mpicomm.allreduce(allnnz, MPI.SUM) #print(" proc {} for field {}, gproc = {}".format(tod.mpicomm.rank, fld, gproc), flush=True) #print("field {}: proc {}, gatherproc = {}, allnnz = {}".format(fld, tod.mpicomm.rank, gproc, allnnz), flush=True) rdata = None if gproc == 0: if gdata is not None: if allnnz == 1: rdata = gdata else: rdata = gdata.reshape((-1, allnnz)) else: # Data not yet on rank 0 if tod.mpicomm.rank == 0: # Receive data from the first process in this row #print(" proc {} for field {}, recv type".format(tod.mpicomm.rank, fld), flush=True) rtype = tod.mpicomm.recv(source=gproc, tag=(mtag + 1)) #print(" proc {} for field {}, recv size".format(tod.mpicomm.rank, fld), flush=True) rsize = tod.mpicomm.recv(source=gproc, tag=(mtag + 2)) #print(" proc {} for field {}, recv data".format(tod.mpicomm.rank, fld), flush=True) rdata = np.zeros(rsize, dtype=np.dtype(rtype)) tod.mpicomm.Recv(rdata, source=gproc, tag=mtag) # Reshape if needed if allnnz > 1: rdata = rdata.reshape((-1, allnnz)) elif (tod.mpicomm.rank == gproc): # Send our data #print(" proc {} for field {}, send {} samples of {}".format(tod.mpicomm.rank, fld, len(gdata), gdata.dtype.char), flush=True) #print(" proc {} for field {}, send type with tag = {}".format(tod.mpicomm.rank, fld, mtag+1), flush=True) tod.mpicomm.send(gdata.dtype.char, dest=0, tag=(mtag + 1)) #print(" proc {} for field {}, send size with tag = {}".format(tod.mpicomm.rank, fld, mtag+2), flush=True) tod.mpicomm.send(len(gdata), dest=0, tag=(mtag + 2)) #print(" proc {} for field {}, send data with tag {}".format(tod.mpicomm.rank, fld, mtag), flush=True) tod.mpicomm.Send(gdata, 0, tag=mtag) return rdata # For efficiency, we are going to gather the data for all frames at once. # Then we will split those up when doing the write. # Frame offsets relative to the memory buffers we are gathering fdataoff = [0] for f in frame_sizes[:-1]: last = fdataoff[-1] fdataoff.append(last + f) # The list of frames- only on the root process. fdata = None if tod.mpicomm.rank == 0: fdata = [c3g.G3Frame(c3g.G3FrameType.Scan) for f in range(n_frames)] else: fdata = [None for f in range(n_frames)] # Compute the overlap of all frames with the local process. We want to # to find the full sample range that this process overlaps the total set # of frames. cacheoff = None ncache = 0 for f in range(n_frames): # Compute overlap of the frame with the local samples. fcacheoff, froff, nfr = local_frame_indices(local_first, nlocal, frame_offsets[f], frame_sizes[f]) #print("proc {}: frame {} has cache off {}, fr off {}, nfr {}".format(tod.mpicomm.rank, f, fcacheoff, froff, nfr), flush=True) if fcacheoff is not None: if cacheoff is None: cacheoff = fcacheoff ncache = nfr else: ncache += nfr #print("proc {}: cache off now {}, ncache now {}".format(tod.mpicomm.rank, cacheoff, ncache), flush=True) # Now gather the full sample data one field at a time. The root process # splits up the results into frames. # First gather common fields from the first row of the process grid. for findx, (cachefield, g3t, framefield) in enumerate(common): #print("proc {} entering gather_field(0, {}, {}, {}, {})".format(tod.mpicomm.rank, cachefield, findx, cacheoff, ncache), flush=True) data = gather_field(0, cachefield, findx, cacheoff, ncache) if tod.mpicomm.rank == 0: #print("Casting field {} to type {}".format(field, g3t), flush=True) if g3t == c3g.G3VectorTime: # Special case for time values stored as int64_t, but # wrapped in a class. for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] g3times = list() for t in range(ndata): g3times.append(c3g.G3Time(data[dataoff + t])) fdata[f][framefield] = c3g.G3VectorTime(g3times) del g3times else: # The bindings of G3Vector seem to only work with # lists. This is probably horribly inefficient. for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] if len(data.shape) == 1: fdata[f][framefield] = \ g3t(data[dataoff:dataoff+ndata].tolist()) else: # We have a 2D quantity fdata[f][framefield] = \ g3t(data[dataoff:dataoff+ndata,:].flatten().tolist()) del data # Wait for everyone to catch up... tod.mpicomm.barrier() # For each detector field, processes which have the detector # in their local_dets should be in the same process row. # We do the gather over just this process row. if (detector_fields is not None) or (flag_fields is not None): dpats = {d: re.compile(".*{}.*".format(d)) for d in tod.local_dets} detmaps = None if detector_fields is not None: if tod.mpicomm.rank == 0: detmaps = [c3g.G3TimestreamMap() for f in range(n_frames)] for dindx, (cachefield, framefield) in enumerate(detector_fields): pc = -1 for det, pat in dpats.items(): if pat.match(cachefield) is not None: #print("proc {} has field {}".format(tod.mpicomm.rank, field), flush=True) pc = rankdet break # As a sanity check, verify that every process which # has this field is in the same process row. rowcheck = tod.mpicomm.gather(pc, root=0) prow = 0 if tod.mpicomm.rank == 0: rc = np.array([x for x in rowcheck if (x >= 0)], dtype=np.int32) #print(field, rc, flush=True) prow = np.max(rc) if np.min(rc) != prow: msg = "Processes with field {} are not in the "\ "same row\n".format(cachefield) sys.stderr.write(msg) tod.mpicomm.abort() # Every process finds out which process row is participating. prow = tod.mpicomm.bcast(prow, root=0) #print("proc {} got prow = {}".format(tod.mpicomm.rank, prow), flush=True) # Get the data on rank 0 data = gather_field(prow, cachefield, dindx, cacheoff, ncache) if tod.mpicomm.rank == 0: if units is None: # We do this conditional, since we can't use # G3TimestreamUnits.None in python ("None" is # interpreted as python None). for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] detmaps[f][framefield] = \ c3g.G3Timestream(data[dataoff:dataoff+ndata]) else: for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] detmaps[f][framefield] = \ c3g.G3Timestream(data[dataoff:dataoff+ndata], units) if tod.mpicomm.rank == 0: for f in range(n_frames): fdata[f][detector_map] = detmaps[f] flagmaps = None if flag_fields is not None: if tod.mpicomm.rank == 0: flagmaps = [c3g.G3MapVectorInt() for f in range(n_frames)] for dindx, (cachefield, framefield) in enumerate(flag_fields): pc = -1 for det, pat in dpats.items(): if pat.match(cachefield) is not None: pc = rankdet break # As a sanity check, verify that every process which # has this field is in the same process row. rowcheck = tod.mpicomm.gather(pc, root=0) prow = 0 if tod.mpicomm.rank == 0: rc = np.array([x for x in rowcheck if (x >= 0)], dtype=np.int32) prow = np.max(rc) if np.min(rc) != prow: msg = "Processes with field {} are not in the "\ "same row\n".format(cachefield) sys.stderr.write(msg) tod.mpicomm.abort() # Every process finds out which process row is participating. prow = tod.mpicomm.bcast(prow, root=0) # Get the data on rank 0 data = gather_field(prow, cachefield, dindx, cacheoff, ncache) if tod.mpicomm.rank == 0: # The bindings of G3Vector seem to only work with # lists... Also there is no vectormap for unsigned # char, so we have to use int... for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] flagmaps[f][framefield] = \ c3g.G3VectorInt(\ data[dataoff:dataoff+ndata].astype(np.int32)\ .tolist()) if tod.mpicomm.rank == 0: for f in range(n_frames): fdata[f][flag_map] = flagmaps[f] return fdata
def recode_timestream(ts, params, rmstarget=2**10, rmsmode="white"): """ts is a G3Timestream. Returns a new G3Timestream for same samples as ts, but with data scaled and translated with gain and offset, rounded, and with FLAC compression enabled. Args: ts (G3Timestream) : Input signal params (bool or dict) : if True, compress with default parameters. If dict with 'rmstarget' member, override default `rmstarget`. If dict with `gain` and `offset` members, use those instead. params (None, bool or dict) : If None, False or an empty dict, no compression or casting to integers. If True or non-empty dictionary, enable compression. Expected fields in the dictionary ('rmstarget', 'gain', 'offset', 'rmsmode') allow overriding defaults. rmstarget (float) : Scale the iput signal to have this RMS. Should be much smaller then the 24-bit integer range: [-2 ** 23 : 2 ** 23] = [-8,388,608 : 8,388,608]. The gain will be reduced if the scaled signal does not fit within the range of allowed values. rmsmode (string) : "white" or "full", determines how the signal RMS is measured. Returns: new_ts (G3Timestream) : Scaled and translated timestream with the FLAC compression enabled gain (float) : The applied gain offset (float) : The removed offset """ if not params: return ts, 1, 0 gain = None offset = None if isinstance(params, dict): if "rmsmode" in params: rmsmode = params["rmsmode"] if "rmstarget" in params: rmstarget = params["rmstarget"] if "gain" in params: gain = params["gain"] if "offset" in params: offset = params["offset"] v = np.array(ts) vmin = np.amin(v) vmax = np.amax(v) if offset is None: offset = 0.5 * (vmin + vmax) amp = vmax - offset else: amp = np.max(np.abs(vmin - offset), np.abs(vmax - offset)) if gain is None: if rmsmode == "white": rms = np.std(np.diff(v)) / np.sqrt(2) elif rmsmode == "full": rms = np.std(v) else: raise RuntimeError("Unrecognized RMS mode = '{}'".format(rmsmode)) if rms == 0: gain = 1 else: gain = rmstarget / rms # If the data have extreme outliers, we have to reduce the gain # to fit the 24-bit signed integer range while amp * gain >= 2**23: gain *= 0.5 elif amp * gain >= 2**23: raise RuntimeError("The specified gain and offset saturate the band.") v = np.round((v - offset) * gain) new_ts = core3g.G3Timestream(v) new_ts.units = core3g.G3TimestreamUnits.Counts new_ts.SetFLACCompression(True) new_ts.start = ts.start new_ts.stop = ts.stop return new_ts, gain, offset
#!/usr/bin/env python import pickle, numpy, sys from spt3g import core ts = core.G3Timestream(numpy.ones(1200)) ts.units = core.G3TimestreamUnits.Counts ts.SetFLACCompression(5) ts[5] = numpy.nan pickle.dump(ts, open('tsdump.pkl', 'wb')) ts_rehyd = pickle.load(open('tsdump.pkl', 'rb')) print('Original') print(ts.units) print(ts[4], ts[5], ts[6]) print('Rehydrated') print(ts_rehyd.units) print(ts_rehyd[4], ts_rehyd[5], ts_rehyd[6]) if ts_rehyd.units != ts.units: print('Units do not match') sys.exit(1) if numpy.isfinite(ts_rehyd[5]): print('Element 5 finite!') sys.exit(1) if (numpy.asarray(ts_rehyd)[:5] != 1).any() or (numpy.asarray(ts_rehyd)[6:] != 1).any(): print('Elements not 1!')
#!/usr/bin/env python import numpy from spt3g import core # Check that timestream maps being cast with numpy.asarray have the right # content and are indistinguishable from numpy.asarray(list(tsm.values()) tsm = core.G3TimestreamMap() start = core.G3Time.Now() stop = start + 5 * core.G3Units.s for ts in ['A', 'B', 'C', 'D']: i = core.G3Timestream(numpy.random.normal(size=600)) i.start = start i.stop = stop i.units = core.G3TimestreamUnits.Tcmb tsm[ts] = i buffer1d = numpy.asarray(list(tsm.values())) buffer2d = numpy.asarray(tsm) assert (buffer1d.shape == buffer2d.shape) assert (buffer2d.shape == (4, 600)) assert ((buffer1d == buffer2d).all())
#!/usr/bin/env python from __future__ import print_function import random, os, sys, numpy, time from spt3g import core port = random.randint(10000, 60000) frames = [] for i in range(0, 20): f = core.G3Frame() f['Sequence'] = i f['Data'] = core.G3Timestream(numpy.zeros(100000)) frames.append(f) print('Port: ', port) child = os.fork() if child != 0: # Parent print('Parent') send = core.G3NetworkSender(hostname='*', port=port) time.sleep(1) # XXX: how to signal that the remote end is ready? print('Sending') for f in frames: send(f) send(core.G3Frame(core.G3FrameType.EndProcessing)) pid, status = os.wait() print('Child Status: ', status)