def AddBenchData(f):
'''
Add the optical bench positions to the frame.
'''
if f.type != core.G3FrameType.GcpSlow:
return
bench_axes = ['y1', 'y2', 'y3', 'x4', 'x5', 'z6']
benchcom = core.G3TimestreamMap()
benchpos = core.G3TimestreamMap()
benchzero = core.G3TimestreamMap()
for i, key in enumerate(bench_axes):
# As of 2017-08-03, SCU time is not trustworthy
# start = f['antenna0']['scu']['benchSampleTime'][0][0]
# stop = f['antenna0']['scu']['benchSampleTime'][0][-1]
# For now, do this bit of evil
start = f['antenna0']['tracker']['utc'][0][0]
stop = f['antenna0']['tracker']['utc'][0][-1]
benchcom[key] = core.G3Timestream(f['antenna0']['scu']['benchExpected'][i])
benchcom[key].start = start
benchcom[key].stop = stop
benchpos[key] = core.G3Timestream(f['antenna0']['scu']['benchActual'][i])
benchpos[key].start = start
benchpos[key].stop = stop
benchzero[key] = core.G3Timestream(f['antenna0']['scu']['benchZeros'][i])
benchzero[key].start = start
benchzero[key].stop = stop
f['BenchPosition'] = benchpos
f['BenchCommandedPosition'] = benchcom
f['BenchZeros'] = benchzero
def convert_azel_to_radec(az, el, location=spt):
'''
When passed G3Timestreams of azimuth and elevation positions and a
telescope location (where SPT is, by default), return an (RA, Dec)
tuple of timestreams corresponding to the astronomical coordinates
at which the telescope was pointing.
Example:
ra, dec = convert_azel_to_radec(az, el)
'''
assert(az.start == el.start)
assert(az.stop == el.stop)
assert(az.n_samples == el.n_samples)
# record locations of bad elevation values to mark them later
badel_inds = numpy.where((el < -90. * core.G3Units.deg) | (el > 90. * core.G3Units.deg))
el[badel_inds] = 0. * core.G3Units.deg
t = astropy.time.Time(numpy.asarray([i.mjd for i in az.times()]), format='mjd')
k = astropy.coordinates.AltAz(az=numpy.asarray(az)/core.G3Units.deg*astropy.units.deg, alt=numpy.asarray(el)/core.G3Units.deg*astropy.units.deg, obstime=t, location=location, pressure=0)
kt = k.transform_to(astropy.coordinates.FK5)
ra = core.G3Timestream(numpy.asarray(kt.ra/astropy.units.deg)*core.G3Units.deg)
dec = core.G3Timestream(numpy.asarray(kt.dec/astropy.units.deg)*core.G3Units.deg)
dec[badel_inds] = numpy.nan
ra.start = dec.start = az.start
ra.stop = dec.stop = az.stop
return (ra, dec)
def convert_radec_to_azel(ra, dec, location=spt):
'''
When passed G3Timestreams of RA and declination positions and a
telescope location (where SPT is, by default), return an (Az, El)
tuple of timestreams corresponding to the local coordinates
at which the telescope was pointing.
Example:
az, el = convert_radec_to_azel(az, el)
'''
assert(ra.start == dec.start)
assert(ra.stop == dec.stop)
assert(ra.n_samples == dec.n_samples)
t = astropy.time.Time(numpy.asarray([i.mjd for i in ra.times()]), format='mjd')
k = astropy.coordinates.FK5(ra=numpy.asarray(ra)/core.G3Units.deg*astropy.units.deg, dec=numpy.asarray(dec)/core.G3Units.deg*astropy.units.deg)
kt = k.transform_to(astropy.coordinates.AltAz(obstime=t, location=location, pressure=0))
az = core.G3Timestream(numpy.asarray(kt.az/astropy.units.deg)*core.G3Units.deg)
el = core.G3Timestream(numpy.asarray(kt.alt/astropy.units.deg)*core.G3Units.deg)
az.start = el.start = ra.start
az.stop = el.stop = ra.stop
return (az, el)
def process(self, data, det_name):
locs = np.random.randint(data.n_samples,
size=(np.random.randint(self.max_glitches), ))
heights = np.random.randn(
len(locs)) * self.height_std_sigma * np.std(data)
glitches = np.zeros((data.n_samples, ))
glitches[locs] += heights
self.glitch_map[det_name] = core.G3Timestream(glitches)
return core.G3Timestream(data + glitches)
def convert_radec_to_gal(ra, dec):
"""
Convert timestreams of right ascension and declination to Galactic
longitude and latitude.
Arguments
---------
ra, dec : np.ndarray or G3Timestream
Array of Equatorial sky coordinates. If inputs are G3Timestream
objects, G3Timestreams are also returned.
Returns
-------
glon, glat : np.ndarray or G3Timestream
"""
singleton = False
if isinstance(ra, core.G3Timestream):
assert ra.start == dec.start
assert ra.stop == dec.stop
assert ra.n_samples == dec.n_samples
else:
try:
len(ra)
except TypeError:
singleton = True
if singleton:
ra = np.atleast_1d(ra)
dec = np.atleast_1d(dec)
assert len(ra) == len(dec)
k = astropy.coordinates.FK5(
ra=np.asarray(ra) / core.G3Units.deg * astropy.units.deg,
dec=np.asarray(dec) / core.G3Units.deg * astropy.units.deg,
)
kt = k.transform_to(astropy.coordinates.Galactic)
glon = np.asarray(kt.l / astropy.units.deg) * core.G3Units.deg
glat = np.asarray(kt.b / astropy.units.deg) * core.G3Units.deg
if isinstance(ra, core.G3Timestream):
glon = core.G3Timestream(glon)
glat = core.G3Timestream(glat)
glon.start = glat.start = ra.start
glon.stop = glat.stop = ra.stop
elif singleton:
glon = glon[0]
glat = glat[0]
return (glon, glat)
def convert_gal_to_radec(glon, glat):
"""
Convert timestreams of Galactic longitude and latitude to right ascension
and declination.
Arguments
---------
glon, glat : np.ndarray or G3Timestream
Array of Galactic sky coordinates. If inputs are G3Timestream
objects, G3Timestreams are also returned.
Returns
-------
ra, dec : np.ndarray or G3Timestream
"""
singleton = False
if isinstance(glon, core.G3Timestream):
assert glon.start == glat.start
assert glon.stop == glat.stop
assert glon.n_samples == glat.n_samples
else:
try:
len(glon)
except TypeError:
singleton = True
if singleton:
glon = np.atleast_1d(glon)
glat = np.atleast_1d(glat)
assert len(glon) == len(glat)
k = astropy.coordinates.Galactic(
l=np.asarray(glon) / core.G3Units.deg * astropy.units.deg,
b=np.asarray(glat) / core.G3Units.deg * astropy.units.deg,
)
kt = k.transform_to(astropy.coordinates.FK5)
ra = np.asarray(kt.ra / astropy.units.deg) * core.G3Units.deg
dec = np.asarray(kt.dec / astropy.units.deg) * core.G3Units.deg
if isinstance(ra, core.G3Timestream):
ra = core.G3Timestream(ra)
dec = core.G3Timestream(dec)
ra.start = dec.start = glon.start
ra.stop = dec.stop = glon.stop
elif singleton:
ra = ra[0]
dec = dec[0]
return (ra, dec)
def __call__(self, f):
if f.type == FT.Calibration and f['cal_type'] == 'focal_plane':
self.focal_plane = f
if f.type != FT.Scan:
return [f]
# As long as we have a focal_plane, we can create signal vectors.
if self.focal_plane is None:
return [f]
f['signal'] = core.G3TimestreamMap()
# Determine time samples we will be covering.
if self.start_time is None:
first = f['vertex_enc_raw'].t[0] * core.G3Units.sec
self.start_time = core.G3Time(
np.ceil(first / self.tick_step) * self.tick_step)
# And we will end before...
last = core.G3Time(f['vertex_enc_raw'].t[-1] * core.G3Units.sec)
n = int((last.time - self.start_time.time) / self.tick_step)
end_time = core.G3Time(self.start_time.time + n * self.tick_step)
z = np.zeros(n)
for k in self.focal_plane['signal_names']:
f['signal'][k] = core.G3Timestream(z)
# You can't broadcast-set the start and end time unless the
# elements are already populated.
f['signal'].start = self.start_time
f['signal'].stop = end_time
self.start_time = end_time
return [f]
def noise_scan_frames(n_frames=3,
n_dets=20,
input='signal',
n_samps=200,
samp_rate=0.005 * core.G3Units.second,
t_start=core.G3Time('2020-1-1T00:00:00')):
"""
Generate a list of frames filled with noise data and nothing else.
Args:
n_frames (int): number of frames to make
n_dets (int): number of detectors per frame
input (str): name of G3TimestreamMap for detectors, should be some form of 'signal'
n_samps (int): number of samples per detector timestream
samp_rate (G3Unit.second): detector sampling rate
t_start (G3Time): start time of the set of frames
"""
frame_list = []
for n in range(n_frames):
f = core.G3Frame()
f.type = core.G3FrameType.Scan
tsm = core.G3TimestreamMap()
z = np.zeros((n_samps, ))
for d in enumerate_det_id(n_dets):
tsm[d] = core.G3Timestream(z)
tsm.start = t_start
tsm.stop = t_start + n_samps * samp_rate
tsm = MakeNoiseData().apply(tsm)
f[input] = tsm
t_start += n_samps * samp_rate
frame_list.append(f)
return frame_list
def __call__(self, frame):
if 'DfMuxTransferFunction' in frame:
self.default_tf = frame['DfMuxTransferFunction']
if frame.type == core.G3FrameType.Wiring:
self.wiringmap = frame['WiringMap']
self.system = frame['ReadoutSystem']
self.convfactors = {}
return
if frame.type == core.G3FrameType.Housekeeping:
self.hkmap = frame['DfMuxHousekeeping']
self.convfactors = {}
return
if self.keepconversions and frame.type == core.G3FrameType.Observation:
self.convfactors = {}
self.hkmap = None
return
if frame.type != core.G3FrameType.Scan:
return
# Housekeeping data can also be in Scan frames
if 'DfMuxHousekeeping' in frame:
if self.hkmap is None or (frame['DfMuxHousekeeping'].values()[0].timestamp != self.hkmap.values()[0].timestamp and not self.keepconversions):
# XXX: finer-grained check for same values?
self.hkmap = frame['DfMuxHousekeeping']
self.convfactors = {}
# Now the meat
newts = core.G3TimestreamMap()
oldts = frame[self.input]
for bolo,ts in oldts.items():
if ts.units not in self.convfactors:
self.convfactors[ts.units] = {}
if ts.units == self.units:
convfactor = 1.
elif bolo in self.convfactors[ts.units]:
convfactor = self.convfactors[ts.units][bolo]
else:
tf = self.default_tf # XXX: might get from HK data
try:
convfactor = get_timestream_unit_conversion(ts.units, self.units, bolo, wiringmap=self.wiringmap, hkmap=self.hkmap, system=self.system, tf=tf)
except KeyError:
if not self.skiperrors:
raise
newts[bolo] = ts
continue
self.convfactors[ts.units][bolo] = convfactor # And cache it
# Convert timestream and store results
convts = core.G3Timestream(ts)
if core.G3TimestreamUnits.Resistance in [self.units, ts.units] and self.units != ts.units:
convts = 1. / convts
convts.units = self.units
convts *= convfactor
if convts.units != core.G3TimestreamUnits.Counts:
convts.SetFLACCompression(False)
newts[bolo] = convts
frame[self.output] = newts
def process(self, data, det_name):
locs = np.random.randint(data.n_samples,
size=(np.random.randint(self.max_jumps), ))
heights = np.random.randn(
len(locs)) * self.height_std_sigma * np.std(data)
jumps = np.zeros((data.n_samples, ))
for i in range(len(locs)):
jumps[locs[i]:] += heights[i]
self.jump_map[det_name] = core.G3Timestream(jumps)
return data + jumps
def start_stream(self, session, params=None):
if params is None:
params = {}
delay = params.get('delay', 1)
ts_len = params.get('ts_len', 100)
# Writes status frame
f = core.G3Frame(core.G3FrameType.Housekeeping)
f['session_id'] = 0
f['start_time'] = time.time()
self.writer.Process(f)
self.is_streaming = True
frame_num = 0
while self.is_streaming:
f = core.G3Frame(core.G3FrameType.Scan)
t1 = time.time()
t0 = t1 - delay
ts = np.arange(t0, t1, 1 / self.freq)
f['session_id'] = 0
f['frame_num'] = frame_num
f['data'] = core.G3TimestreamMap()
for k, c in self.channels.items():
fparams = copy.copy(c)
bg = np.random.normal(0, fparams.get('stdev', 0), len(ts))
if fparams['type'] == 'const':
xs = bg + fparams['val']
elif fparams['type'] in ['lin', 'linear']:
xs = bg + ts * fparams['slope'] + fparams.get('offset', 0)
# Wraps from -pi to pi
xs = np.mod(xs + np.pi, 2 * np.pi) - np.pi
f['data'][k] = core.G3Timestream(xs)
f['data'][k].start = core.G3Time(t0 * core.G3Units.sec)
f['data'][k].stop = core.G3Time(t1 * core.G3Units.sec)
self.log.info("Writing G3 Frame")
self.writer.Process(f)
frame_num += 1
time.sleep(delay)
print("Writing EndProcessingFrame")
f = core.G3Frame(core.G3FrameType.EndProcessing)
self.writer.Process(f)
return True, "Finished streaming"
def start_stream(self, session, params=None):
"""
Task to stream fake detector data as G3Frames
Args:
frame_rate (float, optional):
Frequency [Hz] at which G3Frames are sent over the network.
Defaults to 1 frame pers sec.
sample_rate (float, optional):
Sample rate [Hz] for each channel.
Defaults to 10 Hz.
"""
if params is None:
params = {}
frame_rate = params.get('frame_rate', 1.)
sample_rate = params.get('sample_rate', 10.)
f = core.G3Frame(core.G3FrameType.Observation)
f['session_id'] = 0
f['start_time'] = time.time()
self.writer.Process(f)
frame_num = 0
self.is_streaming = True
while self.is_streaming:
frame_start = time.time()
time.sleep(1. / frame_rate)
frame_stop = time.time()
times = np.arange(frame_start, frame_stop, 1. / sample_rate)
f = core.G3Frame(core.G3FrameType.Scan)
f['session_id'] = 0
f['frame_num'] = frame_num
f['data'] = core.G3TimestreamMap()
for i, chan in enumerate(self.channels):
ts = core.G3Timestream([chan.read(t) for t in times])
ts.start = core.G3Time(frame_start * core.G3Units.sec)
ts.stop = core.G3Time(frame_stop * core.G3Units.sec)
f['data'][str(i)] = ts
self.writer.Process(f)
self.log.info("Writing frame...")
frame_num += 1
return True, "Finished streaming"
def check_for_sim_keys(fr, valid_ids_key = 'valid_ids',
out_tsm_key = 'valid_ids_tsm'):
'''
CalculatePointing expects a TimestreamMap, so make sure there's one
in the frame when doing mock-observations.
valid_ids_key should already exist in simstub and point to a list.
'''
if fr.type == core.G3FrameType.Scan:
if not valid_ids_key in fr:
raise KeyError(
"%s not found in frame. "%valid_ids_key +
"List of valid bolometer ids required for mock-observing.")
tsm = core.G3TimestreamMap()
for bid in fr[valid_ids_key]:
tsm[bid]=core.G3Timestream()
fr[out_tsm_key] = tsm
def __call__(self, f):
if f.type == core.G3FrameType.Scan:
if self.input not in f.keys() or type(f[self.input]) != core.G3TimestreamMap:
raise ValueError("""Frame is a Scan but does not have a G3Timestream map
named {}""".format(self.input))
processing = core.G3TimestreamMap()
for k in f[self.input].keys():
processing[k] = core.G3Timestream( self.process(f[self.input][k], k) )
processing.start = f[self.input].start
processing.stop = f[self.input].stop
if self.input == self.output:
f.pop(self.input)
f[self.output] = processing
def convert_azel_to_radec(az, el, location=spt, mjd=None):
"""
Convert timestreams of local azimuth and elevation to right ascension and
declination.
Arguments
---------
az, el : np.ndarray or G3Timestream
Array of local coordinates. If inputs are G3Timestream objects,
G3Timestreams are also returned.
location : astropy.coordinates.EarthLocation instance
The telescope location on Earth.
mjd : np.ndarray
An array of times for each az/el sample. If input az and el
are not G3Timestreams, this argument is required.
Returns
-------
ra, dec : np.ndarray or G3Timestream
"""
singleton = False
if isinstance(az, core.G3Timestream):
assert az.start == el.start
assert az.stop == el.stop
assert az.n_samples == el.n_samples
t = astropy.time.Time(np.asarray([i.mjd for i in az.times()]),
format="mjd")
else:
try:
len(az)
except TypeError:
singleton = True
if singleton:
az = np.atleast_1d(az)
el = np.atleast_1d(el)
mjd = np.atleast_1d(mjd)
assert len(az) == len(el)
t = astropy.time.Time(mjd, format="mjd")
check_iers(az.stop)
# record locations of bad elevation values to mark them later
badel_inds = np.where((el < -90.0 * core.G3Units.deg)
| (el > 90.0 * core.G3Units.deg))
el[badel_inds] = 0.0 * core.G3Units.deg
k = astropy.coordinates.AltAz(
az=np.asarray(az) / core.G3Units.deg * astropy.units.deg,
alt=np.asarray(el) / core.G3Units.deg * astropy.units.deg,
obstime=t,
location=location,
pressure=0,
)
kt = k.transform_to(astropy.coordinates.FK5)
ra = np.asarray(kt.ra / astropy.units.deg) * core.G3Units.deg
dec = np.asarray(kt.dec / astropy.units.deg) * core.G3Units.deg
dec[badel_inds] = np.nan
if isinstance(az, core.G3Timestream):
ra = core.G3Timestream(ra)
dec = core.G3Timestream(dec)
ra.start = dec.start = az.start
ra.stop = dec.stop = az.stop
elif singleton:
ra = ra[0]
dec = dec[0]
return (ra, dec)
def __call__(self, f):
if f.type == FT.Calibration and f['cal_type'] == 'focal_plane':
self.focal_plane = f
if f.type != FT.Scan:
return [f]
if f.type == FT.EndProcessing:
flush = True
else:
flush = False
self.frame_buffer.append(f)
self.raw_buffer.append(f[self.enc_name])
# Figure out what frames we're able to process, given info we have.
frames_out = []
# Work in units of seconds.
raw_t0, raw_t1 = self.raw_buffer[0].t[0], self.raw_buffer[-1].t[-1]
# Process any frame that ends before raw_t1.
frame_index = 0
while len(self.frame_buffer) > 0:
f = self.frame_buffer[0]
if not flush and (f['signal'].stop.time / core.G3Units.sec >
raw_t1):
break
sig = f[self.signal_name] # f['signal']
frame_t0 = sig.start.time / core.G3Units.sec
frame_t1 = sig.stop.time / core.G3Units.sec
tick_rate = sig.sample_rate / core.G3Units.Hz
# Figure out what range of samples we will be able to set.
start_index = np.ceil((raw_t0 - frame_t0) * tick_rate)
end_index = np.floor((raw_t1 - frame_t0) * tick_rate) + 1
start_index = max(0, int(start_index))
end_index = min(int(end_index), sig.n_samples)
if end_index != sig.n_samples and not flush:
# Buffer.
break
# Otherwise, do the interpolations...
frames_out.append(self.frame_buffer.pop(0))
t_raw = np.hstack([r.t for r in self.raw_buffer])
t_int = frame_t0 + np.arange(start_index, end_index) / tick_rate
boresight = core.G3TimestreamMap()
vs = {}
for k in ['az', 'el', 'corot']:
interp = spline1d(
t_raw, np.hstack([r.data[k] for r in self.raw_buffer]))
v = np.empty(sig.n_samples)
v[:start_index] = np.nan
v[start_index:end_index] = interp(t_int)
v[end_index:] = np.nan
vs[k] = v
boresight[k] = core.G3Timestream(vs[k])
boresight.start = sig.start
boresight.stop = sig.stop
f[self.boresight_name] = boresight # f['boresight']
# Compute quaternion.
q = ( # Sky <-- near (el, az=0)
coords.q_euler(2, -vs['az'] * np.pi / 180) *
# ... sky at az=0 <-- near (el=0,az=0)
coords.q_euler(1, -vs['el'] * np.pi / 180) *
# ... (1,-xi,eta) <-- (-eta,-xi,1)
coords.q_euler(1, np.pi / 2) *
# ... (-eta,-xi,1) <-- (eta,xi,1)
coords.q_euler(2, np.pi))
# Note that there's no "TimestreamQuat" class. So no timestamps.
f[self.boresight_name + '_q'] = q # f['boresight_q']
# Discard raw data we're not using any more. Out of caution,
# keep one more frame than we have buffered.
while len(self.raw_buffer) - len(self.frame_buffer) > 2:
self.raw_buffer.pop(0)
return frames_out
def convert_radec_to_azel(ra, dec, location=spt, mjd=None):
"""
Convert timestreams of right ascension and declination to local
azimuth and elevation.
Arguments
---------
ra, dec : np.ndarray or G3Timestream
Array of Equatorial sky coordinates. If inputs are G3Timestream
objects, G3Timestreams are also returned.
location : astropy.coordinates.EarthLocation instance
The telescope location on Earth.
mjd : np.ndarray
An array of times for each ra/dec sample. If input ra and dec
are not G3Timestreams, this argument is required.
Returns
-------
az, el : np.ndarray or G3Timestream
"""
singleton = False
if isinstance(ra, core.G3Timestream):
assert ra.start == dec.start
assert ra.stop == dec.stop
assert ra.n_samples == dec.n_samples
t = astropy.time.Time(np.asarray([i.mjd for i in ra.times()]),
format="mjd")
else:
try:
len(ra)
except TypeError:
singleton = True
if singleton:
ra = np.atleast_1d(ra)
dec = np.atleast_1d(dec)
mjd = np.atleast_1d(mjd)
assert len(ra) == len(dec)
t = astropy.time.Time(mjd, format="mjd")
check_iers(ra.stop)
k = astropy.coordinates.FK5(
ra=np.asarray(ra) / core.G3Units.deg * astropy.units.deg,
dec=np.asarray(dec) / core.G3Units.deg * astropy.units.deg,
)
kt = k.transform_to(
astropy.coordinates.AltAz(obstime=t, location=location, pressure=0))
az = np.asarray(kt.az / astropy.units.deg) * core.G3Units.deg
el = np.asarray(kt.alt / astropy.units.deg) * core.G3Units.deg
if isinstance(ra, core.G3Timestream):
az = core.G3Timestream(az)
el = core.G3Timestream(el)
az.start = el.start = ra.start
az.stop = el.stop = ra.stop
elif singleton:
az = az[0]
el = el[0]
return (az, el)
#!/usr/bin/env python import numpy from spt3g import core ts = core.G3Timestream(numpy.zeros(50)) ts.units = core.G3TimestreamUnits.Power ts.start = core.G3Time(0) ts.stop = core.G3Time(10 * core.G3Units.s) assert (ts[5] == 0) # Check scalar getitem ts[12] = 13.4 # Check scalar setitem assert (ts[12] == 13.4) ts[:] = numpy.arange(50) # Test vector setitem assert (ts[12]) == 12.0 # Test units preserved by slicing assert (ts[:5].units == ts.units) # Test length with slicing assert (len(ts[::2]) == len(ts) / 2) assert (len(ts[5:]) == len(ts) - 5) # Test consistency with numpy slicing with steps that do and do not divide evenly into array length assert ((numpy.asarray(ts[::2]) == numpy.asarray(ts)[::2]).all()) assert ((numpy.asarray(ts[::3]) == numpy.asarray(ts)[::3]).all()) assert ((numpy.asarray(ts[5::2]) == numpy.asarray(ts)[5::2]).all()) assert ((numpy.asarray(ts[5::3]) == numpy.asarray(ts)[5::3]).all()) assert ((numpy.asarray(ts[5:-4:2]) == numpy.asarray(ts)[5:-4:2]).all())
def AddBenchData(f):
'''
Add the optical bench positions to the frame.
'''
if f.type != core.G3FrameType.GcpSlow:
return
bench_axes = ['y1', 'y2', 'y3', 'x4', 'x5', 'z6']
benchcom = core.G3TimestreamMap()
benchpos = core.G3TimestreamMap()
benchzero = core.G3TimestreamMap()
benchoff = core.G3TimestreamMap()
bencherr = core.G3TimestreamMap()
bench_info = core.G3TimestreamMap()
for i, key in enumerate(bench_axes):
# As of 2017-08-03, SCU time is not trustworthy
# start = f['antenna0']['scu']['benchSampleTime'][0][0]
# stop = f['antenna0']['scu']['benchSampleTime'][0][-1]
# For now, do this bit of evil
start = f['antenna0']['tracker']['utc'][0][0]
stop = f['antenna0']['tracker']['utc'][0][-1]
benchcom[key] = core.G3Timestream(
f['antenna0']['scu']['benchExpected'][i])
benchcom[key].start = start
benchcom[key].stop = stop
benchpos[key] = core.G3Timestream(
f['antenna0']['scu']['benchActual'][i])
benchpos[key].start = start
benchpos[key].stop = stop
benchzero[key] = core.G3Timestream(
f['antenna0']['scu']['benchZeros'][i])
benchzero[key].start = start
benchzero[key].stop = stop
benchoff[key] = core.G3Timestream(
f['antenna0']['scu']['benchOffsets'][i])
benchoff[key].start = start
benchoff[key].stop = stop
bencherr[key] = core.G3Timestream(
f['antenna0']['scu']['benchErrors'][i])
bencherr[key].start = start
bencherr[key].stop = stop
info_items = [
'benchFocus', 'benchDeadBand', 'benchAcquiredThreshold',
'benchPrimaryState', 'benchSecondaryState', 'benchFault', 'timeLocked'
]
bench_info = core.G3TimestreamMap()
for i, key in enumerate(info_items):
start = f['antenna0']['tracker']['utc'][0][0]
stop = f['antenna0']['tracker']['utc'][0][-1]
bench_info[key] = core.G3Timestream(f['antenna0']['scu'][key][0])
bench_info[key].start = start
bench_info[key].stop = stop
f['BenchPosition'] = benchpos
f['BenchCommandedPosition'] = benchcom
f['BenchZeros'] = benchzero
f['BenchOffsets'] = benchoff
f['BenchErrors'] = bencherr
f['BenchInfo'] = bench_info
f['BenchSampleTime'] = f['antenna0']['scu']['benchSampleTime'][0]
def start_background_streamer(self, session, params=None):
"""start_background_streamer(params=None)
Process to run streaming process. A data stream is started
automatically. It can be stopped and started by the start and stop
tasks. Either way keep alive flow control frames are being sent.
Parameters
----------
frame_rate : float, optional
Frequency [Hz] at which G3Frames are sent over the network.
Defaults to 1 frame pers sec.
sample_rate : float, optional
Sample rate [Hz] for each channel. Defaults to 10 Hz.
"""
if params is None:
params = {}
self.writer = core.G3NetworkSender(hostname=self.target_host,
port=self.port)
frame_rate = params.get('frame_rate', 1.)
sample_rate = params.get('sample_rate', 10.)
frame_num = 0
self.running_in_background = True
# Control flags FIFO stack to keep Writer single threaded
self.flags = deque([FlowControl.START])
while self.running_in_background:
# Send START frame
if next(iter(self.flags), None) is FlowControl.START:
self._set_stream_on() # sends start flowcontrol
self.is_streaming = True
self.flags.popleft()
print("stream running in background")
self.log.debug("control flags: {f}", f=self.flags)
# Send keep alive flow control frame
f = core.G3Frame(core.G3FrameType.none)
f['sostream_flowcontrol'] = FlowControl.ALIVE.value
self.writer.Process(f)
if self.is_streaming:
frame_start = time.time()
time.sleep(1. / frame_rate)
frame_stop = time.time()
times = np.arange(frame_start, frame_stop, 1. / sample_rate)
f = core.G3Frame(core.G3FrameType.Scan)
f['session_id'] = 0
f['frame_num'] = frame_num
f['sostream_id'] = self.stream_id
f['data'] = core.G3TimestreamMap()
for i, chan in enumerate(self.channels):
ts = core.G3Timestream([chan.read() for t in times])
ts.start = core.G3Time(frame_start * core.G3Units.sec)
ts.stop = core.G3Time(frame_stop * core.G3Units.sec)
f['data'][f"r{i:04}"] = ts
self.writer.Process(f)
self.log.info("Writing frame...")
frame_num += 1
# Send END frame
if next(iter(self.flags), None) is FlowControl.END:
self._send_end_flowcontrol_frame()
self._send_cleanse_flowcontrol_frame()
self.is_streaming = False
self.flags.popleft()
else:
# Don't send keep alive frames too quickly
time.sleep(1)
# Shutdown streamer
if next(iter(self.flags), None) is SHUTDOWN:
self.running_in_background = False
self.flags.popleft()
# Teardown writer
self.writer.Close()
self.writer = None
return True, "Finished streaming"
#!/usr/bin/env python
from __future__ import print_function
import numpy, sys
from spt3g import core
then = core.G3Time.Now()
now = core.G3Time(then.time + 3 * core.G3Units.second)
data = numpy.zeros(200)
ts = core.G3Timestream(data)
ts.start = then
ts.stop = now
assert (numpy.abs(ts.sample_rate / core.G3Units.Hz - 66.33333) < 1e-5)
times = ts.times()
assert (ts.times()[0] == ts.start)
print(ts.times()[-1].time, ts.stop.time)
assert (numpy.abs(ts.times()[-1].time - ts.stop.time) < 2
) # Max 1 tick roundoff error
tsm = core.G3TimestreamMap()
tsm['Test'] = ts
assert (numpy.abs(tsm.sample_rate / core.G3Units.Hz - 66.33333) < 1e-5)
times = tsm.times()
#!/usr/bin/env python
from spt3g import core, coordinateutils
import numpy as np
np.random.seed(42)
n_samps = int(1e3)
az_0 = core.G3Timestream(np.random.rand(n_samps) * 2.0 * np.pi - np.pi)
pole_avoidance = 0.7
el_0 = core.G3Timestream(
np.random.rand(n_samps) * np.pi * pole_avoidance -
np.pi / 2.0 * pole_avoidance)
az_0.start = core.G3Time('20170329_000001')
el_0.start = core.G3Time('20170329_000001')
az_0.stop = core.G3Time('20170329_100001')
el_0.stop = core.G3Time('20170329_100001')
az_1 = az_0 + 10 * core.G3Units.arcmin
el_1 = el_0 + 10 * core.G3Units.arcmin
ra_0, dec_0 = coordinateutils.azel.convert_azel_to_radec(az_0, el_0)
ra_1, dec_1 = coordinateutils.azel.convert_azel_to_radec(az_1, el_1)
o_az_0 = az_0 + (np.random.rand() - 0.5) * 2 * core.G3Units.deg
o_el_0 = el_0 + (np.random.rand() - 0.5) * 2 * core.G3Units.deg
o_ra_0, o_dec_0 = coordinateutils.azel.convert_azel_to_radec(o_az_0, o_el_0)
import astropy.coordinates
def cache_to_frames(tod,
start_frame,
n_frames,
frame_offsets,
frame_sizes,
common=None,
detector_fields=None,
flag_fields=None,
detector_map="detectors",
flag_map="flags",
units=None):
"""Gather all data from the distributed cache for a single frame.
Args:
tod (toast.TOD): instance of a TOD class.
start_frame (int): the first frame index.
n_frames (int): the number of frames.
frame_offsets (list): list of the first samples of all frames.
frame_sizes (list): list of the number of samples in each frame.
common (tuple): (cache name, G3 type, frame name) of each common
field.
detector_fields (tuple): (cache name, frame name) of each detector
field.
flag_fields (tuple): (cache name, frame name) of each flag field.
detector_map (str): the name of the frame timestream map.
flag_map (str): then name of the frame flag map.
units: G3 units of the detector data.
"""
# Local sample range
local_first = tod.local_samples[0]
nlocal = tod.local_samples[1]
# The process grid
detranks, sampranks = tod.grid_size
rankdet, ranksamp = tod.grid_ranks
# Helper function:
# For a given timestream, the gather is done across the
# process row which contains the specific detector, or across
# the first process row for common telescope data.
def gather_field(prow, fld, indx, cacheoff, ncache):
gproc = 0
gdata = None
# We are going to allreduce this later, so that every process
# knows the dimensions of the field.
allnnz = 0
if rankdet == prow:
#print(" proc {} doing gather of {}".format(tod.mpicomm.rank, fld), flush=True)
# This process is in the process row that has this field,
# participate in the gather operation.
pdata = None
# Find the data type and shape from the cache object
mtype = None
ref = tod.cache.reference(fld)
nnz = 1
if (len(ref.shape) > 1) and (ref.shape[1] > 0):
nnz = ref.shape[1]
if ref.dtype == np.dtype(np.float64):
mtype = MPI.DOUBLE
elif ref.dtype == np.dtype(np.int64):
mtype = MPI.INT64_T
elif ref.dtype == np.dtype(np.int32):
mtype = MPI.INT32_T
elif ref.dtype == np.dtype(np.uint8):
mtype = MPI.UINT8_T
else:
msg = "Cannot gather cache field {} of type {}"\
.format(fld, ref.dtype)
raise RuntimeError(msg)
#print("field {}: proc {} has nnz = {}".format(fld, tod.mpicomm.rank, nnz), flush=True)
pz = 0
if cacheoff is not None:
pdata = ref.flatten()[nnz * cacheoff:nnz * (cacheoff + ncache)]
pz = nnz * ncache
else:
pdata = np.zeros(0, dtype=ref.dtype)
psizes = tod.grid_comm_row.gather(pz, root=0)
disp = None
totsize = None
if ranksamp == 0:
#print("Gathering field {} with type {}".format(fld, mtype), flush=True)
# We are the process collecting the gathered data.
gproc = tod.mpicomm.rank
allnnz = nnz
# Compute the displacements into the receive buffer.
disp = [0]
for ps in psizes[:-1]:
last = disp[-1]
disp.append(last + ps)
totsize = np.sum(psizes)
# allocate receive buffer
gdata = np.zeros(totsize, dtype=ref.dtype)
#print("Gatherv psizes = {}, disp = {}".format(psizes, disp), flush=True)
#print("field {}: proc {} start Gatherv".format(fld, tod.mpicomm.rank), flush=True)
tod.grid_comm_row.Gatherv(pdata, [gdata, psizes, disp, mtype],
root=0)
#print("field {}: proc {} finish Gatherv".format(fld, tod.mpicomm.rank), flush=True)
del disp
del psizes
del pdata
del ref
# Now send this data to the root process of the whole communicator.
# Only one process (the first one in process row "prow") has data
# to send.
# Create a unique message tag
mtag = 10 * indx
#print(" proc {} hit allreduce of gproc".format(tod.mpicomm.rank), flush=True)
# All processes find out which one did the gather
gproc = tod.mpicomm.allreduce(gproc, MPI.SUM)
# All processes find out the field dimensions
allnnz = tod.mpicomm.allreduce(allnnz, MPI.SUM)
#print(" proc {} for field {}, gproc = {}".format(tod.mpicomm.rank, fld, gproc), flush=True)
#print("field {}: proc {}, gatherproc = {}, allnnz = {}".format(fld, tod.mpicomm.rank, gproc, allnnz), flush=True)
rdata = None
if gproc == 0:
if gdata is not None:
if allnnz == 1:
rdata = gdata
else:
rdata = gdata.reshape((-1, allnnz))
else:
# Data not yet on rank 0
if tod.mpicomm.rank == 0:
# Receive data from the first process in this row
#print(" proc {} for field {}, recv type".format(tod.mpicomm.rank, fld), flush=True)
rtype = tod.mpicomm.recv(source=gproc, tag=(mtag + 1))
#print(" proc {} for field {}, recv size".format(tod.mpicomm.rank, fld), flush=True)
rsize = tod.mpicomm.recv(source=gproc, tag=(mtag + 2))
#print(" proc {} for field {}, recv data".format(tod.mpicomm.rank, fld), flush=True)
rdata = np.zeros(rsize, dtype=np.dtype(rtype))
tod.mpicomm.Recv(rdata, source=gproc, tag=mtag)
# Reshape if needed
if allnnz > 1:
rdata = rdata.reshape((-1, allnnz))
elif (tod.mpicomm.rank == gproc):
# Send our data
#print(" proc {} for field {}, send {} samples of {}".format(tod.mpicomm.rank, fld, len(gdata), gdata.dtype.char), flush=True)
#print(" proc {} for field {}, send type with tag = {}".format(tod.mpicomm.rank, fld, mtag+1), flush=True)
tod.mpicomm.send(gdata.dtype.char, dest=0, tag=(mtag + 1))
#print(" proc {} for field {}, send size with tag = {}".format(tod.mpicomm.rank, fld, mtag+2), flush=True)
tod.mpicomm.send(len(gdata), dest=0, tag=(mtag + 2))
#print(" proc {} for field {}, send data with tag {}".format(tod.mpicomm.rank, fld, mtag), flush=True)
tod.mpicomm.Send(gdata, 0, tag=mtag)
return rdata
# For efficiency, we are going to gather the data for all frames at once.
# Then we will split those up when doing the write.
# Frame offsets relative to the memory buffers we are gathering
fdataoff = [0]
for f in frame_sizes[:-1]:
last = fdataoff[-1]
fdataoff.append(last + f)
# The list of frames- only on the root process.
fdata = None
if tod.mpicomm.rank == 0:
fdata = [c3g.G3Frame(c3g.G3FrameType.Scan) for f in range(n_frames)]
else:
fdata = [None for f in range(n_frames)]
# Compute the overlap of all frames with the local process. We want to
# to find the full sample range that this process overlaps the total set
# of frames.
cacheoff = None
ncache = 0
for f in range(n_frames):
# Compute overlap of the frame with the local samples.
fcacheoff, froff, nfr = local_frame_indices(local_first, nlocal,
frame_offsets[f],
frame_sizes[f])
#print("proc {}: frame {} has cache off {}, fr off {}, nfr {}".format(tod.mpicomm.rank, f, fcacheoff, froff, nfr), flush=True)
if fcacheoff is not None:
if cacheoff is None:
cacheoff = fcacheoff
ncache = nfr
else:
ncache += nfr
#print("proc {}: cache off now {}, ncache now {}".format(tod.mpicomm.rank, cacheoff, ncache), flush=True)
# Now gather the full sample data one field at a time. The root process
# splits up the results into frames.
# First gather common fields from the first row of the process grid.
for findx, (cachefield, g3t, framefield) in enumerate(common):
#print("proc {} entering gather_field(0, {}, {}, {}, {})".format(tod.mpicomm.rank, cachefield, findx, cacheoff, ncache), flush=True)
data = gather_field(0, cachefield, findx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
#print("Casting field {} to type {}".format(field, g3t), flush=True)
if g3t == c3g.G3VectorTime:
# Special case for time values stored as int64_t, but
# wrapped in a class.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
g3times = list()
for t in range(ndata):
g3times.append(c3g.G3Time(data[dataoff + t]))
fdata[f][framefield] = c3g.G3VectorTime(g3times)
del g3times
else:
# The bindings of G3Vector seem to only work with
# lists. This is probably horribly inefficient.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
if len(data.shape) == 1:
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata].tolist())
else:
# We have a 2D quantity
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata,:].flatten().tolist())
del data
# Wait for everyone to catch up...
tod.mpicomm.barrier()
# For each detector field, processes which have the detector
# in their local_dets should be in the same process row.
# We do the gather over just this process row.
if (detector_fields is not None) or (flag_fields is not None):
dpats = {d: re.compile(".*{}.*".format(d)) for d in tod.local_dets}
detmaps = None
if detector_fields is not None:
if tod.mpicomm.rank == 0:
detmaps = [c3g.G3TimestreamMap() for f in range(n_frames)]
for dindx, (cachefield, framefield) in enumerate(detector_fields):
pc = -1
for det, pat in dpats.items():
if pat.match(cachefield) is not None:
#print("proc {} has field {}".format(tod.mpicomm.rank, field), flush=True)
pc = rankdet
break
# As a sanity check, verify that every process which
# has this field is in the same process row.
rowcheck = tod.mpicomm.gather(pc, root=0)
prow = 0
if tod.mpicomm.rank == 0:
rc = np.array([x for x in rowcheck if (x >= 0)],
dtype=np.int32)
#print(field, rc, flush=True)
prow = np.max(rc)
if np.min(rc) != prow:
msg = "Processes with field {} are not in the "\
"same row\n".format(cachefield)
sys.stderr.write(msg)
tod.mpicomm.abort()
# Every process finds out which process row is participating.
prow = tod.mpicomm.bcast(prow, root=0)
#print("proc {} got prow = {}".format(tod.mpicomm.rank, prow), flush=True)
# Get the data on rank 0
data = gather_field(prow, cachefield, dindx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
if units is None:
# We do this conditional, since we can't use
# G3TimestreamUnits.None in python ("None" is
# interpreted as python None).
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
detmaps[f][framefield] = \
c3g.G3Timestream(data[dataoff:dataoff+ndata])
else:
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
detmaps[f][framefield] = \
c3g.G3Timestream(data[dataoff:dataoff+ndata],
units)
if tod.mpicomm.rank == 0:
for f in range(n_frames):
fdata[f][detector_map] = detmaps[f]
flagmaps = None
if flag_fields is not None:
if tod.mpicomm.rank == 0:
flagmaps = [c3g.G3MapVectorInt() for f in range(n_frames)]
for dindx, (cachefield, framefield) in enumerate(flag_fields):
pc = -1
for det, pat in dpats.items():
if pat.match(cachefield) is not None:
pc = rankdet
break
# As a sanity check, verify that every process which
# has this field is in the same process row.
rowcheck = tod.mpicomm.gather(pc, root=0)
prow = 0
if tod.mpicomm.rank == 0:
rc = np.array([x for x in rowcheck if (x >= 0)],
dtype=np.int32)
prow = np.max(rc)
if np.min(rc) != prow:
msg = "Processes with field {} are not in the "\
"same row\n".format(cachefield)
sys.stderr.write(msg)
tod.mpicomm.abort()
# Every process finds out which process row is participating.
prow = tod.mpicomm.bcast(prow, root=0)
# Get the data on rank 0
data = gather_field(prow, cachefield, dindx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
# The bindings of G3Vector seem to only work with
# lists... Also there is no vectormap for unsigned
# char, so we have to use int...
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
flagmaps[f][framefield] = \
c3g.G3VectorInt(\
data[dataoff:dataoff+ndata].astype(np.int32)\
.tolist())
if tod.mpicomm.rank == 0:
for f in range(n_frames):
fdata[f][flag_map] = flagmaps[f]
return fdata
def recode_timestream(ts, params, rmstarget=2**10, rmsmode="white"):
"""ts is a G3Timestream. Returns a new
G3Timestream for same samples as ts, but with data
scaled and translated with gain and offset,
rounded, and with FLAC compression enabled.
Args:
ts (G3Timestream) : Input signal
params (bool or dict) : if True, compress with default
parameters. If dict with 'rmstarget' member, override
default `rmstarget`. If dict with `gain` and `offset`
members, use those instead.
params (None, bool or dict) : If None, False or an empty dict,
no compression or casting to integers. If True or
non-empty dictionary, enable compression. Expected fields
in the dictionary ('rmstarget', 'gain', 'offset', 'rmsmode')
allow overriding defaults.
rmstarget (float) : Scale the iput signal to have this RMS.
Should be much smaller then the 24-bit integer range:
[-2 ** 23 : 2 ** 23] = [-8,388,608 : 8,388,608].
The gain will be reduced if the scaled signal does
not fit within the range of allowed values.
rmsmode (string) : "white" or "full", determines how the
signal RMS is measured.
Returns:
new_ts (G3Timestream) : Scaled and translated timestream
with the FLAC compression enabled
gain (float) : The applied gain
offset (float) : The removed offset
"""
if not params:
return ts, 1, 0
gain = None
offset = None
if isinstance(params, dict):
if "rmsmode" in params:
rmsmode = params["rmsmode"]
if "rmstarget" in params:
rmstarget = params["rmstarget"]
if "gain" in params:
gain = params["gain"]
if "offset" in params:
offset = params["offset"]
v = np.array(ts)
vmin = np.amin(v)
vmax = np.amax(v)
if offset is None:
offset = 0.5 * (vmin + vmax)
amp = vmax - offset
else:
amp = np.max(np.abs(vmin - offset), np.abs(vmax - offset))
if gain is None:
if rmsmode == "white":
rms = np.std(np.diff(v)) / np.sqrt(2)
elif rmsmode == "full":
rms = np.std(v)
else:
raise RuntimeError("Unrecognized RMS mode = '{}'".format(rmsmode))
if rms == 0:
gain = 1
else:
gain = rmstarget / rms
# If the data have extreme outliers, we have to reduce the gain
# to fit the 24-bit signed integer range
while amp * gain >= 2**23:
gain *= 0.5
elif amp * gain >= 2**23:
raise RuntimeError("The specified gain and offset saturate the band.")
v = np.round((v - offset) * gain)
new_ts = core3g.G3Timestream(v)
new_ts.units = core3g.G3TimestreamUnits.Counts
new_ts.SetFLACCompression(True)
new_ts.start = ts.start
new_ts.stop = ts.stop
return new_ts, gain, offset
#!/usr/bin/env python
import pickle, numpy, sys
from spt3g import core
ts = core.G3Timestream(numpy.ones(1200))
ts.units = core.G3TimestreamUnits.Counts
ts.SetFLACCompression(5)
ts[5] = numpy.nan
pickle.dump(ts, open('tsdump.pkl', 'wb'))
ts_rehyd = pickle.load(open('tsdump.pkl', 'rb'))
print('Original')
print(ts.units)
print(ts[4], ts[5], ts[6])
print('Rehydrated')
print(ts_rehyd.units)
print(ts_rehyd[4], ts_rehyd[5], ts_rehyd[6])
if ts_rehyd.units != ts.units:
print('Units do not match')
sys.exit(1)
if numpy.isfinite(ts_rehyd[5]):
print('Element 5 finite!')
sys.exit(1)
if (numpy.asarray(ts_rehyd)[:5] != 1).any() or (numpy.asarray(ts_rehyd)[6:] !=
1).any():
print('Elements not 1!')
#!/usr/bin/env python
import numpy
from spt3g import core
# Check that timestream maps being cast with numpy.asarray have the right
# content and are indistinguishable from numpy.asarray(list(tsm.values())
tsm = core.G3TimestreamMap()
start = core.G3Time.Now()
stop = start + 5 * core.G3Units.s
for ts in ['A', 'B', 'C', 'D']:
i = core.G3Timestream(numpy.random.normal(size=600))
i.start = start
i.stop = stop
i.units = core.G3TimestreamUnits.Tcmb
tsm[ts] = i
buffer1d = numpy.asarray(list(tsm.values()))
buffer2d = numpy.asarray(tsm)
assert (buffer1d.shape == buffer2d.shape)
assert (buffer2d.shape == (4, 600))
assert ((buffer1d == buffer2d).all())
#!/usr/bin/env python
from __future__ import print_function
import random, os, sys, numpy, time
from spt3g import core
port = random.randint(10000, 60000)
frames = []
for i in range(0, 20):
f = core.G3Frame()
f['Sequence'] = i
f['Data'] = core.G3Timestream(numpy.zeros(100000))
frames.append(f)
print('Port: ', port)
child = os.fork()
if child != 0:
# Parent
print('Parent')
send = core.G3NetworkSender(hostname='*', port=port)
time.sleep(1) # XXX: how to signal that the remote end is ready?
print('Sending')
for f in frames:
send(f)
send(core.G3Frame(core.G3FrameType.EndProcessing))
pid, status = os.wait()
print('Child Status: ', status)
