def __call__(self, f):
if f.type == FT.Calibration and f['cal_type'] == 'focal_plane':
self.focal_plane = f
if f.type != FT.Scan:
return [f]
# As long as we have a focal_plane, we can create signal vectors.
if self.focal_plane is None:
return [f]
f['signal'] = core.G3TimestreamMap()
# Determine time samples we will be covering.
if self.start_time is None:
first = f['vertex_enc_raw'].t[0] * core.G3Units.sec
self.start_time = core.G3Time(
np.ceil(first / self.tick_step) * self.tick_step)
# And we will end before...
last = core.G3Time(f['vertex_enc_raw'].t[-1] * core.G3Units.sec)
n = int((last.time - self.start_time.time) / self.tick_step)
end_time = core.G3Time(self.start_time.time + n * self.tick_step)
z = np.zeros(n)
for k in self.focal_plane['signal_names']:
f['signal'][k] = core.G3Timestream(z)
# You can't broadcast-set the start and end time unless the
# elements are already populated.
f['signal'].start = self.start_time
f['signal'].stop = end_time
self.start_time = end_time
return [f]
def stream_fake_data(self, session, params=None):
"""stream_fake_data()
**Process** - Process for streaming fake data. This will queue up
G3Frames full of fake data to be sent to lyrebird.
"""
self._run_fake_stream = True
ndets = self.fp.num_dets
chans = np.arange(ndets)
frame_start = time.time()
while self._run_fake_stream:
time.sleep(2)
frame_stop = time.time()
ts = np.arange(frame_start, frame_stop, 1. / self.target_rate)
frame_start = frame_stop
nframes = len(ts)
data_out = np.random.normal(0, 1, (nframes, ndets))
data_out += np.sin(2 * np.pi * ts[:, None] + .2 * chans[None, :])
for t, d in zip(ts, data_out):
fr = core.G3Frame(core.G3FrameType.Scan)
fr['idx'] = 0
fr['data'] = core.G3VectorDouble(d)
fr['timestamp'] = core.G3Time(t * core.G3Units.s)
self.out_queue.put(fr)
fr = core.G3Frame(core.G3FrameType.Scan)
fr['idx'] = 1
fr['data'] = core.G3VectorDouble(np.sin(d))
fr['timestamp'] = core.G3Time(t * core.G3Units.s)
self.out_queue.put(fr)
return True, "Stopped fake stream process"
def test_from_numpy_array(self):
t0 = core.G3Time('2019-01-01T12:30:00')
timestamps = np.linspace(t0.time, t0.time + 1e7 * SEC, 3000)
vectime = core.G3VectorTime(timestamps)
assert (vectime[0] == t0)
assert (vectime[-1] == core.G3Time(timestamps[-1]))
assert (len(vectime) == len(timestamps))
def get_g3_time(unix_time):
"""Convert a double or numpy array of floats to G3Time or
G3VectorTime."""
src = None
if isinstance(unix_time, core.G3VectorDouble):
src = (np.array(unix_time) * core.G3Units.seconds).astype('int')
elif isinstance(unix_time, np.ndarray) and unix_time.ndim == 1:
src = (unix_time * core.G3Units.seconds).astype('int')
if src is not None:
return core.G3VectorTime([core.G3Time(t) for t in src])
return core.G3Time(int(unix_time * core.G3Units.seconds))
def start_stream(self, session, params=None):
if params is None:
params = {}
delay = params.get('delay', 1)
ts_len = params.get('ts_len', 100)
# Writes status frame
f = core.G3Frame(core.G3FrameType.Housekeeping)
f['session_id'] = 0
f['start_time'] = time.time()
self.writer.Process(f)
self.is_streaming = True
frame_num = 0
while self.is_streaming:
f = core.G3Frame(core.G3FrameType.Scan)
t1 = time.time()
t0 = t1 - delay
ts = np.arange(t0, t1, 1 / self.freq)
f['session_id'] = 0
f['frame_num'] = frame_num
f['data'] = core.G3TimestreamMap()
for k, c in self.channels.items():
fparams = copy.copy(c)
bg = np.random.normal(0, fparams.get('stdev', 0), len(ts))
if fparams['type'] == 'const':
xs = bg + fparams['val']
elif fparams['type'] in ['lin', 'linear']:
xs = bg + ts * fparams['slope'] + fparams.get('offset', 0)
# Wraps from -pi to pi
xs = np.mod(xs + np.pi, 2 * np.pi) - np.pi
f['data'][k] = core.G3Timestream(xs)
f['data'][k].start = core.G3Time(t0 * core.G3Units.sec)
f['data'][k].stop = core.G3Time(t1 * core.G3Units.sec)
self.log.info("Writing G3 Frame")
self.writer.Process(f)
frame_num += 1
time.sleep(delay)
print("Writing EndProcessingFrame")
f = core.G3Frame(core.G3FrameType.EndProcessing)
self.writer.Process(f)
return True, "Finished streaming"
def start_stream(self, session, params=None):
"""
Task to stream fake detector data as G3Frames
Args:
frame_rate (float, optional):
Frequency [Hz] at which G3Frames are sent over the network.
Defaults to 1 frame pers sec.
sample_rate (float, optional):
Sample rate [Hz] for each channel.
Defaults to 10 Hz.
"""
if params is None:
params = {}
frame_rate = params.get('frame_rate', 1.)
sample_rate = params.get('sample_rate', 10.)
f = core.G3Frame(core.G3FrameType.Observation)
f['session_id'] = 0
f['start_time'] = time.time()
self.writer.Process(f)
frame_num = 0
self.is_streaming = True
while self.is_streaming:
frame_start = time.time()
time.sleep(1. / frame_rate)
frame_stop = time.time()
times = np.arange(frame_start, frame_stop, 1. / sample_rate)
f = core.G3Frame(core.G3FrameType.Scan)
f['session_id'] = 0
f['frame_num'] = frame_num
f['data'] = core.G3TimestreamMap()
for i, chan in enumerate(self.channels):
ts = core.G3Timestream([chan.read(t) for t in times])
ts.start = core.G3Time(frame_start * core.G3Units.sec)
ts.stop = core.G3Time(frame_stop * core.G3Units.sec)
f['data'][str(i)] = ts
self.writer.Process(f)
self.log.info("Writing frame...")
frame_num += 1
return True, "Finished streaming"
def addinfo(fr):
global n
if fr.type == core.G3FrameType.EndProcessing:
return
fr['time'] = core.G3Time(int(time.time() * core.G3Units.s))
fr['count'] = n
n += 1
def noise_scan_frames(n_frames=3,
n_dets=20,
input='signal',
n_samps=200,
samp_rate=0.005 * core.G3Units.second,
t_start=core.G3Time('2020-1-1T00:00:00')):
"""
Generate a list of frames filled with noise data and nothing else.
Args:
n_frames (int): number of frames to make
n_dets (int): number of detectors per frame
input (str): name of G3TimestreamMap for detectors, should be some form of 'signal'
n_samps (int): number of samples per detector timestream
samp_rate (G3Unit.second): detector sampling rate
t_start (G3Time): start time of the set of frames
"""
frame_list = []
for n in range(n_frames):
f = core.G3Frame()
f.type = core.G3FrameType.Scan
tsm = core.G3TimestreamMap()
z = np.zeros((n_samps, ))
for d in enumerate_det_id(n_dets):
tsm[d] = core.G3Timestream(z)
tsm.start = t_start
tsm.stop = t_start + n_samps * samp_rate
tsm = MakeNoiseData().apply(tsm)
f[input] = tsm
t_start += n_samps * samp_rate
frame_list.append(f)
return frame_list
def addinfo(fr):
global n
if fr.type != core.G3FrameType.Timepoint:
return
fr['time'] = core.G3Time(int(time.time() * core.G3Units.s))
fr['count'] = n
n += 1
def test_to_numpy_array(self):
t0 = core.G3Time('2019-01-01T12:30:00')
timestamps = np.linspace(t0.time,
t0.time + 1e7 * SEC,
3000,
dtype='int64')
vectime = core.G3VectorTime(timestamps)
assert ((np.asarray(vectime) == timestamps).all())
def tag_frame(self, fr):
fr['frame_num'] = self.frame_num
fr['session_id'] = self.session_id
fr['sostream_id'] = self.stream_id
fr['sostream_version'] = SOSTREAM_VERSION
fr['time'] = core.G3Time(time.time() * core.G3Units.s)
self.frame_num += 1
return fr
def g3_cast(data, time=False):
"""
Casts a generic datatype into a corresponding G3 type. With:
int -> G3Int
str -> G3String
float -> G3Double
and lists of type X will go to G3VectorX. If ``time`` is set to True, will
convert to G3Time or G3VectorTime with the assumption that ``data`` consists
of unix timestamps.
Args:
data (int, str, float, or list):
Generic data to be converted to a corresponding G3Type.
time (bool, optional):
If True, will assume data contains unix timestamps and try to cast
to G3Time or G3VectorTime.
Returns:
g3_data:
Corresponding G3 datatype.
"""
is_list = isinstance(data, list)
if is_list:
dtype = type(data[0])
if not all(isinstance(d, dtype) for d in data):
raise TypeError("Data list contains varying types!")
else:
dtype = type(data)
if dtype not in _g3_casts.keys():
raise TypeError("g3_cast does not support type {}. Type must"
"be one of {}".format(dtype, _g3_casts.keys()))
if is_list:
if time:
return core.G3VectorTime(
list(map(lambda t: core.G3Time(t * core.G3Units.s), data)))
else:
cast = _g3_list_casts[type(data[0])]
return cast(data)
else:
if time:
return core.G3Time(data * core.G3Units.s)
else:
cast = _g3_casts[type(data)]
return cast(data)
def addinfo(fr):
if fr.type == core.G3FrameType.EndProcessing:
return
global n
fr['time'] = core.G3Time(int(time.time() * core.G3Units.s))
fr['count'] = n
fr2 = core.G3Frame(core.G3FrameType.Housekeeping)
fr2['count'] = n
n += 1
return [fr, fr2]
def get_v2_stream():
"""Generate some example HK data, in schema version 2.
Returns a list of frames constituting a valid version 2 HK stream.
"""
# Create something to help us track the aggregator session.
hksess = so3g.hk.HKSessionHelper(session_id=1234,
hkagg_version=2,
description="Test HK data.")
# Register a data provider.
prov_id = hksess.add_provider(
description='Fake data for the real world.')
# Start the stream -- write the initial session and status frames.
frames = [
hksess.session_frame(),
hksess.status_frame(),
]
# Now make a data frame.
f = hksess.data_frame(prov_id=prov_id)
# Add some data blocks.
hk = core.G3TimesampleMap()
hk.times = core.G3VectorTime([core.G3Time(i*core.G3Units.seconds) for i in [0, 1, 2, 3, 4]])
hk['speed'] = core.G3VectorDouble([1.2, 1.2, 1.2, 1.2, 1.2])
f['blocks'].append(hk)
f['block_names'].append('group0')
hk = core.G3TimesampleMap()
hk.times = core.G3VectorTime([core.G3Time(i*core.G3Units.seconds) for i in [0, 1, 2, 3, 4]])
hk['position'] = core.G3VectorInt([1, 2, 3, 4, 5])
hk['mode'] = core.G3VectorString(['going', 'going', 'going', 'going', 'gone/'])
f['blocks'].append(hk)
f['block_names'].append('group1')
frames.append(f)
return frames
def get_test_block(length, keys=['a', 'b', 'c', 'd'], offset=0, ordered=True):
type_cycle = [(core.G3VectorDouble, float), (core.G3VectorInt, int),
(core.G3VectorString, str), (core.G3VectorBool, bool)]
t0 = core.G3Time('2019-01-01T12:30:00') + offset * SEC
m = core.G3TimesampleMap()
times = np.arange(length)
if not ordered:
np.random.shuffle(times)
m.times = core.G3VectorTime(t0 + times * SEC)
for i, k in enumerate(keys):
y = (np.random.uniform(size=length) * 100).astype(int)
constructor, cast_func = type_cycle[i % len(type_cycle)]
vect = constructor(list(map(cast_func, y)))
m[k] = vect
return m
def g3_to_array(g3file, verbose=False):
"""
Takes a G3 file output from the SMuRF archiver and reads to a numpy array.
Parameters
----------
g3file : full path to the G3 file
verbose : OPTIONAL choice for verbose output (-v, --verbosity)
Returns
-------
times : array of G3Time objects
data : array of arrays, where each internal array is a SMuRF channel
"""
frames = [fr for fr in core.G3File(g3file)]
data = []
frametimes = []
for frame in frames:
if frame.type == core.G3FrameType.Scan:
frametime = frame['data'].times()
frametimes.append(frametime)
if frametimes == []:
warnings.warn('Wrong frame type')
strtimes = np.hstack(frametimes)
times = []
for strt in strtimes:
t = core.G3Time(strt).time / core.G3Units.s
times.append(t)
times = np.asarray(times)
channums = []
i = 0
while i < len(frames):
if verbose:
print('Trying frame %i' % i)
frametype = frames[i].type
if frametype == core.G3FrameType.Scan:
for chan in frames[i]['data'].keys():
channums.append(int(chan.strip('r')))
break
else:
i += 1
if verbose:
print('Channel numbers obtained')
channums.sort()
for ch in channums:
if verbose:
print('Adding channel %s' % ch)
chdata = []
for frame in frames:
if frame.type == core.G3FrameType.Scan:
framedata = frame['data']['r' + format(ch, "04")]
chdata.append(framedata)
chdata_all = np.hstack(chdata)
data.append(chdata_all)
biases = []
biasnums = []
for num in frames[i]['tes_biases'].keys():
biasnums.append(int(num))
biasnums.sort()
for b in biasnums:
if verbose:
print('Adding bias number %i' % b)
bias = []
for frame in frames:
if frame.type == core.G3FrameType.Scan:
biasdata = frame['tes_biases'][str(b)]
bias.append(biasdata)
bias_all = np.hstack(bias)
biases.append(bias_all)
biases = np.asarray(biases)
data = np.asarray(data)
return times, data, biases
def HousekeepingFromJSON(cls, dat):
'''
Build HKBoardInfo object from a JSON blob returned by the
_dump_housekeeping call
'''
# Board-global quantities
boardhk = HkBoardInfo()
if 'is128x' in dat:
boardhk.is128x = dat['is128x']
else:
boardhk.is128x = False
year = dat['timestamp']['y']
if year == 0:
# It probably isn't 1900
systime = time.gmtime()
# Check for New Year's, assuming no more than 24 hours clock slew
if dat['timestamp']['d'] == 1 and systime.tm_yday >= 365:
year = systime.tm_year + 1
elif dat['timestamp']['d'] >= 365 and systime.tm_yday == 1:
year = systime.tm_year - 1
else:
year = systime.tm_year
year -= 2000
boardhk.timestamp = core.G3Time(y=year,d=dat['timestamp']['d'],h=dat['timestamp']['h'],m=dat['timestamp']['m'],s=dat['timestamp']['s'],ss=dat['timestamp']['ss'])
boardhk.timestamp_port = str(dat['timestamp_port'])
boardhk.serial = str(dat['serial'])
boardhk.fir_stage = dat['fir_stage']
for i in dat['currents'].items():
boardhk.currents[str(i[0])] = i[1]
for i in dat['voltages'].items():
boardhk.voltages[str(i[0])] = i[1]
for i in dat['temperatures'].items():
boardhk.temperatures[str(i[0])] = i[1]
# Mezzanines
for n, mezz in enumerate(dat['mezzanines']):
mezzhk = HkMezzanineInfo()
mezzhk.present = mezz['present']
mezzhk.power = mezz['power']
if mezzhk.present:
mezzhk.serial = str(mezz['ipmi']['product']['serial_number'])
mezzhk.part_number = str(mezz['ipmi']['product']['part_number'])
mezzhk.revision = str(mezz['ipmi']['product']['version_number'])
for i in mezz['currents'].items():
mezzhk.currents[str(i[0])] = i[1]
for i in mezz['voltages'].items():
mezzhk.voltages[str(i[0])] = i[1]
if mezzhk.present and mezzhk.power:
mezzhk.temperature = mezz['temperature']
# these parameters are not in the 64x housekeeping tuber
mezzhk.squid_heater = mezz.get('squid_heater', 0.0)
mezzhk.squid_controller_power = mezz.get('squid_controller_power', False)
mezzhk.squid_controller_temperature = mezz.get('squid_controller_temperature', 0.0)
# Modules
for m, mod in enumerate(mezz['modules']):
modhk = HkModuleInfo()
modhk.routing = str(mod['routing'][0])
modhk.module_number = m+1
if mezzhk.present and mezzhk.power:
if 'gains' in mod:
modhk.carrier_gain = mod['gains']['carrier']
modhk.nuller_gain = mod['gains']['nuller']
if 'overload' in mod:
modhk.carrier_railed = mod['overload']['carrier']
modhk.nuller_railed = mod['overload']['nuller']
modhk.demod_railed = mod['overload']['demod']
if 'squid_current_bias' in mod:
modhk.squid_current_bias = mod['squid_current_bias']
if 'squid_flux_bias' in mod:
modhk.squid_current_bias = mod['squid_flux_bias']
if 'squid_feedback' in mod:
modhk.squid_feedback = str(mod['squid_feedback'])
if 'squid_tuning' in mod and mod['squid_tuning'] is not None:
modhk.squid_state = str(mod['squid_tuning']['state'])
modhk.squid_transimpedance = mod['squid_tuning']['transimpedance'] if mod['squid_tuning']['transimpedance'] is not None else numpy.nan
modhk.squid_p2p = mod['squid_tuning']['p2p'] if mod['squid_tuning']['p2p'] is not None else numpy.nan
for k, chan in enumerate(mod['channels']):
chanhk = HkChannelInfo()
chanhk.channel_number = k+1
chanhk.carrier_amplitude = chan['carrier_amplitude']
chanhk.nuller_amplitude = chan['nuller_amplitude']
chanhk.dan_gain = chan['dan_gain']
chanhk.dan_streaming_enable = chan['dan_streaming_enable']
if boardhk.is128x:
chanhk.carrier_frequency = chan['frequency']*core.G3Units.Hz
chanhk.demod_frequency = chan['frequency']*core.G3Units.Hz
else:
chanhk.carrier_frequency = chan['carrier_frequency']*core.G3Units.Hz
chanhk.demod_frequency = chan['demod_frequency']*core.G3Units.Hz
chanhk.dan_accumulator_enable = chan['dan_accumulator_enable']
chanhk.dan_feedback_enable = chan['dan_feedback_enable']
if 'dan_railed' in chan:
chanhk.dan_railed = chan['dan_railed']
if 'tuning' in chan and chan['tuning'] is not None:
chanhk.state = str(chan['tuning']['state'])
if ('rlatched' in chan['tuning'] and
chan['tuning']['rlatched'] is not None):
chanhk.rlatched = chan['tuning']['rlatched']
if ('rnormal' in chan['tuning'] and
chan['tuning']['rnormal'] is not None):
chanhk.rnormal = chan['tuning']['rnormal']
if ('rfrac_achieved' in chan['tuning'] and
chan['tuning']['rfrac_achieved'] is not None):
chanhk.rfrac_achieved = chan['tuning']['rfrac_achieved']
if ('loopgain' in chan['tuning'] and
chan['tuning']['loopgain'] is not None):
chanhk.loopgain = chan['tuning']['loopgain']
modhk.channels[k+1] = chanhk
mezzhk.modules[m+1] = modhk
boardhk.mezz[n+1] = mezzhk
return boardhk
def addinfo(fr):
global n
fr['time'] = core.G3Time(int(time.time() * core.G3Units.s))
fr['count'] = n
n += 1
def HousekeepingFromJSON(cls, dat):
'''
Build HKBoardInfo object from the union of the JSON blobs
returned by the classic_housekeeping and class_dfmux_dump
Tuber calls
'''
# Board-global quantities
boardhk = HkBoardInfo()
if dat['ts']['port'] == 'IRIG test':
boardhk.timestamp = core.G3Time(dat['ts']['s'] * core.G3Units.s)
else:
year = dat['ts']['y']
if year == 0:
# It probably isn't 1900
systime = time.gmtime()
# Check for New Year's, assuming no more than 24 hours clock slew
if dat['ts']['d'] == 1 and systime.tm_yday >= 365:
year = systime.tm_year + 1
elif dat['ts']['d'] >= 365 and systime.tm_yday == 1:
year = systime.tm_year - 1
else:
year = systime.tm_year
year -= 2000
boardhk.timestamp = core.G3Time(y=year,
d=dat['ts']['d'],
h=dat['ts']['h'],
m=dat['ts']['m'],
s=dat['ts']['s'],
ss=dat['ts']['ss'])
boardhk.timestamp_port = str(dat['ts']['port'])
boardhk.serial = 'N/A'
boardhk.fir_stage = dat['fir_stage']
for i in dat['voltages']['mb'].items():
boardhk.voltages[str(i[0])] = i[1]
for i in dat['temperatures']['mb'].items():
boardhk.temperatures[str(i[0])] = i[1]
# Mezzanines
for mezz in [1, 2]:
mezzhk = HkMezzanineInfo()
mezzhk.power = dat['voltages']['mezz%d' % mezz]['v3p'] > 0
mezzhk.present = True
mezzhk.serial = 'N/A'
mezzhk.part_number = 'SPTpol Mezz'
mezzhk.revision = 'N/A'
for i in dat['voltages']['mezz%d' % mezz].items():
if isinstance(i[1], float):
mezzhk.voltages[str(i[0])] = i[1]
# Modules
for mod in [1, 2]:
wire = (mezz - 1) * 2 + mod
# XXX: SQUID data (bias point, FLL) not in HK call!
# XXX: SQUID tuning state
modhk = HkModuleInfo()
modhk.carrier_gain = dat['mezz_gains']['wire%d' %
wire]['carrier']
modhk.nuller_gain = dat['mezz_gains']['wire%d' %
wire]['nuller']
modhk.demod_gain = dat['mezz_gains']['wire%d' % wire]['demod']
modhk.carrier_railed = dat['voltages']['mezz%d' % mezz][
'overload_dmfs_car_%s' % (['a', 'b'][mod - 1])]
modhk.nuller_railed = dat['voltages']['mezz%d' % mezz][
'overload_dmfs_nul_%s' % (['a', 'b'][mod - 1])]
modhk.demod_railed = dat['voltages']['mezz%d' %
mezz]['overload_dmfd_%s' %
(['a', 'b'
][mod - 1])]
modhk.module_number = mod
for chan in range(1,
dat['config']['dmfd_channels_per_wire'] + 1):
chanhk = HkChannelInfo()
chanhk.channel_number = chan
chanhk.dan_gain = dat['dan']['gain'][wire - 1][chan - 1]
chanhk.dan_accumulator_enable = dat['dan'][
'accumulator_enable'][wire - 1][chan - 1]
chanhk.dan_feedback_enable = dat['dan']['feedback_enable'][
wire - 1][chan - 1]
chanhk.dan_streaming_enable = dat['dan'][
'streaming_enable'][wire - 1][chan - 1]
chanhk.dan_railed = dat['dan']['accumulator_railed'][
wire - 1][chan - 1]
# Channel params: (freq, phase, amp)
carrier = dat['dmfs']['carrier'][wire - 1][chan - 1]
nuller = dat['dmfs']['nuller'][wire - 1][chan - 1]
# Demod params: (freq, phase)
demod = dat['dmfd']['demod'][wire - 1][chan - 1]
chanhk.carrier_amplitude = carrier[2] / (2.**23 - 1.)
chanhk.nuller_amplitude = nuller[2] / (2.**23 - 1.)
chanhk.carrier_frequency = carrier[0] * 25.e6 / (
2.**32 - 1) * core.G3Units.Hz
chanhk.demod_frequency = demod[0] * 25.e6 / (
2.**32 - 1) * core.G3Units.Hz
modhk.channels[chan] = chanhk
mezzhk.modules[mod] = modhk
boardhk.mezz[mezz] = mezzhk
return boardhk
for i in range(10):
# Number of samples
n = int(halfscan / v_az / dt)
# Vector of unix timestamps
t = frame_time + dt * np.arange(n)
# Vector of az and el
az = v_az * dt * np.arange(n)
if i % 2:
az = -az
el = az * 0 + 50.
# Construct a "block", which is a named G3TimesampleMap.
block = core.G3TimesampleMap()
block.times = core.G3VectorTime(
[core.G3Time(_t * core.G3Units.s) for _t in t])
block['az'] = core.G3VectorDouble(az)
block['el'] = core.G3VectorDouble(el)
# Create an output data frame template associated with this
# provider.
frame = session.data_frame(prov_id)
# Add the block and block name to the frame, and write it.
frame['block_names'].append('pointing')
frame['blocks'].append(block)
writer.Process(frame)
# For next iteration.
frame_time += n * dt
def cache_to_frames(tod,
start_frame,
n_frames,
frame_offsets,
frame_sizes,
common=None,
detector_fields=None,
flag_fields=None,
detector_map="detectors",
flag_map="flags",
units=None):
"""Gather all data from the distributed cache for a single frame.
Args:
tod (toast.TOD): instance of a TOD class.
start_frame (int): the first frame index.
n_frames (int): the number of frames.
frame_offsets (list): list of the first samples of all frames.
frame_sizes (list): list of the number of samples in each frame.
common (tuple): (cache name, G3 type, frame name) of each common
field.
detector_fields (tuple): (cache name, frame name) of each detector
field.
flag_fields (tuple): (cache name, frame name) of each flag field.
detector_map (str): the name of the frame timestream map.
flag_map (str): then name of the frame flag map.
units: G3 units of the detector data.
"""
# Local sample range
local_first = tod.local_samples[0]
nlocal = tod.local_samples[1]
# The process grid
detranks, sampranks = tod.grid_size
rankdet, ranksamp = tod.grid_ranks
# Helper function:
# For a given timestream, the gather is done across the
# process row which contains the specific detector, or across
# the first process row for common telescope data.
def gather_field(prow, fld, indx, cacheoff, ncache):
gproc = 0
gdata = None
# We are going to allreduce this later, so that every process
# knows the dimensions of the field.
allnnz = 0
if rankdet == prow:
#print(" proc {} doing gather of {}".format(tod.mpicomm.rank, fld), flush=True)
# This process is in the process row that has this field,
# participate in the gather operation.
pdata = None
# Find the data type and shape from the cache object
mtype = None
ref = tod.cache.reference(fld)
nnz = 1
if (len(ref.shape) > 1) and (ref.shape[1] > 0):
nnz = ref.shape[1]
if ref.dtype == np.dtype(np.float64):
mtype = MPI.DOUBLE
elif ref.dtype == np.dtype(np.int64):
mtype = MPI.INT64_T
elif ref.dtype == np.dtype(np.int32):
mtype = MPI.INT32_T
elif ref.dtype == np.dtype(np.uint8):
mtype = MPI.UINT8_T
else:
msg = "Cannot gather cache field {} of type {}"\
.format(fld, ref.dtype)
raise RuntimeError(msg)
#print("field {}: proc {} has nnz = {}".format(fld, tod.mpicomm.rank, nnz), flush=True)
pz = 0
if cacheoff is not None:
pdata = ref.flatten()[nnz * cacheoff:nnz * (cacheoff + ncache)]
pz = nnz * ncache
else:
pdata = np.zeros(0, dtype=ref.dtype)
psizes = tod.grid_comm_row.gather(pz, root=0)
disp = None
totsize = None
if ranksamp == 0:
#print("Gathering field {} with type {}".format(fld, mtype), flush=True)
# We are the process collecting the gathered data.
gproc = tod.mpicomm.rank
allnnz = nnz
# Compute the displacements into the receive buffer.
disp = [0]
for ps in psizes[:-1]:
last = disp[-1]
disp.append(last + ps)
totsize = np.sum(psizes)
# allocate receive buffer
gdata = np.zeros(totsize, dtype=ref.dtype)
#print("Gatherv psizes = {}, disp = {}".format(psizes, disp), flush=True)
#print("field {}: proc {} start Gatherv".format(fld, tod.mpicomm.rank), flush=True)
tod.grid_comm_row.Gatherv(pdata, [gdata, psizes, disp, mtype],
root=0)
#print("field {}: proc {} finish Gatherv".format(fld, tod.mpicomm.rank), flush=True)
del disp
del psizes
del pdata
del ref
# Now send this data to the root process of the whole communicator.
# Only one process (the first one in process row "prow") has data
# to send.
# Create a unique message tag
mtag = 10 * indx
#print(" proc {} hit allreduce of gproc".format(tod.mpicomm.rank), flush=True)
# All processes find out which one did the gather
gproc = tod.mpicomm.allreduce(gproc, MPI.SUM)
# All processes find out the field dimensions
allnnz = tod.mpicomm.allreduce(allnnz, MPI.SUM)
#print(" proc {} for field {}, gproc = {}".format(tod.mpicomm.rank, fld, gproc), flush=True)
#print("field {}: proc {}, gatherproc = {}, allnnz = {}".format(fld, tod.mpicomm.rank, gproc, allnnz), flush=True)
rdata = None
if gproc == 0:
if gdata is not None:
if allnnz == 1:
rdata = gdata
else:
rdata = gdata.reshape((-1, allnnz))
else:
# Data not yet on rank 0
if tod.mpicomm.rank == 0:
# Receive data from the first process in this row
#print(" proc {} for field {}, recv type".format(tod.mpicomm.rank, fld), flush=True)
rtype = tod.mpicomm.recv(source=gproc, tag=(mtag + 1))
#print(" proc {} for field {}, recv size".format(tod.mpicomm.rank, fld), flush=True)
rsize = tod.mpicomm.recv(source=gproc, tag=(mtag + 2))
#print(" proc {} for field {}, recv data".format(tod.mpicomm.rank, fld), flush=True)
rdata = np.zeros(rsize, dtype=np.dtype(rtype))
tod.mpicomm.Recv(rdata, source=gproc, tag=mtag)
# Reshape if needed
if allnnz > 1:
rdata = rdata.reshape((-1, allnnz))
elif (tod.mpicomm.rank == gproc):
# Send our data
#print(" proc {} for field {}, send {} samples of {}".format(tod.mpicomm.rank, fld, len(gdata), gdata.dtype.char), flush=True)
#print(" proc {} for field {}, send type with tag = {}".format(tod.mpicomm.rank, fld, mtag+1), flush=True)
tod.mpicomm.send(gdata.dtype.char, dest=0, tag=(mtag + 1))
#print(" proc {} for field {}, send size with tag = {}".format(tod.mpicomm.rank, fld, mtag+2), flush=True)
tod.mpicomm.send(len(gdata), dest=0, tag=(mtag + 2))
#print(" proc {} for field {}, send data with tag {}".format(tod.mpicomm.rank, fld, mtag), flush=True)
tod.mpicomm.Send(gdata, 0, tag=mtag)
return rdata
# For efficiency, we are going to gather the data for all frames at once.
# Then we will split those up when doing the write.
# Frame offsets relative to the memory buffers we are gathering
fdataoff = [0]
for f in frame_sizes[:-1]:
last = fdataoff[-1]
fdataoff.append(last + f)
# The list of frames- only on the root process.
fdata = None
if tod.mpicomm.rank == 0:
fdata = [c3g.G3Frame(c3g.G3FrameType.Scan) for f in range(n_frames)]
else:
fdata = [None for f in range(n_frames)]
# Compute the overlap of all frames with the local process. We want to
# to find the full sample range that this process overlaps the total set
# of frames.
cacheoff = None
ncache = 0
for f in range(n_frames):
# Compute overlap of the frame with the local samples.
fcacheoff, froff, nfr = local_frame_indices(local_first, nlocal,
frame_offsets[f],
frame_sizes[f])
#print("proc {}: frame {} has cache off {}, fr off {}, nfr {}".format(tod.mpicomm.rank, f, fcacheoff, froff, nfr), flush=True)
if fcacheoff is not None:
if cacheoff is None:
cacheoff = fcacheoff
ncache = nfr
else:
ncache += nfr
#print("proc {}: cache off now {}, ncache now {}".format(tod.mpicomm.rank, cacheoff, ncache), flush=True)
# Now gather the full sample data one field at a time. The root process
# splits up the results into frames.
# First gather common fields from the first row of the process grid.
for findx, (cachefield, g3t, framefield) in enumerate(common):
#print("proc {} entering gather_field(0, {}, {}, {}, {})".format(tod.mpicomm.rank, cachefield, findx, cacheoff, ncache), flush=True)
data = gather_field(0, cachefield, findx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
#print("Casting field {} to type {}".format(field, g3t), flush=True)
if g3t == c3g.G3VectorTime:
# Special case for time values stored as int64_t, but
# wrapped in a class.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
g3times = list()
for t in range(ndata):
g3times.append(c3g.G3Time(data[dataoff + t]))
fdata[f][framefield] = c3g.G3VectorTime(g3times)
del g3times
else:
# The bindings of G3Vector seem to only work with
# lists. This is probably horribly inefficient.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
if len(data.shape) == 1:
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata].tolist())
else:
# We have a 2D quantity
fdata[f][framefield] = \
g3t(data[dataoff:dataoff+ndata,:].flatten().tolist())
del data
# Wait for everyone to catch up...
tod.mpicomm.barrier()
# For each detector field, processes which have the detector
# in their local_dets should be in the same process row.
# We do the gather over just this process row.
if (detector_fields is not None) or (flag_fields is not None):
dpats = {d: re.compile(".*{}.*".format(d)) for d in tod.local_dets}
detmaps = None
if detector_fields is not None:
if tod.mpicomm.rank == 0:
detmaps = [c3g.G3TimestreamMap() for f in range(n_frames)]
for dindx, (cachefield, framefield) in enumerate(detector_fields):
pc = -1
for det, pat in dpats.items():
if pat.match(cachefield) is not None:
#print("proc {} has field {}".format(tod.mpicomm.rank, field), flush=True)
pc = rankdet
break
# As a sanity check, verify that every process which
# has this field is in the same process row.
rowcheck = tod.mpicomm.gather(pc, root=0)
prow = 0
if tod.mpicomm.rank == 0:
rc = np.array([x for x in rowcheck if (x >= 0)],
dtype=np.int32)
#print(field, rc, flush=True)
prow = np.max(rc)
if np.min(rc) != prow:
msg = "Processes with field {} are not in the "\
"same row\n".format(cachefield)
sys.stderr.write(msg)
tod.mpicomm.abort()
# Every process finds out which process row is participating.
prow = tod.mpicomm.bcast(prow, root=0)
#print("proc {} got prow = {}".format(tod.mpicomm.rank, prow), flush=True)
# Get the data on rank 0
data = gather_field(prow, cachefield, dindx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
if units is None:
# We do this conditional, since we can't use
# G3TimestreamUnits.None in python ("None" is
# interpreted as python None).
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
detmaps[f][framefield] = \
c3g.G3Timestream(data[dataoff:dataoff+ndata])
else:
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
detmaps[f][framefield] = \
c3g.G3Timestream(data[dataoff:dataoff+ndata],
units)
if tod.mpicomm.rank == 0:
for f in range(n_frames):
fdata[f][detector_map] = detmaps[f]
flagmaps = None
if flag_fields is not None:
if tod.mpicomm.rank == 0:
flagmaps = [c3g.G3MapVectorInt() for f in range(n_frames)]
for dindx, (cachefield, framefield) in enumerate(flag_fields):
pc = -1
for det, pat in dpats.items():
if pat.match(cachefield) is not None:
pc = rankdet
break
# As a sanity check, verify that every process which
# has this field is in the same process row.
rowcheck = tod.mpicomm.gather(pc, root=0)
prow = 0
if tod.mpicomm.rank == 0:
rc = np.array([x for x in rowcheck if (x >= 0)],
dtype=np.int32)
prow = np.max(rc)
if np.min(rc) != prow:
msg = "Processes with field {} are not in the "\
"same row\n".format(cachefield)
sys.stderr.write(msg)
tod.mpicomm.abort()
# Every process finds out which process row is participating.
prow = tod.mpicomm.bcast(prow, root=0)
# Get the data on rank 0
data = gather_field(prow, cachefield, dindx, cacheoff, ncache)
if tod.mpicomm.rank == 0:
# The bindings of G3Vector seem to only work with
# lists... Also there is no vectormap for unsigned
# char, so we have to use int...
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
flagmaps[f][framefield] = \
c3g.G3VectorInt(\
data[dataoff:dataoff+ndata].astype(np.int32)\
.tolist())
if tod.mpicomm.rank == 0:
for f in range(n_frames):
fdata[f][flag_map] = flagmaps[f]
return fdata
iv1 = so3g.IntervalsDouble()\
.add_interval(0., 1.)\
.add_interval(2., 3.)\
.add_interval(4., 5.)
iv2 = so3g.IntervalsDouble()\
.add_interval(1., 2.5)
assert (len((iv1 + iv2).array()) == 2)
assert (len((iv1 * iv2).array()) == 1)
assert (len((iv1 - iv2).array()) == 2)
assert (len((iv2 - iv1).array()) == 4)
print('Sanity check on G3Time')
ti = so3g.IntervalsTime()\
.add_interval(core.G3Time('2018-1-1T00:00:00'),
core.G3Time('2018-1-2T00:00:00'))
print(' ', ti)
print(' ', ti.array())
print(' ', (-ti).array())
print()
print('Interval <-> mask testing')
mask = np.zeros(20, 'uint16')
n_bit, target_bit = 16, 12
for ikill, nint in [(None, 0), (19, 1), (0, 2), (10, 3), (11, 3)]:
if ikill is not None:
mask[ikill] = (1 << target_bit)
iv = so3g.IntervalsInt.from_mask(mask, n_bit)
assert (len(iv) == n_bit)
for i in range(n_bit):
def WriteDB(fr, client, fields=None):
'''
Write points to the database for each field
Arguments
---------
client :
InfluxDB client
fields :
Which gcp fields to add to database. See parse_field for options. If
None, add all.
'''
from influxdb.exceptions import InfluxDBClientError
from influxdb.exceptions import InfluxDBServerError
if fr.type != core.G3FrameType.GcpSlow:
return
all_fields = build_field_list(fr)
if fields is None:
fields = all_fields.keys()
dict_list = []
for f in fields:
field_dat = all_fields[f]
if len(field_dat) == 4:
stat, attr, ind, unit = field_dat
try:
dat = getattr(fr[stat], attr)[ind]
time = getattr(fr[stat], 'time')
except AttributeError:
# OnlinePointingModel
dat = fr[stat][attr][ind]
time = fr[stat]['time']
elif len(field_dat) == 3:
stat, attr, unit = field_dat
if stat not in fr:
# Field only exists in live data stream
continue
try:
dat = getattr(fr[stat], attr)
except AttributeError:
try:
dat = fr[stat][attr]
except KeyError: # Field only exists in live data stream
continue
if 'Bench' in stat: # funny time field for bench positions
time = fr['BenchSampleTime']
elif 'Mux' in stat:
time = fr['MuxTime']
elif stat in ['CryoStatus', 'Weather', 'PTStatus']:
time = fr['{}Time'.format(stat)]
else:
try:
time = getattr(fr[stat], 'time')
except AttributeError:
time = fr[stat]['time']
elif len(field_dat) == 2:
stat, unit = field_dat
try:
dat = fr[stat]
except KeyError: #eg, no obsid
core.log_warn('No key {}'.format(stat), unit='InfluxDB')
continue
try:
time = getattr(fr[stat], 'time')
except AttributeError as err:
time = [tm for tm in fr['antenna0']['tracker']['utc'][0]]
# InfluxDB wants time in nanoseconds since the UNIX epoch in UTC
try:
time = [x.time / U.nanosecond for x in np.atleast_1d(time)]
except AttributeError:
time = [
core.G3Time(t0).time / U.nanosecond
for t0 in np.atleast_1d(time)
]
if dat is None:
core.log_warn('{} dat is None'.format(f), unit='InfluxDB')
continue
dat = np.atleast_1d(dat)
try:
dlen = len(dat)
except TypeError:
# sometimes source_name is a weird non-none value
continue
if unit is not None:
if unit == 'C':
zeropt_K = 273.15
cal_dat = dat / U.K - zeropt_K
else:
cal_dat = dat / unit
else:
cal_dat = dat
try:
if np.any(np.isnan(cal_dat)):
continue
except TypeError:
pass
if 'heat' not in f:
tag = f
else:
tag = f.replace('heat_', '')
# for fields that have az/el components
az_el_names = [
'az', 'el', 'az', 'el', 'ra', 'dec', 'x', 'y', 'hr_angle', 'sin',
'cos', 'lat'
]
tag2 = f
for name in az_el_names:
# require name_ at beginning or _name at end
match1 = re.findall('^{}_'.format(name), f)
match2 = re.findall('_{}$'.format(name), f)
if len(match1):
tag2 = f.replace(match1[0], '')
if len(match2):
tag2 = f.replace(match2[0], '')
# also group source names
if 'source' in f:
tag2 = 'source'
stat = 'TrackerPointing'
if stat == 'PTStatus':
groups = ['now', 'min', 'max']
for g in groups:
match = re.findall('_{}$'.format(g), f)
if len(match):
tag2 = f.replace(match[0], '')
# group bench positions
# require bench_ at beginning
match = re.findall('^bench', f)
if len(match):
tag2 = attr # y1, y2, etc
stat = 'Bench'
# group Mux properties
if 'Mux' in stat:
stat = 'muxHousekeeping'
tag2 = 'ib' + f.split('ib')[-1]
dict_list += make_lines(
measurement=stat,
field=f,
time=time,
dat=cal_dat,
tags={
'label': tag,
'label2': tag2
},
)
try:
now = core.G3Time.Now()
delay = float(now.time / U.nanosecond - time[-1]) / 1e9
if delay > 5:
core.log_info('{} Delay: {} s'.format(now.isoformat(), delay),
unit='InfluxDB')
except RuntimeError: # sometimes timestamp gets screwed up
pass
try:
client.write_points(dict_list,
batch_size=len(dict_list),
protocol='line')
except (InfluxDBClientError, InfluxDBServerError) as v:
core.log_error('Error writing to database. {}'.format(v),
unit='InfluxDB')
#!/usr/bin/env python
from spt3g import core, coordinateutils
import numpy as np
np.random.seed(42)
n_samps = int(1e3)
az_0 = core.G3Timestream(np.random.rand(n_samps) * 2.0 * np.pi - np.pi)
pole_avoidance = 0.7
el_0 = core.G3Timestream(
np.random.rand(n_samps) * np.pi * pole_avoidance -
np.pi / 2.0 * pole_avoidance)
az_0.start = core.G3Time('20170329_000001')
el_0.start = core.G3Time('20170329_000001')
az_0.stop = core.G3Time('20170329_100001')
el_0.stop = core.G3Time('20170329_100001')
az_1 = az_0 + 10 * core.G3Units.arcmin
el_1 = el_0 + 10 * core.G3Units.arcmin
ra_0, dec_0 = coordinateutils.azel.convert_azel_to_radec(az_0, el_0)
ra_1, dec_1 = coordinateutils.azel.convert_azel_to_radec(az_1, el_1)
o_az_0 = az_0 + (np.random.rand() - 0.5) * 2 * core.G3Units.deg
o_el_0 = el_0 + (np.random.rand() - 0.5) * 2 * core.G3Units.deg
o_ra_0, o_dec_0 = coordinateutils.azel.convert_azel_to_radec(o_az_0, o_el_0)
import astropy.coordinates
def _process_irig_packet(self, parse_index):
# Unpack the IRIG data
start_ind = parse_index
end_ind = start_ind + self._irig_packet_size
unpacked_data = np.array(
struct.unpack(self._irig_unpack_str,
self._data[start_ind:end_ind]))
# Unpack the IRIG header
ind1 = 0
ind2 = ind1 + self._irig_header_units
header = unpacked_data[ind1:ind2][0]
if header != self._irig_header:
raise RuntimeError("%s: IRIG header error: 0x%x" %
(self._error_msg, header))
# Unpack the IRIG clock
ind1 = ind2
ind2 = ind1 + self._irig_clock_units
clock = unpacked_data[ind1:ind2][0]
# Unpack the IRIG clock overflows
ind1 = ind2
ind2 = ind1 + self._irig_overflow_units
overflow = unpacked_data[ind1:ind2][0]
# Adjust the IRIG clock for overflows
clock_adjusted = clock + (overflow << self._num_overflow_bits)
# Unpack the IRIG info
ind1 = ind2
ind2 = ind1 + self._irig_data_length
info = unpacked_data[ind1:ind2]
# Unpack the IRIG synch pulses
ind1 = ind2
ind2 = ind1 + self._irig_data_length
synch = unpacked_data[ind1:ind2]
# Unpack the IRIG synch pulse overflows
ind1 = ind2
ind2 = ind1 + self._irig_data_length
synch_overflow = unpacked_data[ind1:ind2]
synch_adjusted = (synch + (synch_overflow << self._num_overflow_bits))
# Convert raw IRIG bits to a meaningful time
year, yday, hour, mins, secs = self._irig_time_conversion(info)
# Obtain the time using the G3 object
irig_time = core.G3Time(y=int(year),
d=int(yday),
h=int(hour),
m=int(mins),
s=int(secs),
ss=0) # no subseconds
# Store the time as a G3Int
time_s = core.G3Double(
float(irig_time.time) / float(core.G3Units.seconds))
# time_s = core.G3UInt(irig_time.time)
# Store clock value as a G3UInt
clock_adjusted = core.G3UInt(int(clock_adjusted))
# Store the synchronization pulse clock values as a
# synch_adjusted = core.G3VectorUInt(np.array(synch_adjusted))
# Create and return a G3 timepoint frame with clock and irig data
irig_frame = core.G3Frame(core.G3FrameType.Timepoint)
irig_frame['chwp_irig_time'] = time_s
irig_frame['chwp_irig_clock'] = clock_adjusted
# irig_frame['chwp_irig_synch'] = synch_adjusted
return irig_frame
def split_field(data,
g3t,
framefield,
mapfield=None,
g3units=units,
times=None):
"""Split a gathered data buffer into frames- only on root process.
"""
if data is None:
return
if g3t == core3g.G3VectorTime:
# Special case for time values stored as int64_t, but
# wrapped in a class.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
g3times = list()
for t in range(ndata):
g3times.append(core3g.G3Time(data[dataoff + t]))
if mapfield is None:
fdata[f][framefield] = core3g.G3VectorTime(g3times)
else:
fdata[f][framefield][mapfield] = \
core3g.G3VectorTime(g3times)
del g3times
elif g3t == so3g.IntervalsInt:
# Flag vector is written as a simple boolean.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
# Extract flag vector (0 or 1) for this frame
frame_flags = (data[dataoff:dataoff + ndata] != 0).astype(int)
# Convert bit 0 to an IntervalsInt.
ival = so3g.IntervalsInt.from_mask(frame_flags, 1)[0]
if mapfield is None:
fdata[f][framefield] = ival
else:
fdata[f][framefield][mapfield] = ival
elif g3t == core3g.G3Timestream:
if times is None:
raise RuntimeError(
"You must provide the time stamp vector with a "
"Timestream object")
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
timeslice = times[cacheoff + dataoff:cacheoff + dataoff +
ndata]
tstart = timeslice[0] * 1e8
tstop = timeslice[-1] * 1e8
if mapfield is None:
if g3units is None:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata])
else:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata], g3units)
fdata[f][framefield].start = core3g.G3Time(tstart)
fdata[f][framefield].stop = core3g.G3Time(tstop)
else:
# Individual detector data. The only fields that
# we (optionally) compress.
if g3units is None:
tstream = g3t(data[dataoff:dataoff + ndata])
else:
tstream = g3t(data[dataoff:dataoff + ndata], g3units)
if compress and "compressor_gain_" + framefield in fdata[f]:
(tstream, gain,
offset) = recode_timestream(tstream, compress)
fdata[f]["compressor_gain_" +
framefield][mapfield] = gain
fdata[f]["compressor_offset_" +
framefield][mapfield] = offset
fdata[f][framefield][mapfield] = tstream
fdata[f][framefield][mapfield].start = core3g.G3Time(
tstart)
fdata[f][framefield][mapfield].stop = core3g.G3Time(tstop)
else:
# The bindings of G3Vector seem to only work with
# lists. This is probably horribly inefficient.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
if len(data.shape) == 1:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata].tolist())
else:
# We have a 2D quantity
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata, :].flatten()
.tolist())
return
def start_background_streamer(self, session, params=None):
"""start_background_streamer(params=None)
Process to run streaming process. A data stream is started
automatically. It can be stopped and started by the start and stop
tasks. Either way keep alive flow control frames are being sent.
Parameters
----------
frame_rate : float, optional
Frequency [Hz] at which G3Frames are sent over the network.
Defaults to 1 frame pers sec.
sample_rate : float, optional
Sample rate [Hz] for each channel. Defaults to 10 Hz.
"""
if params is None:
params = {}
self.writer = core.G3NetworkSender(hostname=self.target_host,
port=self.port)
frame_rate = params.get('frame_rate', 1.)
sample_rate = params.get('sample_rate', 10.)
frame_num = 0
self.running_in_background = True
# Control flags FIFO stack to keep Writer single threaded
self.flags = deque([FlowControl.START])
while self.running_in_background:
# Send START frame
if next(iter(self.flags), None) is FlowControl.START:
self._set_stream_on() # sends start flowcontrol
self.is_streaming = True
self.flags.popleft()
print("stream running in background")
self.log.debug("control flags: {f}", f=self.flags)
# Send keep alive flow control frame
f = core.G3Frame(core.G3FrameType.none)
f['sostream_flowcontrol'] = FlowControl.ALIVE.value
self.writer.Process(f)
if self.is_streaming:
frame_start = time.time()
time.sleep(1. / frame_rate)
frame_stop = time.time()
times = np.arange(frame_start, frame_stop, 1. / sample_rate)
f = core.G3Frame(core.G3FrameType.Scan)
f['session_id'] = 0
f['frame_num'] = frame_num
f['sostream_id'] = self.stream_id
f['data'] = core.G3TimestreamMap()
for i, chan in enumerate(self.channels):
ts = core.G3Timestream([chan.read() for t in times])
ts.start = core.G3Time(frame_start * core.G3Units.sec)
ts.stop = core.G3Time(frame_stop * core.G3Units.sec)
f['data'][f"r{i:04}"] = ts
self.writer.Process(f)
self.log.info("Writing frame...")
frame_num += 1
# Send END frame
if next(iter(self.flags), None) is FlowControl.END:
self._send_end_flowcontrol_frame()
self._send_cleanse_flowcontrol_frame()
self.is_streaming = False
self.flags.popleft()
else:
# Don't send keep alive frames too quickly
time.sleep(1)
# Shutdown streamer
if next(iter(self.flags), None) is SHUTDOWN:
self.running_in_background = False
self.flags.popleft()
# Teardown writer
self.writer.Close()
self.writer = None
return True, "Finished streaming"
def test_from_list(self):
t0 = core.G3Time('2019-01-01T12:30:00')
vectime = core.G3VectorTime([t0, t0 + 10 * SEC])
assert (vectime[0].time == t0.time)
assert (vectime[1].time == t0.time + 10 * SEC)
#!/usr/bin/env python import numpy from spt3g import core ts = core.G3Timestream(numpy.zeros(50)) ts.units = core.G3TimestreamUnits.Power ts.start = core.G3Time(0) ts.stop = core.G3Time(10 * core.G3Units.s) assert (ts[5] == 0) # Check scalar getitem ts[12] = 13.4 # Check scalar setitem assert (ts[12] == 13.4) ts[:] = numpy.arange(50) # Test vector setitem assert (ts[12]) == 12.0 # Test units preserved by slicing assert (ts[:5].units == ts.units) # Test length with slicing assert (len(ts[::2]) == len(ts) / 2) assert (len(ts[5:]) == len(ts) - 5) # Test consistency with numpy slicing with steps that do and do not divide evenly into array length assert ((numpy.asarray(ts[::2]) == numpy.asarray(ts)[::2]).all()) assert ((numpy.asarray(ts[::3]) == numpy.asarray(ts)[::3]).all()) assert ((numpy.asarray(ts[5::2]) == numpy.asarray(ts)[5::2]).all()) assert ((numpy.asarray(ts[5::3]) == numpy.asarray(ts)[5::3]).all()) assert ((numpy.asarray(ts[5:-4:2]) == numpy.asarray(ts)[5:-4:2]).all())
from adama_utils import get_obsids_from_times
from spt3g import core
import pandas as pd
obsids = get_obsids_from_times(core.G3Time("20190801_000000"),
core.G3Time("20190901_000000"),
sources=['ra0hdec-44.75', 'ra0hdec-52.25',
'ra0hdec-59.75', 'ra0hdec-67.25'])
obsids_refactored = {'source':[], 'obsid':[]}
for source in obsids:
for obsid in obsids[source]:
obsids_refactored['source'].append(source)
obsids_refactored['obsid'].append(obsid)
df = pd.DataFrame(obsids_refactored)
df.to_csv('obsids_to_process.txt', sep='\t', index=False)
