def DecryptFeatureBit(f):
'''
Unpacks the GCP feature flags
'''
if f.type != core.G3FrameType.GcpSlow:
return
flag_array = core.G3VectorString()
feature_bit = f['array']['frame']['features'].value
flags = [
'analyze', 'source_scan', 'cabin_shutter', 'elnod', 'pol_cal',
'calibrator', 'every_pixel_on_src', 'skydip', 'optical', 'noise',
'trail', 'el_scan', None, None, None, None, None, None, None, 'debug'
]
# Sorry... NDH
for i in enumerate(flags):
if feature_bit & (1 << i[0]):
if i[1] is None:
core.log_error('Got an unused feature bit: {:d}'.format(i[0]))
flag_array.append(i[1])
f['GCPFeatureBits'] = flag_array
def WiringFromJSON(cls, dat, ip, crate, slot):
'''
Build WiringMap object from a JSON blob returned by the
_dump_housekeeping call
'''
found = False
hwm = DfMuxWiringMap()
serial = int(dat['serial'])
for imezz, mezz in enumerate(dat['mezzanines']):
for imod, mod in enumerate(mezz['modules']):
for ichan, chan in enumerate(mod['channels']):
name = (chan.get('tuning', {}) or {}).get('name', None)
if not name:
continue
mapping = DfMuxChannelMapping()
mapping.board_ip = ip
mapping.board_serial = serial
mapping.board_slot = slot
mapping.crate_serial = crate
mapping.module = imod + 4 * imezz
mapping.channel = ichan
try:
name = str(name)
hwm[name] = mapping
found = True
except:
core.log_error("Invalid channel name %r" % (name))
if not found:
core.log_error("No mapped channels found on iceboard%04d. "
"You may need to update pydfmux to a newer version of pydfmux "
"that stores mapped channel names on the boards, and reload "
"the hardware map." % (serial),
unit='HousekeepingConsumer')
return hwm
def yellanddrop(fr):
global n_badpkts
global n_goodpkts
global logtweaked
if fr.type != core.G3FrameType.Timepoint:
return
if len(fr['DfMux']) < nboards - 2:
if n_badpkts > 0 and n_badpkts % 100 == 0:
core.log_error(
'Only %d/%d boards (%s) responding for %d samples -- check for sample misalignment. Temporarily suppressing DfMuxBuilder logging and disabling data archiving.'
% (len(fr['DfMux']), nboards, ', '.join(
[str(k) for k in fr['DfMux'].keys()]), n_badpkts),
unit='Data Acquisition')
# Turn up the threshold on DfMuxBuilder to prevent flooding the console
core.set_log_level(core.G3LogLevel.LOG_ERROR, 'DfMuxBuilder')
logtweaked = True
n_badpkts += 1
n_goodpkts = 0
return []
else:
n_goodpkts += 1
if n_goodpkts > 5 and logtweaked:
# Turn the threshold back down
core.set_log_level(core.G3LogLevel.LOG_NOTICE, 'DfMuxBuilder')
core.log_notice(
'Gross board misalignment resolved. Re-enabling DfMuxBuilder logging and data archiving.',
unit='Data Acquisition')
logtweaked = False
n_badpkts = 0
def __call__(self, frame):
if frame.type == core.G3FrameType.Wiring:
wmap = frame['WiringMap']
self.board_ips = set([m.board_ip for m in wmap.values()])
self.tuber = [(ip, TuberClient(socket.inet_ntoa(struct.pack('i', ip)), timeout=5.0)) for ip in self.board_ips]
if frame.type == core.G3FrameType.Housekeeping:
if self.tuber is None:
self.tuber = [(ip, TuberClient(socket.inet_ntoa(struct.pack('i', ip)), timeout=5.0)) for ip in self.board_ips]
hkdata = DfMuxHousekeepingMap()
try:
for board,board_tuber in self.tuber:
board_tuber.CallMethod('Dfmux', '_dump_housekeeping', False)
time.sleep(0.02) # Stagger return data transfer a little to
# avoid overloading the network on the return
for board,board_tuber in self.tuber:
dat = board_tuber.GetReply()[0]['result']
boardhk = self.HousekeepingFromJSON(dat)
hkdata[int(boardhk.serial)] = boardhk
frame['DfMuxHousekeeping'] = hkdata
except socket.timeout:
core.log_error('Timeout collecting housekeeping data from mux boards. Dropping housekeeping sample', unit='HousekeepingConsumer')
return []
except Exception as e:
core.log_error('Error (%s) collecting housekeeping data from mux boards. Dropping housekeeping sample' % e, unit='HousekeepingConsumer')
return []
def ping(self):
"""
Send a watchdog ping message to the GCP pager process. This method is
called by the `run` method at regular intervals whenever the
`data_valid` method returns True.
"""
try:
if not self.sim:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(self.timeout)
sock.connect((self.host, self.port))
sock.send('watchdog {}'.format(self.name).encode())
resp = sock.recv(4096)
if resp:
core.log_debug(
'Sent watchdog ping, got response {}'.format(resp.decode()),
unit=self.unit,
)
sock.close()
except Exception as e:
core.log_error('Error sending watchdog ping: {}'.format(e), unit=self.unit)
# try again in ten seconds
self.last_ping = time.time() - self.interval + 10
else:
core.log_info('Sent watchdog ping', unit=self.unit)
self.last_ping = time.time()
def __call__(self, frame):
# only ping on Timepoint frames
if not self.sim and 'DfMux' not in frame:
return
# only ping if another ping isn't already running
if self.thread is not None:
if not self.thread.is_alive():
del self.thread
self.thread = None
else:
return
# only ping on the appropriate interval
now = time.time()
if self.last_ping and (now - self.last_ping < self.interval):
return
# only ping if all the modules are returning data
if not self.sim:
data = frame['DfMux']
nmods_expected = 8 * len(data.keys())
nmods = sum([v.nmodules for v in data.values()])
if nmods < nmods_expected:
core.log_error("Missing {} modules in DAQ data stream".format(nmods_expected - nmods),
unit='GCPWatchdog')
self.last_missing = now
return
else:
# only ping if normal data acquisition has been going for a bit
if self.last_missing and now - self.last_missing < 10:
return
# only ping if calibrator is returning data
if self.calibrator:
if 'CalibratorOn' not in frame:
core.log_error("Missing calibrator signal in DAQ data stream",
unit='GCPWatchdog')
self.last_missing_calibrator = now
return
else:
# only ping if normal data acquisition has been going for a bit
if self.last_missing_calibrator and now - self.last_missing_calibrator < 10:
return
# spawn thread
self.thread = threading.Thread(target=self.ping)
self.thread.start()
def data_valid(self, frame):
"""
Check the incoming frame for completeness.
* Ensure that all modules in the listed iceboards are reporting.
* If `calibrator` is True, ensure that the calibrator sync signal is in the frame.
"""
# always ready in sim mode
if self.sim:
return True
# only ping on Timepoint frames
if 'DfMux' not in frame:
return False
now = time.time()
# only ping if all expected modules are present
data = frame['DfMux']
nmods_expected = 8 * len(data.keys())
nmods = sum([v.nmodules for v in data.values()])
if nmods < nmods_expected:
core.log_error(
"Missing {} modules in DAQ data stream".format(nmods_expected -
nmods),
unit=self.unit,
)
self.last_missing = now
return False
# only ping if the calibrator sync signal is present
if self.calibrator and 'CalibratorOn' not in frame:
core.log_error(
"Missing calibrator signal in DAQ data stream",
unit=self.unit,
)
self.last_missing = now
return False
# only ping if normal data acquisition has been going for a bit
if self.last_missing and now - self.last_missing < 10:
return False
return True
def ping(self):
# send a watchdog command to the pager server port
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(self.timeout)
sock.connect((self.host, self.port))
sock.send('watchdog daq'.encode())
resp = sock.recv(4096)
if resp:
core.log_debug("Sent DAQ watchdog ping, got response {}".format(resp.decode()),
unit='GCPWatchdog')
sock.close()
except Exception as e:
core.log_error("Error sending watchdog ping: {}".format(e), unit='GCPWatchdog')
# try again in ten seconds
self.last_ping = time.time() - self.interval + 10
else:
core.log_info('Sent DAQ watchdog ping', unit='GCPWatchdog')
self.last_ping = time.time()
def ProcessBuffered(self, frame):
'''
Process frames in the buffered order. Returned value should
always be a list of frames, possibly empty.
'''
if frame.type == core.G3FrameType.Timepoint:
self.board_serials = [('%04d' % k) for k in frame['DfMux'].keys()]
return [frame]
if frame.type == core.G3FrameType.Wiring:
core.log_fatal("Received spurious wiring frame. Do not use "
"PyDfMuxHardwareMapInjector with the HousekeepingConsumer. "
"You may update pydfmux to a newer version of pydfmux "
"that stores mapped channel names on the boards, and "
"rearrange your data acquisition script.",
unit='HousekeepingConsumer')
if frame.type == core.G3FrameType.Housekeeping:
self.map_boards()
hwm = DfMuxWiringMap()
hkdata = DfMuxHousekeepingMap()
try:
for board in self.board_serials:
self.tuber[board].CallMethod('Dfmux', '_dump_housekeeping', False)
time.sleep(0.02) # Stagger return data transfer a little to
# avoid overloading the network on the return
found = False
for board in self.board_serials:
dat = self.tuber[board].GetReply()[0]['result']
boardhk = self.HousekeepingFromJSON(dat)
hkdata[int(boardhk.serial)] = boardhk
ip, crate, slot = self.board_map[board]
boardw = self.WiringFromJSON(dat, ip, crate, slot)
for key in boardw.keys():
hwm[key] = boardw[key]
found = True
if not found:
core.log_fatal("No mapped channels found on any IceBoards. "
"You may need to update pydfmux to a newer version of pydfmux "
"that stores mapped channel names on the boards, and reload "
"the hardware map.",
unit='HousekeepingConsumer')
frame['DfMuxHousekeeping'] = hkdata
hwmf = core.G3Frame(core.G3FrameType.Wiring)
hwmf['WiringMap'] = hwm
hwmf['ReadoutSystem'] = 'ICE'
if self.hwmf is None:
self.hwmf = hwmf
# If this is the first time the consumer is triggered, make sure
# a Housekeeping and WiringMap frame are issued before any
# Timepoint frames
frames = [hwmf, frame]
return frames
else:
# Compare wiring maps and re-issue frame if anything has changed
old_hwm = self.hwmf['WiringMap']
if (set(hwm.keys()) ^ set(old_hwm.keys())):
self.hwmf = hwmf
return [hwmf, frame]
for k in hwm.keys():
try:
if vars(hwm[str(k)]) != vars(old_hwm[str(k)]):
self.hwmf = hwmf
return [hwmf, frame]
except:
core.log_error("Invalid HWM key %r" % k)
# If we get here then the wiring map hasn't changed,
# so return the populated Housekeeping frame as it is
return [frame]
except socket.timeout:
core.log_error('Timeout collecting housekeeping data from mux boards. Dropping housekeeping sample', unit='HousekeepingConsumer')
return []
except Exception as e:
core.log_error('Error (%s) collecting housekeeping data from mux boards. Dropping housekeeping sample' % e, unit='HousekeepingConsumer')
return []
return [frame]
def __call__(self, f):
"""Processes a frame. Only Housekeeping frames will be examined;
other frames will simply be counted. All frames are passed
through unmodified.
"""
if f.type == core.G3FrameType.EndProcessing:
self.report_and_reset()
return [f]
if f.type != core.G3FrameType.Housekeeping:
self.stats['n_other'] += 1
return f
self.stats['n_hk'] += 1
if f['hkagg_type'] == so3g.HKFrameType.session:
session_id = f['session_id']
if self.session_id is not None:
if self.session_id != session_id:
self.report_and_reset(
) # note this does clear self.session_id.
if self.session_id is None:
core.log_info('New HK Session id = %i, timestamp = %i' %
(session_id, f['start_time']),
unit='HKScanner')
self.session_id = session_id
self.stats['n_session'] += 1
elif f['hkagg_type'] == so3g.HKFrameType.status:
# Have any providers disappeared?
now_prov_id = [p['prov_id'].value for p in f['providers']]
for p, info in self.providers.items():
if p not in now_prov_id:
info['active'] = False
# New providers?
for p in now_prov_id:
info = self.providers.get(p)
if info is not None:
if not info['active']:
core.log_warn('prov_id %i came back to life.' % p,
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
info['n_active'] += 1
info['active'] = True
else:
self.providers[p] = {
'active':
True, # Currently active (during processing).
'n_active':
1, # Number of times this provider id became active.
'n_frames': 0, # Number of data frames.
'timestamp_init':
f['timestamp'], # Timestamp of provider appearance
'timestamp_data':
None, # Timestamp of most recent data frame.
'ticks':
0, # Total number of timestamps in all blocks.
'span': None, # (earliest_time, latest_time)
}
elif f['hkagg_type'] == so3g.HKFrameType.data:
info = self.providers[f['prov_id']]
info['n_frames'] += 1
t_this = f['timestamp']
if info['timestamp_data'] is None:
t_ref = info['timestamp_init']
if t_this < t_ref:
core.log_warn('data timestamp (%.1f) precedes provider '
'timestamp by %f seconds.' %
(t_this, t_this - t_ref),
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
elif t_this <= info['timestamp_data']:
core.log_warn(
'data frame timestamps are not strictly ordered.',
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
info['timestamp_data'] = t_this # update
t_check = []
for b in f['blocks']:
if len(b.t):
if info['span'] is None:
info['span'] = b.t[0], b.t[-1]
else:
t0, t1 = info['span']
info['span'] = min(b.t[0], t0), max(b.t[-1], t1)
t_check.append(b.t[0])
info['ticks'] += len(b.t)
for k, v in b.data.items():
if len(v) != len(b.t):
core.log_error(
'Field "%s" has %i samples but .t has %i samples.'
% (k, len(v), len(b.t)))
self.stats['concerns']['n_error'] += 1
if len(t_check) and abs(min(t_check) - t_this) > 60:
core.log_warn(
'data frame timestamp (%.1f) does not correspond to '
'data timestamp vectors (%s) .' % (t_this, t_check),
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
else:
core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'],
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
return [f]
def WriteDB(fr, client, fields=None):
'''
Write points to the database for each field
Arguments
---------
client :
InfluxDB client
fields :
Which gcp fields to add to database. See parse_field for options. If
None, add all.
'''
from influxdb.exceptions import InfluxDBClientError
from influxdb.exceptions import InfluxDBServerError
if fr.type != core.G3FrameType.GcpSlow:
return
all_fields = build_field_list(fr)
if fields is None:
fields = all_fields.keys()
dict_list = []
for f in fields:
field_dat = all_fields[f]
if len(field_dat) == 4:
stat, attr, ind, unit = field_dat
try:
dat = getattr(fr[stat], attr)[ind]
time = getattr(fr[stat], 'time')
except AttributeError:
# OnlinePointingModel
dat = fr[stat][attr][ind]
time = fr[stat]['time']
elif len(field_dat) == 3:
stat, attr, unit = field_dat
if stat not in fr:
# Field only exists in live data stream
continue
try:
dat = getattr(fr[stat], attr)
except AttributeError:
try:
dat = fr[stat][attr]
except KeyError: # Field only exists in live data stream
continue
if 'Bench' in stat: # funny time field for bench positions
time = fr['BenchSampleTime']
elif 'Mux' in stat:
time = fr['MuxTime']
elif stat in ['CryoStatus', 'Weather', 'PTStatus']:
time = fr['{}Time'.format(stat)]
else:
try:
time = getattr(fr[stat], 'time')
except AttributeError:
time = fr[stat]['time']
elif len(field_dat) == 2:
stat, unit = field_dat
try:
dat = fr[stat]
except KeyError: #eg, no obsid
core.log_warn('No key {}'.format(stat), unit='InfluxDB')
continue
try:
time = getattr(fr[stat], 'time')
except AttributeError as err:
time = [tm for tm in fr['antenna0']['tracker']['utc'][0]]
# InfluxDB wants time in nanoseconds since the UNIX epoch in UTC
try:
time = [x.time / U.nanosecond for x in np.atleast_1d(time)]
except AttributeError:
time = [
core.G3Time(t0).time / U.nanosecond
for t0 in np.atleast_1d(time)
]
if dat is None:
core.log_warn('{} dat is None'.format(f), unit='InfluxDB')
continue
dat = np.atleast_1d(dat)
try:
dlen = len(dat)
except TypeError:
# sometimes source_name is a weird non-none value
continue
if unit is not None:
if unit == 'C':
zeropt_K = 273.15
cal_dat = dat / U.K - zeropt_K
else:
cal_dat = dat / unit
else:
cal_dat = dat
try:
if np.any(np.isnan(cal_dat)):
continue
except TypeError:
pass
if 'heat' not in f:
tag = f
else:
tag = f.replace('heat_', '')
# for fields that have az/el components
az_el_names = [
'az', 'el', 'az', 'el', 'ra', 'dec', 'x', 'y', 'hr_angle', 'sin',
'cos', 'lat'
]
tag2 = f
for name in az_el_names:
# require name_ at beginning or _name at end
match1 = re.findall('^{}_'.format(name), f)
match2 = re.findall('_{}$'.format(name), f)
if len(match1):
tag2 = f.replace(match1[0], '')
if len(match2):
tag2 = f.replace(match2[0], '')
# also group source names
if 'source' in f:
tag2 = 'source'
stat = 'TrackerPointing'
if stat == 'PTStatus':
groups = ['now', 'min', 'max']
for g in groups:
match = re.findall('_{}$'.format(g), f)
if len(match):
tag2 = f.replace(match[0], '')
# group bench positions
# require bench_ at beginning
match = re.findall('^bench', f)
if len(match):
tag2 = attr # y1, y2, etc
stat = 'Bench'
# group Mux properties
if 'Mux' in stat:
stat = 'muxHousekeeping'
tag2 = 'ib' + f.split('ib')[-1]
dict_list += make_lines(
measurement=stat,
field=f,
time=time,
dat=cal_dat,
tags={
'label': tag,
'label2': tag2
},
)
try:
now = core.G3Time.Now()
delay = float(now.time / U.nanosecond - time[-1]) / 1e9
if delay > 5:
core.log_info('{} Delay: {} s'.format(now.isoformat(), delay),
unit='InfluxDB')
except RuntimeError: # sometimes timestamp gets screwed up
pass
try:
client.write_points(dict_list,
batch_size=len(dict_list),
protocol='line')
except (InfluxDBClientError, InfluxDBServerError) as v:
core.log_error('Error writing to database. {}'.format(v),
unit='InfluxDB')
def __call__(self, f):
"""Processes a frame. Only Housekeeping frames will be examined;
other frames will simply be counted. All frames are passed
through unmodified.
"""
if f.type == core.G3FrameType.EndProcessing:
self.report_and_reset()
return [f]
if f.type != core.G3FrameType.Housekeeping:
self.stats['n_other'] += 1
return f
self.stats['n_hk'] += 1
vers = f.get('hkagg_version', 0)
self.stats['versions'][vers] = self.stats['versions'].get(vers, 0) + 1
if f['hkagg_type'] == so3g.HKFrameType.session:
session_id = f['session_id']
if self.session_id is not None:
if self.session_id != session_id:
self.report_and_reset(
) # note this does clear self.session_id.
if self.session_id is None:
core.log_info('New HK Session id = %i, timestamp = %i' %
(session_id, f['start_time']),
unit='HKScanner')
self.session_id = session_id
self.stats['n_session'] += 1
elif f['hkagg_type'] == so3g.HKFrameType.status:
# Have any providers disappeared?
now_prov_id = [p['prov_id'].value for p in f['providers']]
for p, info in self.providers.items():
if p not in now_prov_id:
info['active'] = False
# New providers?
for p in now_prov_id:
info = self.providers.get(p)
if info is not None:
if not info['active']:
core.log_warn('prov_id %i came back to life.' % p,
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
info['n_active'] += 1
info['active'] = True
else:
self.providers[p] = {
'active':
True, # Currently active (during processing).
'n_active':
1, # Number of times this provider id became active.
'n_frames': 0, # Number of data frames.
'timestamp_init':
f['timestamp'], # Timestamp of provider appearance
'timestamp_data':
None, # Timestamp of most recent data frame.
'ticks':
0, # Total number of timestamps in all blocks.
'span': None, # (earliest_time, latest_time)
'block_streams_map':
{}, # Map from field name to block name.
}
elif f['hkagg_type'] == so3g.HKFrameType.data:
info = self.providers[f['prov_id']]
vers = f.get('hkagg_version', 0)
info['n_frames'] += 1
t_this = f['timestamp']
if info['timestamp_data'] is None:
t_ref = info['timestamp_init']
if t_this < t_ref:
core.log_warn('data timestamp (%.1f) precedes provider '
'timestamp by %f seconds.' %
(t_this, t_this - t_ref),
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
elif t_this <= info['timestamp_data']:
core.log_warn(
'data frame timestamps are not strictly ordered.',
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
info['timestamp_data'] = t_this # update
t_check = []
blocks = f['blocks']
if vers == 0:
block_timef = lambda block: block.t
block_itemf = lambda block: [(k, block.data[k])
for k in block.data.keys()]
elif vers >= 1:
block_timef = lambda block: np.array(
[t.time / core.G3Units.seconds for t in b.times])
block_itemf = lambda block: [(k, block[k])
for k in block.keys()]
if vers in [0]:
block_name = lambda block_idx: list(
sorted(blocks[block_idx].data.keys()))[0]
if vers in [1]:
block_name = lambda block_idx: list(
sorted(blocks[block_idx].keys()))[0]
elif vers >= 2:
block_names = f.get('block_names', [])
if len(block_names) != len(blocks):
# This is a schema error in its own right.
core.log_error(
'Frame does not have "block_names" entry, '
'or it is not the same length as "blocks".',
unit='HKScanner')
self.stats['concerns']['n_error'] += 1
# Fall back on v1 strategy.
block_name = lambda block_idx: list(
sorted(blocks[block_idx].keys()))[0]
else:
block_name = lambda block_idx: f['block_names'][block_idx]
for block_idx, b in enumerate(blocks):
times = block_timef(b)
if len(times):
if info['span'] is None:
info['span'] = times[0], times[-1]
else:
t0, t1 = info['span']
info['span'] = min(times[0], t0), max(times[-1], t1)
t_check.append(times[0])
info['ticks'] += len(times)
bname = block_name(block_idx)
for k, v in block_itemf(b):
if len(v) != len(times):
core.log_error(
'Field "%s" has %i samples but .t has %i samples.'
% (k, len(v), len(times)))
self.stats['concerns']['n_error'] += 1
# Make sure field has a block_stream registered.
if k not in info['block_streams_map']:
info['block_streams_map'][k] = bname
if info['block_streams_map'][k] != bname:
core.log_error(
'Field "%s" appeared in block_name %s '
'and later in block_name %s.' %
(k, info['block_streams_map'][k], bname))
self.stats['concerns']['n_error'] += 1
if len(t_check) and abs(min(t_check) - t_this) > 60:
core.log_warn(
'data frame timestamp (%.1f) does not correspond to '
'data timestamp vectors (%s) .' % (t_this, t_check),
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
else:
core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'],
unit='HKScanner')
self.stats['concerns']['n_warning'] += 1
return [f]
