def DecryptFeatureBit(f): ''' Unpacks the GCP feature flags ''' if f.type != core.G3FrameType.GcpSlow: return flag_array = core.G3VectorString() feature_bit = f['array']['frame']['features'].value flags = [ 'analyze', 'source_scan', 'cabin_shutter', 'elnod', 'pol_cal', 'calibrator', 'every_pixel_on_src', 'skydip', 'optical', 'noise', 'trail', 'el_scan', None, None, None, None, None, None, None, 'debug' ] # Sorry... NDH for i in enumerate(flags): if feature_bit & (1 << i[0]): if i[1] is None: core.log_error('Got an unused feature bit: {:d}'.format(i[0])) flag_array.append(i[1]) f['GCPFeatureBits'] = flag_array
def WiringFromJSON(cls, dat, ip, crate, slot): ''' Build WiringMap object from a JSON blob returned by the _dump_housekeeping call ''' found = False hwm = DfMuxWiringMap() serial = int(dat['serial']) for imezz, mezz in enumerate(dat['mezzanines']): for imod, mod in enumerate(mezz['modules']): for ichan, chan in enumerate(mod['channels']): name = (chan.get('tuning', {}) or {}).get('name', None) if not name: continue mapping = DfMuxChannelMapping() mapping.board_ip = ip mapping.board_serial = serial mapping.board_slot = slot mapping.crate_serial = crate mapping.module = imod + 4 * imezz mapping.channel = ichan try: name = str(name) hwm[name] = mapping found = True except: core.log_error("Invalid channel name %r" % (name)) if not found: core.log_error("No mapped channels found on iceboard%04d. " "You may need to update pydfmux to a newer version of pydfmux " "that stores mapped channel names on the boards, and reload " "the hardware map." % (serial), unit='HousekeepingConsumer') return hwm
def yellanddrop(fr): global n_badpkts global n_goodpkts global logtweaked if fr.type != core.G3FrameType.Timepoint: return if len(fr['DfMux']) < nboards - 2: if n_badpkts > 0 and n_badpkts % 100 == 0: core.log_error( 'Only %d/%d boards (%s) responding for %d samples -- check for sample misalignment. Temporarily suppressing DfMuxBuilder logging and disabling data archiving.' % (len(fr['DfMux']), nboards, ', '.join( [str(k) for k in fr['DfMux'].keys()]), n_badpkts), unit='Data Acquisition') # Turn up the threshold on DfMuxBuilder to prevent flooding the console core.set_log_level(core.G3LogLevel.LOG_ERROR, 'DfMuxBuilder') logtweaked = True n_badpkts += 1 n_goodpkts = 0 return [] else: n_goodpkts += 1 if n_goodpkts > 5 and logtweaked: # Turn the threshold back down core.set_log_level(core.G3LogLevel.LOG_NOTICE, 'DfMuxBuilder') core.log_notice( 'Gross board misalignment resolved. Re-enabling DfMuxBuilder logging and data archiving.', unit='Data Acquisition') logtweaked = False n_badpkts = 0
def __call__(self, frame): if frame.type == core.G3FrameType.Wiring: wmap = frame['WiringMap'] self.board_ips = set([m.board_ip for m in wmap.values()]) self.tuber = [(ip, TuberClient(socket.inet_ntoa(struct.pack('i', ip)), timeout=5.0)) for ip in self.board_ips] if frame.type == core.G3FrameType.Housekeeping: if self.tuber is None: self.tuber = [(ip, TuberClient(socket.inet_ntoa(struct.pack('i', ip)), timeout=5.0)) for ip in self.board_ips] hkdata = DfMuxHousekeepingMap() try: for board,board_tuber in self.tuber: board_tuber.CallMethod('Dfmux', '_dump_housekeeping', False) time.sleep(0.02) # Stagger return data transfer a little to # avoid overloading the network on the return for board,board_tuber in self.tuber: dat = board_tuber.GetReply()[0]['result'] boardhk = self.HousekeepingFromJSON(dat) hkdata[int(boardhk.serial)] = boardhk frame['DfMuxHousekeeping'] = hkdata except socket.timeout: core.log_error('Timeout collecting housekeeping data from mux boards. Dropping housekeeping sample', unit='HousekeepingConsumer') return [] except Exception as e: core.log_error('Error (%s) collecting housekeeping data from mux boards. Dropping housekeeping sample' % e, unit='HousekeepingConsumer') return []
def ping(self): """ Send a watchdog ping message to the GCP pager process. This method is called by the `run` method at regular intervals whenever the `data_valid` method returns True. """ try: if not self.sim: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(self.timeout) sock.connect((self.host, self.port)) sock.send('watchdog {}'.format(self.name).encode()) resp = sock.recv(4096) if resp: core.log_debug( 'Sent watchdog ping, got response {}'.format(resp.decode()), unit=self.unit, ) sock.close() except Exception as e: core.log_error('Error sending watchdog ping: {}'.format(e), unit=self.unit) # try again in ten seconds self.last_ping = time.time() - self.interval + 10 else: core.log_info('Sent watchdog ping', unit=self.unit) self.last_ping = time.time()
def __call__(self, frame): # only ping on Timepoint frames if not self.sim and 'DfMux' not in frame: return # only ping if another ping isn't already running if self.thread is not None: if not self.thread.is_alive(): del self.thread self.thread = None else: return # only ping on the appropriate interval now = time.time() if self.last_ping and (now - self.last_ping < self.interval): return # only ping if all the modules are returning data if not self.sim: data = frame['DfMux'] nmods_expected = 8 * len(data.keys()) nmods = sum([v.nmodules for v in data.values()]) if nmods < nmods_expected: core.log_error("Missing {} modules in DAQ data stream".format(nmods_expected - nmods), unit='GCPWatchdog') self.last_missing = now return else: # only ping if normal data acquisition has been going for a bit if self.last_missing and now - self.last_missing < 10: return # only ping if calibrator is returning data if self.calibrator: if 'CalibratorOn' not in frame: core.log_error("Missing calibrator signal in DAQ data stream", unit='GCPWatchdog') self.last_missing_calibrator = now return else: # only ping if normal data acquisition has been going for a bit if self.last_missing_calibrator and now - self.last_missing_calibrator < 10: return # spawn thread self.thread = threading.Thread(target=self.ping) self.thread.start()
def data_valid(self, frame): """ Check the incoming frame for completeness. * Ensure that all modules in the listed iceboards are reporting. * If `calibrator` is True, ensure that the calibrator sync signal is in the frame. """ # always ready in sim mode if self.sim: return True # only ping on Timepoint frames if 'DfMux' not in frame: return False now = time.time() # only ping if all expected modules are present data = frame['DfMux'] nmods_expected = 8 * len(data.keys()) nmods = sum([v.nmodules for v in data.values()]) if nmods < nmods_expected: core.log_error( "Missing {} modules in DAQ data stream".format(nmods_expected - nmods), unit=self.unit, ) self.last_missing = now return False # only ping if the calibrator sync signal is present if self.calibrator and 'CalibratorOn' not in frame: core.log_error( "Missing calibrator signal in DAQ data stream", unit=self.unit, ) self.last_missing = now return False # only ping if normal data acquisition has been going for a bit if self.last_missing and now - self.last_missing < 10: return False return True
def ping(self): # send a watchdog command to the pager server port try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(self.timeout) sock.connect((self.host, self.port)) sock.send('watchdog daq'.encode()) resp = sock.recv(4096) if resp: core.log_debug("Sent DAQ watchdog ping, got response {}".format(resp.decode()), unit='GCPWatchdog') sock.close() except Exception as e: core.log_error("Error sending watchdog ping: {}".format(e), unit='GCPWatchdog') # try again in ten seconds self.last_ping = time.time() - self.interval + 10 else: core.log_info('Sent DAQ watchdog ping', unit='GCPWatchdog') self.last_ping = time.time()
def ProcessBuffered(self, frame): ''' Process frames in the buffered order. Returned value should always be a list of frames, possibly empty. ''' if frame.type == core.G3FrameType.Timepoint: self.board_serials = [('%04d' % k) for k in frame['DfMux'].keys()] return [frame] if frame.type == core.G3FrameType.Wiring: core.log_fatal("Received spurious wiring frame. Do not use " "PyDfMuxHardwareMapInjector with the HousekeepingConsumer. " "You may update pydfmux to a newer version of pydfmux " "that stores mapped channel names on the boards, and " "rearrange your data acquisition script.", unit='HousekeepingConsumer') if frame.type == core.G3FrameType.Housekeeping: self.map_boards() hwm = DfMuxWiringMap() hkdata = DfMuxHousekeepingMap() try: for board in self.board_serials: self.tuber[board].CallMethod('Dfmux', '_dump_housekeeping', False) time.sleep(0.02) # Stagger return data transfer a little to # avoid overloading the network on the return found = False for board in self.board_serials: dat = self.tuber[board].GetReply()[0]['result'] boardhk = self.HousekeepingFromJSON(dat) hkdata[int(boardhk.serial)] = boardhk ip, crate, slot = self.board_map[board] boardw = self.WiringFromJSON(dat, ip, crate, slot) for key in boardw.keys(): hwm[key] = boardw[key] found = True if not found: core.log_fatal("No mapped channels found on any IceBoards. " "You may need to update pydfmux to a newer version of pydfmux " "that stores mapped channel names on the boards, and reload " "the hardware map.", unit='HousekeepingConsumer') frame['DfMuxHousekeeping'] = hkdata hwmf = core.G3Frame(core.G3FrameType.Wiring) hwmf['WiringMap'] = hwm hwmf['ReadoutSystem'] = 'ICE' if self.hwmf is None: self.hwmf = hwmf # If this is the first time the consumer is triggered, make sure # a Housekeeping and WiringMap frame are issued before any # Timepoint frames frames = [hwmf, frame] return frames else: # Compare wiring maps and re-issue frame if anything has changed old_hwm = self.hwmf['WiringMap'] if (set(hwm.keys()) ^ set(old_hwm.keys())): self.hwmf = hwmf return [hwmf, frame] for k in hwm.keys(): try: if vars(hwm[str(k)]) != vars(old_hwm[str(k)]): self.hwmf = hwmf return [hwmf, frame] except: core.log_error("Invalid HWM key %r" % k) # If we get here then the wiring map hasn't changed, # so return the populated Housekeeping frame as it is return [frame] except socket.timeout: core.log_error('Timeout collecting housekeeping data from mux boards. Dropping housekeeping sample', unit='HousekeepingConsumer') return [] except Exception as e: core.log_error('Error (%s) collecting housekeeping data from mux boards. Dropping housekeeping sample' % e, unit='HousekeepingConsumer') return [] return [frame]
def __call__(self, f): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. """ if f.type == core.G3FrameType.EndProcessing: self.report_and_reset() return [f] if f.type != core.G3FrameType.Housekeeping: self.stats['n_other'] += 1 return f self.stats['n_hk'] += 1 if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.report_and_reset( ) # note this does clear self.session_id. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id self.stats['n_session'] += 1 elif f['hkagg_type'] == so3g.HKFrameType.status: # Have any providers disappeared? now_prov_id = [p['prov_id'].value for p in f['providers']] for p, info in self.providers.items(): if p not in now_prov_id: info['active'] = False # New providers? for p in now_prov_id: info = self.providers.get(p) if info is not None: if not info['active']: core.log_warn('prov_id %i came back to life.' % p, unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['n_active'] += 1 info['active'] = True else: self.providers[p] = { 'active': True, # Currently active (during processing). 'n_active': 1, # Number of times this provider id became active. 'n_frames': 0, # Number of data frames. 'timestamp_init': f['timestamp'], # Timestamp of provider appearance 'timestamp_data': None, # Timestamp of most recent data frame. 'ticks': 0, # Total number of timestamps in all blocks. 'span': None, # (earliest_time, latest_time) } elif f['hkagg_type'] == so3g.HKFrameType.data: info = self.providers[f['prov_id']] info['n_frames'] += 1 t_this = f['timestamp'] if info['timestamp_data'] is None: t_ref = info['timestamp_init'] if t_this < t_ref: core.log_warn('data timestamp (%.1f) precedes provider ' 'timestamp by %f seconds.' % (t_this, t_this - t_ref), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 elif t_this <= info['timestamp_data']: core.log_warn( 'data frame timestamps are not strictly ordered.', unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['timestamp_data'] = t_this # update t_check = [] for b in f['blocks']: if len(b.t): if info['span'] is None: info['span'] = b.t[0], b.t[-1] else: t0, t1 = info['span'] info['span'] = min(b.t[0], t0), max(b.t[-1], t1) t_check.append(b.t[0]) info['ticks'] += len(b.t) for k, v in b.data.items(): if len(v) != len(b.t): core.log_error( 'Field "%s" has %i samples but .t has %i samples.' % (k, len(v), len(b.t))) self.stats['concerns']['n_error'] += 1 if len(t_check) and abs(min(t_check) - t_this) > 60: core.log_warn( 'data frame timestamp (%.1f) does not correspond to ' 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') self.stats['concerns']['n_warning'] += 1 return [f]
def WriteDB(fr, client, fields=None): ''' Write points to the database for each field Arguments --------- client : InfluxDB client fields : Which gcp fields to add to database. See parse_field for options. If None, add all. ''' from influxdb.exceptions import InfluxDBClientError from influxdb.exceptions import InfluxDBServerError if fr.type != core.G3FrameType.GcpSlow: return all_fields = build_field_list(fr) if fields is None: fields = all_fields.keys() dict_list = [] for f in fields: field_dat = all_fields[f] if len(field_dat) == 4: stat, attr, ind, unit = field_dat try: dat = getattr(fr[stat], attr)[ind] time = getattr(fr[stat], 'time') except AttributeError: # OnlinePointingModel dat = fr[stat][attr][ind] time = fr[stat]['time'] elif len(field_dat) == 3: stat, attr, unit = field_dat if stat not in fr: # Field only exists in live data stream continue try: dat = getattr(fr[stat], attr) except AttributeError: try: dat = fr[stat][attr] except KeyError: # Field only exists in live data stream continue if 'Bench' in stat: # funny time field for bench positions time = fr['BenchSampleTime'] elif 'Mux' in stat: time = fr['MuxTime'] elif stat in ['CryoStatus', 'Weather', 'PTStatus']: time = fr['{}Time'.format(stat)] else: try: time = getattr(fr[stat], 'time') except AttributeError: time = fr[stat]['time'] elif len(field_dat) == 2: stat, unit = field_dat try: dat = fr[stat] except KeyError: #eg, no obsid core.log_warn('No key {}'.format(stat), unit='InfluxDB') continue try: time = getattr(fr[stat], 'time') except AttributeError as err: time = [tm for tm in fr['antenna0']['tracker']['utc'][0]] # InfluxDB wants time in nanoseconds since the UNIX epoch in UTC try: time = [x.time / U.nanosecond for x in np.atleast_1d(time)] except AttributeError: time = [ core.G3Time(t0).time / U.nanosecond for t0 in np.atleast_1d(time) ] if dat is None: core.log_warn('{} dat is None'.format(f), unit='InfluxDB') continue dat = np.atleast_1d(dat) try: dlen = len(dat) except TypeError: # sometimes source_name is a weird non-none value continue if unit is not None: if unit == 'C': zeropt_K = 273.15 cal_dat = dat / U.K - zeropt_K else: cal_dat = dat / unit else: cal_dat = dat try: if np.any(np.isnan(cal_dat)): continue except TypeError: pass if 'heat' not in f: tag = f else: tag = f.replace('heat_', '') # for fields that have az/el components az_el_names = [ 'az', 'el', 'az', 'el', 'ra', 'dec', 'x', 'y', 'hr_angle', 'sin', 'cos', 'lat' ] tag2 = f for name in az_el_names: # require name_ at beginning or _name at end match1 = re.findall('^{}_'.format(name), f) match2 = re.findall('_{}$'.format(name), f) if len(match1): tag2 = f.replace(match1[0], '') if len(match2): tag2 = f.replace(match2[0], '') # also group source names if 'source' in f: tag2 = 'source' stat = 'TrackerPointing' if stat == 'PTStatus': groups = ['now', 'min', 'max'] for g in groups: match = re.findall('_{}$'.format(g), f) if len(match): tag2 = f.replace(match[0], '') # group bench positions # require bench_ at beginning match = re.findall('^bench', f) if len(match): tag2 = attr # y1, y2, etc stat = 'Bench' # group Mux properties if 'Mux' in stat: stat = 'muxHousekeeping' tag2 = 'ib' + f.split('ib')[-1] dict_list += make_lines( measurement=stat, field=f, time=time, dat=cal_dat, tags={ 'label': tag, 'label2': tag2 }, ) try: now = core.G3Time.Now() delay = float(now.time / U.nanosecond - time[-1]) / 1e9 if delay > 5: core.log_info('{} Delay: {} s'.format(now.isoformat(), delay), unit='InfluxDB') except RuntimeError: # sometimes timestamp gets screwed up pass try: client.write_points(dict_list, batch_size=len(dict_list), protocol='line') except (InfluxDBClientError, InfluxDBServerError) as v: core.log_error('Error writing to database. {}'.format(v), unit='InfluxDB')
def __call__(self, f): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. """ if f.type == core.G3FrameType.EndProcessing: self.report_and_reset() return [f] if f.type != core.G3FrameType.Housekeeping: self.stats['n_other'] += 1 return f self.stats['n_hk'] += 1 vers = f.get('hkagg_version', 0) self.stats['versions'][vers] = self.stats['versions'].get(vers, 0) + 1 if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.report_and_reset( ) # note this does clear self.session_id. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id self.stats['n_session'] += 1 elif f['hkagg_type'] == so3g.HKFrameType.status: # Have any providers disappeared? now_prov_id = [p['prov_id'].value for p in f['providers']] for p, info in self.providers.items(): if p not in now_prov_id: info['active'] = False # New providers? for p in now_prov_id: info = self.providers.get(p) if info is not None: if not info['active']: core.log_warn('prov_id %i came back to life.' % p, unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['n_active'] += 1 info['active'] = True else: self.providers[p] = { 'active': True, # Currently active (during processing). 'n_active': 1, # Number of times this provider id became active. 'n_frames': 0, # Number of data frames. 'timestamp_init': f['timestamp'], # Timestamp of provider appearance 'timestamp_data': None, # Timestamp of most recent data frame. 'ticks': 0, # Total number of timestamps in all blocks. 'span': None, # (earliest_time, latest_time) 'block_streams_map': {}, # Map from field name to block name. } elif f['hkagg_type'] == so3g.HKFrameType.data: info = self.providers[f['prov_id']] vers = f.get('hkagg_version', 0) info['n_frames'] += 1 t_this = f['timestamp'] if info['timestamp_data'] is None: t_ref = info['timestamp_init'] if t_this < t_ref: core.log_warn('data timestamp (%.1f) precedes provider ' 'timestamp by %f seconds.' % (t_this, t_this - t_ref), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 elif t_this <= info['timestamp_data']: core.log_warn( 'data frame timestamps are not strictly ordered.', unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['timestamp_data'] = t_this # update t_check = [] blocks = f['blocks'] if vers == 0: block_timef = lambda block: block.t block_itemf = lambda block: [(k, block.data[k]) for k in block.data.keys()] elif vers >= 1: block_timef = lambda block: np.array( [t.time / core.G3Units.seconds for t in b.times]) block_itemf = lambda block: [(k, block[k]) for k in block.keys()] if vers in [0]: block_name = lambda block_idx: list( sorted(blocks[block_idx].data.keys()))[0] if vers in [1]: block_name = lambda block_idx: list( sorted(blocks[block_idx].keys()))[0] elif vers >= 2: block_names = f.get('block_names', []) if len(block_names) != len(blocks): # This is a schema error in its own right. core.log_error( 'Frame does not have "block_names" entry, ' 'or it is not the same length as "blocks".', unit='HKScanner') self.stats['concerns']['n_error'] += 1 # Fall back on v1 strategy. block_name = lambda block_idx: list( sorted(blocks[block_idx].keys()))[0] else: block_name = lambda block_idx: f['block_names'][block_idx] for block_idx, b in enumerate(blocks): times = block_timef(b) if len(times): if info['span'] is None: info['span'] = times[0], times[-1] else: t0, t1 = info['span'] info['span'] = min(times[0], t0), max(times[-1], t1) t_check.append(times[0]) info['ticks'] += len(times) bname = block_name(block_idx) for k, v in block_itemf(b): if len(v) != len(times): core.log_error( 'Field "%s" has %i samples but .t has %i samples.' % (k, len(v), len(times))) self.stats['concerns']['n_error'] += 1 # Make sure field has a block_stream registered. if k not in info['block_streams_map']: info['block_streams_map'][k] = bname if info['block_streams_map'][k] != bname: core.log_error( 'Field "%s" appeared in block_name %s ' 'and later in block_name %s.' % (k, info['block_streams_map'][k], bname)) self.stats['concerns']['n_error'] += 1 if len(t_check) and abs(min(t_check) - t_this) > 60: core.log_warn( 'data frame timestamp (%.1f) does not correspond to ' 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') self.stats['concerns']['n_warning'] += 1 return [f]