def ping(self): """ Send a watchdog ping message to the GCP pager process. This method is called by the `run` method at regular intervals whenever the `data_valid` method returns True. """ try: if not self.sim: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(self.timeout) sock.connect((self.host, self.port)) sock.send('watchdog {}'.format(self.name).encode()) resp = sock.recv(4096) if resp: core.log_debug( 'Sent watchdog ping, got response {}'.format(resp.decode()), unit=self.unit, ) sock.close() except Exception as e: core.log_error('Error sending watchdog ping: {}'.format(e), unit=self.unit) # try again in ten seconds self.last_ping = time.time() - self.interval + 10 else: core.log_info('Sent watchdog ping', unit=self.unit) self.last_ping = time.time()
def __init__(self, port=50029, chunk_secs=1): # Store passed attributes self._port = port self._chunk_sec = int(chunk_secs) # Definitions for reading data strings self._endian = '<' self._uint_str = 'Q' self._double_str = 'd' # String to describe data length for 'struct' module self._len_str = self._endian + 'i' # String for packing values from G3DoubleObjects # as long long ints self._pack_str = self._endian + 'q' # Establish socket connection to slowDAQ publisher self.socket = socket.socket() self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.bind(('', port)) self.socket.listen(5) self.socket.setblocking(False) core.log_info( "Listening for requests from slowdaq on port %d'" % (self._port), unit='CHWPSlowDAQTee') # Store and send data as a dictionary self.data = {} self.time_chunk = None
def __call__(self, f): """Processes a frame. Only Housekeeping frames will be manipulated; others will be passed through untouched. """ if f.type == core.G3FrameType.EndProcessing: return self.flush() + [f] if f.type != core.G3FrameType.Housekeeping: return f output = [] if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: output += self.flush() # this clears self.session._id. output.append(f) else: pass # Don't re-emit an on-going session frame. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKReframer') self.session_id = session_id output.append(f) elif f['hkagg_type'] == so3g.HKFrameType.status: # Only issue status if something has changed. changes = False # Flush any providers that are now expired. now_prov_id = [p['prov_id'].value for p in f['providers']] for p in list(self.providers.keys()): if p not in now_prov_id: output += self.flush(p) changes = True # Create bundlers for any new providers. for p in now_prov_id: if p not in self.providers: t0 = f['timestamp'] self.providers[p] = _HKProvBundle(t0, self.target) changes = True if changes: output.append(f) elif f['hkagg_type'] == so3g.HKFrameType.data: fb = self.providers[f['prov_id']] fb.add(f) if fb.ready(): output += fb.rebundle() else: raise ValueError('Invalid hkagg_type') return output
def __init__(self, port=50010, verbose=False): # make some noise at startup core.log_info("Initialize gcp.GCPHousekeepingTee on port %d" % port, unit='GCPHousekeepingTee') self.hkblob = self.PackHKToGCP(dfmux.DfMuxHousekeepingMap()) self.socket = socket.socket() self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.bind(('', port)) self.socket.listen(25) self.socket.setblocking(False) self.verbose = verbose # flag for printing debugging statements
def __init__(self, port=50020, verbose=False, bolometers=[]): ''' Send data from the given list of bolometer logical IDs to the GCP. ''' core.log_info('Listening for requests from GCP on port %d' % port, unit='GCPBoloDataTee') core.log_info('Selected bolometers: %s' % bolometers, unit='GCPBoloDataTee') self.bololist = bolometers self.socket = socket.socket() self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.bind(('', port)) self.socket.listen(5) self.socket.setblocking(False) self.data = {}
def flush(self, prov_id=None): """Flush all buffers; empty the provider list; reset session_id. Returns a list of flushed output frames. If prov_id is specified, then only that provider is flushed and popped from provider list. """ if prov_id is not None: return self.providers.pop(prov_id).rebundle(True) core.log_info('Flushing session id %i' % (self.session_id), unit='HKReframer') output = [] for p in self.providers.values(): output += p.rebundle(True) self.providers = {} self.session_id = None return output
def ping(self): # send a watchdog command to the pager server port try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(self.timeout) sock.connect((self.host, self.port)) sock.send('watchdog daq'.encode()) resp = sock.recv(4096) if resp: core.log_debug("Sent DAQ watchdog ping, got response {}".format(resp.decode()), unit='GCPWatchdog') sock.close() except Exception as e: core.log_error("Error sending watchdog ping: {}".format(e), unit='GCPWatchdog') # try again in ten seconds self.last_ping = time.time() - self.interval + 10 else: core.log_info('Sent DAQ watchdog ping', unit='GCPWatchdog') self.last_ping = time.time()
def Process(self, f, index_info=None): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. The index_info will be stored along with a description of the frame's data; see the .process_file function. """ self.counter += 1 if index_info is None: index_info = {'counter': self.counter} f = self.translator(f) assert(len(f) == 1) f = f[0] if f.type == core.G3FrameType.EndProcessing: return [f] if f.type != core.G3FrameType.Housekeeping: return [f] vers = f.get('hkagg_version', 0) assert(vers == 2) if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.flush() # Note that this sets self.session_id=None. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id elif f['hkagg_type'] == so3g.HKFrameType.status: # If a provider has disappeared, flush its information into a # FieldGroup. prov_cands = [_HKProvider.from_g3(p) for p in f['providers']] to_flush = list(self.providers.keys()) # prov_ids... for p in prov_cands: if p.prov_id in to_flush: to_flush.remove(p.prov_id) # no, don't. else: self.providers[p.prov_id] = p for prov_id in to_flush: self.flush([prov_id]) elif f['hkagg_type'] == so3g.HKFrameType.data: # Data frame -- merge info for this provider. prov = self.providers[f['prov_id']] representatives = prov.blocks.keys() for bidx, (bname, b) in enumerate(zip(f['block_names'], f['blocks'])): assert(isinstance(b, core.G3TimesampleMap)) if bname not in prov.blocks: prov.blocks[bname] = {'fields': list(b.keys()), 'start': b.times[0].time / core.G3Units.seconds, 'index_info': []} # To ensure that the last sample is actually included # in the semi-open intervals we use to track frames, # the "end" time has to be after the final sample. prov.blocks[bname]['end'] = b.times[-1].time / core.G3Units.seconds + SPAN_BUFFER_SECONDS ii = {'block_index': bidx, 'timestamp': b.times[0].time, 'count': len(b.times)} ii.update(index_info) prov.blocks[bname]['index_info'].append(ii) else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') return [f]
def __call__(self, f): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. """ if f.type == core.G3FrameType.EndProcessing: self.report_and_reset() return [f] if f.type != core.G3FrameType.Housekeeping: self.stats['n_other'] += 1 return f self.stats['n_hk'] += 1 if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.report_and_reset( ) # note this does clear self.session_id. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id self.stats['n_session'] += 1 elif f['hkagg_type'] == so3g.HKFrameType.status: # Have any providers disappeared? now_prov_id = [p['prov_id'].value for p in f['providers']] for p, info in self.providers.items(): if p not in now_prov_id: info['active'] = False # New providers? for p in now_prov_id: info = self.providers.get(p) if info is not None: if not info['active']: core.log_warn('prov_id %i came back to life.' % p, unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['n_active'] += 1 info['active'] = True else: self.providers[p] = { 'active': True, # Currently active (during processing). 'n_active': 1, # Number of times this provider id became active. 'n_frames': 0, # Number of data frames. 'timestamp_init': f['timestamp'], # Timestamp of provider appearance 'timestamp_data': None, # Timestamp of most recent data frame. 'ticks': 0, # Total number of timestamps in all blocks. 'span': None, # (earliest_time, latest_time) } elif f['hkagg_type'] == so3g.HKFrameType.data: info = self.providers[f['prov_id']] info['n_frames'] += 1 t_this = f['timestamp'] if info['timestamp_data'] is None: t_ref = info['timestamp_init'] if t_this < t_ref: core.log_warn('data timestamp (%.1f) precedes provider ' 'timestamp by %f seconds.' % (t_this, t_this - t_ref), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 elif t_this <= info['timestamp_data']: core.log_warn( 'data frame timestamps are not strictly ordered.', unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['timestamp_data'] = t_this # update t_check = [] for b in f['blocks']: if len(b.t): if info['span'] is None: info['span'] = b.t[0], b.t[-1] else: t0, t1 = info['span'] info['span'] = min(b.t[0], t0), max(b.t[-1], t1) t_check.append(b.t[0]) info['ticks'] += len(b.t) for k, v in b.data.items(): if len(v) != len(b.t): core.log_error( 'Field "%s" has %i samples but .t has %i samples.' % (k, len(v), len(b.t))) self.stats['concerns']['n_error'] += 1 if len(t_check) and abs(min(t_check) - t_this) > 60: core.log_warn( 'data frame timestamp (%.1f) does not correspond to ' 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') self.stats['concerns']['n_warning'] += 1 return [f]
def report_and_reset(self): core.log_info('Report for session_id %i:\n' % self.session_id + str(self.stats) + '\n' + str(self.providers) + '\nEnd report.', unit='HKScanner') self.session_id = None
def WriteDB(fr, client, fields=None): ''' Write points to the database for each field Arguments --------- client : InfluxDB client fields : Which gcp fields to add to database. See parse_field for options. If None, add all. ''' from influxdb.exceptions import InfluxDBClientError from influxdb.exceptions import InfluxDBServerError if fr.type != core.G3FrameType.GcpSlow: return all_fields = build_field_list(fr) if fields is None: fields = all_fields.keys() dict_list = [] for f in fields: field_dat = all_fields[f] if len(field_dat) == 4: stat, attr, ind, unit = field_dat try: dat = getattr(fr[stat], attr)[ind] time = getattr(fr[stat], 'time') except AttributeError: # OnlinePointingModel dat = fr[stat][attr][ind] time = fr[stat]['time'] elif len(field_dat) == 3: stat, attr, unit = field_dat if stat not in fr: # Field only exists in live data stream continue try: dat = getattr(fr[stat], attr) except AttributeError: try: dat = fr[stat][attr] except KeyError: # Field only exists in live data stream continue if 'Bench' in stat: # funny time field for bench positions time = fr['BenchSampleTime'] elif 'Mux' in stat: time = fr['MuxTime'] elif stat in ['CryoStatus', 'Weather', 'PTStatus']: time = fr['{}Time'.format(stat)] else: try: time = getattr(fr[stat], 'time') except AttributeError: time = fr[stat]['time'] elif len(field_dat) == 2: stat, unit = field_dat try: dat = fr[stat] except KeyError: #eg, no obsid core.log_warn('No key {}'.format(stat), unit='InfluxDB') continue try: time = getattr(fr[stat], 'time') except AttributeError as err: time = [tm for tm in fr['antenna0']['tracker']['utc'][0]] # InfluxDB wants time in nanoseconds since the UNIX epoch in UTC try: time = [x.time / U.nanosecond for x in np.atleast_1d(time)] except AttributeError: time = [ core.G3Time(t0).time / U.nanosecond for t0 in np.atleast_1d(time) ] if dat is None: core.log_warn('{} dat is None'.format(f), unit='InfluxDB') continue dat = np.atleast_1d(dat) try: dlen = len(dat) except TypeError: # sometimes source_name is a weird non-none value continue if unit is not None: if unit == 'C': zeropt_K = 273.15 cal_dat = dat / U.K - zeropt_K else: cal_dat = dat / unit else: cal_dat = dat try: if np.any(np.isnan(cal_dat)): continue except TypeError: pass if 'heat' not in f: tag = f else: tag = f.replace('heat_', '') # for fields that have az/el components az_el_names = [ 'az', 'el', 'az', 'el', 'ra', 'dec', 'x', 'y', 'hr_angle', 'sin', 'cos', 'lat' ] tag2 = f for name in az_el_names: # require name_ at beginning or _name at end match1 = re.findall('^{}_'.format(name), f) match2 = re.findall('_{}$'.format(name), f) if len(match1): tag2 = f.replace(match1[0], '') if len(match2): tag2 = f.replace(match2[0], '') # also group source names if 'source' in f: tag2 = 'source' stat = 'TrackerPointing' if stat == 'PTStatus': groups = ['now', 'min', 'max'] for g in groups: match = re.findall('_{}$'.format(g), f) if len(match): tag2 = f.replace(match[0], '') # group bench positions # require bench_ at beginning match = re.findall('^bench', f) if len(match): tag2 = attr # y1, y2, etc stat = 'Bench' # group Mux properties if 'Mux' in stat: stat = 'muxHousekeeping' tag2 = 'ib' + f.split('ib')[-1] dict_list += make_lines( measurement=stat, field=f, time=time, dat=cal_dat, tags={ 'label': tag, 'label2': tag2 }, ) try: now = core.G3Time.Now() delay = float(now.time / U.nanosecond - time[-1]) / 1e9 if delay > 5: core.log_info('{} Delay: {} s'.format(now.isoformat(), delay), unit='InfluxDB') except RuntimeError: # sometimes timestamp gets screwed up pass try: client.write_points(dict_list, batch_size=len(dict_list), protocol='line') except (InfluxDBClientError, InfluxDBServerError) as v: core.log_error('Error writing to database. {}'.format(v), unit='InfluxDB')
def Process(self, f): """Translates one frame to the target schema. Irrelevant frames are passed through unmodified. Args: f: a G3Frame Returns: A list containing only the translated frame. G3Pipeline compatibility would permit us to return a single frame here, instead of a length-1 list. But we also sometimes call Process outside of a G3Pipeline, where a consistent output type is desirable. Returning lists is most future-compatible; consumers that want to assume length-1 should assert it to be true. """ if f.type == core.G3FrameType.EndProcessing: core.log_info(str(self.stats)) return [f] if f.type != core.G3FrameType.Housekeeping: self.stats['n_other'] += 1 return [f] # It is an HK frame. orig_version = f.get('hkagg_version', 0) self.stats['n_hk'] += 1 self.stats['versions'][orig_version] = self.stats['versions'].get(orig_version, 0) + 1 if orig_version > self.target_version and not self.future_tolerant: raise ValueError( ('Translator to v%i encountered v%i, but future_tolerant=False.') % (self.TARGET_VERSION, orig_version)) if orig_version >= self.target_version: return [f] # Always update the version, even if that's our only change... if 'hkagg_version' in f: if 'hkagg_version_orig' not in f: f['hkagg_version_orig'] = orig_version del f['hkagg_version'] f['hkagg_version'] = self.target_version # No difference in Session/Status for v0, v1, v2. if f.get('hkagg_type') != so3g.HKFrameType.data: return [f] if self.target_version == 0: return [f] if orig_version == 0: # Pop the data blocks out of the frame. orig_blocks = f.pop('blocks') f['blocks'] = core.G3VectorFrameObject() # Now process the data blocks. for block in orig_blocks: new_block = core.G3TimesampleMap() new_block.times = so3g.hk.util.get_g3_time(block.t) for k in block.data.keys(): v = block.data[k] new_block[k] = core.G3VectorDouble(v) f['blocks'].append(new_block) if self.target_version == 1: return [f] if orig_version <= 1: # Add 'block_names'. Since we don't want to start # caching Block Stream information, just compute a good # block name based on the alphabetically first field in # the block. block_names = [] for block in f['blocks']: field_names = list(sorted(block.keys())) block_names.append('block_for_%s' % field_names[0]) assert(len(block_names[-1]) < 256) # What have you done. orig_block_names = [] f['block_names'] = core.G3VectorString(block_names) return [f]
def __call__(self, f): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. """ if f.type == core.G3FrameType.EndProcessing: self.report_and_reset() return [f] if f.type != core.G3FrameType.Housekeeping: self.stats['n_other'] += 1 return f self.stats['n_hk'] += 1 vers = f.get('hkagg_version', 0) self.stats['versions'][vers] = self.stats['versions'].get(vers, 0) + 1 if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.report_and_reset( ) # note this does clear self.session_id. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id self.stats['n_session'] += 1 elif f['hkagg_type'] == so3g.HKFrameType.status: # Have any providers disappeared? now_prov_id = [p['prov_id'].value for p in f['providers']] for p, info in self.providers.items(): if p not in now_prov_id: info['active'] = False # New providers? for p in now_prov_id: info = self.providers.get(p) if info is not None: if not info['active']: core.log_warn('prov_id %i came back to life.' % p, unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['n_active'] += 1 info['active'] = True else: self.providers[p] = { 'active': True, # Currently active (during processing). 'n_active': 1, # Number of times this provider id became active. 'n_frames': 0, # Number of data frames. 'timestamp_init': f['timestamp'], # Timestamp of provider appearance 'timestamp_data': None, # Timestamp of most recent data frame. 'ticks': 0, # Total number of timestamps in all blocks. 'span': None, # (earliest_time, latest_time) 'block_streams_map': {}, # Map from field name to block name. } elif f['hkagg_type'] == so3g.HKFrameType.data: info = self.providers[f['prov_id']] vers = f.get('hkagg_version', 0) info['n_frames'] += 1 t_this = f['timestamp'] if info['timestamp_data'] is None: t_ref = info['timestamp_init'] if t_this < t_ref: core.log_warn('data timestamp (%.1f) precedes provider ' 'timestamp by %f seconds.' % (t_this, t_this - t_ref), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 elif t_this <= info['timestamp_data']: core.log_warn( 'data frame timestamps are not strictly ordered.', unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['timestamp_data'] = t_this # update t_check = [] blocks = f['blocks'] if vers == 0: block_timef = lambda block: block.t block_itemf = lambda block: [(k, block.data[k]) for k in block.data.keys()] elif vers >= 1: block_timef = lambda block: np.array( [t.time / core.G3Units.seconds for t in b.times]) block_itemf = lambda block: [(k, block[k]) for k in block.keys()] if vers in [0]: block_name = lambda block_idx: list( sorted(blocks[block_idx].data.keys()))[0] if vers in [1]: block_name = lambda block_idx: list( sorted(blocks[block_idx].keys()))[0] elif vers >= 2: block_names = f.get('block_names', []) if len(block_names) != len(blocks): # This is a schema error in its own right. core.log_error( 'Frame does not have "block_names" entry, ' 'or it is not the same length as "blocks".', unit='HKScanner') self.stats['concerns']['n_error'] += 1 # Fall back on v1 strategy. block_name = lambda block_idx: list( sorted(blocks[block_idx].keys()))[0] else: block_name = lambda block_idx: f['block_names'][block_idx] for block_idx, b in enumerate(blocks): times = block_timef(b) if len(times): if info['span'] is None: info['span'] = times[0], times[-1] else: t0, t1 = info['span'] info['span'] = min(times[0], t0), max(times[-1], t1) t_check.append(times[0]) info['ticks'] += len(times) bname = block_name(block_idx) for k, v in block_itemf(b): if len(v) != len(times): core.log_error( 'Field "%s" has %i samples but .t has %i samples.' % (k, len(v), len(times))) self.stats['concerns']['n_error'] += 1 # Make sure field has a block_stream registered. if k not in info['block_streams_map']: info['block_streams_map'][k] = bname if info['block_streams_map'][k] != bname: core.log_error( 'Field "%s" appeared in block_name %s ' 'and later in block_name %s.' % (k, info['block_streams_map'][k], bname)) self.stats['concerns']['n_error'] += 1 if len(t_check) and abs(min(t_check) - t_this) > 60: core.log_warn( 'data frame timestamp (%.1f) does not correspond to ' 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') self.stats['concerns']['n_warning'] += 1 return [f]
def Process(self, f, index_info=None): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. The index_info will be stored along with a description of the frame's data; see the .process_file function. """ self.counter += 1 if index_info is None: index_info = {'counter': self.counter} if f.type == core.G3FrameType.EndProcessing: return [f] if f.type != core.G3FrameType.Housekeeping: return f if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.flush() # Note that this sets self.session_id=None. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id elif f['hkagg_type'] == so3g.HKFrameType.status: # If a provider has disappeared, flush its information into a # FieldGroup. prov_cands = [_HKProvider.from_g3(p) for p in f['providers']] to_flush = list(self.providers.keys()) # prov_ids... for p in prov_cands: if p.prov_id in to_flush: to_flush.remove(p.prov_id) # no, don't. else: self.providers[p.prov_id] = p for prov_id in to_flush: self.flush([prov_id]) elif f['hkagg_type'] == so3g.HKFrameType.data: # Data frame -- merge info for this provider. prov = self.providers[f['prov_id']] representatives = [block['fields'][0] for block in prov.blocks] for b in f['blocks']: fields = b.data.keys() if len(b.t) == 0 or len(fields) == 0: continue for block_index, rep in enumerate(representatives): if rep in fields: break else: block_index = len(prov.blocks) prov.blocks.append({ 'fields': fields, 'start': b.t[0], 'index_info': [] }) # To ensure that the last sample is actually included # in the semi-open intervals we use to track frames, # the "end" time has to be after the final sample. prov.blocks[block_index]['end'] = b.t[-1] + SPAN_BUFFER_SECONDS prov.blocks[block_index]['index_info'].append(index_info) else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') return [f]