def UnitValue(caldict_entry): '''Turn unit name into floating point unit value''' try: uname = caldict_entry['UnitName'] if uname and uname != 'None': try: if '/' in uname: unames = list(filter(None, uname.split('/'))) uvalue1 = getattr( core.G3Units, list(filter(None, unames[0].split(' ')))[0]) uvalue2 = getattr( core.G3Units, list(filter(None, unames[1].split(' ')))[0]) uvalue = uvalue1 / uvalue2 else: uvalue = getattr(core.G3Units, uname) except AttributeError: uvalue = 1. core.log_warn('No entry in G3Units for ' + uname + '. Setting UnitValue to 1.0\n') else: uvalue = 1. except KeyError: uvalue = 1. return uvalue
def _concatenate_timestreams(cls, ts_lst, ts_rounding_error=0.6, ts_interp_threshold=0): """ Concatenate G3Timestream objects together. Arguments --------- ts_lst : list list of G3Timestream objects. ts_rounding_error : float allowed error in timestream separation such that timestreams are contiguous, as a fraction of the sample rate. This should be 0 by default, but is 0.5 to allow for downsampler shifting, and then bumpted again to 0.6 to allow for floating-point errors in what 0.5 is. ts_interp_threshold : float allowed timestream separation below which gaps between timestreams are interpolated to be made continuous Returns ------- ts : G3Timestream instance The concatenation of the input list of G3Timestream objects """ #check for contiguous timestreams for i in range(1, len(ts_lst)): ts_sep = (ts_lst[i].start.time - ts_lst[i - 1].stop.time) * ts_lst[i].sample_rate if numpy.abs(ts_sep - 1) > ts_rounding_error: if (ts_interp_threshold > 0) and ( (ts_sep - 1) < ts_interp_threshold): log_warn( "Timestreams are not contiguous: timestreams %d and %d " "separated by %f samples (%s). Interpolating." % (i, i - 1, ts_sep - 1, str(ts_lst[i].start))) v = numpy.linspace(ts_lst[i - 1][-1], ts_lst[i][0], ts_sep + 1)[1:-1] ts_interp = cls(v) ts_interp.units = ts_lst[0].units ts_interp.start = ts_lst[i - 1].stop + 1. / ts_lst[i].sample_rate ts_interp.stop = ts_lst[i].start - 1. / ts_lst[i].sample_rate ts_lst = ts_lst[:i] + [ts_interp] + ts_lst[i:] else: log_fatal( "Timestreams are not contiguous: timestreams %d and %d " "separated by %f samples (%s)" % (i, i - 1, ts_sep - 1, str(ts_lst[i].start))) if (ts_lst[i].units != ts_lst[0].units): log_fatal("Timestreams are not the same units") out_ts = cls(numpy.concatenate(ts_lst)) out_ts.units = ts_lst[0].units out_ts.start = ts_lst[0].start out_ts.stop = ts_lst[-1].stop return out_ts
def __init__(self, input='CalTimestreams', output_root=None, bands=None, bpm='BolometerProperties'): core.log_warn( "SplitTimestreamsByBand is deprecated, use SplitByBand instead") super(SplitTimestreamsByBand, self).__init__(input=input, output_root=output_root, bands=bands, bpm=bpm)
def BolometerPropertiesInjector(frame): if frame.type == core.G3FrameType.Wiring: nchan = len(frame['WiringMap'].keys()) core.log_notice("Collecting data from %d mapped channels." % (nchan), unit='Data Acquisition') bpm = os.path.join(os.path.dirname(args.hardware_map), 'nominal_online_cal.g3') try: if not os.path.exists(bpm): raise IOError('Missing file %s' % bpm) fr = list(core.G3File(bpm))[0] return [frame, fr] except Exception as e: core.log_warn('Error loading BolometerPropertiesMap: %s' % (str(e)), unit='Data Acquisition')
def check_iers(g3_time): """ Check whether IERS calculations will work, and load an IERS database file from backup if all else fails. Arguments --------- g3_time : G3Time instance Most recent time for which an IERS calculation must be computed. """ t = astropy.time.Time(g3_time.mjd, format="mjd") # check if accessing the IERS table outright works. try: t.ut1 return except: pass # if that fails, allow extrapolation iers.conf.auto_max_age = None t = astropy.time.Time(g3_time.mjd, format="mjd") try: t.ut1 core.log_warn("IERS auto update failed, allowing extrapolation", unit="IERS") return except: pass # and if that fails, use a locally cached file that is hopefully setup correctly. fname = os.path.join(os.path.dirname(os.path.abspath(__file__)), "finals2000A.all") iers.conf.auto_download = False iers.IERS.iers_table = iers.IERS_A.open(fname) t = astropy.time.Time(g3_time.mjd, format="mjd") t.ut1 core.log_warn("Using IERS table from local cache {}".format(fname), unit="IERS")
def ValidateMaps(frame, ignore_missing_weights=False): """ Validate that the input map frame has all the necessary keys. If ignore_missing_weights is False (default), a warning is issued when the frame contains weighted Stokes maps without a weights map. Set this option to True when feeding single bolometer map frames with common weights through a pipeline. """ if isinstance(frame, core.G3Frame) and frame.type != core.G3FrameType.Map: return map_id = frame.get("Id", None) if "T" not in frame: core.log_fatal("Map frame %s: Missing T map" % map_id, unit="ValidateMaps") if ("Q" in frame and not "U" in frame) or ("U" in frame and not "Q" in frame): core.log_fatal("Map frame %s: Missing Q or U map" % map_id, unit="ValidateMaps") if "Wpol" in frame and "Wunpol" in frame: core.log_fatal( "Map frame %s: Found both polarized and unpolarized weights" % map_id, unit="ValidateMaps", ) stub = frame["T"].clone(False) for k in ["T", "Q", "U", "Wpol", "Wunpol"]: if k not in frame: continue if not frame[k].compatible(stub): core.log_fatal( "Map frame %s: Map %s not compatible with T map" % (map_id, k), unit="ValidateMaps", ) if k in "TQU": if k == "U" and frame[k].pol_conv is maps.MapPolConv.none: core.log_warn( "Map frame %s: U map polarization convention not set" % map_id, unit="ValidateMaps", ) if frame[k].weighted and not ignore_missing_weights: if "Wpol" not in frame and "Wunpol" not in frame: core.log_warn( "Map frame %s: Missing weights" % map_id, unit="ValidateMaps" ) if k == "T" and "Q" not in frame and "Wunpol" not in frame: core.log_warn( "Map frame %s: Missing unpolarized weights" % map_id, unit="ValidateMaps", ) if k in "QU" and "Wpol" not in frame: core.log_warn( "Map frame %s: Missing polarized weights" % map_id, unit="ValidateMaps", ) else: if frame[k].polarized and ("Q" not in frame or "U" not in frame): core.log_fatal( "Map frame %s: Found unpolarized maps with polarized weights" % map_id, unit="ValidateMaps", ) elif not frame[k].polarized and ("Q" in frame or "U" in frame): core.log_fatal( "Map frame %s: Found polarized maps with unpolarized weights" % map_id, unit="ValidateMaps", )
def load_skymap_fits(filename, hdu=None, keys=None, memmap=False, apply_units=False): """ Load a fits file containing a sky map. Arguments --------- filename : str Path to fits file hdu : int, optional If supplied, the data are extract from the given HDU index. keys : list of strings, optional If supplied, return only these keys in the output dictionary. Options are: T, Q, U, W. memmap : bool, optional Argument passed to astropy.io.fits.open. If True, the map is not read into memory, but only the required pixels are read when needed. Default: False. apply_units : bool, optional If True, and input maps have known units, multiply by the appropriate conversion factor to return maps in G3Units. Returns ------- a dictionary of maps keyed with e.g. 'T', 'Q', 'U' and 'W'. """ import astropy.io.fits assert (os.path.exists(filename)) map_type = None pol = None map_opts = {} output = {} # defaults for missing header entries pol_conv = None units = 'Tcmb' coord_ref = 'Equatorial' proj = 'Proj5' weighted = False alpha_center = None delta_center = None res = None xres = None unit_dict = { 'k_cmb': ('Tcmb', core.G3Units.K), 'kcmb': ('Tcmb', core.G3Units.K), 'kcmb^2': ('Tcmb', core.G3Units.K**2), } if keys is None: keys = ['T', 'Q', 'U', 'W'] with astropy.io.fits.open(filename, memmap=memmap) as hdulist: for hidx, H in enumerate(hdulist): hdr = H.header mtype = hdr.get('MAPTYPE', None) if mtype == 'FLAT' or 'PROJ' in hdr or 'WCSAXES' in hdr or 'CTYPE1' in hdr: # flat map if not map_type: map_type = 'flat' elif map_type != 'flat': raise ValueError( "Expected a {} sky map in HDU {}, found a flat map". format(map_type, hidx)) # expect that flat sky maps on disk are probably in IAU if not pol_conv: pol_conv = 'IAU' elif mtype == 'HEALPIX' or hdr.get( 'PIXTYPE', None) == 'HEALPIX' or 'NSIDE' in hdr: # healpix map if not map_type: map_type = 'healpix' elif map_type != 'healpix': raise ValueError( "Expected a {} sky map in HDU {}, found a healpix map". format(map_type, hidx)) # expect that healpix maps on disk are probably in COSMO if not pol_conv: pol_conv = 'COSMO' if 'POLAR' in hdr: pdict = {'T': True, 'F': False} pol = pdict.get(hdr['POLAR'], hdr['POLAR']) if pol and map_type is not None: if hdr.get('POLCCONV', '').upper() not in ['IAU', 'COSMO']: core.log_warn( 'Polarization convention not set, assuming %s' % pol_conv, unit='load_skymap_fits') else: pol_conv = hdr['POLCCONV'].upper() uconv = None if 'TUNIT' in hdr: u = hdr['TUNIT'].lower() if u in unit_dict: units, uconv = unit_dict[u] else: units = hdr.get('UNITS', units) if 'COORDSYS' in hdr: cdict = {'C': 'Equatorial', 'G': 'Galactic', 'L': 'Local'} coord_ref = cdict.get(hdr['COORDSYS'], coord_ref) else: coord_ref = hdr.get('COORDREF', coord_ref) overflow = hdr.get('OVERFLOW', 0) map_opts.update( units=getattr(core.G3TimestreamUnits, units), coord_ref=getattr(MapCoordReference, coord_ref), pol_conv=getattr(MapPolConv, pol_conv) if pol_conv else None, ) if map_type == 'flat': map_opts.update(flat_pol=hdr.get('FLATPOL', False)) map_opts.update(parse_wcs_header(hdr)) elif map_type == 'healpix': nside = hdr['NSIDE'] nested = hdr['ORDERING'].strip().lower() in ['nest', 'nested'] map_opts.update(nested=nested) # primary HDU if H.data is None: continue # extracting a particular HDU if map_type is not None and hdu is not None and hidx != hdu: continue if map_type == 'flat': if hdr.get('ISWEIGHT', None): if 'W' not in keys: continue else: if hdr.get('POLTYPE', 'T') not in keys: continue data = np.array(H.data, dtype=float) if map_opts.pop('transpose', False): data = np.array(data.T) if uconv is not None: data *= uconv # map type must be known if the HDU contains map data if map_type not in ['flat', 'healpix']: raise ValueError("Unknown map type in HDU {}".format(hidx)) if map_type == 'flat' and hdr.get('ISWEIGHT', None): # flat map weights assert ('T' in output) if pol is None: pol = True if ('Q' in output and 'U' in output) else False weight_map = output.setdefault( 'W', G3SkyMapWeights(output['T'], polarized=pol)) fm = FlatSkyMap(data, **map_opts) fm.overflow = overflow setattr(weight_map, hdr['WTYPE'], fm) del data elif map_type == 'flat' and not hdr.get('ISWEIGHT', None): # flat map data ptype = hdr.get('POLTYPE', 'T') pol_type = getattr(MapPolType, ptype, None) fm = FlatSkyMap(data, pol_type=pol_type, **map_opts) fm.overflow = overflow output[ptype] = fm del data elif map_type == 'healpix': # healpix map data col_dict = { 'TEMPERATURE': 'T', 'Q_POLARISATION': 'Q', 'U_POLARISATION': 'U', 'Q-POLARISATION': 'Q', 'U-POLARISATION': 'U', 'I_STOKES': 'T', 'Q_STOKES': 'Q', 'U_STOKES': 'U', 'I': 'T', 'II': 'TT', 'IQ': 'TQ', 'IU': 'TU', 'II_COV': 'TT', 'IQ_COV': 'TQ', 'IU_COV': 'TU', 'QQ_COV': 'QQ', 'QU_COV': 'QU', 'UU_COV': 'UU', } pix = None partial = hdr.get('INDXSCHM') == 'EXPLICIT' or hdr.get( 'OBJECT') == 'PARTIAL' for cidx, hcol in enumerate(H.data.names): col = col_dict.get(hcol, hcol) if col in 'TQU' and col not in keys: continue elif col in ['TT', 'TQ', 'TU', 'QQ', 'QU', 'UU' ] and 'W' not in keys: continue data = np.array(H.data[hcol], dtype=float).ravel() if col == 'PIXEL' or (partial and cidx == 0): pix = np.array(data, dtype=int).ravel() del data continue uconv = None u = 'TUNIT{:d}'.format(cidx + 1) if u in hdr: u = hdr[u].lower() if u in unit_dict: units, uconv = unit_dict[u.lower()] if uconv is not None: data *= uconv overflow = hdr.get('TOFLW{:d}'.format(cidx + 1), overflow) weighted = hdr.get( 'TISWGT{:d}'.format(cidx + 1), col in ['TT', 'TQ', 'TU', 'QQ', 'QU', 'UU'], ) if weighted: assert ('T' in output) if pol is None: pol = True if ('Q' in output and 'U' in output) else False weight_map = output.setdefault( 'W', G3SkyMapWeights(output['T'], polarized=pol)) mdata = (pix, data, nside) if pix is not None else data hm = HealpixSkyMap(mdata, **map_opts) hm.overflow = overflow setattr(weight_map, col, hm) else: pol_type = getattr(MapPolType, col, None) mdata = (pix, data, nside) if pix is not None else data hm = HealpixSkyMap(mdata, pol_type=pol_type, **map_opts) output[col] = hm del mdata, data del pix break del H.data for k, m in output.items(): m.pol_conv = map_opts['pol_conv'] if k == 'W': continue m.weighted = 'W' in output return output
def Process(self, f, index_info=None): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. The index_info will be stored along with a description of the frame's data; see the .process_file function. """ self.counter += 1 if index_info is None: index_info = {'counter': self.counter} f = self.translator(f) assert(len(f) == 1) f = f[0] if f.type == core.G3FrameType.EndProcessing: return [f] if f.type != core.G3FrameType.Housekeeping: return [f] vers = f.get('hkagg_version', 0) assert(vers == 2) if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.flush() # Note that this sets self.session_id=None. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id elif f['hkagg_type'] == so3g.HKFrameType.status: # If a provider has disappeared, flush its information into a # FieldGroup. prov_cands = [_HKProvider.from_g3(p) for p in f['providers']] to_flush = list(self.providers.keys()) # prov_ids... for p in prov_cands: if p.prov_id in to_flush: to_flush.remove(p.prov_id) # no, don't. else: self.providers[p.prov_id] = p for prov_id in to_flush: self.flush([prov_id]) elif f['hkagg_type'] == so3g.HKFrameType.data: # Data frame -- merge info for this provider. prov = self.providers[f['prov_id']] representatives = prov.blocks.keys() for bidx, (bname, b) in enumerate(zip(f['block_names'], f['blocks'])): assert(isinstance(b, core.G3TimesampleMap)) if bname not in prov.blocks: prov.blocks[bname] = {'fields': list(b.keys()), 'start': b.times[0].time / core.G3Units.seconds, 'index_info': []} # To ensure that the last sample is actually included # in the semi-open intervals we use to track frames, # the "end" time has to be after the final sample. prov.blocks[bname]['end'] = b.times[-1].time / core.G3Units.seconds + SPAN_BUFFER_SECONDS ii = {'block_index': bidx, 'timestamp': b.times[0].time, 'count': len(b.times)} ii.update(index_info) prov.blocks[bname]['index_info'].append(ii) else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') return [f]
def __call__(self, f): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. """ if f.type == core.G3FrameType.EndProcessing: self.report_and_reset() return [f] if f.type != core.G3FrameType.Housekeeping: self.stats['n_other'] += 1 return f self.stats['n_hk'] += 1 if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.report_and_reset( ) # note this does clear self.session_id. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id self.stats['n_session'] += 1 elif f['hkagg_type'] == so3g.HKFrameType.status: # Have any providers disappeared? now_prov_id = [p['prov_id'].value for p in f['providers']] for p, info in self.providers.items(): if p not in now_prov_id: info['active'] = False # New providers? for p in now_prov_id: info = self.providers.get(p) if info is not None: if not info['active']: core.log_warn('prov_id %i came back to life.' % p, unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['n_active'] += 1 info['active'] = True else: self.providers[p] = { 'active': True, # Currently active (during processing). 'n_active': 1, # Number of times this provider id became active. 'n_frames': 0, # Number of data frames. 'timestamp_init': f['timestamp'], # Timestamp of provider appearance 'timestamp_data': None, # Timestamp of most recent data frame. 'ticks': 0, # Total number of timestamps in all blocks. 'span': None, # (earliest_time, latest_time) } elif f['hkagg_type'] == so3g.HKFrameType.data: info = self.providers[f['prov_id']] info['n_frames'] += 1 t_this = f['timestamp'] if info['timestamp_data'] is None: t_ref = info['timestamp_init'] if t_this < t_ref: core.log_warn('data timestamp (%.1f) precedes provider ' 'timestamp by %f seconds.' % (t_this, t_this - t_ref), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 elif t_this <= info['timestamp_data']: core.log_warn( 'data frame timestamps are not strictly ordered.', unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['timestamp_data'] = t_this # update t_check = [] for b in f['blocks']: if len(b.t): if info['span'] is None: info['span'] = b.t[0], b.t[-1] else: t0, t1 = info['span'] info['span'] = min(b.t[0], t0), max(b.t[-1], t1) t_check.append(b.t[0]) info['ticks'] += len(b.t) for k, v in b.data.items(): if len(v) != len(b.t): core.log_error( 'Field "%s" has %i samples but .t has %i samples.' % (k, len(v), len(b.t))) self.stats['concerns']['n_error'] += 1 if len(t_check) and abs(min(t_check) - t_this) > 60: core.log_warn( 'data frame timestamp (%.1f) does not correspond to ' 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') self.stats['concerns']['n_warning'] += 1 return [f]
def WriteDB(fr, client, fields=None): ''' Write points to the database for each field Arguments --------- client : InfluxDB client fields : Which gcp fields to add to database. See parse_field for options. If None, add all. ''' from influxdb.exceptions import InfluxDBClientError from influxdb.exceptions import InfluxDBServerError if fr.type != core.G3FrameType.GcpSlow: return all_fields = build_field_list(fr) if fields is None: fields = all_fields.keys() dict_list = [] for f in fields: field_dat = all_fields[f] if len(field_dat) == 4: stat, attr, ind, unit = field_dat try: dat = getattr(fr[stat], attr)[ind] time = getattr(fr[stat], 'time') except AttributeError: # OnlinePointingModel dat = fr[stat][attr][ind] time = fr[stat]['time'] elif len(field_dat) == 3: stat, attr, unit = field_dat if stat not in fr: # Field only exists in live data stream continue try: dat = getattr(fr[stat], attr) except AttributeError: try: dat = fr[stat][attr] except KeyError: # Field only exists in live data stream continue if 'Bench' in stat: # funny time field for bench positions time = fr['BenchSampleTime'] elif 'Mux' in stat: time = fr['MuxTime'] elif stat in ['CryoStatus', 'Weather', 'PTStatus']: time = fr['{}Time'.format(stat)] else: try: time = getattr(fr[stat], 'time') except AttributeError: time = fr[stat]['time'] elif len(field_dat) == 2: stat, unit = field_dat try: dat = fr[stat] except KeyError: #eg, no obsid core.log_warn('No key {}'.format(stat), unit='InfluxDB') continue try: time = getattr(fr[stat], 'time') except AttributeError as err: time = [tm for tm in fr['antenna0']['tracker']['utc'][0]] # InfluxDB wants time in nanoseconds since the UNIX epoch in UTC try: time = [x.time / U.nanosecond for x in np.atleast_1d(time)] except AttributeError: time = [ core.G3Time(t0).time / U.nanosecond for t0 in np.atleast_1d(time) ] if dat is None: core.log_warn('{} dat is None'.format(f), unit='InfluxDB') continue dat = np.atleast_1d(dat) try: dlen = len(dat) except TypeError: # sometimes source_name is a weird non-none value continue if unit is not None: if unit == 'C': zeropt_K = 273.15 cal_dat = dat / U.K - zeropt_K else: cal_dat = dat / unit else: cal_dat = dat try: if np.any(np.isnan(cal_dat)): continue except TypeError: pass if 'heat' not in f: tag = f else: tag = f.replace('heat_', '') # for fields that have az/el components az_el_names = [ 'az', 'el', 'az', 'el', 'ra', 'dec', 'x', 'y', 'hr_angle', 'sin', 'cos', 'lat' ] tag2 = f for name in az_el_names: # require name_ at beginning or _name at end match1 = re.findall('^{}_'.format(name), f) match2 = re.findall('_{}$'.format(name), f) if len(match1): tag2 = f.replace(match1[0], '') if len(match2): tag2 = f.replace(match2[0], '') # also group source names if 'source' in f: tag2 = 'source' stat = 'TrackerPointing' if stat == 'PTStatus': groups = ['now', 'min', 'max'] for g in groups: match = re.findall('_{}$'.format(g), f) if len(match): tag2 = f.replace(match[0], '') # group bench positions # require bench_ at beginning match = re.findall('^bench', f) if len(match): tag2 = attr # y1, y2, etc stat = 'Bench' # group Mux properties if 'Mux' in stat: stat = 'muxHousekeeping' tag2 = 'ib' + f.split('ib')[-1] dict_list += make_lines( measurement=stat, field=f, time=time, dat=cal_dat, tags={ 'label': tag, 'label2': tag2 }, ) try: now = core.G3Time.Now() delay = float(now.time / U.nanosecond - time[-1]) / 1e9 if delay > 5: core.log_info('{} Delay: {} s'.format(now.isoformat(), delay), unit='InfluxDB') except RuntimeError: # sometimes timestamp gets screwed up pass try: client.write_points(dict_list, batch_size=len(dict_list), protocol='line') except (InfluxDBClientError, InfluxDBServerError) as v: core.log_error('Error writing to database. {}'.format(v), unit='InfluxDB')
def readCalFile(self, calfile=None): cal_dict = {} if calfile is None: self.calibration_file = self.default_calibration_file else: self.calibration_file = calfile try: f = open(self.calibration_file, 'r') except: core.log_warn('3G calibration file ' + self.calibration_file + ' not found.\n') return cal_dict ninvalid = 0 for line in f: if line[0] != '#' and line[0] != '\n' and len(line) > 0: try: line = line.replace('\n', '') fline = list(filter(None, line.split(' '))) #list() for py3 name = fline[0] linedict = {} linedict['Offset'] = np.float(fline[1]) linedict['ReciprocalFactor'] = np.float(fline[2]) try: linedict['UnitName'] = fline[3] except: linedict['UnitName'] = 'None' cal_dict[name] = linedict except: ninvalid += 1 if ninvalid > 0: if ninvalid == 1: core.log_warn( 'One invalid line found and ignored in calibration file.\n' ) else: core.log_warn( str(ninvalid) + ' invalid lines found and ignored in calibration file.\n') f.close() # now parse into dictionary structure NW uses in ARCExtractor cal_dict_orig = cal_dict.copy() cal_dict = {} origkeys = cal_dict_orig.keys() origkeys = sorted(origkeys) names = [key for key in origkeys] regmaps = [(list(filter(None, (tname).split('.'))))[0] for tname in names] uregmaps = np.unique(np.asarray(regmaps)) for urm in uregmaps: cal_dict[urm] = {} regblocks = [(list(filter(None, (tname).split('.'))))[1] for tname in names] uregblocks, argurb = np.unique(np.asarray(regblocks), return_index=True) for j in np.arange(len(uregblocks)): cal_dict[regmaps[argurb[j]]][uregblocks[j]] = {} regs = [(list(filter(None, (tname).split('.'))))[2] for tname in names] for j in np.arange(len(regs)): reg = regs[j] if '[' in reg: reg_plus_indices = reg.split('[') realreg = reg_plus_indices[0] indices = ((reg_plus_indices[1]).split(']'))[0] if '-' in indices: cal_dict[regmaps[j]][regblocks[j]][realreg] = {} siindices = indices.split('-') iindices = (np.array(siindices)).astype(int) for i in np.arange(np.max(iindices)-np.min(iindices)+1) + \ np.min(iindices): cal_dict[regmaps[j]][regblocks[j]]\ [realreg][i] = cal_dict_orig[names[j]] else: if indices == '0': cal_dict[regmaps[j]][regblocks[j]][realreg] = {} cal_dict[regmaps[j]][regblocks[j]][realreg]\ [np.int(indices)] = cal_dict_orig[names[j]] else: cal_dict[regmaps[j]][regblocks[j]][reg] = \ cal_dict_orig[names[j]] return cal_dict
def create_g3_cal_file(path, read_from_gcp=True, extra_dict=None, use_extra_info=True, gcp_cal_file=None): ''' Create a G3 register calibration file. Usually reads in calibration and units from a GCP cal file then adds extra information that isn't in the GCP one. This extra information can be handed to the routine as a dict (note format below); otherwise, the hard-coded extra info will be used. (Feel free to edit this hard-coded info.) ''' cal_dict = {} status = 0 if read_from_gcp: try: default_calibration_file = \ os.environ['GCP']+'/control/conf/spt/cal' except: default_calibration_file = \ '/home/sptdat/gcproot/control/conf/spt/cal' if gcp_cal_file is None: calibration_file = default_calibration_file else: calibration_file = gcp_cal_file try: f = open(calibration_file) except: core.log_warn('GCP calibration file ' + calibration_file + ' not found.\n') f = [] ninvalid = 0 for line in f: if line[0] != '#' and line[0] != '\n' and len(line) > 0: line = line.replace('\n', '') linedict = {} # cal file has spaces AND tabs (and comment characters), whee! info_and_comment = list(filter(None, line.split('#'))) fline = list(filter(None, (info_and_comment[0]).split(' '))) if len(fline) == 1 and '\t' in fline[0]: fline = list( filter(None, (info_and_comment[0]).split('\t'))) name = (fline[0]).replace('*', '0') try: linedict['Offset'] = np.float(fline[1]) if '/' in fline[2]: gainfacs = (fline[2]).split('/') gainfac = np.float(gainfacs[1]) / np.float(gainfacs[0]) else: gainfac = 1. / np.float(fline[2]) linedict['ReciprocalFactor'] = gainfac linedict['UnitName'] = '' # try to figure out units. currently cal file # only has 2 forms, but this is not robust. if len(info_and_comment) > 1: comment = info_and_comment[1] if '->' in comment: comments = list(filter(None, (comment).split('->'))) unitname = \ list(filter(None,(comments[1]).split(' ')))[0] unitname = list(filter(None, (unitname).split('/')))[0] else: comments = \ list(filter(None,(comment).split('display in '))) unitname = \ list(filter(None,(comments[0]).split(' ')))[0] unitname = unitname.replace('\n', '') linedict['UnitName'] = unitname # another hudge kludge if 'rate' in name: if '/' not in unitname: unitname = unitname + '/s' linedict['UnitName'] = unitname cal_dict[name] = linedict except: ninvalid += 1 if ninvalid > 0: if ninvalid == 1: core.log_warn( 'One invalid line found and ignored in calibration file.\n' ) else: core.log_warn( str(ninvalid) + ' invalid lines found and ignored in calibration file.\n') f.close() # now add extra info if use_extra_info: if extra_dict is None: cal_dict['antenna0.acu.az_pos'] = \ {'ReciprocalFactor':1.,'Offset':0,'UnitName':'deg'} cal_dict['antenna0.acu.el_pos'] = \ {'ReciprocalFactor':1.,'Offset':0,'UnitName':'deg'} cal_dict['antenna0.acu.az_err'] = \ {'ReciprocalFactor':1.,'Offset':0,'UnitName':'deg'} cal_dict['antenna0.acu.el_err'] = \ {'ReciprocalFactor':1.,'Offset':0,'UnitName':'deg'} cal_dict['antenna0.scu.temp'] = \ {'ReciprocalFactor':1.,'Offset':-273.15,'UnitName':'K'} cal_dict['array.weather.airTemperature'] = \ {'ReciprocalFactor':1.,'Offset':-273.15,'UnitName':'K'} else: try: for key in extra_dict: cal_dict[key] = {} cal_dict[key]['Offset'] = extra_dict[key]['Offset'] cal_dict[key]['ReciprocalFactor'] = \ extra_dict[key]['ReciprocalFactor'] cal_dict[key]['UnitName'] = extra_dict[key]['UnitName'] except: core.log_warn( 'extra_dict input format not recognized, ignoring') del cal_dict[key] # write cal file fcal = open(path, 'w') fcal.write( '#-----------------------------------------------------------------------\n' ) fcal.write('# This is an SPT-3G register calibration file. \n') fcal.write('#\n') fcal.write('# It was created on ' + time.ctime() + ' by user ' + os.getlogin() + '\n') fcal.write( '# with the spt3g_software function gcp.CalFile.create_g3_cal_file. \n' ) fcal.write('#\n') fcal.write('# Each line has the following format:\n') fcal.write('#\n') fcal.write('# register offset 1/factor units\n') fcal.write('#\n') fcal.write( '# (Note that factors are stored as reciprocals, because they are most \n' ) fcal.write('# often one over a whole number.) \n') fcal.write('#\n') fcal.write('# The calibrated value of a register is given by:\n') fcal.write('#\n') fcal.write( '# reg_cal = factor * (offset + register_value) * core.G3Units.$UNIT$\n' ) fcal.write('#\n') fcal.write( '#-----------------------------------------------------------------------\n' ) fcal.write('# Register Offset 1/Factor Units \n') fcal.write( '#-----------------------------------------------------------------------\n' ) fcal.write('\n') ktemp = cal_dict.keys() ktemp.sort() for key in ktemp: kstr = (str(key)).ljust(42) gstr = '%e' % cal_dict[key]['ReciprocalFactor'] ostr = '%10.4f' % cal_dict[key]['Offset'] fcal.write(kstr + ' ' + ostr + ' ' + gstr + ' ' + cal_dict[key]['UnitName'] + '\n') fcal.close() return status
def __call__(self, f): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. """ if f.type == core.G3FrameType.EndProcessing: self.report_and_reset() return [f] if f.type != core.G3FrameType.Housekeeping: self.stats['n_other'] += 1 return f self.stats['n_hk'] += 1 vers = f.get('hkagg_version', 0) self.stats['versions'][vers] = self.stats['versions'].get(vers, 0) + 1 if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.report_and_reset( ) # note this does clear self.session_id. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id self.stats['n_session'] += 1 elif f['hkagg_type'] == so3g.HKFrameType.status: # Have any providers disappeared? now_prov_id = [p['prov_id'].value for p in f['providers']] for p, info in self.providers.items(): if p not in now_prov_id: info['active'] = False # New providers? for p in now_prov_id: info = self.providers.get(p) if info is not None: if not info['active']: core.log_warn('prov_id %i came back to life.' % p, unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['n_active'] += 1 info['active'] = True else: self.providers[p] = { 'active': True, # Currently active (during processing). 'n_active': 1, # Number of times this provider id became active. 'n_frames': 0, # Number of data frames. 'timestamp_init': f['timestamp'], # Timestamp of provider appearance 'timestamp_data': None, # Timestamp of most recent data frame. 'ticks': 0, # Total number of timestamps in all blocks. 'span': None, # (earliest_time, latest_time) 'block_streams_map': {}, # Map from field name to block name. } elif f['hkagg_type'] == so3g.HKFrameType.data: info = self.providers[f['prov_id']] vers = f.get('hkagg_version', 0) info['n_frames'] += 1 t_this = f['timestamp'] if info['timestamp_data'] is None: t_ref = info['timestamp_init'] if t_this < t_ref: core.log_warn('data timestamp (%.1f) precedes provider ' 'timestamp by %f seconds.' % (t_this, t_this - t_ref), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 elif t_this <= info['timestamp_data']: core.log_warn( 'data frame timestamps are not strictly ordered.', unit='HKScanner') self.stats['concerns']['n_warning'] += 1 info['timestamp_data'] = t_this # update t_check = [] blocks = f['blocks'] if vers == 0: block_timef = lambda block: block.t block_itemf = lambda block: [(k, block.data[k]) for k in block.data.keys()] elif vers >= 1: block_timef = lambda block: np.array( [t.time / core.G3Units.seconds for t in b.times]) block_itemf = lambda block: [(k, block[k]) for k in block.keys()] if vers in [0]: block_name = lambda block_idx: list( sorted(blocks[block_idx].data.keys()))[0] if vers in [1]: block_name = lambda block_idx: list( sorted(blocks[block_idx].keys()))[0] elif vers >= 2: block_names = f.get('block_names', []) if len(block_names) != len(blocks): # This is a schema error in its own right. core.log_error( 'Frame does not have "block_names" entry, ' 'or it is not the same length as "blocks".', unit='HKScanner') self.stats['concerns']['n_error'] += 1 # Fall back on v1 strategy. block_name = lambda block_idx: list( sorted(blocks[block_idx].keys()))[0] else: block_name = lambda block_idx: f['block_names'][block_idx] for block_idx, b in enumerate(blocks): times = block_timef(b) if len(times): if info['span'] is None: info['span'] = times[0], times[-1] else: t0, t1 = info['span'] info['span'] = min(times[0], t0), max(times[-1], t1) t_check.append(times[0]) info['ticks'] += len(times) bname = block_name(block_idx) for k, v in block_itemf(b): if len(v) != len(times): core.log_error( 'Field "%s" has %i samples but .t has %i samples.' % (k, len(v), len(times))) self.stats['concerns']['n_error'] += 1 # Make sure field has a block_stream registered. if k not in info['block_streams_map']: info['block_streams_map'][k] = bname if info['block_streams_map'][k] != bname: core.log_error( 'Field "%s" appeared in block_name %s ' 'and later in block_name %s.' % (k, info['block_streams_map'][k], bname)) self.stats['concerns']['n_error'] += 1 if len(t_check) and abs(min(t_check) - t_this) > 60: core.log_warn( 'data frame timestamp (%.1f) does not correspond to ' 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') self.stats['concerns']['n_warning'] += 1 return [f]
def Process(self, f, index_info=None): """Processes a frame. Only Housekeeping frames will be examined; other frames will simply be counted. All frames are passed through unmodified. The index_info will be stored along with a description of the frame's data; see the .process_file function. """ self.counter += 1 if index_info is None: index_info = {'counter': self.counter} if f.type == core.G3FrameType.EndProcessing: return [f] if f.type != core.G3FrameType.Housekeeping: return f if f['hkagg_type'] == so3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: self.flush() # Note that this sets self.session_id=None. if self.session_id is None: core.log_info('New HK Session id = %i, timestamp = %i' % (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id elif f['hkagg_type'] == so3g.HKFrameType.status: # If a provider has disappeared, flush its information into a # FieldGroup. prov_cands = [_HKProvider.from_g3(p) for p in f['providers']] to_flush = list(self.providers.keys()) # prov_ids... for p in prov_cands: if p.prov_id in to_flush: to_flush.remove(p.prov_id) # no, don't. else: self.providers[p.prov_id] = p for prov_id in to_flush: self.flush([prov_id]) elif f['hkagg_type'] == so3g.HKFrameType.data: # Data frame -- merge info for this provider. prov = self.providers[f['prov_id']] representatives = [block['fields'][0] for block in prov.blocks] for b in f['blocks']: fields = b.data.keys() if len(b.t) == 0 or len(fields) == 0: continue for block_index, rep in enumerate(representatives): if rep in fields: break else: block_index = len(prov.blocks) prov.blocks.append({ 'fields': fields, 'start': b.t[0], 'index_info': [] }) # To ensure that the last sample is actually included # in the semi-open intervals we use to track frames, # the "end" time has to be after the final sample. prov.blocks[block_index]['end'] = b.t[-1] + SPAN_BUFFER_SECONDS prov.blocks[block_index]['index_info'].append(index_info) else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') return [f]