def _concatenate_timestreams(cls, ts_lst, ts_rounding_error=0.6): """ Concatenate G3Timestream objects together. Arguments --------- ts_lst : list list of G3Timestream objects. ts_rounding_error : float allowed error in timestream separation such that timestreams are contiguous, as a fraction of the sample rate. This should be 0 by default, but is 0.5 to allow for downsampler shifting, and then bumpted again to 0.6 to allow for floating-point errors in what 0.5 is. Returns ------- ts : G3Timestream instance The concatenation of the input list of G3Timestream objects """ #check for contiguous timestreams for i in range(1, len(ts_lst)): ts_sep = (ts_lst[i].start.time - ts_lst[i - 1].stop.time) * ts_lst[i].sample_rate if numpy.abs(ts_sep - 1) > ts_rounding_error: log_fatal("Timestreams are not contiguous: timestreams %d and %d " "separated by %f samples" % (i, i - 1, ts_sep - 1)) if (ts_lst[i].units != ts_lst[0].units): log_fatal("Timestreams are not the same units") out_ts = cls(numpy.concatenate(ts_lst)) out_ts.units = ts_lst[0].units out_ts.start = ts_lst[0].start out_ts.stop = ts_lst[-1].stop return out_ts
def _concatenate_timestream_maps(cls, ts_map_lst, ts_rounding_error=0.6): """ Concatenate G3TimestreamMap objects together. Arguments --------- ts_map_lst : list list of G3TimestreamMap objects. ts_rounding_error : float allowed error in timestream separation such that timestreams are contiguous, as a fraction of the sample rate. This should be 0 by default, but is 0.5 to allow for downsampler shifting, and then bumpted again to 0.6 to allow for floating-point errors in what 0.5 is. Returns ------- tsm : G3TimestreamMap instance The concatenation of the input list of G3TimestreamMap objects """ for tsm in ts_map_lst[1:]: if set(tsm.keys()) != set(ts_map_lst[0].keys()): log_fatal("Timestream maps do not have the same keys") out_tsm = cls() for k in ts_map_lst[0].keys(): ts_lst = [tsm[k] for tsm in ts_map_lst] out_tsm[k] = G3Timestream.concatenate(ts_lst, ts_rounding_error) return out_tsm
def counts_to_rms_amps(wiringmap, hkmap, bolo, system, tf=None): boardhk, mezzhk, modhk, chanhk = HousekeepingForBolo(hkmap, wiringmap, bolo, True) if system == 'ICE': if chanhk.dan_streaming_enable: tf_I = IceboardConversions.convert_TF(modhk.nuller_gain, target='nuller', custom_TF=tf, unit='RAW', frequency=chanhk.carrier_frequency/core.G3Units.Hz) else: tf_V = IceboardConversions.convert_adc_samples('Streamer') tf_I = tf_V/modhk.squid_transimpedance if chanhk.carrier_frequency != 0: tf_I /= numpy.sqrt(2) # Use RMS amps elif system == 'DfMux': if chanhk.dan_streaming_enable: squid_converter = convert_squid(units='Normalized') tf_I = 1. / (2.**24) * squid_converter.theo_AnToIbolo(1., Gnuller=modhk.nuller_gain, eff=1.) else: dfmux_converter = convert_demod('gcp') resistor='10K' # Assume this for DAN Off # Handle the three exceptions for SPTpol if bolo in ['Sq3SBpol23Ch7', 'Sq3SBpol23Ch3', 'Sq3SBpol23Ch5']: resistor='5K' # XXX above list is wrong for 100d data if boardhk.timestamp.mjd < 56293: core.log_fatal('Set correct resistor values for 2012 SPTpol data', unit='dfmux.unittransforms') tf_I = dfmux_converter.theo_AdcToIbolo(1., modhk.demod_gain, resistor, f_carrier_equals_f_demod=True) tf_I *= 256 # 3G software shifts 8 bits right compared to SPTpol, so account for difference in meaning of "counts" else: core.log_fatal('Unknown transfer function profile %s' % system, unit='dfmux.unittransforms') return tf_I * core.G3Units.amp
def _concatenate_timestreams(cls, ts_lst, ts_rounding_error=0.6, ts_interp_threshold=0): """ Concatenate G3Timestream objects together. Arguments --------- ts_lst : list list of G3Timestream objects. ts_rounding_error : float allowed error in timestream separation such that timestreams are contiguous, as a fraction of the sample rate. This should be 0 by default, but is 0.5 to allow for downsampler shifting, and then bumpted again to 0.6 to allow for floating-point errors in what 0.5 is. ts_interp_threshold : float allowed timestream separation below which gaps between timestreams are interpolated to be made continuous Returns ------- ts : G3Timestream instance The concatenation of the input list of G3Timestream objects """ #check for contiguous timestreams for i in range(1, len(ts_lst)): ts_sep = (ts_lst[i].start.time - ts_lst[i - 1].stop.time) * ts_lst[i].sample_rate if numpy.abs(ts_sep - 1) > ts_rounding_error: if (ts_interp_threshold > 0) and ( (ts_sep - 1) < ts_interp_threshold): log_warn( "Timestreams are not contiguous: timestreams %d and %d " "separated by %f samples (%s). Interpolating." % (i, i - 1, ts_sep - 1, str(ts_lst[i].start))) v = numpy.linspace(ts_lst[i - 1][-1], ts_lst[i][0], ts_sep + 1)[1:-1] ts_interp = cls(v) ts_interp.units = ts_lst[0].units ts_interp.start = ts_lst[i - 1].stop + 1. / ts_lst[i].sample_rate ts_interp.stop = ts_lst[i].start - 1. / ts_lst[i].sample_rate ts_lst = ts_lst[:i] + [ts_interp] + ts_lst[i:] else: log_fatal( "Timestreams are not contiguous: timestreams %d and %d " "separated by %f samples (%s)" % (i, i - 1, ts_sep - 1, str(ts_lst[i].start))) if (ts_lst[i].units != ts_lst[0].units): log_fatal("Timestreams are not the same units") out_ts = cls(numpy.concatenate(ts_lst)) out_ts.units = ts_lst[0].units out_ts.start = ts_lst[0].start out_ts.stop = ts_lst[-1].stop return out_ts
def bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf=None): boardhk, mezzhk, modhk, chanhk = HousekeepingForBolo(hkmap, wiringmap, bolo, True) if system == 'ICE': tf_V = IceboardConversions.convert_TF(modhk.carrier_gain, target='carrier', custom_TF=tf, unit='NORMALIZED', frequency=chanhk.carrier_frequency/core.G3Units.Hz) volts = tf_V * chanhk.carrier_amplitude * core.G3Units.V if chanhk.carrier_frequency != 0: volts /= numpy.sqrt(2) # Use RMS amps elif system == 'DfMux': squid_converter = convert_squid(units='Normalized') volts = squid_converter.theo_AcToVbias(chanhk.carrier_amplitude, modhk.carrier_gain) * core.G3Units.V else: core.log_fatal('Unknown transfer function profile %s' % system, unit='dfmux.unittransforms') return volts
def get_timestream_unit_conversion(from_units, to_units, bolo, wiringmap=None, hkmap=None, system=None, tf=None): ''' Return the scalar conversion factor to move timestream data from a given system of units (Watts, Amps, Counts) to another one. Requires a wiring map and recent housekeeping data. Returned quantities are RMS for currents and time-averaged for power. Note that this does not handle conversions to on-sky quantities (e.g. K_cmb) ''' # Rather than special-casing the full mesh of conversions, # first calculate a conversion to watts from whatever units # we have, then a conversion from watts if wiringmap is None: core.log_fatal('Try to convert data with no wiring map', unit='dfmux.unittransforms') if hkmap is None: core.log_fatal('Try to convert data with no housekeeping', unit='dfmux.unittransforms') # First, convert to watts if from_units == core.G3TimestreamUnits.Counts: to_watts = counts_to_rms_amps(wiringmap, hkmap, bolo, system, tf) to_watts *= bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf) elif from_units == core.G3TimestreamUnits.Amps: to_watts = bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf) elif from_units == core.G3TimestreamUnits.Watts: to_watts = 1. else: core.log_fatal('Unknown units scheme %s in timestream for bolo %s' % (repr(from_units), bolo), unit='dfmux.unittransforms') # Now the conversion from watts if to_units == core.G3TimestreamUnits.Counts: from_watts = 1. / counts_to_rms_amps(wiringmap, hkmap, bolo, system, tf) from_watts /= bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf) elif to_units == core.G3TimestreamUnits.Amps: from_watts = bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf) if from_watts != 0: from_watts = 1. / from_watts elif to_units == core.G3TimestreamUnits.Watts: from_watts = 1. else: core.log_fatal('Trying to convert timestreams to unknown units %s' % repr(to_units), unit='dfmux.unittransforms') # Multiply them together return to_watts * from_watts
def build_pymodule(pycallable, *args, **kwargs): '''Convert a python callable and arguments into a core.G3Module by hook or by crook''' if isinstance(pycallable, G3Module): return pycallable if not callable(pycallable): log_fatal('Argument not a python callable', unit='G3Pipeline') if type(pycallable) == types.FunctionType: class PyFuncModule(G3Module): def Process(self, fr): return pycallable(fr, *args, **kwargs) return PyFuncModule() # If this is not a function, and it is callable, it is a class. If it is # a non-instantiated class, instantiate it. try: # issubclass() throws TypeError if argument is instantiated issubclass(pycallable, G3Module) isclass = True except TypeError: isclass = False # Instantiate if necessary. Not in try block so we don't miss TypeError if isclass: pycallable = pycallable(*args, **kwargs) else: # No way to handle arguments in this call; assert there are none if len(args) != 0 or len(kwargs) != 0: log_fatal( 'Cannot pass through arguments when passed instantiated class', unit='G3Pipeline') # See if it was a Python G3Module subclass if isinstance(pycallable, G3Module): return pycallable # This is a python callable that is not a module, so wrap it class PyCallObjModule(G3Module): def Process(self, fr): return pycallable(fr) return PyCallObjModule()
def __init__(self, input='CalTimestreams', property=None, property_list=None, output_root=None, bpm='BolometerProperties'): ''' Split the input map given by input into several output maps named output_root + key (e.g. CalTimestreams + str(property)) with the default options). Arguments --------- input : str Key name of the input map to split. property : str Attribute name to extract from the BolometerProperties object. Required. property_list : list of properties Properties to include in the output keys. Entries that are not strings will be converted to strings using the `SplitByProperty.converter` method. If property_list is not None, use only the names in the list (possibly writing empty timestream maps to the frame). Otherwise, creates maps for every that exists in the input. output_root : str Prefix for the output keys. If None (default), use `input` as the output root. bpm : str The key name of the BolometerPropertiesMap from which to extract the requested `property` for splitting the input map. ''' if property is None: core.log_fatal("Property is a required argument") self.bpmattr = property self.input = input self.output_root = output_root if output_root is not None else input if property_list is not None: self.props = [ self.converter(x) if not isinstance(x, str) else x for x in property_list ] else: self.props = None self.bpmkey = bpm self.bpm = None
def ValidateMaps(frame, ignore_missing_weights=False): """ Validate that the input map frame has all the necessary keys. If ignore_missing_weights is False (default), a warning is issued when the frame contains weighted Stokes maps without a weights map. Set this option to True when feeding single bolometer map frames with common weights through a pipeline. """ if isinstance(frame, core.G3Frame) and frame.type != core.G3FrameType.Map: return map_id = frame.get("Id", None) if "T" not in frame: core.log_fatal("Map frame %s: Missing T map" % map_id, unit="ValidateMaps") if ("Q" in frame and not "U" in frame) or ("U" in frame and not "Q" in frame): core.log_fatal("Map frame %s: Missing Q or U map" % map_id, unit="ValidateMaps") if "Wpol" in frame and "Wunpol" in frame: core.log_fatal( "Map frame %s: Found both polarized and unpolarized weights" % map_id, unit="ValidateMaps", ) stub = frame["T"].clone(False) for k in ["T", "Q", "U", "Wpol", "Wunpol"]: if k not in frame: continue if not frame[k].compatible(stub): core.log_fatal( "Map frame %s: Map %s not compatible with T map" % (map_id, k), unit="ValidateMaps", ) if k in "TQU": if k == "U" and frame[k].pol_conv is maps.MapPolConv.none: core.log_warn( "Map frame %s: U map polarization convention not set" % map_id, unit="ValidateMaps", ) if frame[k].weighted and not ignore_missing_weights: if "Wpol" not in frame and "Wunpol" not in frame: core.log_warn( "Map frame %s: Missing weights" % map_id, unit="ValidateMaps" ) if k == "T" and "Q" not in frame and "Wunpol" not in frame: core.log_warn( "Map frame %s: Missing unpolarized weights" % map_id, unit="ValidateMaps", ) if k in "QU" and "Wpol" not in frame: core.log_warn( "Map frame %s: Missing polarized weights" % map_id, unit="ValidateMaps", ) else: if frame[k].polarized and ("Q" not in frame or "U" not in frame): core.log_fatal( "Map frame %s: Found unpolarized maps with polarized weights" % map_id, unit="ValidateMaps", ) elif not frame[k].polarized and ("Q" in frame or "U" in frame): core.log_fatal( "Map frame %s: Found polarized maps with unpolarized weights" % map_id, unit="ValidateMaps", )
def ProcessBuffered(self, frame): ''' Process frames in the buffered order. Returned value should always be a list of frames, possibly empty. ''' if frame.type == core.G3FrameType.Timepoint: self.board_serials = [('%04d' % k) for k in frame['DfMux'].keys()] return [frame] if frame.type == core.G3FrameType.Wiring: core.log_fatal("Received spurious wiring frame. Do not use " "PyDfMuxHardwareMapInjector with the HousekeepingConsumer. " "You may update pydfmux to a newer version of pydfmux " "that stores mapped channel names on the boards, and " "rearrange your data acquisition script.", unit='HousekeepingConsumer') if frame.type == core.G3FrameType.Housekeeping: self.map_boards() hwm = DfMuxWiringMap() hkdata = DfMuxHousekeepingMap() try: for board in self.board_serials: self.tuber[board].CallMethod('Dfmux', '_dump_housekeeping', False) time.sleep(0.02) # Stagger return data transfer a little to # avoid overloading the network on the return found = False for board in self.board_serials: dat = self.tuber[board].GetReply()[0]['result'] boardhk = self.HousekeepingFromJSON(dat) hkdata[int(boardhk.serial)] = boardhk ip, crate, slot = self.board_map[board] boardw = self.WiringFromJSON(dat, ip, crate, slot) for key in boardw.keys(): hwm[key] = boardw[key] found = True if not found: core.log_fatal("No mapped channels found on any IceBoards. " "You may need to update pydfmux to a newer version of pydfmux " "that stores mapped channel names on the boards, and reload " "the hardware map.", unit='HousekeepingConsumer') frame['DfMuxHousekeeping'] = hkdata hwmf = core.G3Frame(core.G3FrameType.Wiring) hwmf['WiringMap'] = hwm hwmf['ReadoutSystem'] = 'ICE' if self.hwmf is None: self.hwmf = hwmf # If this is the first time the consumer is triggered, make sure # a Housekeeping and WiringMap frame are issued before any # Timepoint frames frames = [hwmf, frame] return frames else: # Compare wiring maps and re-issue frame if anything has changed old_hwm = self.hwmf['WiringMap'] if (set(hwm.keys()) ^ set(old_hwm.keys())): self.hwmf = hwmf return [hwmf, frame] for k in hwm.keys(): try: if vars(hwm[str(k)]) != vars(old_hwm[str(k)]): self.hwmf = hwmf return [hwmf, frame] except: core.log_error("Invalid HWM key %r" % k) # If we get here then the wiring map hasn't changed, # so return the populated Housekeeping frame as it is return [frame] except socket.timeout: core.log_error('Timeout collecting housekeeping data from mux boards. Dropping housekeeping sample', unit='HousekeepingConsumer') return [] except Exception as e: core.log_error('Error (%s) collecting housekeeping data from mux boards. Dropping housekeeping sample' % e, unit='HousekeepingConsumer') return [] return [frame]
def __call__(self, frame): if self.num_frames >= self.n_desired_frames: log_fatal("Manual Abort Triggered") if frame.type == self.type: self.num_frames += 1 return