def _concatenate_timestreams(cls, ts_lst, ts_rounding_error=0.6):
"""
Concatenate G3Timestream objects together.
Arguments
---------
ts_lst : list
list of G3Timestream objects.
ts_rounding_error : float
allowed error in timestream separation such that timestreams are
contiguous, as a fraction of the sample rate. This should be
0 by default, but is 0.5 to allow for downsampler shifting,
and then bumpted again to 0.6 to allow for floating-point
errors in what 0.5 is.
Returns
-------
ts : G3Timestream instance
The concatenation of the input list of G3Timestream objects
"""
#check for contiguous timestreams
for i in range(1, len(ts_lst)):
ts_sep = (ts_lst[i].start.time -
ts_lst[i - 1].stop.time) * ts_lst[i].sample_rate
if numpy.abs(ts_sep - 1) > ts_rounding_error:
log_fatal("Timestreams are not contiguous: timestreams %d and %d "
"separated by %f samples" % (i, i - 1, ts_sep - 1))
if (ts_lst[i].units != ts_lst[0].units):
log_fatal("Timestreams are not the same units")
out_ts = cls(numpy.concatenate(ts_lst))
out_ts.units = ts_lst[0].units
out_ts.start = ts_lst[0].start
out_ts.stop = ts_lst[-1].stop
return out_ts
def _concatenate_timestream_maps(cls, ts_map_lst, ts_rounding_error=0.6):
"""
Concatenate G3TimestreamMap objects together.
Arguments
---------
ts_map_lst : list
list of G3TimestreamMap objects.
ts_rounding_error : float
allowed error in timestream separation such that timestreams are
contiguous, as a fraction of the sample rate. This should be
0 by default, but is 0.5 to allow for downsampler shifting,
and then bumpted again to 0.6 to allow for floating-point
errors in what 0.5 is.
Returns
-------
tsm : G3TimestreamMap instance
The concatenation of the input list of G3TimestreamMap objects
"""
for tsm in ts_map_lst[1:]:
if set(tsm.keys()) != set(ts_map_lst[0].keys()):
log_fatal("Timestream maps do not have the same keys")
out_tsm = cls()
for k in ts_map_lst[0].keys():
ts_lst = [tsm[k] for tsm in ts_map_lst]
out_tsm[k] = G3Timestream.concatenate(ts_lst, ts_rounding_error)
return out_tsm
def counts_to_rms_amps(wiringmap, hkmap, bolo, system, tf=None):
boardhk, mezzhk, modhk, chanhk = HousekeepingForBolo(hkmap, wiringmap, bolo, True)
if system == 'ICE':
if chanhk.dan_streaming_enable:
tf_I = IceboardConversions.convert_TF(modhk.nuller_gain, target='nuller', custom_TF=tf, unit='RAW', frequency=chanhk.carrier_frequency/core.G3Units.Hz)
else:
tf_V = IceboardConversions.convert_adc_samples('Streamer')
tf_I = tf_V/modhk.squid_transimpedance
if chanhk.carrier_frequency != 0:
tf_I /= numpy.sqrt(2) # Use RMS amps
elif system == 'DfMux':
if chanhk.dan_streaming_enable:
squid_converter = convert_squid(units='Normalized')
tf_I = 1. / (2.**24) * squid_converter.theo_AnToIbolo(1., Gnuller=modhk.nuller_gain, eff=1.)
else:
dfmux_converter = convert_demod('gcp')
resistor='10K' # Assume this for DAN Off
# Handle the three exceptions for SPTpol
if bolo in ['Sq3SBpol23Ch7', 'Sq3SBpol23Ch3', 'Sq3SBpol23Ch5']:
resistor='5K'
# XXX above list is wrong for 100d data
if boardhk.timestamp.mjd < 56293:
core.log_fatal('Set correct resistor values for 2012 SPTpol data', unit='dfmux.unittransforms')
tf_I = dfmux_converter.theo_AdcToIbolo(1., modhk.demod_gain, resistor, f_carrier_equals_f_demod=True)
tf_I *= 256 # 3G software shifts 8 bits right compared to SPTpol, so account for difference in meaning of "counts"
else:
core.log_fatal('Unknown transfer function profile %s' % system, unit='dfmux.unittransforms')
return tf_I * core.G3Units.amp
def _concatenate_timestreams(cls,
ts_lst,
ts_rounding_error=0.6,
ts_interp_threshold=0):
"""
Concatenate G3Timestream objects together.
Arguments
---------
ts_lst : list
list of G3Timestream objects.
ts_rounding_error : float
allowed error in timestream separation such that timestreams are
contiguous, as a fraction of the sample rate. This should be
0 by default, but is 0.5 to allow for downsampler shifting,
and then bumpted again to 0.6 to allow for floating-point
errors in what 0.5 is.
ts_interp_threshold : float
allowed timestream separation below which gaps between timestreams are
interpolated to be made continuous
Returns
-------
ts : G3Timestream instance
The concatenation of the input list of G3Timestream objects
"""
#check for contiguous timestreams
for i in range(1, len(ts_lst)):
ts_sep = (ts_lst[i].start.time -
ts_lst[i - 1].stop.time) * ts_lst[i].sample_rate
if numpy.abs(ts_sep - 1) > ts_rounding_error:
if (ts_interp_threshold > 0) and (
(ts_sep - 1) < ts_interp_threshold):
log_warn(
"Timestreams are not contiguous: timestreams %d and %d "
"separated by %f samples (%s). Interpolating." %
(i, i - 1, ts_sep - 1, str(ts_lst[i].start)))
v = numpy.linspace(ts_lst[i - 1][-1], ts_lst[i][0],
ts_sep + 1)[1:-1]
ts_interp = cls(v)
ts_interp.units = ts_lst[0].units
ts_interp.start = ts_lst[i -
1].stop + 1. / ts_lst[i].sample_rate
ts_interp.stop = ts_lst[i].start - 1. / ts_lst[i].sample_rate
ts_lst = ts_lst[:i] + [ts_interp] + ts_lst[i:]
else:
log_fatal(
"Timestreams are not contiguous: timestreams %d and %d "
"separated by %f samples (%s)" %
(i, i - 1, ts_sep - 1, str(ts_lst[i].start)))
if (ts_lst[i].units != ts_lst[0].units):
log_fatal("Timestreams are not the same units")
out_ts = cls(numpy.concatenate(ts_lst))
out_ts.units = ts_lst[0].units
out_ts.start = ts_lst[0].start
out_ts.stop = ts_lst[-1].stop
return out_ts
def bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf=None):
boardhk, mezzhk, modhk, chanhk = HousekeepingForBolo(hkmap, wiringmap, bolo, True)
if system == 'ICE':
tf_V = IceboardConversions.convert_TF(modhk.carrier_gain, target='carrier', custom_TF=tf, unit='NORMALIZED', frequency=chanhk.carrier_frequency/core.G3Units.Hz)
volts = tf_V * chanhk.carrier_amplitude * core.G3Units.V
if chanhk.carrier_frequency != 0:
volts /= numpy.sqrt(2) # Use RMS amps
elif system == 'DfMux':
squid_converter = convert_squid(units='Normalized')
volts = squid_converter.theo_AcToVbias(chanhk.carrier_amplitude, modhk.carrier_gain) * core.G3Units.V
else:
core.log_fatal('Unknown transfer function profile %s' % system, unit='dfmux.unittransforms')
return volts
def get_timestream_unit_conversion(from_units,
to_units,
bolo,
wiringmap=None,
hkmap=None,
system=None,
tf=None):
'''
Return the scalar conversion factor to move timestream data from a
given system of units (Watts, Amps, Counts) to another one.
Requires a wiring map and recent housekeeping data.
Returned quantities are RMS for currents and time-averaged for power.
Note that this does not handle conversions to on-sky quantities (e.g. K_cmb)
'''
# Rather than special-casing the full mesh of conversions,
# first calculate a conversion to watts from whatever units
# we have, then a conversion from watts
if wiringmap is None:
core.log_fatal('Try to convert data with no wiring map',
unit='dfmux.unittransforms')
if hkmap is None:
core.log_fatal('Try to convert data with no housekeeping',
unit='dfmux.unittransforms')
# First, convert to watts
if from_units == core.G3TimestreamUnits.Counts:
to_watts = counts_to_rms_amps(wiringmap, hkmap, bolo, system, tf)
to_watts *= bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf)
elif from_units == core.G3TimestreamUnits.Amps:
to_watts = bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf)
elif from_units == core.G3TimestreamUnits.Watts:
to_watts = 1.
else:
core.log_fatal('Unknown units scheme %s in timestream for bolo %s' %
(repr(from_units), bolo),
unit='dfmux.unittransforms')
# Now the conversion from watts
if to_units == core.G3TimestreamUnits.Counts:
from_watts = 1. / counts_to_rms_amps(wiringmap, hkmap, bolo, system,
tf)
from_watts /= bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf)
elif to_units == core.G3TimestreamUnits.Amps:
from_watts = bolo_bias_voltage_rms(wiringmap, hkmap, bolo, system, tf)
if from_watts != 0:
from_watts = 1. / from_watts
elif to_units == core.G3TimestreamUnits.Watts:
from_watts = 1.
else:
core.log_fatal('Trying to convert timestreams to unknown units %s' %
repr(to_units),
unit='dfmux.unittransforms')
# Multiply them together
return to_watts * from_watts
def build_pymodule(pycallable, *args, **kwargs):
'''Convert a python callable and arguments into a core.G3Module by hook or by crook'''
if isinstance(pycallable, G3Module):
return pycallable
if not callable(pycallable):
log_fatal('Argument not a python callable', unit='G3Pipeline')
if type(pycallable) == types.FunctionType:
class PyFuncModule(G3Module):
def Process(self, fr):
return pycallable(fr, *args, **kwargs)
return PyFuncModule()
# If this is not a function, and it is callable, it is a class. If it is
# a non-instantiated class, instantiate it.
try:
# issubclass() throws TypeError if argument is instantiated
issubclass(pycallable, G3Module)
isclass = True
except TypeError:
isclass = False
# Instantiate if necessary. Not in try block so we don't miss TypeError
if isclass:
pycallable = pycallable(*args, **kwargs)
else:
# No way to handle arguments in this call; assert there are none
if len(args) != 0 or len(kwargs) != 0:
log_fatal(
'Cannot pass through arguments when passed instantiated class',
unit='G3Pipeline')
# See if it was a Python G3Module subclass
if isinstance(pycallable, G3Module):
return pycallable
# This is a python callable that is not a module, so wrap it
class PyCallObjModule(G3Module):
def Process(self, fr):
return pycallable(fr)
return PyCallObjModule()
def __init__(self,
input='CalTimestreams',
property=None,
property_list=None,
output_root=None,
bpm='BolometerProperties'):
'''
Split the input map given by input into several output
maps named output_root + key (e.g. CalTimestreams + str(property)) with
the default options).
Arguments
---------
input : str
Key name of the input map to split.
property : str
Attribute name to extract from the BolometerProperties object.
Required.
property_list : list of properties
Properties to include in the output keys. Entries that are not strings
will be converted to strings using the `SplitByProperty.converter` method.
If property_list is not None, use only the names in the
list (possibly writing empty timestream maps to the frame). Otherwise,
creates maps for every that exists in the input.
output_root : str
Prefix for the output keys.
If None (default), use `input` as the output root.
bpm : str
The key name of the BolometerPropertiesMap from which to extract
the requested `property` for splitting the input map.
'''
if property is None:
core.log_fatal("Property is a required argument")
self.bpmattr = property
self.input = input
self.output_root = output_root if output_root is not None else input
if property_list is not None:
self.props = [
self.converter(x) if not isinstance(x, str) else x
for x in property_list
]
else:
self.props = None
self.bpmkey = bpm
self.bpm = None
def ValidateMaps(frame, ignore_missing_weights=False):
"""
Validate that the input map frame has all the necessary keys.
If ignore_missing_weights is False (default), a warning is issued when the
frame contains weighted Stokes maps without a weights map. Set this option
to True when feeding single bolometer map frames with common weights through
a pipeline.
"""
if isinstance(frame, core.G3Frame) and frame.type != core.G3FrameType.Map:
return
map_id = frame.get("Id", None)
if "T" not in frame:
core.log_fatal("Map frame %s: Missing T map" % map_id, unit="ValidateMaps")
if ("Q" in frame and not "U" in frame) or ("U" in frame and not "Q" in frame):
core.log_fatal("Map frame %s: Missing Q or U map" % map_id, unit="ValidateMaps")
if "Wpol" in frame and "Wunpol" in frame:
core.log_fatal(
"Map frame %s: Found both polarized and unpolarized weights" % map_id,
unit="ValidateMaps",
)
stub = frame["T"].clone(False)
for k in ["T", "Q", "U", "Wpol", "Wunpol"]:
if k not in frame:
continue
if not frame[k].compatible(stub):
core.log_fatal(
"Map frame %s: Map %s not compatible with T map" % (map_id, k),
unit="ValidateMaps",
)
if k in "TQU":
if k == "U" and frame[k].pol_conv is maps.MapPolConv.none:
core.log_warn(
"Map frame %s: U map polarization convention not set" % map_id,
unit="ValidateMaps",
)
if frame[k].weighted and not ignore_missing_weights:
if "Wpol" not in frame and "Wunpol" not in frame:
core.log_warn(
"Map frame %s: Missing weights" % map_id, unit="ValidateMaps"
)
if k == "T" and "Q" not in frame and "Wunpol" not in frame:
core.log_warn(
"Map frame %s: Missing unpolarized weights" % map_id,
unit="ValidateMaps",
)
if k in "QU" and "Wpol" not in frame:
core.log_warn(
"Map frame %s: Missing polarized weights" % map_id,
unit="ValidateMaps",
)
else:
if frame[k].polarized and ("Q" not in frame or "U" not in frame):
core.log_fatal(
"Map frame %s: Found unpolarized maps with polarized weights"
% map_id,
unit="ValidateMaps",
)
elif not frame[k].polarized and ("Q" in frame or "U" in frame):
core.log_fatal(
"Map frame %s: Found polarized maps with unpolarized weights"
% map_id,
unit="ValidateMaps",
)
def ProcessBuffered(self, frame):
'''
Process frames in the buffered order. Returned value should
always be a list of frames, possibly empty.
'''
if frame.type == core.G3FrameType.Timepoint:
self.board_serials = [('%04d' % k) for k in frame['DfMux'].keys()]
return [frame]
if frame.type == core.G3FrameType.Wiring:
core.log_fatal("Received spurious wiring frame. Do not use "
"PyDfMuxHardwareMapInjector with the HousekeepingConsumer. "
"You may update pydfmux to a newer version of pydfmux "
"that stores mapped channel names on the boards, and "
"rearrange your data acquisition script.",
unit='HousekeepingConsumer')
if frame.type == core.G3FrameType.Housekeeping:
self.map_boards()
hwm = DfMuxWiringMap()
hkdata = DfMuxHousekeepingMap()
try:
for board in self.board_serials:
self.tuber[board].CallMethod('Dfmux', '_dump_housekeeping', False)
time.sleep(0.02) # Stagger return data transfer a little to
# avoid overloading the network on the return
found = False
for board in self.board_serials:
dat = self.tuber[board].GetReply()[0]['result']
boardhk = self.HousekeepingFromJSON(dat)
hkdata[int(boardhk.serial)] = boardhk
ip, crate, slot = self.board_map[board]
boardw = self.WiringFromJSON(dat, ip, crate, slot)
for key in boardw.keys():
hwm[key] = boardw[key]
found = True
if not found:
core.log_fatal("No mapped channels found on any IceBoards. "
"You may need to update pydfmux to a newer version of pydfmux "
"that stores mapped channel names on the boards, and reload "
"the hardware map.",
unit='HousekeepingConsumer')
frame['DfMuxHousekeeping'] = hkdata
hwmf = core.G3Frame(core.G3FrameType.Wiring)
hwmf['WiringMap'] = hwm
hwmf['ReadoutSystem'] = 'ICE'
if self.hwmf is None:
self.hwmf = hwmf
# If this is the first time the consumer is triggered, make sure
# a Housekeeping and WiringMap frame are issued before any
# Timepoint frames
frames = [hwmf, frame]
return frames
else:
# Compare wiring maps and re-issue frame if anything has changed
old_hwm = self.hwmf['WiringMap']
if (set(hwm.keys()) ^ set(old_hwm.keys())):
self.hwmf = hwmf
return [hwmf, frame]
for k in hwm.keys():
try:
if vars(hwm[str(k)]) != vars(old_hwm[str(k)]):
self.hwmf = hwmf
return [hwmf, frame]
except:
core.log_error("Invalid HWM key %r" % k)
# If we get here then the wiring map hasn't changed,
# so return the populated Housekeeping frame as it is
return [frame]
except socket.timeout:
core.log_error('Timeout collecting housekeeping data from mux boards. Dropping housekeeping sample', unit='HousekeepingConsumer')
return []
except Exception as e:
core.log_error('Error (%s) collecting housekeeping data from mux boards. Dropping housekeeping sample' % e, unit='HousekeepingConsumer')
return []
return [frame]
def __call__(self, frame):
if self.num_frames >= self.n_desired_frames:
log_fatal("Manual Abort Triggered")
if frame.type == self.type:
self.num_frames += 1
return
