def AddVbiasAndCurrentConv(frame, wiring_map):
hk_map = frame['DfMuxHousekeeping']
v_bias = core.G3MapDouble()
i_conv = core.G3MapDouble()
for k in wiring_map.keys():
vb = dfmux.unittransforms.bolo_bias_voltage_rms(wiring_map, hk_map,
bolo = k, tf = tf, system = system) / core.G3Units.V
ic = dfmux.unittransforms.counts_to_rms_amps(wiring_map, hk_map,
bolo = k, tf = tf, system = system) / core.G3Units.amp
v_bias[k] = vb
i_conv[k] = ic
frame['VoltageBias'] = v_bias
frame['CurrentConv'] = i_conv
def CalibrateValue(data, caldict_entry):
'''Apply gain / offset units from G3 cal file to register'''
uvalue = UnitValue(caldict_entry)
g3type = type(data)
# make a copy
if np.size(data) == 1:
data = data.value
data2 = np.array(data, dtype='float64')
thisdtype = data2.dtype
# calibrate units
data2 += np.array(caldict_entry['Offset'], dtype=thisdtype)
data2 *= np.array(uvalue / caldict_entry['ReciprocalFactor'],
dtype=thisdtype)
if not data2.shape:
data2 = data2.tolist()
# if a register has units, it can't be an int anymore. well, actually,
# it can't be an int if we're adding floats to it or multiplying it by
# floats either, so convert everything that has an entry in the cal file
# to float/double.
if g3type == core.G3VectorInt:
return core.G3VectorDouble(data2)
elif g3type == core.G3MapInt:
return core.G3MapDouble(data2)
elif g3type == core.G3Int:
return core.G3Double(data2)
else:
return g3type(data2)
def UnpackMuxData(f):
'''
Add the DFMux data to the frame.
'''
if f.type != core.G3FrameType.GcpSlow:
return
try:
mux = f['array']['muxHousekeeping']
boards = mux['boardname']
except KeyError:
return
fpga_temp = core.G3MapDouble()
board_name = core.G3MapString()
for i, bn in enumerate(boards):
bn = str(bn).replace('"', '') # get rid of extra quotes in board name
if bn != "":
board_name[str(i)] = bn
fpga_temp[str(i)] = mux['MB_TEMPERATURE_FPGA_DIE'][i]
fpga_temp.time = mux['utc']
board_name.time = mux['utc']
f['MuxFPGATemp'] = fpga_temp
f['MuxBoardName'] = board_name
def UnpackWeatherData(f):
'''Extracts weather status information to Weather key
in frame'''
if f.type != core.G3FrameType.GcpSlow:
return
if 'weather' not in f['array']:
return
board = f['array']['weather']
t = core.G3MapDouble()
t['telescope_temp'] = board['airTemperature'].value
t['telescope_pressure'] = board['pressure'].value
t['inside_dsl_temp'] = board['internalTemperature'].value
t['wind_speed'] = board['windSpeed'].value
t['wind_direction'] = board['windDirection'].value
t['battery'] = board['battery'].value
t['rel_humidity'] = board['relativeHumidity'].value
t['power'] = board['power'].value
t['tau'] = f['array']['tipper']['tau'].value
t['tatm'] = f['array']['tipper']['tatm'].value
f['Weather'] = t
f['WeatherTime'] = board['utc']
def calc_avg_current(frame, ts_key='TimestreamAmps',
output_key='AvgCurrent', bolo_props=None, wiring_map=None,
average_per_wafer=True, average_per_band=True, average_per_squid=False):
if frame.type == core.G3FrameType.Scan and \
ts_key in frame.keys() and bolo_props is not None:
pixel_tgroups = get_template_groups(bolo_props, wiring_map=wiring_map,
per_band = average_per_band,
per_wafer = average_per_wafer,
per_squid = average_per_squid,
include_keys = True)
frame[output_key] = core.G3MapDouble()
n_bolos = {}
for group_key, bolonames in pixel_tgroups.items():
for bolo in bolonames:
if bolo in frame[ts_key].keys():
current = np.median(frame[ts_key][bolo])
# cut psds that are not finite or are zero
if np.isfinite(current) & (current>0):
if group_key not in frame[output_key].keys():
n_bolos[group_key] = 0
frame[output_key][group_key] = current
else:
frame[output_key][group_key] += current
n_bolos[group_key] += 1
for group_key in n_bolos.keys():
frame[output_key][group_key] /= float(n_bolos[group_key])
def calc_asd(frame, ts_key, asd_key='ASD', units='temperature'):
if frame.type == core.G3FrameType.Scan and \
ts_key in frame.keys():
if units == 'temperature':
# 1/rt(2) for uK rtHz to uK rtsec
units_factor = (core.G3Units.microkelvin * np.sqrt(core.G3Units.sec) * np.sqrt(2.))
elif units == 'current':
# rt(2) because `dfmux.ConvertTimestreamUnits` converts to pA_RMS
# when using current units (see spt3g_software/calibration/python/noise_analysis.py)
# for more info on this annoying convention.
units_factor = (core.G3Units.amp*1e-12 / np.sqrt(core.G3Units.Hz)) / np.sqrt(2.)
asd_integral_key = asd_key + '_integral'
frame[asd_integral_key] = core.G3MapDouble()
frame[asd_key] = core.G3MapVectorDouble()
ts = frame[ts_key]
psds, freqs = dftutils.get_psd_of_ts_map(ts, pad=False, window_function=windows.boxcar)
for bolo in psds.keys():
frame[asd_key][bolo] = np.sqrt(psds[bolo]) / units_factor
for bolo in psds.keys():
frame[asd_integral_key][bolo] = np.sum(np.array(psds[bolo])[(freqs>0.01) & (freqs<0.5)])
frame[asd_key]['frequency'] = freqs
def ExplodeBolometerProperties(frame, bpmname='NominalBolometerProperties'):
'''
Take a bolometer properties map (usually the nominal one) and convert it
into its constituent keys as though they came from real calibration. This
is the inverse of BuildBoloPropertiesMap and mostly is useful when combining
hardware map information with real calibration.
'''
if bpmname not in frame:
return
bpm = frame[bpmname]
polangle = core.G3MapDouble()
poleff = core.G3MapDouble()
bands = core.G3MapDouble()
names = core.G3MapString()
pixels = core.G3MapString()
wafers = core.G3MapString()
xoff = core.G3MapDouble()
yoff = core.G3MapDouble()
for bolo, p in bpm.iteritems():
polangle[bolo] = p.pol_angle
poleff[bolo] = p.pol_efficiency
bands[bolo] = p.band
names[bolo] = p.physical_name
xoff[bolo] = p.x_offset
yoff[bolo] = p.y_offset
pixels[bolo] = p.pixel_id
wafers[bolo] = p.wafer_id
frame['PolarizationAngle'] = polangle
frame['PolarizationEfficiency'] = poleff
frame['PhysicalBoloIDs'] = names
frame['BoloBands'] = bands
frame['PointingOffsetX'] = xoff
frame['PointingOffsetY'] = yoff
frame['PixelIDs'] = pixels
frame['WaferIDs'] = wafers
def UnpackPTData(f):
'''Extracts pulse tube status information to PTStatus key
in frame'''
if f.type != core.G3FrameType.GcpSlow:
return
if 'pt415' not in f['array']:
return
board = f['array']['pt415']
p = core.G3MapDouble()
p['optics_lowp'] = board['pressure_low'][0]
p['min_optics_lowp'] = board['min_pressure_low'][0]
p['max_optics_lowp'] = board['max_pressure_low'][0]
p['optics_highp'] = board['pressure_high'][0]
p['min_optics_highp'] = board['min_pressure_high'][0]
p['max_optics_highp'] = board['max_pressure_high'][0]
p['optics_tempoil'] = board['temp_oil'][0]
p['min_optics_tempoil'] = board['min_temp_oil'][0]
p['max_optics_tempoil'] = board['max_temp_oil'][0]
p['receiver_lowp'] = board['pressure_low'][1]
p['min_receiver_lowp'] = board['min_pressure_low'][1]
p['max_receiver_lowp'] = board['max_pressure_low'][1]
p['receiver_highp'] = board['pressure_high'][1]
p['min_receiver_highp'] = board['min_pressure_high'][1]
p['max_receiver_highp'] = board['max_pressure_high'][1]
p['receiver_tempoil'] = board['temp_oil'][1]
p['min_receiver_tempoil'] = board['min_temp_oil'][1]
p['max_receiver_tempoil'] = board['max_temp_oil'][1]
p['optics_is_valid'] = board['deviceIsValid'][0]
p['receiver_is_valid'] = board['deviceIsValid'][1]
f['PTStatus'] = p
f['PTStatusTime'] = board['utc']
def __call__(self, frame):
# If 'CalibratorResponse' is in the frame, then we must be looking at
# a calframe and not the RCW38-pixelraster output. Note that this is
# potentially very confusing because the calframe also contains entries
# keyed with 'RCW38FluxCalibration', but these are *not* filled with the
# RCW38 output for the observation of interest. If I understand
# correctly, they contain the RCW38 information used to calibrate the
# preceding very fast point.
if frame.type == core.G3FrameType.Calibration:
if 'CalibratorResponse' in frame.keys():
self.CalibratorResponse = frame['CalibratorResponse']
self.BolometerProperties = frame['BolometerProperties']
elif 'RCW38FluxCalibration' in frame.keys():
self.FluxCalibration = frame['RCW38FluxCalibration']
self.IntegralFlux = frame['RCW38IntegralFlux']
if len(self.CalibratorResponse) > 0 and \
len(self.BolometerProperties) > 0 and \
len(self.FluxCalibration) > 0 and \
len(self.IntegralFlux) > 0:
opteff = core.G3MapDouble()
for bolo in self.CalibratorResponse.keys():
if bolo in self.BolometerProperties.keys() and \
bolo in self.FluxCalibration.keys():
boloprops = self.BolometerProperties[bolo]
cal_response = self.CalibratorResponse[bolo]
flux_cal = self.FluxCalibration[bolo]
int_flux = self.IntegralFlux[bolo]
band = boloprops.band
opteff[bolo] = -1*cal_response * flux_cal * int_flux / rcw38_abs_cal[band] / adama_utils.wattsPerKcmb(band)
newframe = core.G3Frame()
newframe['opteff'] = opteff
newframe['BolometerProperties'] = self.BolometerProperties
return newframe
else:
return []
#!/usr/bin/env python
from spt3g import core
import sys
b = core.G3MapDouble()
b['gh'] = 5
a = core.G3MapDouble(b)
print(a['gh'])
if a['gh'] != b['gh']:
sys.exit(1)
sys.exit(0)
def UnpackCryoData(f):
'''
Extracts cryo information into CryoStatus key
'''
if f.type != core.G3FrameType.GcpSlow:
return
if 'cryo' not in f['array']:
return
t = core.G3MapDouble()
t.time = f['array']['cryo']['utc']
t.cryo_is_valid = f['array']['cryo']['cryoIsValid'][0]
# Measured values
# He10
t.uc_head = f['array']['cryo']['temperature'][0][0]
t.ic_head = f['array']['cryo']['temperature'][0][1]
t.he4_head = f['array']['cryo']['temperature'][0][2]
t.he4_fb = f['array']['cryo']['temperature'][0][3]
t.he4_pump = f['array']['cryo']['temperature'][0][4]
t.ic_pump = f['array']['cryo']['temperature'][0][5]
t.uc_pump = f['array']['cryo']['temperature'][0][6]
t.he4_sw = f['array']['cryo']['temperature'][0][7]
t.ic_sw = f['array']['cryo']['temperature'][0][8]
t.uc_sw = f['array']['cryo']['temperature'][0][9]
t.uc_stage = f['array']['cryo']['temperature'][0][10]
t.lc_tower = f['array']['cryo']['temperature'][0][11]
t.ic_stage = f['array']['cryo']['temperature'][0][12]
t.t4k_head = f['array']['cryo']['temperature'][0][13]
t.t4k_squid_strap = f['array']['cryo']['temperature'][0][14]
t.t50k_head = f['array']['cryo']['temperature'][0][15]
# Optics
t.b1_50k_wbp_near = f['array']['cryo']['temperature'][1][0]
t.b2_50k_wbp_far = f['array']['cryo']['temperature'][1][1]
t.b3_50k_diving_board = f['array']['cryo']['temperature'][1][2]
t.b4_50k_top_bot_ptc = f['array']['cryo']['temperature'][1][3]
t.y1_50k_head = f['array']['cryo']['temperature'][1][4]
t.y2_50k_window_strap_near = f['array']['cryo']['temperature'][1][5]
t.y3_50k_tube_strap_near = f['array']['cryo']['temperature'][1][6]
t.y4_50k_tube = f['array']['cryo']['temperature'][1][7]
t.g1_4k_head = f['array']['cryo']['temperature'][1][8]
t.g2_4k_strap = f['array']['cryo']['temperature'][1][9]
t.g3_4k_lens_tab = f['array']['cryo']['temperature'][1][10]
t.g4_4k_lens_tab_far = f['array']['cryo']['temperature'][1][11]
t.r1_4k_top_top_ptc = f['array']['cryo']['temperature'][1][12]
t.r2_50k_midop_bot_ptc = f['array']['cryo']['temperature'][1][13]
t.r3_4k_lyot_flange = f['array']['cryo']['temperature'][1][14]
t.r4_4k_lyot = f['array']['cryo']['temperature'][1][15]
# Receiver
t.t4k_plate_far = f['array']['cryo']['temperature'][2][0]
t.t4k_strap_optics = f['array']['cryo']['temperature'][2][1]
t.t4k_plate_mid = f['array']['cryo']['temperature'][2][2]
t.t4k_plate_top = f['array']['cryo']['temperature'][2][3]
t.t4k_plate_ptc = f['array']['cryo']['temperature'][2][4]
t.t50k_harness_middle = f['array']['cryo']['temperature'][2][6]
t.t50k_strap = f['array']['cryo']['temperature'][2][7]
t.squid_wh1_sl1 = f['array']['cryo']['temperature'][2][8]
t.squid_wh5_sl1 = f['array']['cryo']['temperature'][2][9]
t.squid_wh3_sl7 = f['array']['cryo']['temperature'][2][10]
t.cal_filament = f['array']['cryo']['temperature'][2][11]
t.cal_ambient1 = f['array']['cryo']['temperature'][2][12]
t.cal_ambient2 = f['array']['cryo']['temperature'][2][13]
t.cal_ambient3 = f['array']['cryo']['temperature'][2][14]
# Heaters
t.heat_he4_pump = f['array']['cryo']['heater_dac'][0][3]
t.heat_ic_pump = f['array']['cryo']['heater_dac'][0][4]
t.heat_uc_pump = f['array']['cryo']['heater_dac'][0][5]
t.heat_he4_sw = f['array']['cryo']['heater_dac'][0][0]
t.heat_ic_sw = f['array']['cryo']['heater_dac'][0][1]
t.heat_uc_sw = f['array']['cryo']['heater_dac'][0][2]
f['CryoStatus'] = t
def CalibrateFrame(f, calibration_file=None):
'''Apply gain / offset / units from G3 cal file'''
if f.type != core.G3FrameType.GcpSlow:
return
try:
if f['Calibrated'] == True:
print('Already calibrated!\n')
return
except KeyError:
f['Calibrated'] = True
cf = CalFile.CalFileReader()
cd = cf.readCalFile(calibration_file)
for board in f.keys():
if board == 'Calibrated':
continue
cboard = copy.deepcopy(f[board])
for rmap in cboard.keys():
for reg in cboard[rmap].keys():
try:
rcd = cd[board][rmap][reg]
except KeyError:
continue
rsize = numpy.size(cboard[rmap][reg])
if rsize > 1:
rshape = numpy.shape(cboard[rmap][reg])
if len(rshape) > 1:
for i in range(rshape[0]):
try:
rcdi = rcd[i]
except KeyError:
rcdi = rcd
uvalue = UnitValue(rcdi)
datatemp = numpy.asarray(cboard[rmap][reg][i])
datatemp2 = datatemp.copy()
# if a register has units, it can't be an
# int anymore.
# well, actually, it can't be an int if
# we're adding floats to it or multiplying
# it by floats either, so convert
# everything that has an entry in the cal
# file to float/double.
datatemp2 = numpy.asarray(datatemp2,
dtype='float64')
thisdtype = datatemp2.dtype
datatemp2 += \
numpy.array(rcdi['Offset'],dtype=thisdtype)
datatemp2 *= numpy.array(uvalue /
rcdi['ReciprocalFactor'],
dtype=thisdtype)
if type(cboard[rmap][reg][i]) \
is core.G3VectorInt:
regitemp = core.G3VectorDouble(datatemp2)
elif type(cboard[rmap][reg][i]) \
is core.G3MapInt:
regitemp = core.G3MapDouble(datatemp2)
elif type(cboard[rmap][reg][i]) \
is core.G3Int:
regitemp = core.G3Double(datatemp2)
else:
regitemp = \
(type(cboard[rmap][reg][i]))(datatemp2)
cboard[rmap][reg][i] = regitemp
else:
try:
rcdi = rcd[0]
except KeyError:
rcdi = rcd
uvalue = UnitValue(rcdi)
datatemp = numpy.asarray(cboard[rmap][reg])
datatemp2 = datatemp.copy()
# if a register has units, it can't be an
# int anymore. well, actually (see above)...
datatemp2 = numpy.asarray(datatemp2, dtype='float64')
thisdtype = datatemp2.dtype
datatemp2 += \
numpy.array(rcdi['Offset'],dtype=thisdtype)
datatemp2 *= numpy.array(uvalue /
rcdi['ReciprocalFactor'],
dtype=thisdtype)
if type(cboard[rmap][reg]) \
is core.G3VectorInt:
regtemp = core.G3VectorDouble(datatemp2)
elif type(cboard[rmap][reg]) \
is core.G3MapInt:
regtemp = core.G3MapDouble(datatemp2)
elif type(cboard[rmap][reg]) \
is core.G3Int:
regtemp = core.G3Double(datatemp2)
else:
regtemp = \
(type(cboard[rmap][reg]))(datatemp2)
cboard[rmap][reg] = regtemp
else:
try:
rcdi = rcd[0]
except KeyError:
rcdi = rcd
uvalue = UnitValue(rcdi)
datatemp = cboard[rmap][reg].value
datatemp2 = datatemp
# if a register has units, it can't be an
# int anymore. well, actually (see above)...
datatemp2 = numpy.float(datatemp2)
datatemp2 = datatemp2 + rcdi['Offset']
datatemp2 *= uvalue / rcdi['ReciprocalFactor']
if type(cboard[rmap][reg]) \
is core.G3VectorInt:
regtemp = core.G3VectorDouble(datatemp2)
elif type(cboard[rmap][reg]) \
is core.G3MapInt:
regtemp = core.G3MapDouble(datatemp2)
elif type(cboard[rmap][reg]) \
is core.G3Int:
regtemp = core.G3Double(datatemp2)
else:
regtemp = \
(type(cboard[rmap][reg]))(datatemp2)
cboard[rmap][reg] = regtemp
del f[board]
f[board] = cboard
def tod_to_frames(
tod,
start_frame,
n_frames,
frame_offsets,
frame_sizes,
cache_signal=None,
cache_flags=None,
cache_common_flags=None,
copy_common=None,
copy_detector=None,
mask_flag_common=255,
mask_flag=255,
units=None,
dets=None,
compress=False,
):
"""Gather all data from the distributed TOD cache for a set of frames.
Args:
tod (toast.TOD): instance of a TOD class.
start_frame (int): the first frame index.
n_frames (int): the number of frames.
frame_offsets (array_like): list of the first samples of all frames.
frame_sizes (list): list of the number of samples in each frame.
cache_signal (str): if None, read signal from TOD. Otherwise use this
cache prefix for the detector signal timestreams.
cache_flags (str): if None read det flags from TOD. Otherwise use
this cache prefix for the detector flag timestreams.
cache_common_flags (str): if None, read common flags from TOD.
Otherwise use this cache prefix.
copy_common (tuple): (cache name, G3 type, frame name) of each extra
common field to copy from cache.
copy_detector (tuple): (cache name prefix, G3 type, G3 map type,
frame name) of each distributed detector field (excluding the
"signal") to copy from cache.
mask_flag_common (int): Bitmask to apply to common flags.
mask_flag (int): Bitmask to apply to per-detector flags.
units: G3 units of the detector data.
dets (list): List of detectors to include in the frame. If None,
use all of the detectors in the TOD object.
compress (bool or dict): If True or a dictionary of compression parameters,
store the timestreams as FLAC-compressed, 24-bit integers instead of
uncompressed doubles.
Returns:
(list): List of frames on rank zero. Other processes have a list of
None values.
"""
comm = tod.mpicomm
rank = 0
if comm is not None:
rank = comm.rank
comm_row = tod.grid_comm_row
# Detector names
if dets is None:
detnames = tod.detectors
else:
detnames = []
use_dets = set(dets)
for det in tod.detectors:
if det in use_dets:
detnames.append(det)
# Local sample range
local_first = tod.local_samples[0]
nlocal = tod.local_samples[1]
# The process grid
detranks, sampranks = tod.grid_size
rankdet, ranksamp = tod.grid_ranks
def get_local_cache(prow, field, cacheoff, ncache):
"""Read a local slice of a cache field.
"""
mtype = None
pdata = None
nnz = 0
if rankdet == prow:
ref = tod.cache.reference(field)
nnz = 1
if (len(ref.shape) > 1) and (ref.shape[1] > 0):
nnz = ref.shape[1]
if comm is not None:
if ref.dtype == np.dtype(np.float64):
mtype = MPI.DOUBLE
elif ref.dtype == np.dtype(np.int64):
mtype = MPI.INT64_T
elif ref.dtype == np.dtype(np.int32):
mtype = MPI.INT32_T
elif ref.dtype == np.dtype(np.uint8):
mtype = MPI.UINT8_T
else:
msg = "Cannot use cache field {} of type {}"\
.format(field, ref.dtype)
raise RuntimeError(msg)
if cacheoff is not None:
pdata = ref.flatten()[nnz * cacheoff:nnz * (cacheoff + ncache)]
else:
pdata = np.zeros(0, dtype=ref.dtype)
return (pdata, nnz, mtype)
def gather_field(prow, pdata, nnz, mpitype, cacheoff, ncache, tag):
"""Gather a single timestream buffer to the root process.
"""
is_none = pdata is None
all_none = comm.allreduce(is_none, MPI.LAND)
if all_none:
# This situation arises at least when gathering HWP angle from LAT
return None
gdata = None
# We are going to allreduce this later, so that every process
# knows the dimensions of the field.
gproc = 0
allnnz = 0
# Size of the local buffer
pz = 0
if pdata is not None:
pz = len(pdata)
if rankdet == prow:
psizes = None
if comm_row is None:
psizes = [pz]
else:
psizes = comm_row.gather(pz, root=0)
disp = None
totsize = None
if ranksamp == 0:
# We are the process collecting the gathered data.
allnnz = nnz
gproc = rank
# Compute the displacements into the receive buffer.
disp = [0]
for ps in psizes[:-1]:
last = disp[-1]
disp.append(last + ps)
totsize = np.sum(psizes)
# allocate receive buffer
gdata = np.zeros(totsize, dtype=pdata.dtype)
if comm_row is None:
pdata[:] = gdata
else:
comm_row.Gatherv(pdata, [gdata, psizes, disp, mpitype], root=0)
del disp
del psizes
# Now send this data to the root process of the whole communicator.
# Only one process (the first one in process row "prow") has data
# to send.
if comm is not None:
# All processes find out which one did the gather
gproc = comm.allreduce(gproc, MPI.SUM)
# All processes find out the field dimensions
allnnz = comm.allreduce(allnnz, MPI.SUM)
mtag = 10 * tag
rdata = None
if gproc == 0:
if gdata is not None:
if allnnz == 1:
rdata = gdata
else:
rdata = gdata.reshape((-1, allnnz))
else:
# Data not yet on rank 0
if rank == 0:
# Receive data from the first process in this row
rtype = comm.recv(source=gproc, tag=(mtag + 1))
rsize = comm.recv(source=gproc, tag=(mtag + 2))
rdata = np.zeros(rsize, dtype=np.dtype(rtype))
comm.Recv(rdata, source=gproc, tag=mtag)
# Reshape if needed
if allnnz > 1:
rdata = rdata.reshape((-1, allnnz))
elif (rank == gproc):
# Send our data
comm.send(gdata.dtype.char, dest=0, tag=(mtag + 1))
comm.send(len(gdata), dest=0, tag=(mtag + 2))
comm.Send(gdata, 0, tag=mtag)
return rdata
# For efficiency, we are going to gather the data for all frames at once.
# Then we will split those up when doing the write.
# Frame offsets relative to the memory buffers we are gathering
fdataoff = [0]
for f in frame_sizes[:-1]:
last = fdataoff[-1]
fdataoff.append(last + f)
# The list of frames- only on the root process.
fdata = None
if rank == 0:
fdata = [
core3g.G3Frame(core3g.G3FrameType.Scan) for f in range(n_frames)
]
else:
fdata = [None for f in range(n_frames)]
def split_field(data,
g3t,
framefield,
mapfield=None,
g3units=units,
times=None):
"""Split a gathered data buffer into frames- only on root process.
"""
if data is None:
return
if g3t == core3g.G3VectorTime:
# Special case for time values stored as int64_t, but
# wrapped in a class.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
g3times = list()
for t in range(ndata):
g3times.append(core3g.G3Time(data[dataoff + t]))
if mapfield is None:
fdata[f][framefield] = core3g.G3VectorTime(g3times)
else:
fdata[f][framefield][mapfield] = \
core3g.G3VectorTime(g3times)
del g3times
elif g3t == so3g.IntervalsInt:
# Flag vector is written as a simple boolean.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
# Extract flag vector (0 or 1) for this frame
frame_flags = (data[dataoff:dataoff + ndata] != 0).astype(int)
# Convert bit 0 to an IntervalsInt.
ival = so3g.IntervalsInt.from_mask(frame_flags, 1)[0]
if mapfield is None:
fdata[f][framefield] = ival
else:
fdata[f][framefield][mapfield] = ival
elif g3t == core3g.G3Timestream:
if times is None:
raise RuntimeError(
"You must provide the time stamp vector with a "
"Timestream object")
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
timeslice = times[cacheoff + dataoff:cacheoff + dataoff +
ndata]
tstart = timeslice[0] * 1e8
tstop = timeslice[-1] * 1e8
if mapfield is None:
if g3units is None:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata])
else:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata], g3units)
fdata[f][framefield].start = core3g.G3Time(tstart)
fdata[f][framefield].stop = core3g.G3Time(tstop)
else:
# Individual detector data. The only fields that
# we (optionally) compress.
if g3units is None:
tstream = g3t(data[dataoff:dataoff + ndata])
else:
tstream = g3t(data[dataoff:dataoff + ndata], g3units)
if compress and "compressor_gain_" + framefield in fdata[f]:
(tstream, gain,
offset) = recode_timestream(tstream, compress)
fdata[f]["compressor_gain_" +
framefield][mapfield] = gain
fdata[f]["compressor_offset_" +
framefield][mapfield] = offset
fdata[f][framefield][mapfield] = tstream
fdata[f][framefield][mapfield].start = core3g.G3Time(
tstart)
fdata[f][framefield][mapfield].stop = core3g.G3Time(tstop)
else:
# The bindings of G3Vector seem to only work with
# lists. This is probably horribly inefficient.
for f in range(n_frames):
dataoff = fdataoff[f]
ndata = frame_sizes[f]
if len(data.shape) == 1:
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata].tolist())
else:
# We have a 2D quantity
fdata[f][framefield] = \
g3t(data[dataoff : dataoff + ndata, :].flatten()
.tolist())
return
# Compute the overlap of all frames with the local process. We want to
# to find the full sample range that this process overlaps the total set
# of frames.
cacheoff = None
ncache = 0
for f in range(n_frames):
# Compute overlap of the frame with the local samples.
fcacheoff, froff, nfr = s3utils.local_frame_indices(
local_first, nlocal, frame_offsets[f], frame_sizes[f])
if fcacheoff is not None:
if cacheoff is None:
cacheoff = fcacheoff
ncache = nfr
else:
ncache += nfr
# Now gather the full sample data one field at a time. The root process
# splits up the results into frames.
# First collect boresight data. In addition to quaternions for the Az/El
# pointing, we convert this back into angles that follow the specs
# for telescope pointing.
times = None
if rankdet == 0:
times = tod.local_times()
if comm is not None:
times = gather_field(0, times, 1, MPI.DOUBLE, cacheoff, ncache, 0)
bore = None
if rankdet == 0:
bore = tod.read_boresight(local_start=cacheoff, n=ncache).flatten()
if comm is not None:
bore = gather_field(0, bore, 4, MPI.DOUBLE, cacheoff, ncache, 0)
if rank == 0:
split_field(bore.reshape(-1, 4), core3g.G3VectorDouble,
"boresight_radec")
bore = None
if rankdet == 0:
bore = tod.read_boresight_azel(local_start=cacheoff,
n=ncache).flatten()
if comm is not None:
bore = gather_field(0, bore, 4, MPI.DOUBLE, cacheoff, ncache, 1)
if rank == 0:
split_field(bore.reshape(-1, 4), core3g.G3VectorDouble,
"boresight_azel")
corotator_angle = None
if rankdet == 0:
cache_name = "corotator_angle_deg"
if tod.cache.exists(cache_name):
corotator_angle = np.radians(tod.cache.reference(cache_name))
if comm is not None:
corotator_angle = gather_field(0, corotator_angle, 1, MPI.DOUBLE,
cacheoff, ncache, 0)
if rank == 0:
split_field(corotator_angle,
core3g.G3VectorDouble,
"corotator_angle",
times=times)
if rank == 0:
for f in range(n_frames):
fdata[f]["boresight"] = core3g.G3TimestreamMap()
ang_theta, ang_phi, ang_psi = qa.to_angles(bore)
# Astronomical convention for azimuth is opposite to spherical
# coordinate phi.
ang_az = -ang_phi
ang_el = (np.pi / 2.0) - ang_theta
ang_roll = ang_psi
split_field(ang_az,
core3g.G3Timestream,
"boresight",
"az",
None,
times=times)
split_field(ang_el,
core3g.G3Timestream,
"boresight",
"el",
None,
times=times)
split_field(ang_roll,
core3g.G3Timestream,
"boresight",
"roll",
None,
times=times)
hwp_angle = None
if rankdet == 0:
hwp_angle = tod.local_hwp_angle()
if comm is not None:
hwp_angle = gather_field(0, hwp_angle, 1, MPI.DOUBLE, cacheoff, ncache,
0)
if rank == 0:
split_field(hwp_angle, core3g.G3VectorDouble, "hwp_angle", times=times)
# Now the position and velocity information
pos = None
if rankdet == 0:
pos = tod.read_position(local_start=cacheoff, n=ncache).flatten()
if comm is not None:
pos = gather_field(0, pos, 3, MPI.DOUBLE, cacheoff, ncache, 2)
if rank == 0:
split_field(pos.reshape(-1, 3), core3g.G3VectorDouble, "site_position")
vel = None
if rankdet == 0:
vel = tod.read_velocity(local_start=cacheoff, n=ncache).flatten()
if comm is not None:
vel = gather_field(0, vel, 3, MPI.DOUBLE, cacheoff, ncache, 3)
if rank == 0:
split_field(vel.reshape(-1, 3), core3g.G3VectorDouble, "site_velocity")
# Now handle the common flags- either from a cache object or from the
# TOD methods
cflags = None
nnz = 1
if cache_common_flags is None:
if rankdet == 0:
cflags = np.array(
tod.read_common_flags(local_start=cacheoff, n=ncache))
cflags &= mask_flag_common
else:
cflags, nnz, mtype = get_local_cache(0, cache_common_flags, cacheoff,
ncache)
if cflags is not None:
cflags &= mask_flag_common
if comm is not None:
mtype = MPI.UINT8_T
cflags = gather_field(0, cflags, nnz, mtype, cacheoff, ncache, 4)
if rank == 0:
split_field(cflags, so3g.IntervalsInt, "flags_common")
# Any extra common fields
if comm is not None:
comm.barrier()
if copy_common is not None:
for cindx, (cname, g3typ, fname) in enumerate(copy_common):
cdata, nnz, mtype = get_local_cache(0, cname, cacheoff, ncache)
cdata = gather_field(0, cdata, nnz, mtype, cacheoff, ncache, cindx)
if rank == 0:
split_field(cdata, g3typ, fname)
# Now read all per-detector quantities.
# For each detector field, processes which have the detector
# in their local_dets should be in the same process row.
if rank == 0:
for f in range(n_frames):
fdata[f]["signal"] = core3g.G3TimestreamMap()
if compress:
fdata[f]["compressor_gain_signal"] = core3g.G3MapDouble()
fdata[f]["compressor_offset_signal"] = core3g.G3MapDouble()
fdata[f]["flags"] = so3g.MapIntervalsInt()
if copy_detector is not None:
for cname, g3typ, g3maptyp, fnm in copy_detector:
fdata[f][fnm] = g3maptyp()
if compress:
fdata[f]["compressor_gain_" +
fnm] = core3g.G3MapDouble()
fdata[f]["compressor_offset_" +
fnm] = core3g.G3MapDouble()
for dindx, dname in enumerate(detnames):
drow = -1
# Demodulation may have synthesized new detector names
dnames = []
for x in tod.local_dets:
if x.endswith(dname):
dnames.append(x)
if dnames:
drow = rankdet
# As a sanity check, verify that every process which
# has this detector is in the same process row.
rowcheck = None
if comm is None:
rowcheck = [drow]
else:
rowcheck = comm.gather(drow, root=0)
prow = 0
if rank == 0:
rc = np.array([x for x in rowcheck if (x >= 0)], dtype=np.int32)
prow = np.max(rc)
if np.min(rc) != prow:
msg = "Processes with detector {} are not in the "\
"same row of the process grid\n".format(dname)
sys.stderr.write(msg)
if comm is not None:
comm.abort()
# Every process finds out which process row is participating.
if comm is not None:
prow = comm.bcast(prow, root=0)
all_dnames = comm.allgather(dnames)
dnames = list(set(np.concatenate(all_dnames).flat))
# "signal"
for dname in dnames:
detdata = None
nnz = 1
if cache_signal is None:
if rankdet == prow:
detdata = tod.local_signal(dname)[cacheoff:cacheoff +
ncache]
else:
cache_det = "{}_{}".format(cache_signal, dname)
detdata, nnz, mtype = get_local_cache(prow, cache_det,
cacheoff, ncache)
if comm is not None:
mtype = MPI.DOUBLE
detdata = gather_field(prow, detdata, nnz, mtype, cacheoff,
ncache, dindx)
if rank == 0:
split_field(detdata,
core3g.G3Timestream,
"signal",
mapfield=dname,
times=times)
# "flags"
for dname in dnames:
detdata = None
nnz = 1
if cache_flags is None:
if rankdet == prow:
detdata = tod.local_flags(dname)[cacheoff:cacheoff +
ncache]
detdata &= mask_flag
else:
cache_det = "{}_{}".format(cache_flags, dname)
detdata, nnz, mtype = get_local_cache(prow, cache_det,
cacheoff, ncache)
if detdata is not None:
detdata &= mask_flag
if comm is not None:
mtype = MPI.UINT8_T
detdata = gather_field(prow, detdata, nnz, mtype, cacheoff,
ncache, dindx)
if rank == 0:
split_field(detdata,
so3g.IntervalsInt,
"flags",
mapfield=dname)
# Now copy any additional fields.
for dname in dnames:
if copy_detector is not None:
for cname, g3typ, g3maptyp, fnm in copy_detector:
cache_det = "{}_{}".format(cname, dname)
detdata, nnz, mtype = get_local_cache(
prow, cache_det, cacheoff, ncache)
if comm is not None:
detdata = gather_field(prow, detdata, nnz, mtype,
cacheoff, ncache, dindx)
if rank == 0:
split_field(detdata,
g3typ,
fnm,
mapfield=dname,
times=times)
return fdata
# Test that we can read files written on a variety of platforms. Pass a path
# to generate test data for whatever this platform is.
testpath = os.path.join(os.environ['SPT3G_SOFTWARE_PATH'],
'core/tests/portability')
# Test data. Exercise some complicated things (STL bits) that we don't
# necessarily have control over, mapping a few primitive types.
f = core.G3Frame()
f['Five'] = 5
v = core.G3VectorDouble([2.6, 7.2])
f['Vec'] = v
v = core.G3VectorInt([17, 42, 87])
f['VecInt'] = v
m = core.G3MapDouble()
m['Six'] = 6
m['GoingOnSixteen'] = 15.9
f['Map'] = m
if len(sys.argv) > 1:
core.G3Writer(sys.argv[1])(f)
sys.exit(0)
# For now, we test files from big-endian (PPC64) and little-endian (amd64)
# 64-bit systems. Should include some 32-bit ones.
for test in ['test-be.g3', 'test-le.g3']:
print(test)
testdata = core.G3Reader(os.path.join(testpath, test))(None)[0]
def UnpackCryoData(f):
'''
Extracts cryo information into CryoStatus key
'''
if f.type != core.G3FrameType.GcpSlow:
return
if 'cryo' not in f['array']:
return
board = f['array']['cryo']
temps = board['temperature']
heaters = board['heater_dac']
t = core.G3MapDouble()
t['cryo_is_valid'] = board['cryoIsValid'][0]
# Measured values
# He10
t['uc_head'] = temps[0][0]
t['ic_head'] = temps[0][1]
t['he4_head'] = temps[0][2]
t['he4_fb'] = temps[0][3]
t['he4_pump'] = temps[0][4]
t['ic_pump'] = temps[0][5]
t['uc_pump'] = temps[0][6]
t['he4_sw'] = temps[0][7]
t['ic_sw'] = temps[0][8]
t['uc_sw'] = temps[0][9]
t['uc_stage'] = temps[0][10]
t['lc_tower'] = temps[0][11]
t['ic_stage'] = temps[0][12]
t['t4k_head'] = temps[0][13]
t['t4k_squid_strap'] = temps[0][14]
t['t50k_head'] = temps[0][15]
# Optics
t['b1_50k_wbp_near'] = temps[1][0]
t['b2_50k_wbp_far'] = temps[1][1]
t['b3_50k_diving_board'] = temps[1][2]
t['b4_50k_top_bot_ptc'] = temps[1][3]
t['y1_50k_head'] = temps[1][4]
t['y2_50k_window_strap_near'] = temps[1][5]
t['y3_50k_tube_strap_near'] = temps[1][6]
t['y4_50k_tube'] = temps[1][7]
t['g1_4k_head'] = temps[1][8]
t['g2_4k_strap'] = temps[1][9]
t['g3_4k_lens_tab'] = temps[1][10]
t['g4_4k_lens_tab_far'] = temps[1][11]
t['r1_4k_top_top_ptc'] = temps[1][12]
t['r2_50k_midop_bot_ptc'] = temps[1][13]
t['r3_4k_lyot_flange'] = temps[1][14]
t['r4_4k_lyot'] = temps[1][15]
# Receiver
t['t4k_plate_far'] = temps[2][0]
t['t4k_strap_optics'] = temps[2][1]
t['t4k_plate_mid'] = temps[2][2]
t['t4k_plate_top'] = temps[2][3]
t['t4k_plate_ptc'] = temps[2][4]
t['t50k_harness_middle'] = temps[2][6]
t['t50k_strap'] = temps[2][7]
t['squid_wh1_sl1'] = temps[2][8]
t['squid_wh5_sl1'] = temps[2][9]
t['squid_wh3_sl7'] = temps[2][10]
t['cal_filament'] = temps[2][11]
t['cal_ambient1'] = temps[2][12]
t['cal_ambient2'] = temps[2][13]
t['cal_ambient3'] = temps[2][14]
# Heaters
t['heat_he4_pump'] = heaters[0][3]
t['heat_ic_pump'] = heaters[0][4]
t['heat_uc_pump'] = heaters[0][5]
t['heat_he4_sw'] = heaters[0][0]
t['heat_ic_sw'] = heaters[0][1]
t['heat_uc_sw'] = heaters[0][2]
f['CryoStatus'] = t
f['CryoStatusTime'] = board['utc']
def calc_tod_median(frame, Input='TimestreamsWatts', Output='AvgPower'):
if frame.type == core.G3FrameType.Scan:
frame[Output] = core.G3MapDouble()
for bolo in frame[Input].keys():
frame[Output][bolo] = np.median(frame[Input][bolo][np.isfinite(frame[Input][bolo])])
