def AddVbiasAndCurrentConv(frame, wiring_map): hk_map = frame['DfMuxHousekeeping'] v_bias = core.G3MapDouble() i_conv = core.G3MapDouble() for k in wiring_map.keys(): vb = dfmux.unittransforms.bolo_bias_voltage_rms(wiring_map, hk_map, bolo = k, tf = tf, system = system) / core.G3Units.V ic = dfmux.unittransforms.counts_to_rms_amps(wiring_map, hk_map, bolo = k, tf = tf, system = system) / core.G3Units.amp v_bias[k] = vb i_conv[k] = ic frame['VoltageBias'] = v_bias frame['CurrentConv'] = i_conv
def CalibrateValue(data, caldict_entry): '''Apply gain / offset units from G3 cal file to register''' uvalue = UnitValue(caldict_entry) g3type = type(data) # make a copy if np.size(data) == 1: data = data.value data2 = np.array(data, dtype='float64') thisdtype = data2.dtype # calibrate units data2 += np.array(caldict_entry['Offset'], dtype=thisdtype) data2 *= np.array(uvalue / caldict_entry['ReciprocalFactor'], dtype=thisdtype) if not data2.shape: data2 = data2.tolist() # if a register has units, it can't be an int anymore. well, actually, # it can't be an int if we're adding floats to it or multiplying it by # floats either, so convert everything that has an entry in the cal file # to float/double. if g3type == core.G3VectorInt: return core.G3VectorDouble(data2) elif g3type == core.G3MapInt: return core.G3MapDouble(data2) elif g3type == core.G3Int: return core.G3Double(data2) else: return g3type(data2)
def UnpackMuxData(f): ''' Add the DFMux data to the frame. ''' if f.type != core.G3FrameType.GcpSlow: return try: mux = f['array']['muxHousekeeping'] boards = mux['boardname'] except KeyError: return fpga_temp = core.G3MapDouble() board_name = core.G3MapString() for i, bn in enumerate(boards): bn = str(bn).replace('"', '') # get rid of extra quotes in board name if bn != "": board_name[str(i)] = bn fpga_temp[str(i)] = mux['MB_TEMPERATURE_FPGA_DIE'][i] fpga_temp.time = mux['utc'] board_name.time = mux['utc'] f['MuxFPGATemp'] = fpga_temp f['MuxBoardName'] = board_name
def UnpackWeatherData(f): '''Extracts weather status information to Weather key in frame''' if f.type != core.G3FrameType.GcpSlow: return if 'weather' not in f['array']: return board = f['array']['weather'] t = core.G3MapDouble() t['telescope_temp'] = board['airTemperature'].value t['telescope_pressure'] = board['pressure'].value t['inside_dsl_temp'] = board['internalTemperature'].value t['wind_speed'] = board['windSpeed'].value t['wind_direction'] = board['windDirection'].value t['battery'] = board['battery'].value t['rel_humidity'] = board['relativeHumidity'].value t['power'] = board['power'].value t['tau'] = f['array']['tipper']['tau'].value t['tatm'] = f['array']['tipper']['tatm'].value f['Weather'] = t f['WeatherTime'] = board['utc']
def calc_avg_current(frame, ts_key='TimestreamAmps', output_key='AvgCurrent', bolo_props=None, wiring_map=None, average_per_wafer=True, average_per_band=True, average_per_squid=False): if frame.type == core.G3FrameType.Scan and \ ts_key in frame.keys() and bolo_props is not None: pixel_tgroups = get_template_groups(bolo_props, wiring_map=wiring_map, per_band = average_per_band, per_wafer = average_per_wafer, per_squid = average_per_squid, include_keys = True) frame[output_key] = core.G3MapDouble() n_bolos = {} for group_key, bolonames in pixel_tgroups.items(): for bolo in bolonames: if bolo in frame[ts_key].keys(): current = np.median(frame[ts_key][bolo]) # cut psds that are not finite or are zero if np.isfinite(current) & (current>0): if group_key not in frame[output_key].keys(): n_bolos[group_key] = 0 frame[output_key][group_key] = current else: frame[output_key][group_key] += current n_bolos[group_key] += 1 for group_key in n_bolos.keys(): frame[output_key][group_key] /= float(n_bolos[group_key])
def calc_asd(frame, ts_key, asd_key='ASD', units='temperature'): if frame.type == core.G3FrameType.Scan and \ ts_key in frame.keys(): if units == 'temperature': # 1/rt(2) for uK rtHz to uK rtsec units_factor = (core.G3Units.microkelvin * np.sqrt(core.G3Units.sec) * np.sqrt(2.)) elif units == 'current': # rt(2) because `dfmux.ConvertTimestreamUnits` converts to pA_RMS # when using current units (see spt3g_software/calibration/python/noise_analysis.py) # for more info on this annoying convention. units_factor = (core.G3Units.amp*1e-12 / np.sqrt(core.G3Units.Hz)) / np.sqrt(2.) asd_integral_key = asd_key + '_integral' frame[asd_integral_key] = core.G3MapDouble() frame[asd_key] = core.G3MapVectorDouble() ts = frame[ts_key] psds, freqs = dftutils.get_psd_of_ts_map(ts, pad=False, window_function=windows.boxcar) for bolo in psds.keys(): frame[asd_key][bolo] = np.sqrt(psds[bolo]) / units_factor for bolo in psds.keys(): frame[asd_integral_key][bolo] = np.sum(np.array(psds[bolo])[(freqs>0.01) & (freqs<0.5)]) frame[asd_key]['frequency'] = freqs
def ExplodeBolometerProperties(frame, bpmname='NominalBolometerProperties'): ''' Take a bolometer properties map (usually the nominal one) and convert it into its constituent keys as though they came from real calibration. This is the inverse of BuildBoloPropertiesMap and mostly is useful when combining hardware map information with real calibration. ''' if bpmname not in frame: return bpm = frame[bpmname] polangle = core.G3MapDouble() poleff = core.G3MapDouble() bands = core.G3MapDouble() names = core.G3MapString() pixels = core.G3MapString() wafers = core.G3MapString() xoff = core.G3MapDouble() yoff = core.G3MapDouble() for bolo, p in bpm.iteritems(): polangle[bolo] = p.pol_angle poleff[bolo] = p.pol_efficiency bands[bolo] = p.band names[bolo] = p.physical_name xoff[bolo] = p.x_offset yoff[bolo] = p.y_offset pixels[bolo] = p.pixel_id wafers[bolo] = p.wafer_id frame['PolarizationAngle'] = polangle frame['PolarizationEfficiency'] = poleff frame['PhysicalBoloIDs'] = names frame['BoloBands'] = bands frame['PointingOffsetX'] = xoff frame['PointingOffsetY'] = yoff frame['PixelIDs'] = pixels frame['WaferIDs'] = wafers
def UnpackPTData(f): '''Extracts pulse tube status information to PTStatus key in frame''' if f.type != core.G3FrameType.GcpSlow: return if 'pt415' not in f['array']: return board = f['array']['pt415'] p = core.G3MapDouble() p['optics_lowp'] = board['pressure_low'][0] p['min_optics_lowp'] = board['min_pressure_low'][0] p['max_optics_lowp'] = board['max_pressure_low'][0] p['optics_highp'] = board['pressure_high'][0] p['min_optics_highp'] = board['min_pressure_high'][0] p['max_optics_highp'] = board['max_pressure_high'][0] p['optics_tempoil'] = board['temp_oil'][0] p['min_optics_tempoil'] = board['min_temp_oil'][0] p['max_optics_tempoil'] = board['max_temp_oil'][0] p['receiver_lowp'] = board['pressure_low'][1] p['min_receiver_lowp'] = board['min_pressure_low'][1] p['max_receiver_lowp'] = board['max_pressure_low'][1] p['receiver_highp'] = board['pressure_high'][1] p['min_receiver_highp'] = board['min_pressure_high'][1] p['max_receiver_highp'] = board['max_pressure_high'][1] p['receiver_tempoil'] = board['temp_oil'][1] p['min_receiver_tempoil'] = board['min_temp_oil'][1] p['max_receiver_tempoil'] = board['max_temp_oil'][1] p['optics_is_valid'] = board['deviceIsValid'][0] p['receiver_is_valid'] = board['deviceIsValid'][1] f['PTStatus'] = p f['PTStatusTime'] = board['utc']
def __call__(self, frame): # If 'CalibratorResponse' is in the frame, then we must be looking at # a calframe and not the RCW38-pixelraster output. Note that this is # potentially very confusing because the calframe also contains entries # keyed with 'RCW38FluxCalibration', but these are *not* filled with the # RCW38 output for the observation of interest. If I understand # correctly, they contain the RCW38 information used to calibrate the # preceding very fast point. if frame.type == core.G3FrameType.Calibration: if 'CalibratorResponse' in frame.keys(): self.CalibratorResponse = frame['CalibratorResponse'] self.BolometerProperties = frame['BolometerProperties'] elif 'RCW38FluxCalibration' in frame.keys(): self.FluxCalibration = frame['RCW38FluxCalibration'] self.IntegralFlux = frame['RCW38IntegralFlux'] if len(self.CalibratorResponse) > 0 and \ len(self.BolometerProperties) > 0 and \ len(self.FluxCalibration) > 0 and \ len(self.IntegralFlux) > 0: opteff = core.G3MapDouble() for bolo in self.CalibratorResponse.keys(): if bolo in self.BolometerProperties.keys() and \ bolo in self.FluxCalibration.keys(): boloprops = self.BolometerProperties[bolo] cal_response = self.CalibratorResponse[bolo] flux_cal = self.FluxCalibration[bolo] int_flux = self.IntegralFlux[bolo] band = boloprops.band opteff[bolo] = -1*cal_response * flux_cal * int_flux / rcw38_abs_cal[band] / adama_utils.wattsPerKcmb(band) newframe = core.G3Frame() newframe['opteff'] = opteff newframe['BolometerProperties'] = self.BolometerProperties return newframe else: return []
#!/usr/bin/env python from spt3g import core import sys b = core.G3MapDouble() b['gh'] = 5 a = core.G3MapDouble(b) print(a['gh']) if a['gh'] != b['gh']: sys.exit(1) sys.exit(0)
def UnpackCryoData(f): ''' Extracts cryo information into CryoStatus key ''' if f.type != core.G3FrameType.GcpSlow: return if 'cryo' not in f['array']: return t = core.G3MapDouble() t.time = f['array']['cryo']['utc'] t.cryo_is_valid = f['array']['cryo']['cryoIsValid'][0] # Measured values # He10 t.uc_head = f['array']['cryo']['temperature'][0][0] t.ic_head = f['array']['cryo']['temperature'][0][1] t.he4_head = f['array']['cryo']['temperature'][0][2] t.he4_fb = f['array']['cryo']['temperature'][0][3] t.he4_pump = f['array']['cryo']['temperature'][0][4] t.ic_pump = f['array']['cryo']['temperature'][0][5] t.uc_pump = f['array']['cryo']['temperature'][0][6] t.he4_sw = f['array']['cryo']['temperature'][0][7] t.ic_sw = f['array']['cryo']['temperature'][0][8] t.uc_sw = f['array']['cryo']['temperature'][0][9] t.uc_stage = f['array']['cryo']['temperature'][0][10] t.lc_tower = f['array']['cryo']['temperature'][0][11] t.ic_stage = f['array']['cryo']['temperature'][0][12] t.t4k_head = f['array']['cryo']['temperature'][0][13] t.t4k_squid_strap = f['array']['cryo']['temperature'][0][14] t.t50k_head = f['array']['cryo']['temperature'][0][15] # Optics t.b1_50k_wbp_near = f['array']['cryo']['temperature'][1][0] t.b2_50k_wbp_far = f['array']['cryo']['temperature'][1][1] t.b3_50k_diving_board = f['array']['cryo']['temperature'][1][2] t.b4_50k_top_bot_ptc = f['array']['cryo']['temperature'][1][3] t.y1_50k_head = f['array']['cryo']['temperature'][1][4] t.y2_50k_window_strap_near = f['array']['cryo']['temperature'][1][5] t.y3_50k_tube_strap_near = f['array']['cryo']['temperature'][1][6] t.y4_50k_tube = f['array']['cryo']['temperature'][1][7] t.g1_4k_head = f['array']['cryo']['temperature'][1][8] t.g2_4k_strap = f['array']['cryo']['temperature'][1][9] t.g3_4k_lens_tab = f['array']['cryo']['temperature'][1][10] t.g4_4k_lens_tab_far = f['array']['cryo']['temperature'][1][11] t.r1_4k_top_top_ptc = f['array']['cryo']['temperature'][1][12] t.r2_50k_midop_bot_ptc = f['array']['cryo']['temperature'][1][13] t.r3_4k_lyot_flange = f['array']['cryo']['temperature'][1][14] t.r4_4k_lyot = f['array']['cryo']['temperature'][1][15] # Receiver t.t4k_plate_far = f['array']['cryo']['temperature'][2][0] t.t4k_strap_optics = f['array']['cryo']['temperature'][2][1] t.t4k_plate_mid = f['array']['cryo']['temperature'][2][2] t.t4k_plate_top = f['array']['cryo']['temperature'][2][3] t.t4k_plate_ptc = f['array']['cryo']['temperature'][2][4] t.t50k_harness_middle = f['array']['cryo']['temperature'][2][6] t.t50k_strap = f['array']['cryo']['temperature'][2][7] t.squid_wh1_sl1 = f['array']['cryo']['temperature'][2][8] t.squid_wh5_sl1 = f['array']['cryo']['temperature'][2][9] t.squid_wh3_sl7 = f['array']['cryo']['temperature'][2][10] t.cal_filament = f['array']['cryo']['temperature'][2][11] t.cal_ambient1 = f['array']['cryo']['temperature'][2][12] t.cal_ambient2 = f['array']['cryo']['temperature'][2][13] t.cal_ambient3 = f['array']['cryo']['temperature'][2][14] # Heaters t.heat_he4_pump = f['array']['cryo']['heater_dac'][0][3] t.heat_ic_pump = f['array']['cryo']['heater_dac'][0][4] t.heat_uc_pump = f['array']['cryo']['heater_dac'][0][5] t.heat_he4_sw = f['array']['cryo']['heater_dac'][0][0] t.heat_ic_sw = f['array']['cryo']['heater_dac'][0][1] t.heat_uc_sw = f['array']['cryo']['heater_dac'][0][2] f['CryoStatus'] = t
def CalibrateFrame(f, calibration_file=None): '''Apply gain / offset / units from G3 cal file''' if f.type != core.G3FrameType.GcpSlow: return try: if f['Calibrated'] == True: print('Already calibrated!\n') return except KeyError: f['Calibrated'] = True cf = CalFile.CalFileReader() cd = cf.readCalFile(calibration_file) for board in f.keys(): if board == 'Calibrated': continue cboard = copy.deepcopy(f[board]) for rmap in cboard.keys(): for reg in cboard[rmap].keys(): try: rcd = cd[board][rmap][reg] except KeyError: continue rsize = numpy.size(cboard[rmap][reg]) if rsize > 1: rshape = numpy.shape(cboard[rmap][reg]) if len(rshape) > 1: for i in range(rshape[0]): try: rcdi = rcd[i] except KeyError: rcdi = rcd uvalue = UnitValue(rcdi) datatemp = numpy.asarray(cboard[rmap][reg][i]) datatemp2 = datatemp.copy() # if a register has units, it can't be an # int anymore. # well, actually, it can't be an int if # we're adding floats to it or multiplying # it by floats either, so convert # everything that has an entry in the cal # file to float/double. datatemp2 = numpy.asarray(datatemp2, dtype='float64') thisdtype = datatemp2.dtype datatemp2 += \ numpy.array(rcdi['Offset'],dtype=thisdtype) datatemp2 *= numpy.array(uvalue / rcdi['ReciprocalFactor'], dtype=thisdtype) if type(cboard[rmap][reg][i]) \ is core.G3VectorInt: regitemp = core.G3VectorDouble(datatemp2) elif type(cboard[rmap][reg][i]) \ is core.G3MapInt: regitemp = core.G3MapDouble(datatemp2) elif type(cboard[rmap][reg][i]) \ is core.G3Int: regitemp = core.G3Double(datatemp2) else: regitemp = \ (type(cboard[rmap][reg][i]))(datatemp2) cboard[rmap][reg][i] = regitemp else: try: rcdi = rcd[0] except KeyError: rcdi = rcd uvalue = UnitValue(rcdi) datatemp = numpy.asarray(cboard[rmap][reg]) datatemp2 = datatemp.copy() # if a register has units, it can't be an # int anymore. well, actually (see above)... datatemp2 = numpy.asarray(datatemp2, dtype='float64') thisdtype = datatemp2.dtype datatemp2 += \ numpy.array(rcdi['Offset'],dtype=thisdtype) datatemp2 *= numpy.array(uvalue / rcdi['ReciprocalFactor'], dtype=thisdtype) if type(cboard[rmap][reg]) \ is core.G3VectorInt: regtemp = core.G3VectorDouble(datatemp2) elif type(cboard[rmap][reg]) \ is core.G3MapInt: regtemp = core.G3MapDouble(datatemp2) elif type(cboard[rmap][reg]) \ is core.G3Int: regtemp = core.G3Double(datatemp2) else: regtemp = \ (type(cboard[rmap][reg]))(datatemp2) cboard[rmap][reg] = regtemp else: try: rcdi = rcd[0] except KeyError: rcdi = rcd uvalue = UnitValue(rcdi) datatemp = cboard[rmap][reg].value datatemp2 = datatemp # if a register has units, it can't be an # int anymore. well, actually (see above)... datatemp2 = numpy.float(datatemp2) datatemp2 = datatemp2 + rcdi['Offset'] datatemp2 *= uvalue / rcdi['ReciprocalFactor'] if type(cboard[rmap][reg]) \ is core.G3VectorInt: regtemp = core.G3VectorDouble(datatemp2) elif type(cboard[rmap][reg]) \ is core.G3MapInt: regtemp = core.G3MapDouble(datatemp2) elif type(cboard[rmap][reg]) \ is core.G3Int: regtemp = core.G3Double(datatemp2) else: regtemp = \ (type(cboard[rmap][reg]))(datatemp2) cboard[rmap][reg] = regtemp del f[board] f[board] = cboard
def tod_to_frames( tod, start_frame, n_frames, frame_offsets, frame_sizes, cache_signal=None, cache_flags=None, cache_common_flags=None, copy_common=None, copy_detector=None, mask_flag_common=255, mask_flag=255, units=None, dets=None, compress=False, ): """Gather all data from the distributed TOD cache for a set of frames. Args: tod (toast.TOD): instance of a TOD class. start_frame (int): the first frame index. n_frames (int): the number of frames. frame_offsets (array_like): list of the first samples of all frames. frame_sizes (list): list of the number of samples in each frame. cache_signal (str): if None, read signal from TOD. Otherwise use this cache prefix for the detector signal timestreams. cache_flags (str): if None read det flags from TOD. Otherwise use this cache prefix for the detector flag timestreams. cache_common_flags (str): if None, read common flags from TOD. Otherwise use this cache prefix. copy_common (tuple): (cache name, G3 type, frame name) of each extra common field to copy from cache. copy_detector (tuple): (cache name prefix, G3 type, G3 map type, frame name) of each distributed detector field (excluding the "signal") to copy from cache. mask_flag_common (int): Bitmask to apply to common flags. mask_flag (int): Bitmask to apply to per-detector flags. units: G3 units of the detector data. dets (list): List of detectors to include in the frame. If None, use all of the detectors in the TOD object. compress (bool or dict): If True or a dictionary of compression parameters, store the timestreams as FLAC-compressed, 24-bit integers instead of uncompressed doubles. Returns: (list): List of frames on rank zero. Other processes have a list of None values. """ comm = tod.mpicomm rank = 0 if comm is not None: rank = comm.rank comm_row = tod.grid_comm_row # Detector names if dets is None: detnames = tod.detectors else: detnames = [] use_dets = set(dets) for det in tod.detectors: if det in use_dets: detnames.append(det) # Local sample range local_first = tod.local_samples[0] nlocal = tod.local_samples[1] # The process grid detranks, sampranks = tod.grid_size rankdet, ranksamp = tod.grid_ranks def get_local_cache(prow, field, cacheoff, ncache): """Read a local slice of a cache field. """ mtype = None pdata = None nnz = 0 if rankdet == prow: ref = tod.cache.reference(field) nnz = 1 if (len(ref.shape) > 1) and (ref.shape[1] > 0): nnz = ref.shape[1] if comm is not None: if ref.dtype == np.dtype(np.float64): mtype = MPI.DOUBLE elif ref.dtype == np.dtype(np.int64): mtype = MPI.INT64_T elif ref.dtype == np.dtype(np.int32): mtype = MPI.INT32_T elif ref.dtype == np.dtype(np.uint8): mtype = MPI.UINT8_T else: msg = "Cannot use cache field {} of type {}"\ .format(field, ref.dtype) raise RuntimeError(msg) if cacheoff is not None: pdata = ref.flatten()[nnz * cacheoff:nnz * (cacheoff + ncache)] else: pdata = np.zeros(0, dtype=ref.dtype) return (pdata, nnz, mtype) def gather_field(prow, pdata, nnz, mpitype, cacheoff, ncache, tag): """Gather a single timestream buffer to the root process. """ is_none = pdata is None all_none = comm.allreduce(is_none, MPI.LAND) if all_none: # This situation arises at least when gathering HWP angle from LAT return None gdata = None # We are going to allreduce this later, so that every process # knows the dimensions of the field. gproc = 0 allnnz = 0 # Size of the local buffer pz = 0 if pdata is not None: pz = len(pdata) if rankdet == prow: psizes = None if comm_row is None: psizes = [pz] else: psizes = comm_row.gather(pz, root=0) disp = None totsize = None if ranksamp == 0: # We are the process collecting the gathered data. allnnz = nnz gproc = rank # Compute the displacements into the receive buffer. disp = [0] for ps in psizes[:-1]: last = disp[-1] disp.append(last + ps) totsize = np.sum(psizes) # allocate receive buffer gdata = np.zeros(totsize, dtype=pdata.dtype) if comm_row is None: pdata[:] = gdata else: comm_row.Gatherv(pdata, [gdata, psizes, disp, mpitype], root=0) del disp del psizes # Now send this data to the root process of the whole communicator. # Only one process (the first one in process row "prow") has data # to send. if comm is not None: # All processes find out which one did the gather gproc = comm.allreduce(gproc, MPI.SUM) # All processes find out the field dimensions allnnz = comm.allreduce(allnnz, MPI.SUM) mtag = 10 * tag rdata = None if gproc == 0: if gdata is not None: if allnnz == 1: rdata = gdata else: rdata = gdata.reshape((-1, allnnz)) else: # Data not yet on rank 0 if rank == 0: # Receive data from the first process in this row rtype = comm.recv(source=gproc, tag=(mtag + 1)) rsize = comm.recv(source=gproc, tag=(mtag + 2)) rdata = np.zeros(rsize, dtype=np.dtype(rtype)) comm.Recv(rdata, source=gproc, tag=mtag) # Reshape if needed if allnnz > 1: rdata = rdata.reshape((-1, allnnz)) elif (rank == gproc): # Send our data comm.send(gdata.dtype.char, dest=0, tag=(mtag + 1)) comm.send(len(gdata), dest=0, tag=(mtag + 2)) comm.Send(gdata, 0, tag=mtag) return rdata # For efficiency, we are going to gather the data for all frames at once. # Then we will split those up when doing the write. # Frame offsets relative to the memory buffers we are gathering fdataoff = [0] for f in frame_sizes[:-1]: last = fdataoff[-1] fdataoff.append(last + f) # The list of frames- only on the root process. fdata = None if rank == 0: fdata = [ core3g.G3Frame(core3g.G3FrameType.Scan) for f in range(n_frames) ] else: fdata = [None for f in range(n_frames)] def split_field(data, g3t, framefield, mapfield=None, g3units=units, times=None): """Split a gathered data buffer into frames- only on root process. """ if data is None: return if g3t == core3g.G3VectorTime: # Special case for time values stored as int64_t, but # wrapped in a class. for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] g3times = list() for t in range(ndata): g3times.append(core3g.G3Time(data[dataoff + t])) if mapfield is None: fdata[f][framefield] = core3g.G3VectorTime(g3times) else: fdata[f][framefield][mapfield] = \ core3g.G3VectorTime(g3times) del g3times elif g3t == so3g.IntervalsInt: # Flag vector is written as a simple boolean. for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] # Extract flag vector (0 or 1) for this frame frame_flags = (data[dataoff:dataoff + ndata] != 0).astype(int) # Convert bit 0 to an IntervalsInt. ival = so3g.IntervalsInt.from_mask(frame_flags, 1)[0] if mapfield is None: fdata[f][framefield] = ival else: fdata[f][framefield][mapfield] = ival elif g3t == core3g.G3Timestream: if times is None: raise RuntimeError( "You must provide the time stamp vector with a " "Timestream object") for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] timeslice = times[cacheoff + dataoff:cacheoff + dataoff + ndata] tstart = timeslice[0] * 1e8 tstop = timeslice[-1] * 1e8 if mapfield is None: if g3units is None: fdata[f][framefield] = \ g3t(data[dataoff : dataoff + ndata]) else: fdata[f][framefield] = \ g3t(data[dataoff : dataoff + ndata], g3units) fdata[f][framefield].start = core3g.G3Time(tstart) fdata[f][framefield].stop = core3g.G3Time(tstop) else: # Individual detector data. The only fields that # we (optionally) compress. if g3units is None: tstream = g3t(data[dataoff:dataoff + ndata]) else: tstream = g3t(data[dataoff:dataoff + ndata], g3units) if compress and "compressor_gain_" + framefield in fdata[f]: (tstream, gain, offset) = recode_timestream(tstream, compress) fdata[f]["compressor_gain_" + framefield][mapfield] = gain fdata[f]["compressor_offset_" + framefield][mapfield] = offset fdata[f][framefield][mapfield] = tstream fdata[f][framefield][mapfield].start = core3g.G3Time( tstart) fdata[f][framefield][mapfield].stop = core3g.G3Time(tstop) else: # The bindings of G3Vector seem to only work with # lists. This is probably horribly inefficient. for f in range(n_frames): dataoff = fdataoff[f] ndata = frame_sizes[f] if len(data.shape) == 1: fdata[f][framefield] = \ g3t(data[dataoff : dataoff + ndata].tolist()) else: # We have a 2D quantity fdata[f][framefield] = \ g3t(data[dataoff : dataoff + ndata, :].flatten() .tolist()) return # Compute the overlap of all frames with the local process. We want to # to find the full sample range that this process overlaps the total set # of frames. cacheoff = None ncache = 0 for f in range(n_frames): # Compute overlap of the frame with the local samples. fcacheoff, froff, nfr = s3utils.local_frame_indices( local_first, nlocal, frame_offsets[f], frame_sizes[f]) if fcacheoff is not None: if cacheoff is None: cacheoff = fcacheoff ncache = nfr else: ncache += nfr # Now gather the full sample data one field at a time. The root process # splits up the results into frames. # First collect boresight data. In addition to quaternions for the Az/El # pointing, we convert this back into angles that follow the specs # for telescope pointing. times = None if rankdet == 0: times = tod.local_times() if comm is not None: times = gather_field(0, times, 1, MPI.DOUBLE, cacheoff, ncache, 0) bore = None if rankdet == 0: bore = tod.read_boresight(local_start=cacheoff, n=ncache).flatten() if comm is not None: bore = gather_field(0, bore, 4, MPI.DOUBLE, cacheoff, ncache, 0) if rank == 0: split_field(bore.reshape(-1, 4), core3g.G3VectorDouble, "boresight_radec") bore = None if rankdet == 0: bore = tod.read_boresight_azel(local_start=cacheoff, n=ncache).flatten() if comm is not None: bore = gather_field(0, bore, 4, MPI.DOUBLE, cacheoff, ncache, 1) if rank == 0: split_field(bore.reshape(-1, 4), core3g.G3VectorDouble, "boresight_azel") corotator_angle = None if rankdet == 0: cache_name = "corotator_angle_deg" if tod.cache.exists(cache_name): corotator_angle = np.radians(tod.cache.reference(cache_name)) if comm is not None: corotator_angle = gather_field(0, corotator_angle, 1, MPI.DOUBLE, cacheoff, ncache, 0) if rank == 0: split_field(corotator_angle, core3g.G3VectorDouble, "corotator_angle", times=times) if rank == 0: for f in range(n_frames): fdata[f]["boresight"] = core3g.G3TimestreamMap() ang_theta, ang_phi, ang_psi = qa.to_angles(bore) # Astronomical convention for azimuth is opposite to spherical # coordinate phi. ang_az = -ang_phi ang_el = (np.pi / 2.0) - ang_theta ang_roll = ang_psi split_field(ang_az, core3g.G3Timestream, "boresight", "az", None, times=times) split_field(ang_el, core3g.G3Timestream, "boresight", "el", None, times=times) split_field(ang_roll, core3g.G3Timestream, "boresight", "roll", None, times=times) hwp_angle = None if rankdet == 0: hwp_angle = tod.local_hwp_angle() if comm is not None: hwp_angle = gather_field(0, hwp_angle, 1, MPI.DOUBLE, cacheoff, ncache, 0) if rank == 0: split_field(hwp_angle, core3g.G3VectorDouble, "hwp_angle", times=times) # Now the position and velocity information pos = None if rankdet == 0: pos = tod.read_position(local_start=cacheoff, n=ncache).flatten() if comm is not None: pos = gather_field(0, pos, 3, MPI.DOUBLE, cacheoff, ncache, 2) if rank == 0: split_field(pos.reshape(-1, 3), core3g.G3VectorDouble, "site_position") vel = None if rankdet == 0: vel = tod.read_velocity(local_start=cacheoff, n=ncache).flatten() if comm is not None: vel = gather_field(0, vel, 3, MPI.DOUBLE, cacheoff, ncache, 3) if rank == 0: split_field(vel.reshape(-1, 3), core3g.G3VectorDouble, "site_velocity") # Now handle the common flags- either from a cache object or from the # TOD methods cflags = None nnz = 1 if cache_common_flags is None: if rankdet == 0: cflags = np.array( tod.read_common_flags(local_start=cacheoff, n=ncache)) cflags &= mask_flag_common else: cflags, nnz, mtype = get_local_cache(0, cache_common_flags, cacheoff, ncache) if cflags is not None: cflags &= mask_flag_common if comm is not None: mtype = MPI.UINT8_T cflags = gather_field(0, cflags, nnz, mtype, cacheoff, ncache, 4) if rank == 0: split_field(cflags, so3g.IntervalsInt, "flags_common") # Any extra common fields if comm is not None: comm.barrier() if copy_common is not None: for cindx, (cname, g3typ, fname) in enumerate(copy_common): cdata, nnz, mtype = get_local_cache(0, cname, cacheoff, ncache) cdata = gather_field(0, cdata, nnz, mtype, cacheoff, ncache, cindx) if rank == 0: split_field(cdata, g3typ, fname) # Now read all per-detector quantities. # For each detector field, processes which have the detector # in their local_dets should be in the same process row. if rank == 0: for f in range(n_frames): fdata[f]["signal"] = core3g.G3TimestreamMap() if compress: fdata[f]["compressor_gain_signal"] = core3g.G3MapDouble() fdata[f]["compressor_offset_signal"] = core3g.G3MapDouble() fdata[f]["flags"] = so3g.MapIntervalsInt() if copy_detector is not None: for cname, g3typ, g3maptyp, fnm in copy_detector: fdata[f][fnm] = g3maptyp() if compress: fdata[f]["compressor_gain_" + fnm] = core3g.G3MapDouble() fdata[f]["compressor_offset_" + fnm] = core3g.G3MapDouble() for dindx, dname in enumerate(detnames): drow = -1 # Demodulation may have synthesized new detector names dnames = [] for x in tod.local_dets: if x.endswith(dname): dnames.append(x) if dnames: drow = rankdet # As a sanity check, verify that every process which # has this detector is in the same process row. rowcheck = None if comm is None: rowcheck = [drow] else: rowcheck = comm.gather(drow, root=0) prow = 0 if rank == 0: rc = np.array([x for x in rowcheck if (x >= 0)], dtype=np.int32) prow = np.max(rc) if np.min(rc) != prow: msg = "Processes with detector {} are not in the "\ "same row of the process grid\n".format(dname) sys.stderr.write(msg) if comm is not None: comm.abort() # Every process finds out which process row is participating. if comm is not None: prow = comm.bcast(prow, root=0) all_dnames = comm.allgather(dnames) dnames = list(set(np.concatenate(all_dnames).flat)) # "signal" for dname in dnames: detdata = None nnz = 1 if cache_signal is None: if rankdet == prow: detdata = tod.local_signal(dname)[cacheoff:cacheoff + ncache] else: cache_det = "{}_{}".format(cache_signal, dname) detdata, nnz, mtype = get_local_cache(prow, cache_det, cacheoff, ncache) if comm is not None: mtype = MPI.DOUBLE detdata = gather_field(prow, detdata, nnz, mtype, cacheoff, ncache, dindx) if rank == 0: split_field(detdata, core3g.G3Timestream, "signal", mapfield=dname, times=times) # "flags" for dname in dnames: detdata = None nnz = 1 if cache_flags is None: if rankdet == prow: detdata = tod.local_flags(dname)[cacheoff:cacheoff + ncache] detdata &= mask_flag else: cache_det = "{}_{}".format(cache_flags, dname) detdata, nnz, mtype = get_local_cache(prow, cache_det, cacheoff, ncache) if detdata is not None: detdata &= mask_flag if comm is not None: mtype = MPI.UINT8_T detdata = gather_field(prow, detdata, nnz, mtype, cacheoff, ncache, dindx) if rank == 0: split_field(detdata, so3g.IntervalsInt, "flags", mapfield=dname) # Now copy any additional fields. for dname in dnames: if copy_detector is not None: for cname, g3typ, g3maptyp, fnm in copy_detector: cache_det = "{}_{}".format(cname, dname) detdata, nnz, mtype = get_local_cache( prow, cache_det, cacheoff, ncache) if comm is not None: detdata = gather_field(prow, detdata, nnz, mtype, cacheoff, ncache, dindx) if rank == 0: split_field(detdata, g3typ, fnm, mapfield=dname, times=times) return fdata
# Test that we can read files written on a variety of platforms. Pass a path # to generate test data for whatever this platform is. testpath = os.path.join(os.environ['SPT3G_SOFTWARE_PATH'], 'core/tests/portability') # Test data. Exercise some complicated things (STL bits) that we don't # necessarily have control over, mapping a few primitive types. f = core.G3Frame() f['Five'] = 5 v = core.G3VectorDouble([2.6, 7.2]) f['Vec'] = v v = core.G3VectorInt([17, 42, 87]) f['VecInt'] = v m = core.G3MapDouble() m['Six'] = 6 m['GoingOnSixteen'] = 15.9 f['Map'] = m if len(sys.argv) > 1: core.G3Writer(sys.argv[1])(f) sys.exit(0) # For now, we test files from big-endian (PPC64) and little-endian (amd64) # 64-bit systems. Should include some 32-bit ones. for test in ['test-be.g3', 'test-le.g3']: print(test) testdata = core.G3Reader(os.path.join(testpath, test))(None)[0]
def UnpackCryoData(f): ''' Extracts cryo information into CryoStatus key ''' if f.type != core.G3FrameType.GcpSlow: return if 'cryo' not in f['array']: return board = f['array']['cryo'] temps = board['temperature'] heaters = board['heater_dac'] t = core.G3MapDouble() t['cryo_is_valid'] = board['cryoIsValid'][0] # Measured values # He10 t['uc_head'] = temps[0][0] t['ic_head'] = temps[0][1] t['he4_head'] = temps[0][2] t['he4_fb'] = temps[0][3] t['he4_pump'] = temps[0][4] t['ic_pump'] = temps[0][5] t['uc_pump'] = temps[0][6] t['he4_sw'] = temps[0][7] t['ic_sw'] = temps[0][8] t['uc_sw'] = temps[0][9] t['uc_stage'] = temps[0][10] t['lc_tower'] = temps[0][11] t['ic_stage'] = temps[0][12] t['t4k_head'] = temps[0][13] t['t4k_squid_strap'] = temps[0][14] t['t50k_head'] = temps[0][15] # Optics t['b1_50k_wbp_near'] = temps[1][0] t['b2_50k_wbp_far'] = temps[1][1] t['b3_50k_diving_board'] = temps[1][2] t['b4_50k_top_bot_ptc'] = temps[1][3] t['y1_50k_head'] = temps[1][4] t['y2_50k_window_strap_near'] = temps[1][5] t['y3_50k_tube_strap_near'] = temps[1][6] t['y4_50k_tube'] = temps[1][7] t['g1_4k_head'] = temps[1][8] t['g2_4k_strap'] = temps[1][9] t['g3_4k_lens_tab'] = temps[1][10] t['g4_4k_lens_tab_far'] = temps[1][11] t['r1_4k_top_top_ptc'] = temps[1][12] t['r2_50k_midop_bot_ptc'] = temps[1][13] t['r3_4k_lyot_flange'] = temps[1][14] t['r4_4k_lyot'] = temps[1][15] # Receiver t['t4k_plate_far'] = temps[2][0] t['t4k_strap_optics'] = temps[2][1] t['t4k_plate_mid'] = temps[2][2] t['t4k_plate_top'] = temps[2][3] t['t4k_plate_ptc'] = temps[2][4] t['t50k_harness_middle'] = temps[2][6] t['t50k_strap'] = temps[2][7] t['squid_wh1_sl1'] = temps[2][8] t['squid_wh5_sl1'] = temps[2][9] t['squid_wh3_sl7'] = temps[2][10] t['cal_filament'] = temps[2][11] t['cal_ambient1'] = temps[2][12] t['cal_ambient2'] = temps[2][13] t['cal_ambient3'] = temps[2][14] # Heaters t['heat_he4_pump'] = heaters[0][3] t['heat_ic_pump'] = heaters[0][4] t['heat_uc_pump'] = heaters[0][5] t['heat_he4_sw'] = heaters[0][0] t['heat_ic_sw'] = heaters[0][1] t['heat_uc_sw'] = heaters[0][2] f['CryoStatus'] = t f['CryoStatusTime'] = board['utc']
def calc_tod_median(frame, Input='TimestreamsWatts', Output='AvgPower'): if frame.type == core.G3FrameType.Scan: frame[Output] = core.G3MapDouble() for bolo in frame[Input].keys(): frame[Output][bolo] = np.median(frame[Input][bolo][np.isfinite(frame[Input][bolo])])