def get_metadata(g3file, logger=None): """ Extract metadata from g3file and store in header dictionary """ if not logger: logger = LOGGER # Pre-populate extra metadata that we will need hdr = pre_populate_metadata() hdr['PARENT'] = (os.path.basename(g3file), 'Name of parent file') g3 = core.G3File(g3file) logger.info(f"Loading: {g3file}") for frame in g3: # Extract metadata if frame.type == core.G3FrameType.Observation or frame.type == core.G3FrameType.Map: logger.info(f"Extracting metadata from frame: {frame.type}") hdr = extract_metadata_frame(frame, hdr) if frame.type == core.G3FrameType.Map: logger.info( f"Transforming to FITS: {frame.type} -- Id: {frame['Id']}") # maps.RemoveWeights(frame, zero_nans=True) # Make sure OBS-ID is populated for yearly maps if hdr['OBS-ID'][0] is None and hdr['PARENT'][0].split( "_")[0] == 'yearly': f = hdr['PARENT'][0].split("_") # from basename get for example: 'yearly_winter_2020' OBSID = ("_".join([f[0], f[2], f[3]]), hdr['OBS-ID'][1]) hdr['OBS-ID'] = OBSID hdr['DATE-BEG'] = OBSID logger.info(f"Inserting OBS-ID to header: {hdr['OBS-ID']}") logger.info(f"Inserting DATE-BEG to header: {hdr['DATE-BEG']}") return hdr
def read_spt3g_framesizes(file): sizes = list() for frame in c3g.G3File(file): if frame.type == c3g.G3FrameType.Scan: field = frame.keys()[0] sizes.append(len(frame[field])) return sizes
def load_coadds(args): """ Load coadd g3 files for transients filtering. If specified, apply mask to coadd to save memory """ t0 = time.time() coadds = {} # Loop over all coadd files for g3coaddfile in args.coadd: logger.info(f"Reading coadd file(s): {g3coaddfile}") # Loop over frames for frame in core.G3File(g3coaddfile): logger.debug(f"Reading frame: {frame}") if frame.type != core.G3FrameType.Map: continue if frame["Id"] not in args.band: continue if not args.polarized: maps.map_modules.MakeMapsUnpolarized(frame) maps.map_modules.RemoveWeights(frame, zero_nans=True) del frame["Wunpol"] tmap = frame.pop("T") # apply mask if args.mask is not None: tmap *= args.g3mask # del args.g3mask -- no need to delete tmap.compact(zero_nans=True) coadds["Coadd{}".format(frame["Id"])] = {"T": tmap} subtract_coadd = True logger.info(f"Total time coadd read: {ingstools.elapsed_time(t0)}") return coadds, subtract_coadd
def read_spt3g_framesizes(file): sizes = list() obs = True for frame in c3g.G3File(file): if obs: obs = False continue sizes.append(len(frame[STR_TIME])) return sizes
def convert_to_fits_raw(g3file, fitsfile=None, outpath='', hdr=None, overwrite=True, compress=False, logger=None): t0 = time.time() if not logger: logger = LOGGER # Extract the relevante metadata if not hdr: hdr = get_metadata(g3file, logger=logger) # Define fitsfile name only if undefined if fitsfile is None: basename = get_g3basename(g3file) fitsfile = os.path.join(outpath, get_folder_date(hdr), f"{basename}.fits") # Skip if fitsfile exists and overwrite not True if os.path.isfile(fitsfile) and not overwrite: logger.warning(f"File exists, skipping: {fitsfile}") return # Populate additional metadata for DB hdr['FITSNAME'] = (os.path.basename(fitsfile), 'Name of fits file') # Second loop to write FITS g3 = core.G3File(g3file) logger.info(f"Loading 2: {g3file}") for frame in g3: # Convert to FITS if frame.type == core.G3FrameType.Map: logger.info( f"Transforming to FITS: {frame.type} -- Id: {frame['Id']}") maps.RemoveWeights(frame, zero_nans=True) # Make sure that the folder exists: dirname = os.path.dirname(fitsfile) if not os.path.isdir(dirname): logger.info(f"Creating directory {os.path.dirname(fitsfile)}") os.mkdir(dirname) maps.fitsio.save_skymap_fits(fitsfile, frame['T'], overwrite=overwrite, compress=compress, W=frame['Wunpol'], hdr=hdr) logger.info(f"Created: {fitsfile}") logger.info(f"FITS creation time: {elapsed_time(t0)}") return
def load_frames(self): self._cache_init() for ifile, (ffile, foff) in enumerate(zip(self._files, self._file_frame_offs)): nframes = None if ifile == len(self._files) - 1: # we are at the last file nframes = len(self._frame_sizes) - foff else: # get number of frames in this file nframes = self._file_frame_offs[ifile + 1] - foff #print("load {}: proc {} file {} starts at frame {} and has {} frames".format(self._path, self.mpicomm.rank, ifile, foff, nframes), flush=True) # nframes includes only the scan frames. We add one here to # get the total including the observation frame. gfile = [None for x in range(nframes + 1)] if self.mpicomm.rank == 0: gfile = c3g.G3File(ffile) obs = True for fileframe, fdata in enumerate(gfile): if obs: # Skip the observation frame. obs = False continue scanframe = fileframe - 1 frame = foff + scanframe frame_offset = self._frame_sample_offs[frame] frame_size = self._frame_sizes[frame] # if self.mpicomm.rank == 0: # print("load {}: proc {} working on frame {} (samples {} - {}) data = {}".format(self._path, self.mpicomm.rank, frame, frame_offset, frame_offset+frame_size, fdata), flush=True) s3utils.frame_to_cache(self, frame, frame_offset, frame_size, frame_data=fdata, detector_map=STR_DET, flag_map=STR_FLAG, common_prefix=None, det_prefix="{}_".format(STR_DET), flag_prefix="{}_".format(STR_FLAG)) #print("load {}: proc {} hit barrier for frame {}".format(self._path, self.mpicomm.rank, frame), flush=True) self.mpicomm.barrier() #if self.mpicomm.rank == 0: #print("load {}: proc {} finished frame {}".format(self._path, self.mpicomm.rank, frame), flush=True) del gfile #print("load {}: proc {} finished file {}".format(self._path, self.mpicomm.rank, ifile), flush=True) return
def BolometerPropertiesInjector(frame): if frame.type == core.G3FrameType.Wiring: nchan = len(frame['WiringMap'].keys()) core.log_notice("Collecting data from %d mapped channels." % (nchan), unit='Data Acquisition') bpm = os.path.join(os.path.dirname(args.hardware_map), 'nominal_online_cal.g3') try: if not os.path.exists(bpm): raise IOError('Missing file %s' % bpm) fr = list(core.G3File(bpm))[0] return [frame, fr] except Exception as e: core.log_warn('Error loading BolometerPropertiesMap: %s' % (str(e)), unit='Data Acquisition')
def test_write_obs_frame(self, frame_recorder): """Test writing an observation frame to file.""" f = core.G3Frame(core.G3FrameType.Observation) f['session_id'] = 0 f['start_time'] = time.time() filepath = frame_recorder.create_new_file() frame_recorder.frames = [f] frame_recorder.write_frames_to_file() # Check the frame we wrote frames = [fr for fr in core.G3File(filepath)] assert len(frames) == 1 assert frames[0]['session_id'] == 0 assert frames[0].type is core.G3FrameType.Observation assert frame_recorder.last_meta is f
def test_dump(self): if not toast_available: return # Simulate some noise into multiple cache prefixes. This is used # to test the export of multiple timestream flavors. nse = toast.tod.OpSimNoise(out="signal", realization=0) nse.exec(self.data) nse = toast.tod.OpSimNoise(out="component1", realization=0) nse.exec(self.data) nse = toast.tod.OpSimNoise(out="component2", realization=0) nse.exec(self.data) tod = self.data.obs[0]["tod"] # Dump to disk dumper = ToastExport(self.outdir, prefix="sat3", use_intervals=True, cache_name="signal", cache_copy=["component1", "component2"], mask_flag_common=tod.TURNAROUND, filesize=500000, units=core3g.G3TimestreamUnits.Tcmb) dumper.exec(self.data) # Inspect the dumped frames for root, dirs, files in os.walk(self.outdir): files = sorted(files) for f in files: path = os.path.join(root, f) print("file {}".format(path), flush=True) gf = core3g.G3File(path) for frm in gf: print(frm, flush=True) # if frm.type == core3g.G3FrameType.Scan: # common = frm.get("flags_common") # print(common.array(), flush=True) # print(frm.get("flags")) # print(frm.get("signal"), flush=True) return
def load_mask(args): """Load the mask for filter_transient routines""" t0 = time.time() logger.info(f"Reading mask file: {args.mask}") mask = None for frame in core.G3File(args.mask): if frame.type != core.G3FrameType.Map: continue if "Mask" in frame: mask = frame["Mask"] else: mask = frame["T"] break if mask is None: raise RuntimeError("Missing mask frame") mask_id = "Mask" logger.info( f"Total time reading {args.mask}: {ingstools.elapsed_time(t0)}") return mask, mask_id
def g3_to_array(g3file, verbose=False): """ Takes a G3 file output from the SMuRF archiver and reads to a numpy array. Parameters ---------- g3file : full path to the G3 file verbose : OPTIONAL choice for verbose output (-v, --verbosity) Returns ------- times : array of G3Time objects data : array of arrays, where each internal array is a SMuRF channel """ frames = [fr for fr in core.G3File(g3file)] data = [] frametimes = [] for frame in frames: if frame.type == core.G3FrameType.Scan: frametime = frame['data'].times() frametimes.append(frametime) if frametimes == []: warnings.warn('Wrong frame type') strtimes = np.hstack(frametimes) times = [] for strt in strtimes: t = core.G3Time(strt).time / core.G3Units.s times.append(t) times = np.asarray(times) channums = [] i = 0 while i < len(frames): if verbose: print('Trying frame %i' % i) frametype = frames[i].type if frametype == core.G3FrameType.Scan: for chan in frames[i]['data'].keys(): channums.append(int(chan.strip('r'))) break else: i += 1 if verbose: print('Channel numbers obtained') channums.sort() for ch in channums: if verbose: print('Adding channel %s' % ch) chdata = [] for frame in frames: if frame.type == core.G3FrameType.Scan: framedata = frame['data']['r' + format(ch, "04")] chdata.append(framedata) chdata_all = np.hstack(chdata) data.append(chdata_all) biases = [] biasnums = [] for num in frames[i]['tes_biases'].keys(): biasnums.append(int(num)) biasnums.sort() for b in biasnums: if verbose: print('Adding bias number %i' % b) bias = [] for frame in frames: if frame.type == core.G3FrameType.Scan: biasdata = frame['tes_biases'][str(b)] bias.append(biasdata) bias_all = np.hstack(bias) biases.append(bias_all) biases = np.asarray(biases) data = np.asarray(data) return times, data, biases
return np.sqrt(photon / (1 + 2*np.pi*((x*tau)**2))) def atm_noise(x, A, alpha): return np.sqrt(A * (x)**(-1*alpha)) def noise_model(x, readout, A, alpha, photon, tau): return np.sqrt(readout + (A * (x)**(-1*alpha)) + photon / (1 + 2*np.pi*((x*tau)**2))) def horizon_model(x, readout, A, alpha): return np.sqrt(readout + (A * (x)**(-1*alpha))) def knee_func(x, readout, A, alpha, photon, tau): return (A * (x)**(-1*alpha)) - photon / (1 + 2*np.pi*((x*tau)**2)) - readout def horizon_knee_func(x, readout, A, alpha): return (A * (x)**(-1*alpha)) - readout horizondatafile='/home/adama/SPT/spt_analysis/20190329_gainmatching/horizon_noise_77863968_bender_ltd.g3' fr = list(core.G3File(horizondatafile))[1] band_numbers = {90.: 1, 150.: 2, 220.: 3} subplot_numbers = {90.: 1, 150.: 1, 220.: 1} band_labels = {90:'95', 150:'150', 220:'220'} fig, ax = plt.subplots(1, 3, sharex=True, sharey=True, figsize=(12,4)) fig.subplots_adjust(wspace=0) for jband, band in enumerate([90., 150., 220.]): group = '{:.1f}_w180'.format(band) ff_diff = np.array(fr['AverageASDDiff']['frequency']/core.G3Units.Hz) ff_sum = np.array(fr['AverageASDSum']['frequency']/core.G3Units.Hz) asd_diff = np.array(fr['AverageASDDiff'][group]) / np.sqrt(2.) asd_sum = np.array(fr['AverageASDSum'][group]) / np.sqrt(2.)
matplotlib.use('Agg') import matplotlib.pyplot as plt from scipy.stats import ks_2samp from scipy.signal import lombscargle extract_data = False plot = True results_dict = {90: {'delta_cal_response':{}, 'cal_response_before':{}}, 150:{'delta_cal_response':{}, 'cal_response_before':{}}} if extract_data: cal_path = '/sptgrid/analysis/calibration/calibrator' data_path = '/sptlocal/user/kferguson' boloprops = list(core.G3File('/sptgrid/analysis/calibration/boloproperties/60000000.g3'))[0] angle_data = {} for jband, band in enumerate([90,150]): with open(os.path.join(data_path, 'collated_final_angles_more_realizations_' + '%sGHz.pkl'%band),'rb') as f: angle_data[band] = pickle.load(f) scanify = autoprocessing.ScanifyDatabase(read_only=True) subfields = {} cal_obsids = np.sort([int(os.path.splitext(os.path.basename(path))[0]) \ for path in glob(os.path.join(cal_path, '*.g3'))]) field_obsids = np.sort(list(angle_data[90].keys())) cal_obsids_before = [] cal_obsids_after = [] for obsid in field_obsids: cal_obsids_after.append(np.min(cal_obsids[cal_obsids>obsid]))
map_data_fnames_dict[obsid][band] = fname rho_fit_dict = {} rho_err_dict = {} fnames_test = list(map_data_fnames_dict.keys()) for obsid in fnames_test: #map_data_fnames_dict: if obsid in map_sims_fnames_dict: print(obsid) rho_fit_dict[obsid] = {} rho_err_dict[obsid] = {} for band in [90, 150, 220]: print(band) # load sims for fr in core.G3File(map_sims_fnames_dict[obsid][band]): if fr.type == core.G3FrameType.Map and fr[ 'Id'] == '{}GHz'.format(band): sim_frame = fr # load data data_frame = list(core.G3File( map_data_fnames_dict[obsid][band]))[0] rho, rho_err = axion_utils.calculate_rho(data_frame, sim_frame, freq_factor=8.7, return_err=True) rho_fit_dict[obsid][band] = rho rho_err_dict[obsid][band] = rho_err
# language: python # name: python3 # --- import numpy as np import matplotlib.pyplot as plt from spt3g import core, dfmux, calibration, mapmaker, mapspectra from spt3g.mapmaker.mapmakerutils import remove_weight bands = [90, 150, 220] directions = ['Left', 'Right'] stokes_params = ['Q', 'U', 'T'] plt.bone() d = [ fr for fr in core.G3File( '/spt/user/ddutcher/coadds/coadd2018_noW201wafCMpoly19mhpf300.g3') ] maps = {} for fr in d: for direc in directions: if direc in fr['Id']: direction = direc for b in bands: if str(b) in fr['Id']: band = b if band not in maps.keys(): maps[band] = {} #T, Q, U = remove_weight(fr['T'], fr['Q'], fr['U'], fr['Wpol'])
from spt3g import core, calibration, dfmux import numpy as np import matplotlib.pyplot as plt from glob import glob import os.path import pickle fnames = np.sort(glob('opteff_stdproc_*.g3')) obsid_list = [] opteff_quantiles = {90: {}, 150: {}, 220: {}} opteff_all = {} for fname in fnames: d = [fr for fr in core.G3File(fname)] print(d[0]) obsid = os.path.splitext(fname)[0].split('_')[-1] print(obsid) bp = d[0]["BolometerProperties"] # histograms by observation plt.figure() for band in [90, 150, 220]: opteff = np.array([d[0]["opteff"][bolo] for bolo in d[0]["opteff"].keys() \ if bp[bolo].band / core.G3Units.GHz==band]) plt.hist(opteff[np.isfinite(opteff)], bins=np.linspace(0,0.6,61), label='{} GHz'.format(band), histtype='step') plt.legend() plt.xlabel('optical efficiency') plt.title(obsid)
import numpy as np from glob import glob import os.path from spt3g import core, calibration import matplotlib.pyplot as plt fname_bps = '/sptgrid/data/bolodata/downsampled/ra5hdec-24.5/100005793/nominal_online_cal.g3' bps = list(core.G3File(fname_bps))[0]["NominalBolometerProperties"] bolos_in_bps = np.array(list(bps.keys())) autoproc_dir = '/sptgrid/analysis/calibration/' cal_dir = os.path.join(autoproc_dir, 'calibrator') fields = ['ra5hdec-24.5', 'ra5hdec-31.5' ] #, 'ra5hdec-38.5', 'ra5hdec-45.5', 'ra5hdec-52.5', 'ra5hdec-59.5'] band_list = [90, 150, 220] wafer_list = [ 'w172', 'w174', 'w176', 'w177', 'w180', 'w181', 'w188', 'w203', 'w204', 'w206', 'all' ] band_colors = {90: 'C0', 150: 'C1', 220: 'C2'} for field in fields: fields_dir = os.path.join(autoproc_dir, '{}/pointing'.format(field)) deltat_middle = -234 deltat_bottom = -73 deltat_top = 6264 deltat_tol = 30 frac_cal_all_bottom_middle_10 = { wafer: {band: [] for band in band_list}
def dump(run_number, dir_output): ddir = '/data/pb2/ChileData/g3compressed/20000000/Run2000{:04d}/'.format( run_number) dfiles = sorted(glob(ddir + '*.g3')) plotdir = '{:s}/Run{:04d}/'.format(dir_output, run_number) if not os.path.isdir(plotdir): os.makedirs(plotdir) # peek and get the number of detectors f = core.G3File(dfiles[0]) frame = f.next() while frame.type != core.G3FrameType.Scan: frame = f.next() n_dets = len(frame['RawTimestreams_I'].keys()) # loading lots of tods is ram intensive, so do it in chunks jump_interval = 100 for start in range(n_dets)[::jump_interval]: end = start + jump_interval # detector TOD, in both phases tod_buffs_I = {} tod_buffs_Q = {} # detector sample times times_buff = [] # boresight encoder data antennatime_buff = [] az_buff = [] el_buff = [] # set up pipeline pipe = core.G3Pipeline() pipe.Add(core.G3Reader, filename=dfiles) pipe.Add(get_times, times_buff=times_buff) pipe.Add(get_TOD, start=start, end=end, tod_buffs_I=tod_buffs_I, tod_buffs_Q=tod_buffs_Q) pipe.Add(get_azel, antennatime_buff=antennatime_buff, az_buff=az_buff, el_buff=el_buff) pipe.Run() # group stuff into single arrays times = np.concatenate(times_buff) antennatimes = np.concatenate(antennatime_buff) az = np.concatenate(az_buff) el = np.concatenate(el_buff) # nice date format for plotting bolodates = md.date2num([dt.datetime.fromtimestamp(t) for t in times]) antennadates = md.date2num( [dt.datetime.fromtimestamp(t) for t in antennatimes]) # decide which plots we want here PLOT_ADCVSTOD = False for det in tod_buffs_I: print('Concatenating {:s}...'.format(det)) tod_I = np.concatenate(tod_buffs_I[det]) tod_Q = np.concatenate(tod_buffs_Q[det]) # skip problem TODs if np.all(np.isnan(tod_I)) or np.all(np.isnan(tod_Q)): continue if len(tod_I) != len(times) or len(tod_I) != len(tod_Q): print('skipping because tod is different length') continue # plot_ADCvsTOD(bolodates, tod_I, tod_Q, antennatimes, plotdir, az, el, run_number, det) # fig, ax = plt.subplots(figsize=(12,10), sharex=True, nrows=4, ncols=1) # for iax in range(4): # ax[iax].xaxis.set_major_formatter(md.DateFormatter('%Y/%m/%d %H:%M:%S')) # plt.xticks(rotation=45, fontsize=6) # #ax[iax].xaxis.set_tick_params(which='both', rotation=45) # ax[0].plot(bolodates, tod_I, 'b.', markersize=2) # ax[1].plot(bolodates, tod_Q, 'r.', markersize=2) # ax[2].plot(antennadates, az*180/np.pi, 'c-') # ax[3].plot(antennadates, el*180/np.pi, 'm-') # ax[3].set_xlabel('UTC time') # ax[0].set_ylabel('ADC counts (I)') # ax[1].set_ylabel('ADC counts (Q)') # ax[2].set_ylabel('Boresight Az [deg]') # ax[3].set_ylabel('Boresight El [deg]') # fig.suptitle('Run {:d} - {:s}'.format(run_number, det), fontweight='bold') # fig.savefig(plotdir + '{:s}.png'.format(det)) # #plt.show() # plt.close(fig) # plot_dElvsdAz #fig, ax = plt.subplots(111) print(az)
import matplotlib.pyplot as plt import numpy as np from spt3g import core from glob import glob fname_pattern = '/sptlocal/user/kferguson/updated_noise_jackknife_amp_dists_v2/collated_split_best_fit_amps_150GHz_ra0hdec*' fnames = glob(fname_pattern) for fname in fnames: data = list(core.G3File(fname))[0] plt.hist()
def test_00_basic(self): """Write a stream of HK frames and scan it for errors.""" # Write a stream of HK frames. # (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.) print('Streaming to %s' % test_file) w = core.G3Writer(test_file) # Create something to help us track the aggregator session. hksess = so3g.hk.HKSessionHelper(session_id=None, description="Test HK data.") # Register a data provider. prov_id = hksess.add_provider( description='Fake data for the real world.') # Start the stream -- write the initial session and status frames. w.Process(hksess.session_frame()) w.Process(hksess.status_frame()) # Add a bunch of data frames t_next = time.time() for i in range(10): f = hksess.data_frame(prov_id=prov_id, timestamp=t_next) hk = so3g.IrregBlockDouble() hk.prefix = 'hwp_' hk.data['position'] = [1, 2, 3, 4, 5] hk.data['speed'] = [1.2, 1.2, 1.3, 1.2, 1.3] hk.t = t_next + np.arange(len(hk.data['speed'])) t_next += len(hk.data['speed']) f['blocks'].append(hk) w.Process(f) w.Flush() del w print('Stream closed.\n\n') # Now play them back... print('Reading back:') for f in core.G3File(test_file): ht = f.get('hkagg_type') if ht == so3g.HKFrameType.session: print('Session: %i' % f['session_id']) elif ht == so3g.HKFrameType.status: print(' Status update: %i providers' % (len(f['providers']))) elif ht == so3g.HKFrameType.data: print(' Data: %i blocks' % len(f['blocks'])) for block in f['blocks']: for k, v in block.data.items(): print(' %s%s' % (block.prefix, k), v) # Scan and validate. print() print('Running HKScanner on the test data...') scanner = so3g.hk.HKScanner() pipe = core.G3Pipeline() pipe.Add(core.G3Reader(test_file)) pipe.Add(scanner) pipe.Run() print('Stats: ', scanner.stats) print('Providers: ', scanner.providers) self.assertEqual(scanner.stats['concerns']['n_error'], 0) self.assertEqual(scanner.stats['concerns']['n_warning'], 0)
from spt3g import core, calibration import copy bp_filenames = [{ 'old': '/spt/user/production/calibration/boloproperties/31000000.g3', 'new': '31000000_new.g3' }, { 'old': '/spt/user/production/calibration/boloproperties/32512242.g3', 'new': '32512242_new.g3' }, { 'old': '/spt/user/production/calibration/boloproperties/34300000.g3', 'new': '34300000_new.g3' }] fname_nominal_boloprops = '/home/adama/SPT/hardware_maps_southpole/2018/hwm_pole_run2_include_darks_v2/nominal_online_cal.g3' nominal_boloprops = list(core.G3File(fname_nominal_boloprops))[0] for fnames_dict in bp_filenames: fname_old_boloprops = fnames_dict['old'] old_bp = list(core.G3File(fname_old_boloprops))[0] old_boloprops = copy.deepcopy(old_bp) newframe = core.G3Frame(core.G3FrameType.Calibration) for field in old_boloprops: if field != 'BolometerProperties': newframe[field] = old_boloprops[field] newframe['BolometerProperties'] = calibration.BolometerPropertiesMap() print(len(old_boloprops['BolometerProperties'])) for bolo in old_boloprops['BolometerProperties'].keys():
def load_frames(self): log = Logger.get() rank = 0 if self.mpicomm is not None: rank = self.mpicomm.rank # Timestamps self.cache.create(self.TIMESTAMP_NAME, np.float64, (self.local_samples[1], )) # Boresight pointing self.cache.create("boresight_radec", np.float64, (self.local_samples[1], 4)) self.cache.create("boresight_azel", np.float64, (self.local_samples[1], 4)) self.cache.create(self.HWP_ANGLE_NAME, np.float64, (self.local_samples[1], )) # Common flags self.cache.create(self.COMMON_FLAG_NAME, np.uint8, (self.local_samples[1], )) # Telescope position and velocity self.cache.create(self.POSITION_NAME, np.float64, (self.local_samples[1], 3)) self.cache.create(self.VELOCITY_NAME, np.float64, (self.local_samples[1], 3)) # Detector data and flags for det in self.local_dets: name = "{}_{}".format(self.SIGNAL_NAME, det) self.cache.create(name, np.float64, (self.local_samples[1], )) name = "{}_{}".format(self.FLAG_NAME, det) self.cache.create(name, np.uint8, (self.local_samples[1], )) timer = Timer() for ffile in self._file_names: fnf = self._file_nframes[ffile] frame_offsets = self._frame_sample_offs[ffile] frame_sizes = self._frame_sizes[ffile] if rank == 0: log.debug("Loading {} frames from {}".format(fnf, ffile)) # Loop over all frames- only the root process will actually # read data from disk. if rank == 0: gfile = core3g.G3File(ffile) else: gfile = [None] * fnf timer.clear() timer.start() for fdata, frame_offset, frame_size in zip(gfile, frame_offsets, frame_sizes): is_scan = True if rank == 0: if fdata.type != core3g.G3FrameType.Scan: is_scan = False if self.mpicomm is not None: is_scan = self.mpicomm.bcast(is_scan, root=0) if not is_scan: continue frame_to_tod( self, frame_offset, frame_size, frame_data=fdata, all_flavors=self._all_flavors, ) if self.mpicomm is not None: self.mpicomm.barrier() timer.stop() if rank == 0: log.debug("Translated frames in {}s".format(timer.seconds())) del gfile return
def load_observation(path, dets=None, mpicomm=None, prefix=None, **kwargs): """Loads an observation into memory. Given an observation directory, load frame files into memory. Observation and Calibration frames are stored in corresponding toast objects and Scan frames are loaded and distributed. Further selection of a subset of detectors is done based on an explicit list or a Hardware object. Additional keyword arguments are passed to the SOTOD constructor. Args: path (str): The path to the observation directory. dets (list): Either a list of detectors, a Hardware object, or None. mpicomm (mpi4py.MPI.Comm): The communicator. prefix (str): Only consider frame files with this prefix. Returns: (dict): The observation dictionary. """ log = Logger.get() rank = 0 if mpicomm is not None: rank = mpicomm.rank frame_sizes = {} frame_sizes_by_offset = {} frame_sample_offs = {} file_names = list() file_sample_offs = {} nframes = {} obs = dict() latest_obs = None latest_cal_frames = [] first_offset = None if rank == 0: pat = None if prefix is None: pat = re.compile(r".*_(\d{8}).g3") else: pat = re.compile(r"{}_(\d{{8}}).g3".format(prefix)) for root, dirs, files in os.walk(path, topdown=True): for f in sorted(files): fmat = pat.match(f) if fmat is not None: ffile = os.path.join(path, f) fsampoff = int(fmat.group(1)) if first_offset is None: first_offset = fsampoff file_names.append(ffile) allframes = 0 file_sample_offs[ffile] = fsampoff frame_sizes[ffile] = [] frame_sample_offs[ffile] = [] for frame in core3g.G3File(ffile): allframes += 1 if frame.type == core3g.G3FrameType.Scan: # This is a scan frame, process it. fsz = len(frame["boresight"]["az"]) if fsampoff not in frame_sizes_by_offset: frame_sizes_by_offset[fsampoff] = fsz else: if frame_sizes_by_offset[fsampoff] != fsz: raise RuntimeError( "Frame size at {} changes. {} != {}" "".format( fsampoff, frame_sizes_by_offset[fsampoff], fsz)) frame_sample_offs[ffile].append(fsampoff) frame_sizes[ffile].append(fsz) fsampoff += fsz else: frame_sample_offs[ffile].append(0) frame_sizes[ffile].append(0) if frame.type == core3g.G3FrameType.Observation: latest_obs = frame elif frame.type == core3g.G3FrameType.Calibration: if fsampoff == first_offset: latest_cal_frames.append(frame) else: # Unknown frame type- skip it. pass frame_sample_offs[ffile] = np.array( frame_sample_offs[ffile], dtype=np.int64) nframes[ffile] = allframes log.debug("{} starts at {} and has {} frames".format( ffile, file_sample_offs[ffile], nframes[ffile])) break if len(file_names) == 0: raise RuntimeError( "No frames found at '{}' with prefix '{}'".format( path, prefix)) if mpicomm is not None: latest_obs = mpicomm.bcast(latest_obs, root=0) latest_cal_frames = mpicomm.bcast(latest_cal_frames, root=0) nframes = mpicomm.bcast(nframes, root=0) file_names = mpicomm.bcast(file_names, root=0) file_sample_offs = mpicomm.bcast(file_sample_offs, root=0) frame_sizes = mpicomm.bcast(frame_sizes, root=0) frame_sizes_by_offset = mpicomm.bcast(frame_sizes_by_offset, root=0) frame_sample_offs = mpicomm.bcast(frame_sample_offs, root=0) if latest_obs is None: raise RuntimeError("No observation frame was found!") for k, v in latest_obs.iteritems(): obs[k] = s3utils.from_g3_type(v) if len(latest_cal_frames) == 0: raise RuntimeError("No calibration frame with detector offsets!") detoffset, noise = parse_cal_frames(latest_cal_frames, dets) obs["noise"] = noise tod = SOTOD(path, file_names, nframes, file_sample_offs, frame_sizes, frame_sizes_by_offset, frame_sample_offs, detquats=detoffset, mpicomm=mpicomm, **kwargs) obs["tod"] = tod intervals = [] times = tod.local_times() local_offset, nsample = tod.local_samples for offset, size in frame_sizes_by_offset.items(): if offset >= local_offset and offset + size <= local_offset + nsample: start, stop = times[offset], times[offset + size - 1] else: start, stop = 0, 0 intervals.append( Interval(start=start, stop=stop, first=offset, last=offset + size - 1)) obs["intervals"] = intervals return obs
result = re.match('(.*?)_(.*?)_(.*).g3.gz', os.path.basename(fname)) field = result.group(1) obsid = int(result.group(2)) if field not in factor: factor[field] = {} if field not in ks_test: ks_test[field] = {} if obsid not in factor[field]: factor[field][obsid] = {} if obsid not in ks_test[field]: ks_test[field][obsid] = {} if args.group == '2018ee': real_data = list(core.G3File(fname))[0] maps.RemoveWeights(real_data) q_arr = np.array(real_data['Q']) q_weight = np.array(real_data['Wpol'].QQ) q_arr_finite = q_arr[np.isfinite(q_arr)] q_weight_finite = q_weight[np.isfinite(q_arr)] factor[field][obsid][band] = np.std(q_arr_finite * np.sqrt(q_weight_finite)) ks_result = kstest(q_arr_finite * np.sqrt(q_weight_finite) / factor[field][obsid][band], cdf='norm') ks_test[field][obsid][band] = ks_result
from spt3g import core, calibration import matplotlib.pyplot as plt import pydfmux import numpy as np d = list(core.G3File('/home/adama/SPT/spt3g_software/calibration/scripts/new_boloprops.g3'))[0] #'/spt/user/production/calibration/calframe/RCW38-pixelraster/70228757.g3'))[0] dnominal = list(core.G3File('/spt/data/bolodata/downsampled/RCW38-pixelraster/70228757/nominal_online_cal.g3'))[0] hwm = pydfmux.load_session(open('/home/adama/SPT/hardware_maps_southpole/2019/hwm_pole/hwm.yaml'))['hardware_map'] x = {} y = {} x_nominal = {} y_nominal = {} x_hwm = {} y_hwm = {} for bolo in d['BolometerProperties'].keys(): if d['BolometerProperties'][bolo].wafer_id not in x: x[d['BolometerProperties'][bolo].wafer_id] = {} y[d['BolometerProperties'][bolo].wafer_id] = {} x[d['BolometerProperties'][bolo].wafer_id][bolo] = d['BolometerProperties'][bolo].x_offset y[d['BolometerProperties'][bolo].wafer_id][bolo] = d['BolometerProperties'][bolo].y_offset for bolo in dnominal['NominalBolometerProperties'].keys(): if dnominal['NominalBolometerProperties'][bolo].wafer_id not in x_nominal: x_nominal[dnominal['NominalBolometerProperties'][bolo].wafer_id] = {} y_nominal[dnominal['NominalBolometerProperties'][bolo].wafer_id] = {} x_nominal[dnominal['NominalBolometerProperties'][bolo].wafer_id][bolo] = dnominal['NominalBolometerProperties'][bolo].x_offset y_nominal[dnominal['NominalBolometerProperties'][bolo].wafer_id][bolo] = dnominal['NominalBolometerProperties'][bolo].y_offset bololist = hwm.query(pydfmux.Bolometer)
import matplotlib.pyplot as plt from spt3g import core, maps from scipy.stats import norm import os.path coadd_dir = '/sptlocal/user/kferguson/full_daniel_maps/' with open('weights_var_factors_backup.pkl', 'rb') as f: data_kstest = pickle.load(f) for field in data_kstest['ks_result']: for obsid in data_kstest['ks_result'][field]: for band in data_kstest['ks_result'][field][obsid]: if data_kstest['ks_result'][field][obsid][band].pvalue < 0.01: map_fname = '{}_{}GHz_{}_map.g3.gz'.format(field, band, obsid) map_frame = list(core.G3File(os.path.join(coadd_dir, map_fname)))[0] maps.RemoveWeights(map_frame) q_arr = np.array(map_frame['Q']) q_weight = np.array(map_frame['Wpol'].QQ) q_arr_finite = q_arr[np.isfinite(q_arr)] q_weight_finite = q_weight[np.isfinite(q_arr)] factor = q_arr_finite * np.sqrt(q_weight_finite) plt.figure(1) plt.hist(factor / np.std(factor), bins=np.linspace(-4, 4, 101), density=True) plt.plot(np.linspace(-4, 4, 101), norm.pdf(np.linspace(-4, 4, 101))) plt.xlim([-10,10]) plt.xlabel('$[Q_i \\times \sqrt{W_{qqi}}] / $std')
obsids = [ 63084862, 63098693, 63218920, 63227942, 63305224, 63380372, 63640406, 63650590, 63661173, 63689042, 63728180, 64576620, 64591411, 64606397, 64685912, 64701072 ] data = {} for obsid in obsids: print('Processing {}'.format(obsid)) data[obsid] = {} fname = os.path.join(rawdata_path, str(obsid), '0000.g3') fname_cal = os.path.join(rawdata_path, str(obsid), 'offline_calibration.g3') if os.path.exists(fname) and os.path.exists(fname_cal): d = [fr for fr in core.G3File(fname)] wm = d[1]['WiringMap'] hk = d[2]["DfMuxHousekeeping"] bolos = d[2]['RawTimestreams_I'].keys() vbias = { bolo: bolo_bias_voltage_rms(wm, hk, bolo, 'ICE', 'spt3g_filtering_2017_full') for bolo in bolos } data[obsid]['vbiasrms'] = vbias dcal = [fr for fr in core.G3File(fname_cal)] data[obsid]['boloprops'] = dcal[0]["BolometerProperties"] with open('vbias.pkl', 'wb') as f: pickle.dump(data, f)
# Type failing works? Can't create mask from non-integer Intervals. try: mask3 = so3g.IntervalsDouble.mask([], 8) except ValueError: mask3 = 'failed' assert (mask3 == 'failed') print() print('Map test') tmap = so3g.MapIntervalsTime() tmap['a'] = ti print(' ', tmap) print(' ', tmap['a']) # Can we read and write them? print() test_filename = 'test_intervals.g3' print('Writing to %s' % test_filename) w = core.G3Writer(test_filename) f = core.G3Frame() f['map'] = tmap f['iv'] = iv2 w.Process(f) del w print() print('Reading from %s' % test_filename) for f in core.G3File(test_filename): print(' ', f)
class ShowScanNumber(object): def __init__(self): self.counter = 0 def __call__(self, frame): if frame.type == core.G3FrameType.Scan: print("\nInformation on Scan %s (0-indexed):\n"%self.counter) self.counter += 1 # ============================================================================= # Before pipeline, get list of wafers and bolos to exclude if required # ----------------------------------------------------------------------------- bad_bolos = [] wafer_list = [] for fname in args.input_files: for frame in core.G3File(fname): if 'BolometerProperties' in frame: for bolo, props in frame['BolometerProperties'].items(): if props.wafer_id not in [None,'']: waf = props.wafer_id.capitalize() if waf in bad_wafers: bad_bolos.append(bolo) elif len(good_wafers) > 0 and waf not in good_wafers: bad_bolos.append(bolo) elif waf not in wafer_list: wafer_list.append(waf) if not np.isnan(props.band): if (str(int(props.band/core.G3Units.GHz))+'GHz' not in bands): bad_bolos.append(bolo) break
from spt3g import core, calibration, dfmux import numpy as np import matplotlib.pyplot as plt from scipy.signal import welch, periodogram d = [fr for fr in core.G3File('gain_match_test.g3')] pairnames = np.random.choice(d[4]['PairDiffTimestreams'].keys(), 10) for pairname in pairnames: ts = d[4]['PairDiffTimestreams'][pairname] bolo1 = pairname.split('_')[0] bolo2 = pairname.split('_')[1] plt.figure() ff, psd = periodogram(ts, fs=152.5) plt.loglog(ff, np.sqrt(psd) / (1. / np.sqrt(core.G3Units.Hz)) * 1e6, label='{} - {}'.format(bolo1, bolo2)) ts = d[4]['GainMatchCoeff'][bolo1] * np.array(d[4]['CalTimestreams'][bolo1]) + \ d[4]['GainMatchCoeff'][bolo2] * np.array(d[4]['CalTimestreams'][bolo2]) ff, psd = periodogram(ts, fs=152.5) plt.loglog(ff, np.sqrt(psd) / (1. / np.sqrt(core.G3Units.Hz)) * 1e6, label='{} + {}'.format(bolo1, bolo2)) plt.grid() plt.legend() plt.ylim([1e2, 1e8]) plt.savefig('figures/{}.png'.format(pairname), dpi=200) plt.close() wafer_psds_sum = {} wafer_psds_diff = {}