def get_metadata(g3file, logger=None):
"""
Extract metadata from g3file and store in header dictionary
"""
if not logger:
logger = LOGGER
# Pre-populate extra metadata that we will need
hdr = pre_populate_metadata()
hdr['PARENT'] = (os.path.basename(g3file), 'Name of parent file')
g3 = core.G3File(g3file)
logger.info(f"Loading: {g3file}")
for frame in g3:
# Extract metadata
if frame.type == core.G3FrameType.Observation or frame.type == core.G3FrameType.Map:
logger.info(f"Extracting metadata from frame: {frame.type}")
hdr = extract_metadata_frame(frame, hdr)
if frame.type == core.G3FrameType.Map:
logger.info(
f"Transforming to FITS: {frame.type} -- Id: {frame['Id']}")
# maps.RemoveWeights(frame, zero_nans=True)
# Make sure OBS-ID is populated for yearly maps
if hdr['OBS-ID'][0] is None and hdr['PARENT'][0].split(
"_")[0] == 'yearly':
f = hdr['PARENT'][0].split("_")
# from basename get for example: 'yearly_winter_2020'
OBSID = ("_".join([f[0], f[2], f[3]]), hdr['OBS-ID'][1])
hdr['OBS-ID'] = OBSID
hdr['DATE-BEG'] = OBSID
logger.info(f"Inserting OBS-ID to header: {hdr['OBS-ID']}")
logger.info(f"Inserting DATE-BEG to header: {hdr['DATE-BEG']}")
return hdr
def read_spt3g_framesizes(file):
sizes = list()
for frame in c3g.G3File(file):
if frame.type == c3g.G3FrameType.Scan:
field = frame.keys()[0]
sizes.append(len(frame[field]))
return sizes
def load_coadds(args):
"""
Load coadd g3 files for transients filtering.
If specified, apply mask to coadd to save memory
"""
t0 = time.time()
coadds = {}
# Loop over all coadd files
for g3coaddfile in args.coadd:
logger.info(f"Reading coadd file(s): {g3coaddfile}")
# Loop over frames
for frame in core.G3File(g3coaddfile):
logger.debug(f"Reading frame: {frame}")
if frame.type != core.G3FrameType.Map:
continue
if frame["Id"] not in args.band:
continue
if not args.polarized:
maps.map_modules.MakeMapsUnpolarized(frame)
maps.map_modules.RemoveWeights(frame, zero_nans=True)
del frame["Wunpol"]
tmap = frame.pop("T")
# apply mask
if args.mask is not None:
tmap *= args.g3mask
# del args.g3mask -- no need to delete
tmap.compact(zero_nans=True)
coadds["Coadd{}".format(frame["Id"])] = {"T": tmap}
subtract_coadd = True
logger.info(f"Total time coadd read: {ingstools.elapsed_time(t0)}")
return coadds, subtract_coadd
def read_spt3g_framesizes(file):
sizes = list()
obs = True
for frame in c3g.G3File(file):
if obs:
obs = False
continue
sizes.append(len(frame[STR_TIME]))
return sizes
def convert_to_fits_raw(g3file,
fitsfile=None,
outpath='',
hdr=None,
overwrite=True,
compress=False,
logger=None):
t0 = time.time()
if not logger:
logger = LOGGER
# Extract the relevante metadata
if not hdr:
hdr = get_metadata(g3file, logger=logger)
# Define fitsfile name only if undefined
if fitsfile is None:
basename = get_g3basename(g3file)
fitsfile = os.path.join(outpath, get_folder_date(hdr),
f"{basename}.fits")
# Skip if fitsfile exists and overwrite not True
if os.path.isfile(fitsfile) and not overwrite:
logger.warning(f"File exists, skipping: {fitsfile}")
return
# Populate additional metadata for DB
hdr['FITSNAME'] = (os.path.basename(fitsfile), 'Name of fits file')
# Second loop to write FITS
g3 = core.G3File(g3file)
logger.info(f"Loading 2: {g3file}")
for frame in g3:
# Convert to FITS
if frame.type == core.G3FrameType.Map:
logger.info(
f"Transforming to FITS: {frame.type} -- Id: {frame['Id']}")
maps.RemoveWeights(frame, zero_nans=True)
# Make sure that the folder exists:
dirname = os.path.dirname(fitsfile)
if not os.path.isdir(dirname):
logger.info(f"Creating directory {os.path.dirname(fitsfile)}")
os.mkdir(dirname)
maps.fitsio.save_skymap_fits(fitsfile,
frame['T'],
overwrite=overwrite,
compress=compress,
W=frame['Wunpol'],
hdr=hdr)
logger.info(f"Created: {fitsfile}")
logger.info(f"FITS creation time: {elapsed_time(t0)}")
return
def load_frames(self):
self._cache_init()
for ifile, (ffile,
foff) in enumerate(zip(self._files,
self._file_frame_offs)):
nframes = None
if ifile == len(self._files) - 1:
# we are at the last file
nframes = len(self._frame_sizes) - foff
else:
# get number of frames in this file
nframes = self._file_frame_offs[ifile + 1] - foff
#print("load {}: proc {} file {} starts at frame {} and has {} frames".format(self._path, self.mpicomm.rank, ifile, foff, nframes), flush=True)
# nframes includes only the scan frames. We add one here to
# get the total including the observation frame.
gfile = [None for x in range(nframes + 1)]
if self.mpicomm.rank == 0:
gfile = c3g.G3File(ffile)
obs = True
for fileframe, fdata in enumerate(gfile):
if obs:
# Skip the observation frame.
obs = False
continue
scanframe = fileframe - 1
frame = foff + scanframe
frame_offset = self._frame_sample_offs[frame]
frame_size = self._frame_sizes[frame]
# if self.mpicomm.rank == 0:
# print("load {}: proc {} working on frame {} (samples {} - {}) data = {}".format(self._path, self.mpicomm.rank, frame, frame_offset, frame_offset+frame_size, fdata), flush=True)
s3utils.frame_to_cache(self,
frame,
frame_offset,
frame_size,
frame_data=fdata,
detector_map=STR_DET,
flag_map=STR_FLAG,
common_prefix=None,
det_prefix="{}_".format(STR_DET),
flag_prefix="{}_".format(STR_FLAG))
#print("load {}: proc {} hit barrier for frame {}".format(self._path, self.mpicomm.rank, frame), flush=True)
self.mpicomm.barrier()
#if self.mpicomm.rank == 0:
#print("load {}: proc {} finished frame {}".format(self._path, self.mpicomm.rank, frame), flush=True)
del gfile
#print("load {}: proc {} finished file {}".format(self._path, self.mpicomm.rank, ifile), flush=True)
return
def BolometerPropertiesInjector(frame):
if frame.type == core.G3FrameType.Wiring:
nchan = len(frame['WiringMap'].keys())
core.log_notice("Collecting data from %d mapped channels." % (nchan),
unit='Data Acquisition')
bpm = os.path.join(os.path.dirname(args.hardware_map), 'nominal_online_cal.g3')
try:
if not os.path.exists(bpm):
raise IOError('Missing file %s' % bpm)
fr = list(core.G3File(bpm))[0]
return [frame, fr]
except Exception as e:
core.log_warn('Error loading BolometerPropertiesMap: %s' % (str(e)),
unit='Data Acquisition')
def test_write_obs_frame(self, frame_recorder):
"""Test writing an observation frame to file."""
f = core.G3Frame(core.G3FrameType.Observation)
f['session_id'] = 0
f['start_time'] = time.time()
filepath = frame_recorder.create_new_file()
frame_recorder.frames = [f]
frame_recorder.write_frames_to_file()
# Check the frame we wrote
frames = [fr for fr in core.G3File(filepath)]
assert len(frames) == 1
assert frames[0]['session_id'] == 0
assert frames[0].type is core.G3FrameType.Observation
assert frame_recorder.last_meta is f
def test_dump(self):
if not toast_available:
return
# Simulate some noise into multiple cache prefixes. This is used
# to test the export of multiple timestream flavors.
nse = toast.tod.OpSimNoise(out="signal", realization=0)
nse.exec(self.data)
nse = toast.tod.OpSimNoise(out="component1", realization=0)
nse.exec(self.data)
nse = toast.tod.OpSimNoise(out="component2", realization=0)
nse.exec(self.data)
tod = self.data.obs[0]["tod"]
# Dump to disk
dumper = ToastExport(self.outdir,
prefix="sat3",
use_intervals=True,
cache_name="signal",
cache_copy=["component1", "component2"],
mask_flag_common=tod.TURNAROUND,
filesize=500000,
units=core3g.G3TimestreamUnits.Tcmb)
dumper.exec(self.data)
# Inspect the dumped frames
for root, dirs, files in os.walk(self.outdir):
files = sorted(files)
for f in files:
path = os.path.join(root, f)
print("file {}".format(path), flush=True)
gf = core3g.G3File(path)
for frm in gf:
print(frm, flush=True)
# if frm.type == core3g.G3FrameType.Scan:
# common = frm.get("flags_common")
# print(common.array(), flush=True)
# print(frm.get("flags"))
# print(frm.get("signal"), flush=True)
return
def load_mask(args):
"""Load the mask for filter_transient routines"""
t0 = time.time()
logger.info(f"Reading mask file: {args.mask}")
mask = None
for frame in core.G3File(args.mask):
if frame.type != core.G3FrameType.Map:
continue
if "Mask" in frame:
mask = frame["Mask"]
else:
mask = frame["T"]
break
if mask is None:
raise RuntimeError("Missing mask frame")
mask_id = "Mask"
logger.info(
f"Total time reading {args.mask}: {ingstools.elapsed_time(t0)}")
return mask, mask_id
def g3_to_array(g3file, verbose=False):
"""
Takes a G3 file output from the SMuRF archiver and reads to a numpy array.
Parameters
----------
g3file : full path to the G3 file
verbose : OPTIONAL choice for verbose output (-v, --verbosity)
Returns
-------
times : array of G3Time objects
data : array of arrays, where each internal array is a SMuRF channel
"""
frames = [fr for fr in core.G3File(g3file)]
data = []
frametimes = []
for frame in frames:
if frame.type == core.G3FrameType.Scan:
frametime = frame['data'].times()
frametimes.append(frametime)
if frametimes == []:
warnings.warn('Wrong frame type')
strtimes = np.hstack(frametimes)
times = []
for strt in strtimes:
t = core.G3Time(strt).time / core.G3Units.s
times.append(t)
times = np.asarray(times)
channums = []
i = 0
while i < len(frames):
if verbose:
print('Trying frame %i' % i)
frametype = frames[i].type
if frametype == core.G3FrameType.Scan:
for chan in frames[i]['data'].keys():
channums.append(int(chan.strip('r')))
break
else:
i += 1
if verbose:
print('Channel numbers obtained')
channums.sort()
for ch in channums:
if verbose:
print('Adding channel %s' % ch)
chdata = []
for frame in frames:
if frame.type == core.G3FrameType.Scan:
framedata = frame['data']['r' + format(ch, "04")]
chdata.append(framedata)
chdata_all = np.hstack(chdata)
data.append(chdata_all)
biases = []
biasnums = []
for num in frames[i]['tes_biases'].keys():
biasnums.append(int(num))
biasnums.sort()
for b in biasnums:
if verbose:
print('Adding bias number %i' % b)
bias = []
for frame in frames:
if frame.type == core.G3FrameType.Scan:
biasdata = frame['tes_biases'][str(b)]
bias.append(biasdata)
bias_all = np.hstack(bias)
biases.append(bias_all)
biases = np.asarray(biases)
data = np.asarray(data)
return times, data, biases
return np.sqrt(photon / (1 + 2*np.pi*((x*tau)**2)))
def atm_noise(x, A, alpha):
return np.sqrt(A * (x)**(-1*alpha))
def noise_model(x, readout, A, alpha, photon, tau):
return np.sqrt(readout + (A * (x)**(-1*alpha)) + photon / (1 + 2*np.pi*((x*tau)**2)))
def horizon_model(x, readout, A, alpha):
return np.sqrt(readout + (A * (x)**(-1*alpha)))
def knee_func(x, readout, A, alpha, photon, tau):
return (A * (x)**(-1*alpha)) - photon / (1 + 2*np.pi*((x*tau)**2)) - readout
def horizon_knee_func(x, readout, A, alpha):
return (A * (x)**(-1*alpha)) - readout
horizondatafile='/home/adama/SPT/spt_analysis/20190329_gainmatching/horizon_noise_77863968_bender_ltd.g3'
fr = list(core.G3File(horizondatafile))[1]
band_numbers = {90.: 1, 150.: 2, 220.: 3}
subplot_numbers = {90.: 1, 150.: 1, 220.: 1}
band_labels = {90:'95', 150:'150', 220:'220'}
fig, ax = plt.subplots(1, 3, sharex=True, sharey=True, figsize=(12,4))
fig.subplots_adjust(wspace=0)
for jband, band in enumerate([90., 150., 220.]):
group = '{:.1f}_w180'.format(band)
ff_diff = np.array(fr['AverageASDDiff']['frequency']/core.G3Units.Hz)
ff_sum = np.array(fr['AverageASDSum']['frequency']/core.G3Units.Hz)
asd_diff = np.array(fr['AverageASDDiff'][group]) / np.sqrt(2.)
asd_sum = np.array(fr['AverageASDSum'][group]) / np.sqrt(2.)
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from scipy.stats import ks_2samp
from scipy.signal import lombscargle
extract_data = False
plot = True
results_dict = {90: {'delta_cal_response':{},
'cal_response_before':{}},
150:{'delta_cal_response':{},
'cal_response_before':{}}}
if extract_data:
cal_path = '/sptgrid/analysis/calibration/calibrator'
data_path = '/sptlocal/user/kferguson'
boloprops = list(core.G3File('/sptgrid/analysis/calibration/boloproperties/60000000.g3'))[0]
angle_data = {}
for jband, band in enumerate([90,150]):
with open(os.path.join(data_path, 'collated_final_angles_more_realizations_' + '%sGHz.pkl'%band),'rb') as f:
angle_data[band] = pickle.load(f)
scanify = autoprocessing.ScanifyDatabase(read_only=True)
subfields = {}
cal_obsids = np.sort([int(os.path.splitext(os.path.basename(path))[0]) \
for path in glob(os.path.join(cal_path, '*.g3'))])
field_obsids = np.sort(list(angle_data[90].keys()))
cal_obsids_before = []
cal_obsids_after = []
for obsid in field_obsids:
cal_obsids_after.append(np.min(cal_obsids[cal_obsids>obsid]))
map_data_fnames_dict[obsid][band] = fname
rho_fit_dict = {}
rho_err_dict = {}
fnames_test = list(map_data_fnames_dict.keys())
for obsid in fnames_test: #map_data_fnames_dict:
if obsid in map_sims_fnames_dict:
print(obsid)
rho_fit_dict[obsid] = {}
rho_err_dict[obsid] = {}
for band in [90, 150, 220]:
print(band)
# load sims
for fr in core.G3File(map_sims_fnames_dict[obsid][band]):
if fr.type == core.G3FrameType.Map and fr[
'Id'] == '{}GHz'.format(band):
sim_frame = fr
# load data
data_frame = list(core.G3File(
map_data_fnames_dict[obsid][band]))[0]
rho, rho_err = axion_utils.calculate_rho(data_frame,
sim_frame,
freq_factor=8.7,
return_err=True)
rho_fit_dict[obsid][band] = rho
rho_err_dict[obsid][band] = rho_err
# language: python
# name: python3
# ---
import numpy as np
import matplotlib.pyplot as plt
from spt3g import core, dfmux, calibration, mapmaker, mapspectra
from spt3g.mapmaker.mapmakerutils import remove_weight
bands = [90, 150, 220]
directions = ['Left', 'Right']
stokes_params = ['Q', 'U', 'T']
plt.bone()
d = [
fr for fr in core.G3File(
'/spt/user/ddutcher/coadds/coadd2018_noW201wafCMpoly19mhpf300.g3')
]
maps = {}
for fr in d:
for direc in directions:
if direc in fr['Id']:
direction = direc
for b in bands:
if str(b) in fr['Id']:
band = b
if band not in maps.keys():
maps[band] = {}
#T, Q, U = remove_weight(fr['T'], fr['Q'], fr['U'], fr['Wpol'])
from spt3g import core, calibration, dfmux
import numpy as np
import matplotlib.pyplot as plt
from glob import glob
import os.path
import pickle
fnames = np.sort(glob('opteff_stdproc_*.g3'))
obsid_list = []
opteff_quantiles = {90: {}, 150: {}, 220: {}}
opteff_all = {}
for fname in fnames:
d = [fr for fr in core.G3File(fname)]
print(d[0])
obsid = os.path.splitext(fname)[0].split('_')[-1]
print(obsid)
bp = d[0]["BolometerProperties"]
# histograms by observation
plt.figure()
for band in [90, 150, 220]:
opteff = np.array([d[0]["opteff"][bolo] for bolo in d[0]["opteff"].keys() \
if bp[bolo].band / core.G3Units.GHz==band])
plt.hist(opteff[np.isfinite(opteff)], bins=np.linspace(0,0.6,61),
label='{} GHz'.format(band),
histtype='step')
plt.legend()
plt.xlabel('optical efficiency')
plt.title(obsid)
import numpy as np
from glob import glob
import os.path
from spt3g import core, calibration
import matplotlib.pyplot as plt
fname_bps = '/sptgrid/data/bolodata/downsampled/ra5hdec-24.5/100005793/nominal_online_cal.g3'
bps = list(core.G3File(fname_bps))[0]["NominalBolometerProperties"]
bolos_in_bps = np.array(list(bps.keys()))
autoproc_dir = '/sptgrid/analysis/calibration/'
cal_dir = os.path.join(autoproc_dir, 'calibrator')
fields = ['ra5hdec-24.5', 'ra5hdec-31.5'
] #, 'ra5hdec-38.5', 'ra5hdec-45.5', 'ra5hdec-52.5', 'ra5hdec-59.5']
band_list = [90, 150, 220]
wafer_list = [
'w172', 'w174', 'w176', 'w177', 'w180', 'w181', 'w188', 'w203', 'w204',
'w206', 'all'
]
band_colors = {90: 'C0', 150: 'C1', 220: 'C2'}
for field in fields:
fields_dir = os.path.join(autoproc_dir, '{}/pointing'.format(field))
deltat_middle = -234
deltat_bottom = -73
deltat_top = 6264
deltat_tol = 30
frac_cal_all_bottom_middle_10 = {
wafer: {band: []
for band in band_list}
def dump(run_number, dir_output):
ddir = '/data/pb2/ChileData/g3compressed/20000000/Run2000{:04d}/'.format(
run_number)
dfiles = sorted(glob(ddir + '*.g3'))
plotdir = '{:s}/Run{:04d}/'.format(dir_output, run_number)
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
# peek and get the number of detectors
f = core.G3File(dfiles[0])
frame = f.next()
while frame.type != core.G3FrameType.Scan:
frame = f.next()
n_dets = len(frame['RawTimestreams_I'].keys())
# loading lots of tods is ram intensive, so do it in chunks
jump_interval = 100
for start in range(n_dets)[::jump_interval]:
end = start + jump_interval
# detector TOD, in both phases
tod_buffs_I = {}
tod_buffs_Q = {}
# detector sample times
times_buff = []
# boresight encoder data
antennatime_buff = []
az_buff = []
el_buff = []
# set up pipeline
pipe = core.G3Pipeline()
pipe.Add(core.G3Reader, filename=dfiles)
pipe.Add(get_times, times_buff=times_buff)
pipe.Add(get_TOD,
start=start,
end=end,
tod_buffs_I=tod_buffs_I,
tod_buffs_Q=tod_buffs_Q)
pipe.Add(get_azel,
antennatime_buff=antennatime_buff,
az_buff=az_buff,
el_buff=el_buff)
pipe.Run()
# group stuff into single arrays
times = np.concatenate(times_buff)
antennatimes = np.concatenate(antennatime_buff)
az = np.concatenate(az_buff)
el = np.concatenate(el_buff)
# nice date format for plotting
bolodates = md.date2num([dt.datetime.fromtimestamp(t) for t in times])
antennadates = md.date2num(
[dt.datetime.fromtimestamp(t) for t in antennatimes])
# decide which plots we want here
PLOT_ADCVSTOD = False
for det in tod_buffs_I:
print('Concatenating {:s}...'.format(det))
tod_I = np.concatenate(tod_buffs_I[det])
tod_Q = np.concatenate(tod_buffs_Q[det])
# skip problem TODs
if np.all(np.isnan(tod_I)) or np.all(np.isnan(tod_Q)): continue
if len(tod_I) != len(times) or len(tod_I) != len(tod_Q):
print('skipping because tod is different length')
continue
# plot_ADCvsTOD(bolodates, tod_I, tod_Q, antennatimes, plotdir, az, el, run_number, det)
# fig, ax = plt.subplots(figsize=(12,10), sharex=True, nrows=4, ncols=1)
# for iax in range(4):
# ax[iax].xaxis.set_major_formatter(md.DateFormatter('%Y/%m/%d %H:%M:%S'))
# plt.xticks(rotation=45, fontsize=6)
# #ax[iax].xaxis.set_tick_params(which='both', rotation=45)
# ax[0].plot(bolodates, tod_I, 'b.', markersize=2)
# ax[1].plot(bolodates, tod_Q, 'r.', markersize=2)
# ax[2].plot(antennadates, az*180/np.pi, 'c-')
# ax[3].plot(antennadates, el*180/np.pi, 'm-')
# ax[3].set_xlabel('UTC time')
# ax[0].set_ylabel('ADC counts (I)')
# ax[1].set_ylabel('ADC counts (Q)')
# ax[2].set_ylabel('Boresight Az [deg]')
# ax[3].set_ylabel('Boresight El [deg]')
# fig.suptitle('Run {:d} - {:s}'.format(run_number, det), fontweight='bold')
# fig.savefig(plotdir + '{:s}.png'.format(det))
# #plt.show()
# plt.close(fig)
# plot_dElvsdAz
#fig, ax = plt.subplots(111)
print(az)
import matplotlib.pyplot as plt
import numpy as np
from spt3g import core
from glob import glob
fname_pattern = '/sptlocal/user/kferguson/updated_noise_jackknife_amp_dists_v2/collated_split_best_fit_amps_150GHz_ra0hdec*'
fnames = glob(fname_pattern)
for fname in fnames:
data = list(core.G3File(fname))[0]
plt.hist()
def test_00_basic(self):
"""Write a stream of HK frames and scan it for errors."""
# Write a stream of HK frames.
# (Inspect the output with 'spt3g-dump hk_out.g3 so3g'.)
print('Streaming to %s' % test_file)
w = core.G3Writer(test_file)
# Create something to help us track the aggregator session.
hksess = so3g.hk.HKSessionHelper(session_id=None,
description="Test HK data.")
# Register a data provider.
prov_id = hksess.add_provider(
description='Fake data for the real world.')
# Start the stream -- write the initial session and status frames.
w.Process(hksess.session_frame())
w.Process(hksess.status_frame())
# Add a bunch of data frames
t_next = time.time()
for i in range(10):
f = hksess.data_frame(prov_id=prov_id, timestamp=t_next)
hk = so3g.IrregBlockDouble()
hk.prefix = 'hwp_'
hk.data['position'] = [1, 2, 3, 4, 5]
hk.data['speed'] = [1.2, 1.2, 1.3, 1.2, 1.3]
hk.t = t_next + np.arange(len(hk.data['speed']))
t_next += len(hk.data['speed'])
f['blocks'].append(hk)
w.Process(f)
w.Flush()
del w
print('Stream closed.\n\n')
# Now play them back...
print('Reading back:')
for f in core.G3File(test_file):
ht = f.get('hkagg_type')
if ht == so3g.HKFrameType.session:
print('Session: %i' % f['session_id'])
elif ht == so3g.HKFrameType.status:
print(' Status update: %i providers' % (len(f['providers'])))
elif ht == so3g.HKFrameType.data:
print(' Data: %i blocks' % len(f['blocks']))
for block in f['blocks']:
for k, v in block.data.items():
print(' %s%s' % (block.prefix, k), v)
# Scan and validate.
print()
print('Running HKScanner on the test data...')
scanner = so3g.hk.HKScanner()
pipe = core.G3Pipeline()
pipe.Add(core.G3Reader(test_file))
pipe.Add(scanner)
pipe.Run()
print('Stats: ', scanner.stats)
print('Providers: ', scanner.providers)
self.assertEqual(scanner.stats['concerns']['n_error'], 0)
self.assertEqual(scanner.stats['concerns']['n_warning'], 0)
from spt3g import core, calibration
import copy
bp_filenames = [{
'old': '/spt/user/production/calibration/boloproperties/31000000.g3',
'new': '31000000_new.g3'
}, {
'old': '/spt/user/production/calibration/boloproperties/32512242.g3',
'new': '32512242_new.g3'
}, {
'old': '/spt/user/production/calibration/boloproperties/34300000.g3',
'new': '34300000_new.g3'
}]
fname_nominal_boloprops = '/home/adama/SPT/hardware_maps_southpole/2018/hwm_pole_run2_include_darks_v2/nominal_online_cal.g3'
nominal_boloprops = list(core.G3File(fname_nominal_boloprops))[0]
for fnames_dict in bp_filenames:
fname_old_boloprops = fnames_dict['old']
old_bp = list(core.G3File(fname_old_boloprops))[0]
old_boloprops = copy.deepcopy(old_bp)
newframe = core.G3Frame(core.G3FrameType.Calibration)
for field in old_boloprops:
if field != 'BolometerProperties':
newframe[field] = old_boloprops[field]
newframe['BolometerProperties'] = calibration.BolometerPropertiesMap()
print(len(old_boloprops['BolometerProperties']))
for bolo in old_boloprops['BolometerProperties'].keys():
def load_frames(self):
log = Logger.get()
rank = 0
if self.mpicomm is not None:
rank = self.mpicomm.rank
# Timestamps
self.cache.create(self.TIMESTAMP_NAME, np.float64,
(self.local_samples[1], ))
# Boresight pointing
self.cache.create("boresight_radec", np.float64,
(self.local_samples[1], 4))
self.cache.create("boresight_azel", np.float64,
(self.local_samples[1], 4))
self.cache.create(self.HWP_ANGLE_NAME, np.float64,
(self.local_samples[1], ))
# Common flags
self.cache.create(self.COMMON_FLAG_NAME, np.uint8,
(self.local_samples[1], ))
# Telescope position and velocity
self.cache.create(self.POSITION_NAME, np.float64,
(self.local_samples[1], 3))
self.cache.create(self.VELOCITY_NAME, np.float64,
(self.local_samples[1], 3))
# Detector data and flags
for det in self.local_dets:
name = "{}_{}".format(self.SIGNAL_NAME, det)
self.cache.create(name, np.float64, (self.local_samples[1], ))
name = "{}_{}".format(self.FLAG_NAME, det)
self.cache.create(name, np.uint8, (self.local_samples[1], ))
timer = Timer()
for ffile in self._file_names:
fnf = self._file_nframes[ffile]
frame_offsets = self._frame_sample_offs[ffile]
frame_sizes = self._frame_sizes[ffile]
if rank == 0:
log.debug("Loading {} frames from {}".format(fnf, ffile))
# Loop over all frames- only the root process will actually
# read data from disk.
if rank == 0:
gfile = core3g.G3File(ffile)
else:
gfile = [None] * fnf
timer.clear()
timer.start()
for fdata, frame_offset, frame_size in zip(gfile, frame_offsets,
frame_sizes):
is_scan = True
if rank == 0:
if fdata.type != core3g.G3FrameType.Scan:
is_scan = False
if self.mpicomm is not None:
is_scan = self.mpicomm.bcast(is_scan, root=0)
if not is_scan:
continue
frame_to_tod(
self,
frame_offset,
frame_size,
frame_data=fdata,
all_flavors=self._all_flavors,
)
if self.mpicomm is not None:
self.mpicomm.barrier()
timer.stop()
if rank == 0:
log.debug("Translated frames in {}s".format(timer.seconds()))
del gfile
return
def load_observation(path, dets=None, mpicomm=None, prefix=None, **kwargs):
"""Loads an observation into memory.
Given an observation directory, load frame files into memory. Observation
and Calibration frames are stored in corresponding toast objects and Scan
frames are loaded and distributed. Further selection of a subset of
detectors is done based on an explicit list or a Hardware object.
Additional keyword arguments are passed to the SOTOD constructor.
Args:
path (str): The path to the observation directory.
dets (list): Either a list of detectors, a Hardware object, or None.
mpicomm (mpi4py.MPI.Comm): The communicator.
prefix (str): Only consider frame files with this prefix.
Returns:
(dict): The observation dictionary.
"""
log = Logger.get()
rank = 0
if mpicomm is not None:
rank = mpicomm.rank
frame_sizes = {}
frame_sizes_by_offset = {}
frame_sample_offs = {}
file_names = list()
file_sample_offs = {}
nframes = {}
obs = dict()
latest_obs = None
latest_cal_frames = []
first_offset = None
if rank == 0:
pat = None
if prefix is None:
pat = re.compile(r".*_(\d{8}).g3")
else:
pat = re.compile(r"{}_(\d{{8}}).g3".format(prefix))
for root, dirs, files in os.walk(path, topdown=True):
for f in sorted(files):
fmat = pat.match(f)
if fmat is not None:
ffile = os.path.join(path, f)
fsampoff = int(fmat.group(1))
if first_offset is None:
first_offset = fsampoff
file_names.append(ffile)
allframes = 0
file_sample_offs[ffile] = fsampoff
frame_sizes[ffile] = []
frame_sample_offs[ffile] = []
for frame in core3g.G3File(ffile):
allframes += 1
if frame.type == core3g.G3FrameType.Scan:
# This is a scan frame, process it.
fsz = len(frame["boresight"]["az"])
if fsampoff not in frame_sizes_by_offset:
frame_sizes_by_offset[fsampoff] = fsz
else:
if frame_sizes_by_offset[fsampoff] != fsz:
raise RuntimeError(
"Frame size at {} changes. {} != {}"
"".format(
fsampoff,
frame_sizes_by_offset[fsampoff],
fsz))
frame_sample_offs[ffile].append(fsampoff)
frame_sizes[ffile].append(fsz)
fsampoff += fsz
else:
frame_sample_offs[ffile].append(0)
frame_sizes[ffile].append(0)
if frame.type == core3g.G3FrameType.Observation:
latest_obs = frame
elif frame.type == core3g.G3FrameType.Calibration:
if fsampoff == first_offset:
latest_cal_frames.append(frame)
else:
# Unknown frame type- skip it.
pass
frame_sample_offs[ffile] = np.array(
frame_sample_offs[ffile], dtype=np.int64)
nframes[ffile] = allframes
log.debug("{} starts at {} and has {} frames".format(
ffile, file_sample_offs[ffile], nframes[ffile]))
break
if len(file_names) == 0:
raise RuntimeError(
"No frames found at '{}' with prefix '{}'".format(
path, prefix))
if mpicomm is not None:
latest_obs = mpicomm.bcast(latest_obs, root=0)
latest_cal_frames = mpicomm.bcast(latest_cal_frames, root=0)
nframes = mpicomm.bcast(nframes, root=0)
file_names = mpicomm.bcast(file_names, root=0)
file_sample_offs = mpicomm.bcast(file_sample_offs, root=0)
frame_sizes = mpicomm.bcast(frame_sizes, root=0)
frame_sizes_by_offset = mpicomm.bcast(frame_sizes_by_offset, root=0)
frame_sample_offs = mpicomm.bcast(frame_sample_offs, root=0)
if latest_obs is None:
raise RuntimeError("No observation frame was found!")
for k, v in latest_obs.iteritems():
obs[k] = s3utils.from_g3_type(v)
if len(latest_cal_frames) == 0:
raise RuntimeError("No calibration frame with detector offsets!")
detoffset, noise = parse_cal_frames(latest_cal_frames, dets)
obs["noise"] = noise
tod = SOTOD(path,
file_names,
nframes,
file_sample_offs,
frame_sizes,
frame_sizes_by_offset,
frame_sample_offs,
detquats=detoffset,
mpicomm=mpicomm,
**kwargs)
obs["tod"] = tod
intervals = []
times = tod.local_times()
local_offset, nsample = tod.local_samples
for offset, size in frame_sizes_by_offset.items():
if offset >= local_offset and offset + size <= local_offset + nsample:
start, stop = times[offset], times[offset + size - 1]
else:
start, stop = 0, 0
intervals.append(
Interval(start=start,
stop=stop,
first=offset,
last=offset + size - 1))
obs["intervals"] = intervals
return obs
result = re.match('(.*?)_(.*?)_(.*).g3.gz', os.path.basename(fname))
field = result.group(1)
obsid = int(result.group(2))
if field not in factor:
factor[field] = {}
if field not in ks_test:
ks_test[field] = {}
if obsid not in factor[field]:
factor[field][obsid] = {}
if obsid not in ks_test[field]:
ks_test[field][obsid] = {}
if args.group == '2018ee':
real_data = list(core.G3File(fname))[0]
maps.RemoveWeights(real_data)
q_arr = np.array(real_data['Q'])
q_weight = np.array(real_data['Wpol'].QQ)
q_arr_finite = q_arr[np.isfinite(q_arr)]
q_weight_finite = q_weight[np.isfinite(q_arr)]
factor[field][obsid][band] = np.std(q_arr_finite *
np.sqrt(q_weight_finite))
ks_result = kstest(q_arr_finite * np.sqrt(q_weight_finite) /
factor[field][obsid][band],
cdf='norm')
ks_test[field][obsid][band] = ks_result
from spt3g import core, calibration
import matplotlib.pyplot as plt
import pydfmux
import numpy as np
d = list(core.G3File('/home/adama/SPT/spt3g_software/calibration/scripts/new_boloprops.g3'))[0] #'/spt/user/production/calibration/calframe/RCW38-pixelraster/70228757.g3'))[0]
dnominal = list(core.G3File('/spt/data/bolodata/downsampled/RCW38-pixelraster/70228757/nominal_online_cal.g3'))[0]
hwm = pydfmux.load_session(open('/home/adama/SPT/hardware_maps_southpole/2019/hwm_pole/hwm.yaml'))['hardware_map']
x = {}
y = {}
x_nominal = {}
y_nominal = {}
x_hwm = {}
y_hwm = {}
for bolo in d['BolometerProperties'].keys():
if d['BolometerProperties'][bolo].wafer_id not in x:
x[d['BolometerProperties'][bolo].wafer_id] = {}
y[d['BolometerProperties'][bolo].wafer_id] = {}
x[d['BolometerProperties'][bolo].wafer_id][bolo] = d['BolometerProperties'][bolo].x_offset
y[d['BolometerProperties'][bolo].wafer_id][bolo] = d['BolometerProperties'][bolo].y_offset
for bolo in dnominal['NominalBolometerProperties'].keys():
if dnominal['NominalBolometerProperties'][bolo].wafer_id not in x_nominal:
x_nominal[dnominal['NominalBolometerProperties'][bolo].wafer_id] = {}
y_nominal[dnominal['NominalBolometerProperties'][bolo].wafer_id] = {}
x_nominal[dnominal['NominalBolometerProperties'][bolo].wafer_id][bolo] = dnominal['NominalBolometerProperties'][bolo].x_offset
y_nominal[dnominal['NominalBolometerProperties'][bolo].wafer_id][bolo] = dnominal['NominalBolometerProperties'][bolo].y_offset
bololist = hwm.query(pydfmux.Bolometer)
import matplotlib.pyplot as plt
from spt3g import core, maps
from scipy.stats import norm
import os.path
coadd_dir = '/sptlocal/user/kferguson/full_daniel_maps/'
with open('weights_var_factors_backup.pkl', 'rb') as f:
data_kstest = pickle.load(f)
for field in data_kstest['ks_result']:
for obsid in data_kstest['ks_result'][field]:
for band in data_kstest['ks_result'][field][obsid]:
if data_kstest['ks_result'][field][obsid][band].pvalue < 0.01:
map_fname = '{}_{}GHz_{}_map.g3.gz'.format(field, band, obsid)
map_frame = list(core.G3File(os.path.join(coadd_dir, map_fname)))[0]
maps.RemoveWeights(map_frame)
q_arr = np.array(map_frame['Q'])
q_weight = np.array(map_frame['Wpol'].QQ)
q_arr_finite = q_arr[np.isfinite(q_arr)]
q_weight_finite = q_weight[np.isfinite(q_arr)]
factor = q_arr_finite * np.sqrt(q_weight_finite)
plt.figure(1)
plt.hist(factor / np.std(factor), bins=np.linspace(-4, 4, 101),
density=True)
plt.plot(np.linspace(-4, 4, 101), norm.pdf(np.linspace(-4, 4, 101)))
plt.xlim([-10,10])
plt.xlabel('$[Q_i \\times \sqrt{W_{qqi}}] / $std')
obsids = [
63084862, 63098693, 63218920, 63227942, 63305224, 63380372, 63640406,
63650590, 63661173, 63689042, 63728180, 64576620, 64591411, 64606397,
64685912, 64701072
]
data = {}
for obsid in obsids:
print('Processing {}'.format(obsid))
data[obsid] = {}
fname = os.path.join(rawdata_path, str(obsid), '0000.g3')
fname_cal = os.path.join(rawdata_path, str(obsid),
'offline_calibration.g3')
if os.path.exists(fname) and os.path.exists(fname_cal):
d = [fr for fr in core.G3File(fname)]
wm = d[1]['WiringMap']
hk = d[2]["DfMuxHousekeeping"]
bolos = d[2]['RawTimestreams_I'].keys()
vbias = {
bolo: bolo_bias_voltage_rms(wm, hk, bolo, 'ICE',
'spt3g_filtering_2017_full')
for bolo in bolos
}
data[obsid]['vbiasrms'] = vbias
dcal = [fr for fr in core.G3File(fname_cal)]
data[obsid]['boloprops'] = dcal[0]["BolometerProperties"]
with open('vbias.pkl', 'wb') as f:
pickle.dump(data, f)
# Type failing works? Can't create mask from non-integer Intervals.
try:
mask3 = so3g.IntervalsDouble.mask([], 8)
except ValueError:
mask3 = 'failed'
assert (mask3 == 'failed')
print()
print('Map test')
tmap = so3g.MapIntervalsTime()
tmap['a'] = ti
print(' ', tmap)
print(' ', tmap['a'])
# Can we read and write them?
print()
test_filename = 'test_intervals.g3'
print('Writing to %s' % test_filename)
w = core.G3Writer(test_filename)
f = core.G3Frame()
f['map'] = tmap
f['iv'] = iv2
w.Process(f)
del w
print()
print('Reading from %s' % test_filename)
for f in core.G3File(test_filename):
print(' ', f)
class ShowScanNumber(object):
def __init__(self):
self.counter = 0
def __call__(self, frame):
if frame.type == core.G3FrameType.Scan:
print("\nInformation on Scan %s (0-indexed):\n"%self.counter)
self.counter += 1
# =============================================================================
# Before pipeline, get list of wafers and bolos to exclude if required
# -----------------------------------------------------------------------------
bad_bolos = []
wafer_list = []
for fname in args.input_files:
for frame in core.G3File(fname):
if 'BolometerProperties' in frame:
for bolo, props in frame['BolometerProperties'].items():
if props.wafer_id not in [None,'']:
waf = props.wafer_id.capitalize()
if waf in bad_wafers:
bad_bolos.append(bolo)
elif len(good_wafers) > 0 and waf not in good_wafers:
bad_bolos.append(bolo)
elif waf not in wafer_list:
wafer_list.append(waf)
if not np.isnan(props.band):
if (str(int(props.band/core.G3Units.GHz))+'GHz'
not in bands):
bad_bolos.append(bolo)
break
from spt3g import core, calibration, dfmux
import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import welch, periodogram
d = [fr for fr in core.G3File('gain_match_test.g3')]
pairnames = np.random.choice(d[4]['PairDiffTimestreams'].keys(), 10)
for pairname in pairnames:
ts = d[4]['PairDiffTimestreams'][pairname]
bolo1 = pairname.split('_')[0]
bolo2 = pairname.split('_')[1]
plt.figure()
ff, psd = periodogram(ts, fs=152.5)
plt.loglog(ff,
np.sqrt(psd) / (1. / np.sqrt(core.G3Units.Hz)) * 1e6,
label='{} - {}'.format(bolo1, bolo2))
ts = d[4]['GainMatchCoeff'][bolo1] * np.array(d[4]['CalTimestreams'][bolo1]) + \
d[4]['GainMatchCoeff'][bolo2] * np.array(d[4]['CalTimestreams'][bolo2])
ff, psd = periodogram(ts, fs=152.5)
plt.loglog(ff,
np.sqrt(psd) / (1. / np.sqrt(core.G3Units.Hz)) * 1e6,
label='{} + {}'.format(bolo1, bolo2))
plt.grid()
plt.legend()
plt.ylim([1e2, 1e8])
plt.savefig('figures/{}.png'.format(pairname), dpi=200)
plt.close()
wafer_psds_sum = {}
wafer_psds_diff = {}
