def build_noise_archive(info, force_rebuild=False):
nm = load_noise_pkl(info['files'][0])[0]
chipfname = nm.chip_name.replace(' ','_').replace(',','')
archname = '/home/data/archive/%s.npy' % chipfname
df = None
if not force_rebuild and os.path.exists(archname):
try:
df = load_archive(archname)
print "Loaded noise archive from:",archname
return df
except:
pass
nms = []
for fname in info['files']:
try:
nms.extend(load_noise_pkl(fname))
except Exception,e:
print "couldn't get noise measurements from",fname,"error was:",e
def build_simple_archive(pklnames, index_to_id = None, archive_name=None, archive_path=None):
if not type(pklnames) is list:
pklnames = glob.glob(pklnames)
pklnames.sort()
if archive_name is None:
archive_name = os.path.splitext(os.path.basename(pklnames[0]))[0]
if archive_path is None:
archive_path = os.path.join(os.path.split(os.path.split(pklnames[0])[0])[0],'archive')
archname = os.path.join(archive_path,('%s.npy' % archive_name))
data = []
for pklname in pklnames:
pkl = load_noise_pkl(pklname)
if type(pkl) is list:
nms = pkl
else:
nms = []
for k,v in pkl.items():
nms.extend(v)
for nm in nms:
try:
data.append(nm.to_dataframe())
except AttributeError:
print "skipping non noise measurement for now"
df = pd.concat(data,ignore_index=True)
if index_to_id is None:
indexes = list(set(df.resonator_index))
indexes.sort()
index_to_id = indexes
def set_resonator_id(x):
x['resonator_id'] = index_to_id[x.resonator_index.iloc[0]]
return x
df = df.groupby(['resonator_index']).apply(set_resonator_id).reset_index(drop=True)
save_archive(df,archname)
return df
def build_archive(info, archive_name=None, force_rebuild=False):
info['files'].sort()
if archive_name is None:
archive_name = os.path.splitext(os.path.basename(info['files'][0]))[0]
archname = '/home/data/archive/%s.npy' % archive_name
df = None
if not force_rebuild and os.path.exists(archname):
try:
df = load_archive(archname)
print "Loaded noise archive from:", archname
return df
except:
pass
data = collections.defaultdict(list)
data_source_off = collections.defaultdict(list)
for fname in info['files']:
if data:
l0 = len(data.values()[0])
for k, v in data.items():
if len(v) != l0:
print k, l0, len(v)
print "processing", fname
pkl = load_noise_pkl(fname)
noise_on = pkl['noise_on_measurements']
noise_off = pkl['noise_off_sweeps']
noise_mod = pkl['noise_modulated_measurements']
pnames = noise_on[0].fit_params.keys()
try:
pnames.remove('a')
except:
pass
for pn in pnames:
data[pn].extend([nm.fit_params[pn].value for nm in noise_on])
data[pn + '_err'].extend(
[nm.fit_params[pn].stderr for nm in noise_on])
#data[pn].extend([params[pn].value for params in noise_off])
#data[pn + '_err'].extend([params[pn].stderr for params in noise_off])
avals = []
aerrs = []
for nm in noise_on:
if 'a' in nm.fit_params:
avals.append(nm.fit_params['a'].value)
aerrs.append(nm.fit_params['a'].stderr)
else:
avals.append(np.nan)
aerrs.append(np.nan)
data['a'].extend(avals)
data['a_err'].extend(aerrs)
attrs = noise_on[0].__dict__.keys()
attrs.remove('fit_params')
attrs.remove('zbd_voltage')
private = [x for x in attrs if x.startswith('_')]
for private_var in private:
attrs.remove(private_var)
for pn in attrs:
data[pn].extend([getattr(nm, pn) for nm in noise_on])
# To do: add parameters for noise bands
device_noise = []
amplifier_noise = []
for nm in noise_on:
device_noise_mask = (nm.pca_freq > 1) & (nm.pca_freq < 110)
amplifier_noise_mask = (nm.pca_freq > 5e3)
device_noise.append(np.median(nm.pca_eigvals[1,
device_noise_mask]))
amplifier_noise.append(
np.median(nm.pca_eigvals[1, amplifier_noise_mask]))
data['device_noise'].extend(device_noise)
data['amplifier_noise'].extend(amplifier_noise)
data['resonator_id'].extend(
[info['index_to_resnum'][nm.resonator_index] for nm in noise_on])
# Remove once SweepNoiseMeasurement has these attributes.
# Add modulation frequency at which ZBD voltage was measured, from noise_mod?
data['zbd_voltage'].extend([noise_mod[0].zbd_voltage] * len(noise_on))
data['mmw_mod_duty_cycle'].extend([1.0] * len(noise_on))
data['mmw_mod_phase'].extend([0.0] * len(noise_on))
data['mmw_mod_freq'].extend([0.0] * len(noise_on))
# now deal with the noise off case
pnames = noise_off[0].keys()
try:
pnames.remove('a')
except:
pass
for pn in pnames:
data_source_off[pn].extend(
[fit_params[pn].value for fit_params in noise_off])
data_source_off[pn + '_err'].extend(
[fit_params[pn].stderr for fit_params in noise_off])
avals = []
aerrs = []
for fit_params in noise_off:
if 'a' in fit_params:
avals.append(fit_params['a'].value)
aerrs.append(fit_params['a'].stderr)
else:
avals.append(np.nan)
aerrs.append(np.nan)
data_source_off['a'].extend(avals)
data_source_off['a_err'].extend(aerrs)
# This is a tad sloppy; just realized the noise off here won't have a resonator id, so grab it from noise_mod
data_source_off['resonator_id'].extend(
[info['index_to_resnum'][nm.resonator_index] for nm in noise_mod])
attrs = noise_mod[0].__dict__.keys()
attrs.remove('fit_params')
attrs.remove('zbd_voltage')
to_remove = [x for x in attrs if x.startswith('pca')]
for attr in to_remove:
attrs.remove(attr)
private = [x for x in attrs if x.startswith('_')]
for private_var in private:
attrs.remove(private_var)
for pn in attrs:
if pn == 'timestream_modulation_duty_cycle':
data_source_off[pn].extend([1.0 for nm in noise_mod])
else:
data_source_off[pn].extend(
[getattr(nm, pn) for nm in noise_mod])
for nm in noise_on:
nm._close_files()
for nm in noise_mod:
nm._close_files()
df = pd.DataFrame(data)
df['round_temp'] = np.round(df['end_temp'] * 1000 / 10) * 10
df_off = pd.DataFrame(data_source_off)
df_off['round_temp'] = np.round(df['end_temp'] * 1000 / 10) * 10
df = pd.concat([df, df_off])
df = df.reset_index(drop=True)
try:
np.save(archname, df.to_records())
os.chown(archname, os.getuid(), pwd.getpwnam('readout').pw_gid)
except Exception, e:
print "failed to pickle", e
def build_archive(info, archive_name=None, force_rebuild=False):
info['files'].sort()
if archive_name is None:
archive_name = os.path.splitext(os.path.basename(info['files'][0]))[0]
archname = '/home/data/archive/%s.npy' % archive_name
df = None
if not force_rebuild and os.path.exists(archname):
try:
df = load_archive(archname)
print "Loaded noise archive from:",archname
return df
except:
pass
data = collections.defaultdict(list)
data_source_off = collections.defaultdict(list)
for fname in info['files']:
if data:
l0 = len(data.values()[0])
for k,v in data.items():
if len(v) != l0:
print k, l0, len(v)
print "processing",fname
pkl = load_noise_pkl(fname)
noise_on = pkl['noise_on_measurements']
noise_off = pkl['noise_off_sweeps']
noise_mod = pkl['noise_modulated_measurements']
pnames = noise_on[0].fit_params.keys()
try:
pnames.remove('a')
except:
pass
for pn in pnames:
data[pn].extend([nm.fit_params[pn].value for nm in noise_on])
data[pn + '_err'].extend([nm.fit_params[pn].stderr for nm in noise_on])
#data[pn].extend([params[pn].value for params in noise_off])
#data[pn + '_err'].extend([params[pn].stderr for params in noise_off])
avals = []
aerrs = []
for nm in noise_on:
if 'a' in nm.fit_params:
avals.append(nm.fit_params['a'].value)
aerrs.append(nm.fit_params['a'].stderr)
else:
avals.append(np.nan)
aerrs.append(np.nan)
data['a'].extend(avals)
data['a_err'].extend(aerrs)
attrs = noise_on[0].__dict__.keys()
attrs.remove('fit_params')
attrs.remove('zbd_voltage')
private = [x for x in attrs if x.startswith('_')]
for private_var in private:
attrs.remove(private_var)
for pn in attrs:
data[pn].extend([getattr(nm,pn) for nm in noise_on])
# To do: add parameters for noise bands
device_noise = []
amplifier_noise = []
for nm in noise_on:
device_noise_mask = (nm.pca_freq > 1) & (nm.pca_freq < 110)
amplifier_noise_mask = (nm.pca_freq > 5e3)
device_noise.append(np.median(nm.pca_eigvals[1, device_noise_mask]))
amplifier_noise.append(np.median(nm.pca_eigvals[1, amplifier_noise_mask]))
data['device_noise'].extend(device_noise)
data['amplifier_noise'].extend(amplifier_noise)
data['resonator_id'].extend([info['index_to_resnum'][nm.resonator_index] for nm in noise_on])
# Remove once SweepNoiseMeasurement has these attributes.
# Add modulation frequency at which ZBD voltage was measured, from noise_mod?
#data['zbd_voltage'].extend([noise_mod[0].zbd_voltage] * len(noise_on))
#data['mmw_mod_duty_cycle'].extend([1.0] * len(noise_on))
#data['mmw_mod_phase'].extend([0.0] * len(noise_on))
#data['mmw_mod_freq'].extend([0.0] * len(noise_on))
# now deal with the noise off case
# This is a horrible hack that doesn't get the noise off measurement into the archive.
try:
pnames = noise_off[0].keys()
except AttributeError:
pnames = noise_off[0].fit_params.keys()
noise_off = [snm.fit_params for snm in noise_off]
# End horrible hack
try:
pnames.remove('a')
except:
pass
for pn in pnames:
data_source_off[pn].extend([fit_params[pn].value for fit_params in noise_off])
data_source_off[pn + '_err'].extend([fit_params[pn].stderr for fit_params in noise_off])
avals = []
aerrs = []
for fit_params in noise_off:
if 'a' in fit_params:
avals.append(fit_params['a'].value)
aerrs.append(fit_params['a'].stderr)
else:
avals.append(np.nan)
aerrs.append(np.nan)
data_source_off['a'].extend(avals)
data_source_off['a_err'].extend(aerrs)
# This is a tad sloppy; just realized the noise off here won't have a resonator id, so grab it from noise_mod
data_source_off['resonator_id'].extend([info['index_to_resnum'][nm.resonator_index] for nm in noise_mod])
attrs = noise_mod[0].__dict__.keys()
attrs.remove('fit_params')
attrs.remove('zbd_voltage')
to_remove = [x for x in attrs if x.startswith('pca')]
for attr in to_remove:
attrs.remove(attr)
private = [x for x in attrs if x.startswith('_')]
for private_var in private:
attrs.remove(private_var)
for pn in attrs:
if pn == 'timestream_modulation_duty_cycle':
data_source_off[pn].extend([1.0 for nm in noise_mod])
else:
data_source_off[pn].extend([getattr(nm,pn) for nm in noise_mod])
for nm in noise_on:
nm._close_files()
for nm in noise_mod:
nm._close_files()
df = pd.DataFrame(data)
df['round_temp'] = np.round(df['end_temp']*1000/10)*10
df_off = pd.DataFrame(data_source_off)
df_off['round_temp'] = np.round(df['end_temp']*1000/10)*10
df = pd.concat([df,df_off])
df = df.reset_index(drop=True)
try:
np.save(archname,df.to_records())
os.chown(archname, os.getuid(), pwd.getpwnam('readout').pw_gid)
except Exception,e:
print "failed to pickle",e
def build_archive(info, archive_name=None, force_rebuild=False):
info['files'].sort()
if archive_name is None:
archive_name = os.path.splitext(os.path.basename(info['files'][0]))[0]
archname = '/home/data/archive/%s.npy' % archive_name
df = None
if not force_rebuild and os.path.exists(archname):
try:
df = load_archive(archname)
print "Loaded noise archive from:",archname
return df
except:
pass
data = collections.defaultdict(list)
for fname in info['files']:
if data:
l0 = len(data.values()[0])
for k,v in data.items():
if len(v) != l0:
print k, l0, len(v)
print "processing",fname
pkl = load_noise_pkl(fname)
noise_on = pkl['noise_on_measurements']
noise_on_orig = noise_on
if len(noise_on) > len(info['index_to_resnum']):
noise_on = noise_on[::6]
noise_off = pkl['noise_off_sweeps']
noise_mod = pkl['noise_modulated_measurements']
pnames = noise_on[0].fit_params.keys()
try:
pnames.remove('a')
except:
pass
for pn in pnames:
data['noise_on_'+ pn].extend([nm.fit_params[pn].value for nm in noise_on])
data['noise_on_'+ pn + '_err'].extend([nm.fit_params[pn].stderr for nm in noise_on])
data['noise_off_'+ pn].extend([params[pn].value for params in noise_off])
data['noise_off_'+ pn + '_err'].extend([params[pn].stderr for params in noise_off])
avals = []
aerrs = []
for nm in noise_on:
if nm.fit_params.has_key('a'):
avals.append(nm.fit_params['a'].value)
aerrs.append(nm.fit_params['a'].stderr)
else:
avals.append(np.nan)
aerrs.append(np.nan)
data['noise_on_a'].extend(avals)
data['noise_on_a_err'].extend(aerrs)
avals = []
aerrs = []
for nm in noise_off:
if nm.has_key('a'):
avals.append(nm['a'].value)
aerrs.append(nm['a'].stderr)
else:
avals.append(np.nan)
aerrs.append(np.nan)
data['noise_off_a'].extend(avals)
data['noise_off_a_err'].extend(aerrs)
attrs = noise_on[0].__dict__.keys()
attrs.remove('fit_params')
attrs.remove('zbd_voltage')
if '_resonator_model' in attrs:
attrs.remove('_resonator_model')
private = [x for x in attrs if x.startswith('_')]
for private_var in private:
attrs.remove(private_var)
for pn in attrs:
data[pn].extend([getattr(nm,pn) for nm in noise_on])
pca_fr = data['pca_freq'][0]
noise100 = []
noise7k = []
for nm in noise_on:
mask100 = (nm.pca_freq > 90) & (nm.pca_freq < 110)
mask7k = (nm.pca_freq > 7e3)
noise100.append(nm.pca_eigvals[1,mask100].mean())
noise7k.append(nm.pca_eigvals[1,mask7k].mean())
data['noise_100_Hz'].extend(noise100)
data['noise_7_kHz'].extend(noise7k)
data['resonator_id'].extend([info['index_to_resnum'][nm.resonator_index] for nm in noise_on])
# fixme: this seriously needs to be cleaned up.
for nm in noise_mod:
detuning = kid_readout.analysis.resonator.normalized_s21_to_detuning(nm.normalized_timeseries,
nm.resonator_model)
folded = detuning.reshape((-1,128)).mean(0)
response = folded.ptp()
data['noise_fractional_response'].append(response)
data['folded_noise_fractional_response'].append(folded)
data['zbd_voltage'].append(nm.zbd_voltage)
# data['zbd_power'].append(nm.zbd_power)
for nm in noise_on_orig:
nm._close_files()
for nm in noise_mod:
nm._close_files()
df = pd.DataFrame(data)
df['round_temp'] = np.round(df['end_temp']*1000/10)*10
try:
np.save(archname,df.to_records())
except Exception,e:
print "failed to pickle",e