def build_noise_archive(info, force_rebuild=False): nm = load_noise_pkl(info['files'][0])[0] chipfname = nm.chip_name.replace(' ','_').replace(',','') archname = '/home/data/archive/%s.npy' % chipfname df = None if not force_rebuild and os.path.exists(archname): try: df = load_archive(archname) print "Loaded noise archive from:",archname return df except: pass nms = [] for fname in info['files']: try: nms.extend(load_noise_pkl(fname)) except Exception,e: print "couldn't get noise measurements from",fname,"error was:",e
def build_simple_archive(pklnames, index_to_id = None, archive_name=None, archive_path=None): if not type(pklnames) is list: pklnames = glob.glob(pklnames) pklnames.sort() if archive_name is None: archive_name = os.path.splitext(os.path.basename(pklnames[0]))[0] if archive_path is None: archive_path = os.path.join(os.path.split(os.path.split(pklnames[0])[0])[0],'archive') archname = os.path.join(archive_path,('%s.npy' % archive_name)) data = [] for pklname in pklnames: pkl = load_noise_pkl(pklname) if type(pkl) is list: nms = pkl else: nms = [] for k,v in pkl.items(): nms.extend(v) for nm in nms: try: data.append(nm.to_dataframe()) except AttributeError: print "skipping non noise measurement for now" df = pd.concat(data,ignore_index=True) if index_to_id is None: indexes = list(set(df.resonator_index)) indexes.sort() index_to_id = indexes def set_resonator_id(x): x['resonator_id'] = index_to_id[x.resonator_index.iloc[0]] return x df = df.groupby(['resonator_index']).apply(set_resonator_id).reset_index(drop=True) save_archive(df,archname) return df
def build_archive(info, archive_name=None, force_rebuild=False): info['files'].sort() if archive_name is None: archive_name = os.path.splitext(os.path.basename(info['files'][0]))[0] archname = '/home/data/archive/%s.npy' % archive_name df = None if not force_rebuild and os.path.exists(archname): try: df = load_archive(archname) print "Loaded noise archive from:", archname return df except: pass data = collections.defaultdict(list) data_source_off = collections.defaultdict(list) for fname in info['files']: if data: l0 = len(data.values()[0]) for k, v in data.items(): if len(v) != l0: print k, l0, len(v) print "processing", fname pkl = load_noise_pkl(fname) noise_on = pkl['noise_on_measurements'] noise_off = pkl['noise_off_sweeps'] noise_mod = pkl['noise_modulated_measurements'] pnames = noise_on[0].fit_params.keys() try: pnames.remove('a') except: pass for pn in pnames: data[pn].extend([nm.fit_params[pn].value for nm in noise_on]) data[pn + '_err'].extend( [nm.fit_params[pn].stderr for nm in noise_on]) #data[pn].extend([params[pn].value for params in noise_off]) #data[pn + '_err'].extend([params[pn].stderr for params in noise_off]) avals = [] aerrs = [] for nm in noise_on: if 'a' in nm.fit_params: avals.append(nm.fit_params['a'].value) aerrs.append(nm.fit_params['a'].stderr) else: avals.append(np.nan) aerrs.append(np.nan) data['a'].extend(avals) data['a_err'].extend(aerrs) attrs = noise_on[0].__dict__.keys() attrs.remove('fit_params') attrs.remove('zbd_voltage') private = [x for x in attrs if x.startswith('_')] for private_var in private: attrs.remove(private_var) for pn in attrs: data[pn].extend([getattr(nm, pn) for nm in noise_on]) # To do: add parameters for noise bands device_noise = [] amplifier_noise = [] for nm in noise_on: device_noise_mask = (nm.pca_freq > 1) & (nm.pca_freq < 110) amplifier_noise_mask = (nm.pca_freq > 5e3) device_noise.append(np.median(nm.pca_eigvals[1, device_noise_mask])) amplifier_noise.append( np.median(nm.pca_eigvals[1, amplifier_noise_mask])) data['device_noise'].extend(device_noise) data['amplifier_noise'].extend(amplifier_noise) data['resonator_id'].extend( [info['index_to_resnum'][nm.resonator_index] for nm in noise_on]) # Remove once SweepNoiseMeasurement has these attributes. # Add modulation frequency at which ZBD voltage was measured, from noise_mod? data['zbd_voltage'].extend([noise_mod[0].zbd_voltage] * len(noise_on)) data['mmw_mod_duty_cycle'].extend([1.0] * len(noise_on)) data['mmw_mod_phase'].extend([0.0] * len(noise_on)) data['mmw_mod_freq'].extend([0.0] * len(noise_on)) # now deal with the noise off case pnames = noise_off[0].keys() try: pnames.remove('a') except: pass for pn in pnames: data_source_off[pn].extend( [fit_params[pn].value for fit_params in noise_off]) data_source_off[pn + '_err'].extend( [fit_params[pn].stderr for fit_params in noise_off]) avals = [] aerrs = [] for fit_params in noise_off: if 'a' in fit_params: avals.append(fit_params['a'].value) aerrs.append(fit_params['a'].stderr) else: avals.append(np.nan) aerrs.append(np.nan) data_source_off['a'].extend(avals) data_source_off['a_err'].extend(aerrs) # This is a tad sloppy; just realized the noise off here won't have a resonator id, so grab it from noise_mod data_source_off['resonator_id'].extend( [info['index_to_resnum'][nm.resonator_index] for nm in noise_mod]) attrs = noise_mod[0].__dict__.keys() attrs.remove('fit_params') attrs.remove('zbd_voltage') to_remove = [x for x in attrs if x.startswith('pca')] for attr in to_remove: attrs.remove(attr) private = [x for x in attrs if x.startswith('_')] for private_var in private: attrs.remove(private_var) for pn in attrs: if pn == 'timestream_modulation_duty_cycle': data_source_off[pn].extend([1.0 for nm in noise_mod]) else: data_source_off[pn].extend( [getattr(nm, pn) for nm in noise_mod]) for nm in noise_on: nm._close_files() for nm in noise_mod: nm._close_files() df = pd.DataFrame(data) df['round_temp'] = np.round(df['end_temp'] * 1000 / 10) * 10 df_off = pd.DataFrame(data_source_off) df_off['round_temp'] = np.round(df['end_temp'] * 1000 / 10) * 10 df = pd.concat([df, df_off]) df = df.reset_index(drop=True) try: np.save(archname, df.to_records()) os.chown(archname, os.getuid(), pwd.getpwnam('readout').pw_gid) except Exception, e: print "failed to pickle", e
def build_archive(info, archive_name=None, force_rebuild=False): info['files'].sort() if archive_name is None: archive_name = os.path.splitext(os.path.basename(info['files'][0]))[0] archname = '/home/data/archive/%s.npy' % archive_name df = None if not force_rebuild and os.path.exists(archname): try: df = load_archive(archname) print "Loaded noise archive from:",archname return df except: pass data = collections.defaultdict(list) data_source_off = collections.defaultdict(list) for fname in info['files']: if data: l0 = len(data.values()[0]) for k,v in data.items(): if len(v) != l0: print k, l0, len(v) print "processing",fname pkl = load_noise_pkl(fname) noise_on = pkl['noise_on_measurements'] noise_off = pkl['noise_off_sweeps'] noise_mod = pkl['noise_modulated_measurements'] pnames = noise_on[0].fit_params.keys() try: pnames.remove('a') except: pass for pn in pnames: data[pn].extend([nm.fit_params[pn].value for nm in noise_on]) data[pn + '_err'].extend([nm.fit_params[pn].stderr for nm in noise_on]) #data[pn].extend([params[pn].value for params in noise_off]) #data[pn + '_err'].extend([params[pn].stderr for params in noise_off]) avals = [] aerrs = [] for nm in noise_on: if 'a' in nm.fit_params: avals.append(nm.fit_params['a'].value) aerrs.append(nm.fit_params['a'].stderr) else: avals.append(np.nan) aerrs.append(np.nan) data['a'].extend(avals) data['a_err'].extend(aerrs) attrs = noise_on[0].__dict__.keys() attrs.remove('fit_params') attrs.remove('zbd_voltage') private = [x for x in attrs if x.startswith('_')] for private_var in private: attrs.remove(private_var) for pn in attrs: data[pn].extend([getattr(nm,pn) for nm in noise_on]) # To do: add parameters for noise bands device_noise = [] amplifier_noise = [] for nm in noise_on: device_noise_mask = (nm.pca_freq > 1) & (nm.pca_freq < 110) amplifier_noise_mask = (nm.pca_freq > 5e3) device_noise.append(np.median(nm.pca_eigvals[1, device_noise_mask])) amplifier_noise.append(np.median(nm.pca_eigvals[1, amplifier_noise_mask])) data['device_noise'].extend(device_noise) data['amplifier_noise'].extend(amplifier_noise) data['resonator_id'].extend([info['index_to_resnum'][nm.resonator_index] for nm in noise_on]) # Remove once SweepNoiseMeasurement has these attributes. # Add modulation frequency at which ZBD voltage was measured, from noise_mod? #data['zbd_voltage'].extend([noise_mod[0].zbd_voltage] * len(noise_on)) #data['mmw_mod_duty_cycle'].extend([1.0] * len(noise_on)) #data['mmw_mod_phase'].extend([0.0] * len(noise_on)) #data['mmw_mod_freq'].extend([0.0] * len(noise_on)) # now deal with the noise off case # This is a horrible hack that doesn't get the noise off measurement into the archive. try: pnames = noise_off[0].keys() except AttributeError: pnames = noise_off[0].fit_params.keys() noise_off = [snm.fit_params for snm in noise_off] # End horrible hack try: pnames.remove('a') except: pass for pn in pnames: data_source_off[pn].extend([fit_params[pn].value for fit_params in noise_off]) data_source_off[pn + '_err'].extend([fit_params[pn].stderr for fit_params in noise_off]) avals = [] aerrs = [] for fit_params in noise_off: if 'a' in fit_params: avals.append(fit_params['a'].value) aerrs.append(fit_params['a'].stderr) else: avals.append(np.nan) aerrs.append(np.nan) data_source_off['a'].extend(avals) data_source_off['a_err'].extend(aerrs) # This is a tad sloppy; just realized the noise off here won't have a resonator id, so grab it from noise_mod data_source_off['resonator_id'].extend([info['index_to_resnum'][nm.resonator_index] for nm in noise_mod]) attrs = noise_mod[0].__dict__.keys() attrs.remove('fit_params') attrs.remove('zbd_voltage') to_remove = [x for x in attrs if x.startswith('pca')] for attr in to_remove: attrs.remove(attr) private = [x for x in attrs if x.startswith('_')] for private_var in private: attrs.remove(private_var) for pn in attrs: if pn == 'timestream_modulation_duty_cycle': data_source_off[pn].extend([1.0 for nm in noise_mod]) else: data_source_off[pn].extend([getattr(nm,pn) for nm in noise_mod]) for nm in noise_on: nm._close_files() for nm in noise_mod: nm._close_files() df = pd.DataFrame(data) df['round_temp'] = np.round(df['end_temp']*1000/10)*10 df_off = pd.DataFrame(data_source_off) df_off['round_temp'] = np.round(df['end_temp']*1000/10)*10 df = pd.concat([df,df_off]) df = df.reset_index(drop=True) try: np.save(archname,df.to_records()) os.chown(archname, os.getuid(), pwd.getpwnam('readout').pw_gid) except Exception,e: print "failed to pickle",e
def build_archive(info, archive_name=None, force_rebuild=False): info['files'].sort() if archive_name is None: archive_name = os.path.splitext(os.path.basename(info['files'][0]))[0] archname = '/home/data/archive/%s.npy' % archive_name df = None if not force_rebuild and os.path.exists(archname): try: df = load_archive(archname) print "Loaded noise archive from:",archname return df except: pass data = collections.defaultdict(list) for fname in info['files']: if data: l0 = len(data.values()[0]) for k,v in data.items(): if len(v) != l0: print k, l0, len(v) print "processing",fname pkl = load_noise_pkl(fname) noise_on = pkl['noise_on_measurements'] noise_on_orig = noise_on if len(noise_on) > len(info['index_to_resnum']): noise_on = noise_on[::6] noise_off = pkl['noise_off_sweeps'] noise_mod = pkl['noise_modulated_measurements'] pnames = noise_on[0].fit_params.keys() try: pnames.remove('a') except: pass for pn in pnames: data['noise_on_'+ pn].extend([nm.fit_params[pn].value for nm in noise_on]) data['noise_on_'+ pn + '_err'].extend([nm.fit_params[pn].stderr for nm in noise_on]) data['noise_off_'+ pn].extend([params[pn].value for params in noise_off]) data['noise_off_'+ pn + '_err'].extend([params[pn].stderr for params in noise_off]) avals = [] aerrs = [] for nm in noise_on: if nm.fit_params.has_key('a'): avals.append(nm.fit_params['a'].value) aerrs.append(nm.fit_params['a'].stderr) else: avals.append(np.nan) aerrs.append(np.nan) data['noise_on_a'].extend(avals) data['noise_on_a_err'].extend(aerrs) avals = [] aerrs = [] for nm in noise_off: if nm.has_key('a'): avals.append(nm['a'].value) aerrs.append(nm['a'].stderr) else: avals.append(np.nan) aerrs.append(np.nan) data['noise_off_a'].extend(avals) data['noise_off_a_err'].extend(aerrs) attrs = noise_on[0].__dict__.keys() attrs.remove('fit_params') attrs.remove('zbd_voltage') if '_resonator_model' in attrs: attrs.remove('_resonator_model') private = [x for x in attrs if x.startswith('_')] for private_var in private: attrs.remove(private_var) for pn in attrs: data[pn].extend([getattr(nm,pn) for nm in noise_on]) pca_fr = data['pca_freq'][0] noise100 = [] noise7k = [] for nm in noise_on: mask100 = (nm.pca_freq > 90) & (nm.pca_freq < 110) mask7k = (nm.pca_freq > 7e3) noise100.append(nm.pca_eigvals[1,mask100].mean()) noise7k.append(nm.pca_eigvals[1,mask7k].mean()) data['noise_100_Hz'].extend(noise100) data['noise_7_kHz'].extend(noise7k) data['resonator_id'].extend([info['index_to_resnum'][nm.resonator_index] for nm in noise_on]) # fixme: this seriously needs to be cleaned up. for nm in noise_mod: detuning = kid_readout.analysis.resonator.normalized_s21_to_detuning(nm.normalized_timeseries, nm.resonator_model) folded = detuning.reshape((-1,128)).mean(0) response = folded.ptp() data['noise_fractional_response'].append(response) data['folded_noise_fractional_response'].append(folded) data['zbd_voltage'].append(nm.zbd_voltage) # data['zbd_power'].append(nm.zbd_power) for nm in noise_on_orig: nm._close_files() for nm in noise_mod: nm._close_files() df = pd.DataFrame(data) df['round_temp'] = np.round(df['end_temp']*1000/10)*10 try: np.save(archname,df.to_records()) except Exception,e: print "failed to pickle",e