def import_uvfits_set(path_data_0, path_vex, path_out, out_name, bandname, pipeline_name='hops', tavg='scan', only_parallel=False, filend=".uvfits", incoh_avg=False, out_type='hdf', rescale_noise=False, polrep='circ', old_format=True, path_ehtim='', closure='', tavg_closures='scan', precoh_avg_time=0., fix_sigma=0, scale_sigma=1.): ''' Imports whole dataset of uvfits with HOPS folder structure, or even without structure ''' print('path_data_0 = ', path_data_0) print('path_vex = ', path_vex) print('path_out = ', path_out) print('out_name = ', out_name) print('pipeline_name= ', pipeline_name) print('scale_sigma = ', scale_sigma) if fix_sigma > 0: print('fix_sigma = ', fix_sigma) print('tavg = ', tavg) if not os.path.exists(path_out): os.makedirs(path_out) df = pd.DataFrame({}) path0a = glob.glob(path_data_0 + '/*/*' + filend) path0b = glob.glob(path_data_0 + '/*' + filend) path0 = sorted(path0a + path0b) ########################################################################### # VISIBILITIES ########################################################################### for filen in path0: print("********************************************************") print('processing ', filen) print("********************************************************") try: df_foo = uvfits.get_df_from_uvfit(filen, path_vex=path_vex, force_singlepol='no', band=bandname, round_s=0.1, only_parallel=only_parallel, rescale_noise=rescale_noise, polrep=polrep, path_ehtim=path_ehtim, fix_sigma=fix_sigma, scale_sigma=scale_sigma) print('Found datapoints: ', np.shape(df_foo)[0]) #CONVERT TO OLD DF FORMATTING (SEPARATE DATA RECORD FOR EACH POLARIZATION) if old_format: df_foo = ut.old_format(df_foo) if 'std_by_mean' in df_foo.columns: df_foo.drop('std_by_mean', axis=1, inplace=True) df_foo['std_by_mean'] = df_foo['amp'] if 'amp_moments' in df_foo.columns: df_foo.drop('amp_moments', axis=1, inplace=True) df_foo['amp_moments'] = df_foo['amp'] if 'sig_moments' in df_foo.columns: df_foo.drop('sig_moments', axis=1, inplace=True) df_foo['sig_moments'] = df_foo['amp'] print('Averaging this file...') if incoh_avg == False: print('Averaging coherently for ', str(tavg)) df_scan = ut.coh_avg_vis(df_foo.copy(), tavg=tavg, phase_type='phase') else: if precoh_avg_time > 0: print('Averaging coherently for ', str(precoh_avg_time)) df_coh = ut.coh_avg_vis(df_foo.copy(), tavg=precoh_avg_time, phase_type='phase') print('Averaging incoherently for ', str(tavg)) df_scan = ut.incoh_avg_vis(df_coh.copy(), tavg=tavg, phase_type='phase') else: print('Averaging incoherently for ', str(tavg)) df_scan = ut.incoh_avg_vis(df_foo.copy(), tavg=tavg, phase_type='phase') df = pd.concat([df, df_scan.copy()], ignore_index=True) except: print('Nothing from this file...') try: df.drop(list(df[df.baseline.str.contains('R')].index.values), inplace=True) except: pass try: df['source'] = list(map(str, df['source'])) except: pass try: df.dropna(subset=['snr'], inplace=True) except: pass ########################################################################### # CLOSURES ########################################################################### if (closure == 'cphase') | (closure == 'lcamp'): print("********************************************************") print("******************SAVING CLOSURES***********************") print("********************************************************") print("Saving scan-averaged closure phases...") bsp = cl.all_bispectra(df, phase_type='phase') bsp.drop('fracpols', axis=1, inplace=True) bsp.drop('snrs', axis=1, inplace=True) bsp.drop('amps', axis=1, inplace=True) bsp_sc = ut.coh_avg_bsp(bsp, tavg=tavg_closures) out_name_cp = 'cp_sc_' + out_name if out_type == 'hdf': print('Saving file: ' + path_out + out_name_cp + '.h5') bsp_sc.to_hdf(path_out + out_name_cp + '.h5', key=out_name_cp, mode='w', format='table') elif out_type == 'pic': print('Saving file: ' + path_out + out_name_cp + '.pic') bsp_sc.to_pickle(path_out + out_name_cp + '.pic') elif out_type == 'both': print('Saving file: ' + path_out + out_name_cp + '.h5') bsp_sc.to_hdf(path_out + out_name_cp + '.h5', key=out_name_cp, mode='w', format='table') print('Saving file: ' + path_out + out_name_cp + '.pic') bsp_sc.to_pickle(path_out + out_name_cp + '.pic') print("Saving scan-averaged log closure amplitudes...") quad = cl.all_quadruples_new(df, ctype='logcamp', debias='camp') quad.drop('snrs', axis=1, inplace=True) quad.drop('amps', axis=1, inplace=True) quad_sc = ut.avg_camp(quad, tavg=tavg_closures) out_name_lca = 'lca_sc_' + out_name quad_sc['scan_id'] = list(map(np.int64, quad_sc.scan_id)) if out_type == 'hdf': print('Saving file: ' + path_out + out_name_lca + '.h5') quad_sc.to_hdf(path_out + out_name_lca + '.h5', key=out_name_lca, mode='w', format='table') elif out_type == 'pic': print('Saving file: ' + path_out + out_name_lca + '.pic') quad_sc.to_pickle(path_out + out_name_lca + '.pic') elif out_type == 'both': print('Saving file: ' + path_out + out_name_lca + '.h5') quad_sc.to_hdf(path_out + out_name_lca + '.h5', key=out_name_lca, mode='w', format='table') print('Saving file: ' + path_out + out_name_lca + '.pic') quad_sc.to_pickle(path_out + out_name_lca + '.pic') if out_type == 'hdf': print('Saving file: ' + path_out + out_name + '.h5') df.to_hdf(path_out + out_name + '.h5', key=out_name, mode='w', format='table') elif out_type == 'pic': print('Saving file: ' + path_out + out_name + '.pic') df.to_pickle(path_out + out_name + '.pic') elif out_type == 'both': print('Saving file: ' + path_out + out_name + '.h5') df.to_hdf(path_out + out_name + '.h5', key=out_name, mode='w', format='table') print('Saving file: ' + path_out + out_name + '.pic') df.to_pickle(path_out + out_name + '.pic') else: return df
bsp = cl.all_bispectra(data, phase_type='phase') bsp.drop('fracpols', axis=1, inplace=True) bsp.drop('snrs', axis=1, inplace=True) bsp.drop('amps', axis=1, inplace=True) bsp_sc = ut.coh_avg_bsp(bsp, tavg='scan') out_name = 'cp_sc_' + filen.split('.')[0] bsp_sc.to_hdf(path_out + out_name + '.h5', key=out_name, mode='w', format='table') print("Saving log closure amplitudes from scan-averaged data...") #log closure amplitudes from scan-averaged segments filen = 'apc_sc.h5' data = pd.read_hdf(path_out + filen, filen.split('.')[0]) quad = cl.all_quadruples_new(data, ctype='logcamp', debias='camp') quad.drop('snrs', axis=1, inplace=True) quad.drop('amps', axis=1, inplace=True) out_name = 'lca_' + filen.split('.')[0] quad['scan_id'] = list(map(np.int64, quad.scan_id)) quad.to_hdf(path_out + out_name + '.h5', key=out_name, mode='w', format='table') print("Saving scan-averaged log closure amplitudes from 10s data...") #log closure amplitudes phases in 10s segments >> scan averaged filen = 'apc_10s.h5' data = pd.read_hdf(path_out + filen, filen.split('.')[0]) quad = cl.all_quadruples_new(data, ctype='logcamp', debias='camp') quad.drop('snrs', axis=1, inplace=True)