def import_uvfits_set(path_data_0,
path_vex,
path_out,
out_name,
bandname,
pipeline_name='hops',
tavg='scan',
only_parallel=False,
filend=".uvfits",
incoh_avg=False,
out_type='hdf',
rescale_noise=False,
polrep='circ',
old_format=True,
path_ehtim='',
closure='',
tavg_closures='scan',
precoh_avg_time=0.,
fix_sigma=0,
scale_sigma=1.):
'''
Imports whole dataset of uvfits with HOPS folder structure, or even without structure
'''
print('path_data_0 = ', path_data_0)
print('path_vex = ', path_vex)
print('path_out = ', path_out)
print('out_name = ', out_name)
print('pipeline_name= ', pipeline_name)
print('scale_sigma = ', scale_sigma)
if fix_sigma > 0:
print('fix_sigma = ', fix_sigma)
print('tavg = ', tavg)
if not os.path.exists(path_out):
os.makedirs(path_out)
df = pd.DataFrame({})
path0a = glob.glob(path_data_0 + '/*/*' + filend)
path0b = glob.glob(path_data_0 + '/*' + filend)
path0 = sorted(path0a + path0b)
###########################################################################
# VISIBILITIES
###########################################################################
for filen in path0:
print("********************************************************")
print('processing ', filen)
print("********************************************************")
try:
df_foo = uvfits.get_df_from_uvfit(filen,
path_vex=path_vex,
force_singlepol='no',
band=bandname,
round_s=0.1,
only_parallel=only_parallel,
rescale_noise=rescale_noise,
polrep=polrep,
path_ehtim=path_ehtim,
fix_sigma=fix_sigma,
scale_sigma=scale_sigma)
print('Found datapoints: ', np.shape(df_foo)[0])
#CONVERT TO OLD DF FORMATTING (SEPARATE DATA RECORD FOR EACH POLARIZATION)
if old_format:
df_foo = ut.old_format(df_foo)
if 'std_by_mean' in df_foo.columns:
df_foo.drop('std_by_mean', axis=1, inplace=True)
df_foo['std_by_mean'] = df_foo['amp']
if 'amp_moments' in df_foo.columns:
df_foo.drop('amp_moments', axis=1, inplace=True)
df_foo['amp_moments'] = df_foo['amp']
if 'sig_moments' in df_foo.columns:
df_foo.drop('sig_moments', axis=1, inplace=True)
df_foo['sig_moments'] = df_foo['amp']
print('Averaging this file...')
if incoh_avg == False:
print('Averaging coherently for ', str(tavg))
df_scan = ut.coh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
else:
if precoh_avg_time > 0:
print('Averaging coherently for ', str(precoh_avg_time))
df_coh = ut.coh_avg_vis(df_foo.copy(),
tavg=precoh_avg_time,
phase_type='phase')
print('Averaging incoherently for ', str(tavg))
df_scan = ut.incoh_avg_vis(df_coh.copy(),
tavg=tavg,
phase_type='phase')
else:
print('Averaging incoherently for ', str(tavg))
df_scan = ut.incoh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
df = pd.concat([df, df_scan.copy()], ignore_index=True)
except:
print('Nothing from this file...')
try:
df.drop(list(df[df.baseline.str.contains('R')].index.values),
inplace=True)
except:
pass
try:
df['source'] = list(map(str, df['source']))
except:
pass
try:
df.dropna(subset=['snr'], inplace=True)
except:
pass
###########################################################################
# CLOSURES
###########################################################################
if (closure == 'cphase') | (closure == 'lcamp'):
print("********************************************************")
print("******************SAVING CLOSURES***********************")
print("********************************************************")
print("Saving scan-averaged closure phases...")
bsp = cl.all_bispectra(df, phase_type='phase')
bsp.drop('fracpols', axis=1, inplace=True)
bsp.drop('snrs', axis=1, inplace=True)
bsp.drop('amps', axis=1, inplace=True)
bsp_sc = ut.coh_avg_bsp(bsp, tavg=tavg_closures)
out_name_cp = 'cp_sc_' + out_name
if out_type == 'hdf':
print('Saving file: ' + path_out + out_name_cp + '.h5')
bsp_sc.to_hdf(path_out + out_name_cp + '.h5',
key=out_name_cp,
mode='w',
format='table')
elif out_type == 'pic':
print('Saving file: ' + path_out + out_name_cp + '.pic')
bsp_sc.to_pickle(path_out + out_name_cp + '.pic')
elif out_type == 'both':
print('Saving file: ' + path_out + out_name_cp + '.h5')
bsp_sc.to_hdf(path_out + out_name_cp + '.h5',
key=out_name_cp,
mode='w',
format='table')
print('Saving file: ' + path_out + out_name_cp + '.pic')
bsp_sc.to_pickle(path_out + out_name_cp + '.pic')
print("Saving scan-averaged log closure amplitudes...")
quad = cl.all_quadruples_new(df, ctype='logcamp', debias='camp')
quad.drop('snrs', axis=1, inplace=True)
quad.drop('amps', axis=1, inplace=True)
quad_sc = ut.avg_camp(quad, tavg=tavg_closures)
out_name_lca = 'lca_sc_' + out_name
quad_sc['scan_id'] = list(map(np.int64, quad_sc.scan_id))
if out_type == 'hdf':
print('Saving file: ' + path_out + out_name_lca + '.h5')
quad_sc.to_hdf(path_out + out_name_lca + '.h5',
key=out_name_lca,
mode='w',
format='table')
elif out_type == 'pic':
print('Saving file: ' + path_out + out_name_lca + '.pic')
quad_sc.to_pickle(path_out + out_name_lca + '.pic')
elif out_type == 'both':
print('Saving file: ' + path_out + out_name_lca + '.h5')
quad_sc.to_hdf(path_out + out_name_lca + '.h5',
key=out_name_lca,
mode='w',
format='table')
print('Saving file: ' + path_out + out_name_lca + '.pic')
quad_sc.to_pickle(path_out + out_name_lca + '.pic')
if out_type == 'hdf':
print('Saving file: ' + path_out + out_name + '.h5')
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
elif out_type == 'pic':
print('Saving file: ' + path_out + out_name + '.pic')
df.to_pickle(path_out + out_name + '.pic')
elif out_type == 'both':
print('Saving file: ' + path_out + out_name + '.h5')
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
print('Saving file: ' + path_out + out_name + '.pic')
df.to_pickle(path_out + out_name + '.pic')
else:
return df
bsp = cl.all_bispectra(data, phase_type='phase')
bsp.drop('fracpols', axis=1, inplace=True)
bsp.drop('snrs', axis=1, inplace=True)
bsp.drop('amps', axis=1, inplace=True)
out_name = 'cp_' + filen.split('.')[0]
bsp.to_hdf(path_out + out_name + '.h5', key=out_name, mode='w', format='table')
print("Saving scan-averaged closure phases from 10s data...")
#closure phases in 10s segments >> scan averaged
filen = 'apc_10s.h5'
data = pd.read_hdf(path_out + filen, filen.split('.')[0])
bsp = cl.all_bispectra(data, phase_type='phase')
bsp.drop('fracpols', axis=1, inplace=True)
bsp.drop('snrs', axis=1, inplace=True)
bsp.drop('amps', axis=1, inplace=True)
bsp_sc = ut.coh_avg_bsp(bsp, tavg='scan')
out_name = 'cp_sc_' + filen.split('.')[0]
bsp_sc.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
print("Saving log closure amplitudes from scan-averaged data...")
#log closure amplitudes from scan-averaged segments
filen = 'apc_sc.h5'
data = pd.read_hdf(path_out + filen, filen.split('.')[0])
quad = cl.all_quadruples_new(data, ctype='logcamp', debias='camp')
quad.drop('snrs', axis=1, inplace=True)
quad.drop('amps', axis=1, inplace=True)
out_name = 'lca_' + filen.split('.')[0]
quad['scan_id'] = list(map(np.int64, quad.scan_id))