def import_uvfits_set_netcal(path_data_0,
data_subfolder,
path_vex,
path_out,
out_name,
tavg='scan',
exptL=[3597, 3598, 3599, 3600, 3601],
bandL=['lo', 'hi'],
filend="netcal.uvfits",
incoh_avg=False,
out_type='hdf',
polrep=None):
if not os.path.exists(path_out):
os.makedirs(path_out)
df = pd.DataFrame({})
#first all LL
for band in bandL:
for expt in exptL:
path0 = path_data_0 + 'hops-' + band + '/' + data_subfolder + str(
expt) + '/'
for filen in os.listdir(path0):
if filen.endswith(filend):
df_foo = uvfits.get_df_from_uvfit(path0 + filen,
path_vex=path_vex,
force_singlepol='no',
band=band,
round_s=0.1,
only_parallel=True,
polrep=polrep)
if tavg != -1:
if incoh_avg == False:
df_scan = ut.coh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
else:
df_scan = ut.incoh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
df = pd.concat([df, df_scan], ignore_index=True)
else:
print('no averaging')
df = pd.concat([df, df_foo], ignore_index=True)
df.drop(list(df[df.baseline.str.contains('R')].index.values), inplace=True)
df['source'] = list(map(str, df.source))
if out_type == 'hdf':
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
elif out_type == 'pic':
df.to_pickle(path_out + out_name + '.pic')
else:
return df
def import_uvfits_set(path_data_0,
path_vex,
path_out,
out_name,
bandname,
pipeline_name='hops',
tavg='scan',
only_parallel=False,
filend=".uvfits",
incoh_avg=False,
out_type='hdf',
rescale_noise=False,
polrep='circ',
old_format=True,
path_ehtim='',
closure='',
tavg_closures='scan',
precoh_avg_time=0.,
fix_sigma=0,
scale_sigma=1.):
'''
Imports whole dataset of uvfits with HOPS folder structure, or even without structure
'''
print('path_data_0 = ', path_data_0)
print('path_vex = ', path_vex)
print('path_out = ', path_out)
print('out_name = ', out_name)
print('pipeline_name= ', pipeline_name)
print('scale_sigma = ', scale_sigma)
if fix_sigma > 0:
print('fix_sigma = ', fix_sigma)
print('tavg = ', tavg)
if not os.path.exists(path_out):
os.makedirs(path_out)
df = pd.DataFrame({})
path0a = glob.glob(path_data_0 + '/*/*' + filend)
path0b = glob.glob(path_data_0 + '/*' + filend)
path0 = sorted(path0a + path0b)
###########################################################################
# VISIBILITIES
###########################################################################
for filen in path0:
print("********************************************************")
print('processing ', filen)
print("********************************************************")
try:
df_foo = uvfits.get_df_from_uvfit(filen,
path_vex=path_vex,
force_singlepol='no',
band=bandname,
round_s=0.1,
only_parallel=only_parallel,
rescale_noise=rescale_noise,
polrep=polrep,
path_ehtim=path_ehtim,
fix_sigma=fix_sigma,
scale_sigma=scale_sigma)
print('Found datapoints: ', np.shape(df_foo)[0])
#CONVERT TO OLD DF FORMATTING (SEPARATE DATA RECORD FOR EACH POLARIZATION)
if old_format:
df_foo = ut.old_format(df_foo)
if 'std_by_mean' in df_foo.columns:
df_foo.drop('std_by_mean', axis=1, inplace=True)
df_foo['std_by_mean'] = df_foo['amp']
if 'amp_moments' in df_foo.columns:
df_foo.drop('amp_moments', axis=1, inplace=True)
df_foo['amp_moments'] = df_foo['amp']
if 'sig_moments' in df_foo.columns:
df_foo.drop('sig_moments', axis=1, inplace=True)
df_foo['sig_moments'] = df_foo['amp']
print('Averaging this file...')
if incoh_avg == False:
print('Averaging coherently for ', str(tavg))
df_scan = ut.coh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
else:
if precoh_avg_time > 0:
print('Averaging coherently for ', str(precoh_avg_time))
df_coh = ut.coh_avg_vis(df_foo.copy(),
tavg=precoh_avg_time,
phase_type='phase')
print('Averaging incoherently for ', str(tavg))
df_scan = ut.incoh_avg_vis(df_coh.copy(),
tavg=tavg,
phase_type='phase')
else:
print('Averaging incoherently for ', str(tavg))
df_scan = ut.incoh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
df = pd.concat([df, df_scan.copy()], ignore_index=True)
except:
print('Nothing from this file...')
try:
df.drop(list(df[df.baseline.str.contains('R')].index.values),
inplace=True)
except:
pass
try:
df['source'] = list(map(str, df['source']))
except:
pass
try:
df.dropna(subset=['snr'], inplace=True)
except:
pass
###########################################################################
# CLOSURES
###########################################################################
if (closure == 'cphase') | (closure == 'lcamp'):
print("********************************************************")
print("******************SAVING CLOSURES***********************")
print("********************************************************")
print("Saving scan-averaged closure phases...")
bsp = cl.all_bispectra(df, phase_type='phase')
bsp.drop('fracpols', axis=1, inplace=True)
bsp.drop('snrs', axis=1, inplace=True)
bsp.drop('amps', axis=1, inplace=True)
bsp_sc = ut.coh_avg_bsp(bsp, tavg=tavg_closures)
out_name_cp = 'cp_sc_' + out_name
if out_type == 'hdf':
print('Saving file: ' + path_out + out_name_cp + '.h5')
bsp_sc.to_hdf(path_out + out_name_cp + '.h5',
key=out_name_cp,
mode='w',
format='table')
elif out_type == 'pic':
print('Saving file: ' + path_out + out_name_cp + '.pic')
bsp_sc.to_pickle(path_out + out_name_cp + '.pic')
elif out_type == 'both':
print('Saving file: ' + path_out + out_name_cp + '.h5')
bsp_sc.to_hdf(path_out + out_name_cp + '.h5',
key=out_name_cp,
mode='w',
format='table')
print('Saving file: ' + path_out + out_name_cp + '.pic')
bsp_sc.to_pickle(path_out + out_name_cp + '.pic')
print("Saving scan-averaged log closure amplitudes...")
quad = cl.all_quadruples_new(df, ctype='logcamp', debias='camp')
quad.drop('snrs', axis=1, inplace=True)
quad.drop('amps', axis=1, inplace=True)
quad_sc = ut.avg_camp(quad, tavg=tavg_closures)
out_name_lca = 'lca_sc_' + out_name
quad_sc['scan_id'] = list(map(np.int64, quad_sc.scan_id))
if out_type == 'hdf':
print('Saving file: ' + path_out + out_name_lca + '.h5')
quad_sc.to_hdf(path_out + out_name_lca + '.h5',
key=out_name_lca,
mode='w',
format='table')
elif out_type == 'pic':
print('Saving file: ' + path_out + out_name_lca + '.pic')
quad_sc.to_pickle(path_out + out_name_lca + '.pic')
elif out_type == 'both':
print('Saving file: ' + path_out + out_name_lca + '.h5')
quad_sc.to_hdf(path_out + out_name_lca + '.h5',
key=out_name_lca,
mode='w',
format='table')
print('Saving file: ' + path_out + out_name_lca + '.pic')
quad_sc.to_pickle(path_out + out_name_lca + '.pic')
if out_type == 'hdf':
print('Saving file: ' + path_out + out_name + '.h5')
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
elif out_type == 'pic':
print('Saving file: ' + path_out + out_name + '.pic')
df.to_pickle(path_out + out_name + '.pic')
elif out_type == 'both':
print('Saving file: ' + path_out + out_name + '.h5')
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
print('Saving file: ' + path_out + out_name + '.pic')
df.to_pickle(path_out + out_name + '.pic')
else:
return df
def import_uvfits_folder(path_folder,
path_vex,
path_out,
out_name,
pipeline_name='hops',
tavg='scan',
force_singlepol='no',
band='none',
only_parallel=True,
filend=".uvfits",
incoh_avg=False,
out_type='hdf',
rescale_noise=False,
polrep=None,
polrep_path_ehtwork=''):
#if polrep_path_ehtwork!='':
# sys.path.remove('/usr/local/src/ehtim')
# #sys.path.append('/home/maciek/polrep/eht-polrep')
# sys.path.append(polrep_path_ehtwork)
# importlib.reload(eh)
# sys.path.append('/usr/local/src/ehtim')
if not os.path.exists(path_out):
os.makedirs(path_out)
df = pd.DataFrame({})
path0 = path_folder
for filen in os.listdir(path0):
if filen.endswith(filend):
print('processing ', filen)
#try:
df_foo = uvfits.get_df_from_uvfit(path0 + filen,
path_vex=path_vex,
force_singlepol=force_singlepol,
band=band,
round_s=0.1,
only_parallel=only_parallel,
rescale_noise=rescale_noise,
polrep=polrep)
if 'std_by_mean' in df_foo.columns:
df_foo.drop('std_by_mean', axis=1, inplace=True)
df_foo['std_by_mean'] = df_foo['amp']
if incoh_avg == False:
df_scan = ut.coh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
else:
df_scan = ut.incoh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
df = pd.concat([df, df_scan.copy()], ignore_index=True)
df.drop(list(df[df.baseline.str.contains('R')].index.values),
inplace=True)
#except ValueError: pass
else:
pass
print(df.columns)
df.drop(list(df[df.baseline.str.contains('R')].index.values), inplace=True)
df['source'] = list(map(str, df['source']))
if band != 'none':
out_name = out_name + '_' + band
if out_type == 'hdf':
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
elif out_type == 'pic':
df.to_pickle(path_out + out_name + '.pic')
elif out_type == 'both':
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
df.to_pickle(path_out + out_name + '.pic')
else:
return df
def import_uvfits_set(path_data_0,
data_subfolder,
path_vex,
path_out,
out_name,
pipeline_name='hops',
tavg='scan',
exptL=[3597, 3598, 3599, 3600, 3601],
bandL=['lo', 'hi'],
only_parallel=True,
filend=".uvfits",
incoh_avg=False,
out_type='hdf',
rescale_noise=False,
polrep=None,
old_format=True):
if not os.path.exists(path_out):
os.makedirs(path_out)
df = pd.DataFrame({})
for band in bandL:
for expt in exptL:
path0 = path_data_0 + pipeline_name + '-' + band + '/' + data_subfolder + str(
expt) + '/'
for filen in os.listdir(path0):
if filen.endswith(filend):
print('processing ', filen)
try:
df_foo = uvfits.get_df_from_uvfit(
path0 + filen,
path_vex=path_vex,
force_singlepol='no',
band=band,
round_s=0.1,
only_parallel=only_parallel,
rescale_noise=rescale_noise,
polrep=polrep)
if 'std_by_mean' in df_foo.columns:
df_foo.drop('std_by_mean', axis=1, inplace=True)
df_foo['std_by_mean'] = df_foo['amp']
if incoh_avg == False:
df_scan = ut.coh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
else:
df_scan = ut.incoh_avg_vis(df_foo.copy(),
tavg=tavg,
phase_type='phase')
df = pd.concat([df, df_scan.copy()], ignore_index=True)
df.drop(list(
df[df.baseline.str.contains('R')].index.values),
inplace=True)
except:
pass
else:
pass
df.drop(list(df[df.baseline.str.contains('R')].index.values), inplace=True)
df['source'] = list(map(str, df['source']))
if old_format:
df = ut.old_format(df)
if len(bandL) == 1:
out_name = out_name + '_' + bandL[0]
if out_type == 'hdf':
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
elif out_type == 'pic':
df.to_pickle(path_out + out_name + '.pic')
elif out_type == 'both':
df.to_hdf(path_out + out_name + '.h5',
key=out_name,
mode='w',
format='table')
df.to_pickle(path_out + out_name + '.pic')
else:
return df