desc_sum = 0 # low value means it was caught in descending order
clump_sum = np.count_nonzero(
np.absolute(np.diff(events[:, -1])) >
1) # low value here means it was caught in clumps
for i, t in enumerate(events[:, -1]):
asc_sum += np.count_nonzero(events[i:, -1] < t)
desc_sum += np.count_nonzero(events[i:, -1] > t)
asc_sum_total.append(asc_sum)
desc_sum_total.append(desc_sum)
clump_sum_total.append(clump_sum)
asc_hist = np.histogram(asc_sum_total, bins='auto')
desc_hist = np.histogram(desc_sum_total, bins='auto')
clump_hist = np.histogram(clump_sum_total, bins='auto')
if k == 1:
print(clump_hist[0])
assert (np.sum(asc_hist[0]) == np.sum(desc_hist[0]))
assert (np.sum(asc_hist[0]) == np.sum(clump_hist[0]))
fig, ax = plt.subplots(figsize=(14, 8))
for m, hist in enumerate([asc_hist, desc_hist, clump_hist]):
x = hist[1][:-1] + 0.5 * np.diff(hist[1])
pl.error_plot(fig,
ax,
x,
hist[0],
xlabel='Sum Total',
ylabel='counts',
label=labels[m],
title='Lone Run Order Parameter Histogram, %s' %
titles[k])
fig.savefig('%s/%s.png' % (outdir, titles[k]))
fig.savefig('%s/%s_INS_original_data.pdf' % (ss.outpath, obs))
fig_ms.savefig('%s/%s_INS_original_data_ms.pdf' % (ss.outpath, obs))
fig_hist, ax_hist = plt.subplots(figsize=(16, 9))
counts, bins = np.histogram(ss.INS.data_ms[:, 0, :, 0], bins=bins)
exp_counts, exp_var = util.hist_fit(counts, bins)
counts = np.append(counts, 0)
exp_counts = np.append(exp_counts, 0)
exp_var = np.append(exp_var, 0)
plot_lib.error_plot(
fig_hist,
ax_hist,
bins,
counts,
xlabel='Deviation ($\hat{\sigma}$)',
ylabel='Counts',
yscale='log',
drawstyle='steps-post',
ylim=[0.5, 10**(int(np.log10(np.amax(exp_counts))) + 1)],
label='Measurements',
legend=True)
plot_lib.error_plot(
fig_hist,
ax_hist,
bins,
exp_counts,
yerr=np.sqrt(exp_var),
xlabel='Deviation ($\hat{\sigma}$)',
ylabel='Counts',
yscale='log',
drawstyle='steps-post',
ant_locs = np.load(
'/Users/mike_e_dubs/Repositories/MJW-MWA/Useful_Information/MWA_ant_pos.npy'
)
sav = readsav(
'/Users/mike_e_dubs/MWA/FHD/1061313128_Noflag/1061313128_cal.sav')
c = 128 * ['']
c = np.array(c)
print(np.absolute(sav.cal.gain[0][0][:, 6]) < 1)
c[np.absolute(sav.cal.gain[0][0][:, 6]) < 1] = 'r'
c[np.absolute(sav.cal.gain[0][0][:, 6]) > 1] = 'b'
fig, ax = plt.subplots(figsize=(14, 8))
fig_line, ax_line = plt.subplots(figsize=(14, 8))
plot_lib.scatter_plot_2d(fig,
ax,
ant_locs[:, 0],
ant_locs[:, 1],
c=c,
xlabel='X (m)',
ylabel='Y (m)')
plot_lib.error_plot(fig_line,
ax_line,
range(len(ant_locs)),
np.sqrt(ant_locs[:, 0]**2 + ant_locs[:, 1]**2),
ylabel='Radial Distance (m)',
xlabel='Antenna Index')
fig.savefig('/Users/mike_e_dubs/MWA/FHD/1061313128_Noflag/ant_locs.png')
fig_line.savefig(
'/Users/mike_e_dubs/MWA/FHD/1061313128_Noflag/ant_rad_distance.png')
c_TV7 = np.array(c_TV7)
c_broad7 = np.array(c_broad7)
where7 = c_TV7 == 0
where_broad = c_broad7 == 0
c_tv = np.copy(c_TV7)
c_tv[where7] = c_broad7[where7]
counts, bins = np.histogram(c, bins=np.linspace(0, 1, num=50))
Nz = len(c[c == 0])
counts = np.append(counts, 0)
plot_lib.error_plot(fig_hist,
ax_hist,
bins,
counts,
drawstyle='steps-post',
title='Total Occupation Histogram %i$\hat{\sigma}$' %
sig_thresh,
xlabel='Occupation Fraction',
ylabel='Counts',
ylim=[0.1, 10**3],
yscale='log',
label='$N_z = $ %i' % Nz,
legend=True,
leg_size='xx-large')
fig_hist.savefig('%s/%is_hist_total_occ.png' % (basedir, sig_thresh))
plt.close(fig_hist)
fig_scat, ax_scat = plt.subplots(figsize=(8, 4.5))
plot_lib.scatter_plot_2d(
fig_scat,
ax_scat,
x_data[c == 0],
obslist = util.make_obslist(args.obsfile)
edges = [0 + 16 * i for i in range(24)] + [15 + 16 * i for i in range(24)]
freqs = np.load('/Users/mike_e_dubs/python_stuff/MJW-MWA/Useful_Information/MWA_Highband_Freq_Array.npy')
for i, obs in enumerate(obslist):
read_paths = util.read_paths_INS(args.basedir, args.flag_choice, obs)
ins = INS(read_paths=read_paths, flag_choice=args.flag_choice, obs=obs,
outpath=args.outdir, order=args.order)
cp.INS_plot(ins)
mf = MF(ins, sig_thresh=5, shape_dict=shape_dict)
mf.apply_match_test(order=args.order)
cp.MF_plot(mf)
if not i:
occ_num = np.ma.masked_array(mf.INS.data.mask)
occ_den = np.ma.masked_array(np.ones(occ_num.shape))
occ_num[:, 0, edges] = np.ma.masked
occ_den[:, 0, edges] = np.ma.masked
else:
occ_num = occ_num + mf.INS.data.mask
occ_den = occ_den + np.ones(occ_num.shape)
occ_freq = occ_num.sum(axis=(0, 1, 3)) / occ_den.sum(axis=(0, 1, 3)) * 100
occ_total = occ_num.sum() / occ_den.sum() * 100
fig, ax = plt.subplots(figsize=(14, 8))
plot_lib.error_plot(fig, ax, freqs * 10**(-6), occ_freq,
title='Golden Set RFI Frequency Post COTTER',
xlabel='Frequency (Mhz)', ylabel='RFI Occupancy %%',
legend=False)
fig.savefig('%s/figs/RFI_Occupancy.png' % args.outdir)
print('The total RFI occupancy is %s %%' % (occ_total))
pdf1 = np.append(0, pdf1)
pdf2 = np.append(0, pdf2)
pdf_exp = np.append(0, pdf_exp)
var = np.append(0, var)
err = np.sqrt(var)
err /= (Nexp * np.diff(ins2.bins)[0])
# err = np.sqrt(var) / (np.sum(exp) * np.diff(ins2.bins)[0])
# print(err.shape)
fig3, ax3 = plt.subplots(figsize=(8, 9))
#yerr=np.sqrt(var)[:900]
plot_lib.error_plot(fig3,
ax3,
ins2.bins[:600],
pdf2[:600],
drawstyle='steps-post',
xlabel='Deviation ($\hat{\sigma}$)',
label='XRFI Flagging',
legend=True,
yscale='log')
plot_lib.error_plot(fig3,
ax3,
ins2.bins[:600],
pdf_exp[:600],
drawstyle='steps-post',
xlabel='Deviation ($\hat{\sigma}$)',
label='Normal Fit',
legend=True,
yscale='log',
yerr=err[:600])
plot_lib.error_plot(fig3,
from SSINS import SS, plot_lib
import numpy as np
import matplotlib.pyplot as plt
freq_chans = np.load('/Volumes/Faramir/uvfits/freq_chans.npy')
ss = SS(obs='1066742016', inpath='/Volumes/Faramir/uvfits/1066742016.uvfits',
outpath='/Users/mikewilensky/576/1066742016', read_kwargs={'file_type': 'uvfits',
'ant_str': 'cross',
'freq_chans': freq_chans})
freq_chans = np.load('/Volumes/Faramir/uvfits/freq_chans.npy')
counts_1, bins_1 = np.histogram(ss.UV.data_array[2:21], bins='auto')
counts_2, bins_2 = np.histogram(ss.UV.data_array[21:-4], bins='auto')
counts_1 = np.append(counts_1, 0)
counts_2 = np.append(counts_2, 0)
fig, ax = plt.subplots()
plot_lib.error_plot(fig, ax, bins_1, counts_1, label='Contaminated Times', yscale='log',
xscale='linear', drawstyle='steps-post')
plot_lib.error_plot(fig, ax, bins_2, counts_2, label='Clean Times', yscale='log',
xscale='linear', drawstyle='steps-post')
fig.savefig('/Users/mikewilensky/fuck_you.png')
from SSINS import SS, plot_lib
from pyuvdata import UVData
from matplotlib import pyplot as plt
import numpy as np
fig, ax = plt.subplots(figsize=(14, 8), nrows=2)
outpath = '/Users/mike_e_dubs/MWA/1061313128/Compare'
leg_labels = ['AOFlagger Applied', 'No AOFlagger']
titles = ['Visibilities', 'Visibility Differences']
for i, string in enumerate(['', '_noflag']):
inpath = '/Users/mike_e_dubs/MWA/Data/uvfits/1061313128%s.uvfits' % string
UV = UVData()
UV.read(inpath, file_type='uvfits', read_data=False)
freqs = np.logical_or(np.logical_and(1.79e8 < UV.freq_array[0], 1.81e8 > UV.freq_array[0]),
np.logical_and(1.88e8 < UV.freq_array[0], 1.9e8 > UV.freq_array[0]))
UV.read(inpath, file_type='uvfits', read_metadata=False, freq_chans=freqs,
ant_str='cross', times=np.unique(UV.time_array)[19:26])
for k in range(2):
if k:
ss = SS(UV=UV)
n, bins = np.histogram(np.absolute(UV.data_array), bins='auto')
cent = bins[:-1] + 0.5 * np.diff(bins)
plot_lib.error_plot(fig, ax[k], cent, n, title=titles[k], label=leg_labels[i],
xlabel='Amplitude (UNCALIB)', ylabel='counts', yscale='log')
del ss
del UV
fig.savefig('%s/vis_hists.png' % outpath)
ins.counts /= (np.sum(ins.counts) * np.diff(ins.bins))
ins.counts /= (np.sum(ins2.counts) * np.diff(ins2.bins))
ins.counts = np.append(0, ins.counts)
ins2.counts = np.append(0, ins2.counts)
exp, var = util.hist_fit(ins2.counts, ins2.bins)
exp /= (np.sum(exp) * np.diff(ins2.bins))
var /= (np.sum(exp) * np.diff(ins2.bins))**2
exp = np.append(0, exp)
var = np.append(0, var)
print(np.amax(ins.data_ms))
print(np.amax(ins2.data_ms))
plot_lib.error_plot(fig3,
ax3,
ins.bins,
ins.counts,
drawstyle='steps-post',
xlabel='Deviation ($\hat{\sigma}$)',
label='SSINS Flagging',
legend=True,
yscale='log')
plot_lib.error_plot(fig3,
ax3,
ins2.bins,
ins2.counts,
drawstyle='steps-post',
xlabel='Deviation ($\hat{\sigma}$)',
label='XRFI Flagging',
legend=True,
yscale='log')
plot_lib.error_plot(fig3,
ax3,
total_occ_COTTER_num += total_Nbls - COTTER_Nbls
total_occ_den += total_Nbls
total_occ_SSINS_num += SSINS_Nbls
total_occ_COTTER_wf = total_occ_COTTER_num / total_occ_den
total_occ_SSINS_wf = total_occ_SSINS_num / total_occ_den
total_occ_COTTER_freq = total_occ_COTTER_num.sum(
axis=(0, 3)) / total_occ_den.sum(axis=(0, 3)) * 100
total_occ_SSINS_freq = total_occ_SSINS_num.sum(
axis=(0, 3)) / total_occ_den.sum(axis=(0, 3)) * 100
total_occ_COTTER = total_occ_COTTER_num.sum() / total_occ_den.sum() * 100
total_occ_SSINS = total_occ_SSINS_num.sum() / total_occ_den.sum() * 100
print('COTTER occupancy is %f%%' % total_occ_COTTER)
print('SSINS occupancy is %f%%' % total_occ_SSINS)
fig, ax = plt.subplots(figsize=(14, 8))
labels = ['COTTER', 'SSINS']
for i, dat in enumerate([total_occ_COTTER_freq, total_occ_SSINS_freq]):
pl.error_plot(fig,
ax,
freqs,
dat,
xlabel='Frequency (Mhz)',
ylabel='Occupancy (%)',
label=labels[i],
drawstyle='default',
title='Golden Set RFI Occupancy SSINS vs. COTTER')
fig.savefig('%s/COTTER_SSINS_Spectral_Occupancy_Edges_Careful_Mask.png' %
outdir)
# pdf = counts_end / (args.size * np.diff(bins))
# pdf_bars = counts_bars / (args.size * np.diff(bins))
# pdf = np.append(pdf, 0)
# pdf_bars = np.append(pdf_bars, 0)
standard_prob = cdf(bins[1:], **dist_kwargs) - cdf(bins[:-1], **
dist_kwargs)
standard_counts = standard_prob * args.size
standard_errors = np.sqrt(args.size * standard_prob * (1 - standard_prob))
standard_counts = np.append(standard_counts, 0)
standard_errors = np.append(standard_errors, 0)
print('Done making gaussian histogram')
# standard_pdf = np.append(standard_pdf, 0)
fig, ax = plt.subplots(figsize=(16, 9))
plot_lib.error_plot(fig, ax, bins, counts_end, label='Convolved Rayleigh')
plot_lib.error_plot(fig,
ax,
bins,
standard_counts,
yerr=standard_errors,
label='Gaussian',
title='N_avg=%i' % Navg)
fig.savefig('/Users/mike_e_dubs/MWA/Rayleigh_Convergence_Sim_%i.png' %
Navg)
plt.close(fig)
print('Done plotting distributions')
fig_test, ax_test = plt.subplots(figsize=(16, 9), nrows=2)
plot_lib.error_plot(fig_test,
ax_test[0],