def test_obslist():
obsfile = os.path.join(DATA_PATH, 'test_obs_list.txt')
outfile = os.path.join(DATA_PATH, 'test_obs_list_out.txt')
obslist_test = ['1061313008', '1061313128', '1061318864', '1061318984']
obslist = util.make_obslist(obsfile)
util.make_obsfile(obslist, outfile)
obslist_test_2 = util.make_obslist(outfile)
assert obslist_test == obslist, "The lists were not equal"
assert os.path.exists(outfile), "A file was not written"
assert obslist_test == obslist_test_2, "The new file did not read in properly"
os.remove(outfile)
from SSINS.util import make_obslist, make_obsfile
import os
master_list = []
rootdir = '/Users/mike_e_dubs/MWA/Obs_Lists'
for pointing in ['minus_two', 'minus_one', 'zenith', 'plus_one', 'plus_two']:
pointing_list = []
for page_ind in [1, 2, 3]:
path = '%s/2014_EoR_High_RA_350_pointing_%s_obs_page_%i.txt' % (
rootdir, pointing, page_ind)
if os.path.exists(path):
pointing_page_list = make_obslist(path)
pointing_list += pointing_page_list
print(pointing_list)
master_list += pointing_list
make_obsfile(
pointing_list,
'%s/2014_EoR_High_RA_350_Pointing_%s_obs.txt' % (rootdir, pointing))
make_obsfile(master_list, '%s/2014_EoR_High_RA_350_Master_obs.txt' % rootdir)
from SSINS import util as u
import glob
LR = u.make_obslist('/Users/mike_e_dubs/Repositories/MJW-MWA/Obs_Lists/Long_Run_8s_Autos_OBSIDS.txt')
time_dir = '/Users/mike_e_dubs/MWA/INS/Long_Run/time_arrs'
t_list = glob.glob('%s/*lst_arr.npy' % time_dir)
t_list = [path[len(time_dir) + 1:len(time_dir) + 11] for path in t_list]
print(len(t_list))
arr_dir = '/Users/mike_e_dubs/MWA/INS/Long_Run/Original/metadata'
arr_list = glob.glob('%s/*pols.npy' % arr_dir)
arr_list = [path[len(arr_dir) + 1:len(arr_dir) + 11] for path in arr_list]
print(len(arr_list))
missing_obs = []
for obs in LR:
if (obs not in t_list) or (obs not in arr_list):
missing_obs.append(obs)
u.make_obsfile(missing_obs, '/Users/mike_e_dubs/MWA/INS/Long_Run/Original/missing_obs_AWS.txt')
if (TV_bright_dict[obs] > sorted_bright_list[-101])
]
top_100_bright_list = [
TV_bright_dict[obs] for obs in TV_bright_dict
if (TV_bright_dict[obs] > sorted_bright_list[-101])
]
print(sorted(top_100_bright_list) == sorted_bright_list[-100:])
top_200 = [
obs for obs in TV_bright_dict
if (TV_bright_dict[obs] > sorted_bright_list[-201])
]
top_200_bright_list = [
TV_bright_dict[obs] for obs in TV_bright_dict
if (TV_bright_dict[obs] > sorted_bright_list[-201])
]
print(sorted(top_200_bright_list) == sorted_bright_list[-200:])
u.make_obsfile(
top_50,
'/Users/mike_e_dubs/MWA/INS/Long_Run/Original_Jackknife_Revamp_Complete/TV_obs_top50.txt'
)
u.make_obsfile(
top_100,
'/Users/mike_e_dubs/MWA/INS/Long_Run/Original_Jackknife_Revamp_Complete/TV_obs_top100.txt'
)
u.make_obsfile(
top_200,
'/Users/mike_e_dubs/MWA/INS/Long_Run/Original_Jackknife_Revamp_Complete/TV_obs_top200.txt'
)
gridpoint = pnt_dict[args.pointing]
params = {
"minra": 0,
"maxra": 10,
"mindec": -32,
"maxdec": -22,
"projectid": "G0009",
"anychan": 140,
"gridpoint": gridpoint,
"mintime": args.mintime,
"maxtime": args.maxtime
}
faultdict1 = getmeta(params=params)
params.update({"minra": 350, "maxra": 360})
faultdict2 = getmeta(params=params)
obslist = []
for item in faultdict1:
obslist.append(item[0])
for item in faultdict2:
obslist.append(item[0])
outpath = f"{args.outfile}_{args.pointing}.txt"
obslist = sorted(obslist)
make_obsfile(obslist, outpath)
1065276184, 1066568584, 1066741280, 1066741400, 1067084960, 1067085936,
1067086056, 1067086304, 1061318008, 1061662296, 1062351728, 1062523816,
1062525160, 1063126840, 1063644928, 1063645168, 1063645288, 1064764328,
1064764568, 1064936536, 1064937264, 1067091552, 1068814344, 1068814584,
1068814832, 1069762784, 1063128792, 1063130136, 1064939096, 1067092280,
1061315808, 1061316296, 1062351000, 1062522472, 1062522592, 1062954024,
1063126480, 1064763104, 1064763472, 1064763592, 1064763720, 1064935680,
1065279112, 1066743968, 1067089472
]
Hard_List = [
1064589432, 1064761640, 1064933240, 1067087888, 1064932016, 1065275336,
1062352216, 1064764448, 1064765304, 1066573952, 1067090696, 1063128672,
1064765912, 1064766648, 1068816176, 1062350880, 1064934944, 1066572976
]
Streak_List = [
1062176352, 1062176840, 1062348560, 1062779504, 1063123792, 1065449432,
1066569560, 1066741888, 1066743600, 1067259000, 1067259120, 1067259848,
1067260096, 1067260336, 1068810680, 1068810928, 1065878416, 1067257048,
1067257776, 1067257896, 1067258752, 1062351728, 1066573952, 1063128552,
1063128792, 1062780968, 1062781088, 1063126600, 1067089472, 1068811904
]
for lst, label in zip([Soft_List, Hard_List, Streak_List],
['Soft', 'Hard', 'Streak']):
util.make_obsfile(
lst,
'/Users/mike_e_dubs/python_stuff/MJW-MWA/Obs_Lists/Long_Run/Long_Run_%s_Obs.txt'
% label)
else:
prefix = 'cf-'
obslist = u.make_obslist(args.obsfile)
sublist = []
for obs in obslist:
pow = np.load('%s/%s%s.npy' % (args.indir, prefix, obs))
if not args.filter:
cond = np.any(pow > args.thresh)
else:
pow_ds = np.zeros((2, 256 / args.w))
for i in range(2):
for k in range(256 / args.w):
pow_ds[i, k] = np.amax(pow[i, args.w * k:args.w * (k + 1)])
pow_ds = pow_ds[:, 10:20]
cond = np.any(
np.logical_and(pow_ds[:, 1:-1] > pow_ds[:, :-2] + args.thresh,
pow_ds[:, 1:-1] > pow_ds[:, 2:] + args.thresh))
if cond:
sublist.append(obs)
print('The length of sublist is %i' % len(sublist))
if not args.filter:
u.make_obsfile(sublist,
'%s/%s_t%s.txt' % (args.indir, prefix, args.thresh))
else:
u.make_obsfile(
sublist, '%s/%s_filter_w%s_t%s_b10_20.txt' %
(args.indir, prefix, args.w, args.thresh))
from SSINS import util
import glob
GS_file = '/Users/mike_e_dubs/python_stuff/MJW-MWA/Obs_Lists/Golden_Set_OBSIDS.txt'
obslist = util.make_obslist(GS_file)
figpath = '/Users/mike_e_dubs/MWA/INS/Golden_Set/Golden_Set_COTTER_Filter_O3_S5/figs'
L = len(figpath)
dirty_obslist = glob.glob('%s/*data_match.png' % (figpath))
dirty_obslist = [obs[L + 1:L + 11] for obs in dirty_obslist]
for obs in dirty_obslist:
obslist.remove(obs)
outfile = '/Users/mike_e_dubs/python_stuff/MJW-MWA/Obs_Lists/GS_5sig_cubic_clean.txt'
util.make_obsfile(obslist, outfile)
outdir = '/Users/mike_e_dubs/MWA/INS/Long_Run/Original_Jackknife_Revamp_Complete'
if not os.path.exists(outdir):
os.makedirs(outdir)
occ_dict = pickle.load(open(occ_dict_path, 'rb'))
subdict = occ_dict[5][args.shape]
edges = [16 * k for k in range(24)] + [15 + 16 * k for k in range(24)]
good_freqs = np.ones(384, dtype=bool)
good_freqs[edges] = 0
bins = np.linspace(0, 1, num=51)
occlist = []
obslist = []
for obs in subdict:
if args.shape == 'total':
total_occ = np.mean(subdict[obs][good_freqs])
else:
total_occ = subdict[obs]
occlist.append(total_occ)
if (total_occ > args.threshold):
obslist.append(obs)
shutil.copy('%s/%s_original_INS_data.png' % (ins_plots, obs), outdir)
counts, bins = np.histogram(occlist, bins=bins)
cdf = np.cumsum(counts)
cdf = cdf / cdf[-1]
inds = np.array([np.where(cdf > erf(k / np.sqrt(2)))[0][0] for k in range(1, 4)])
bin_edges = bins[inds + 1]
print(obslist)
util.make_obsfile(obslist, '%s/total_occ_gt_%s_obs.txt' % (outdir, args.threshold))
