def test_obslist():
obsfile = os.path.join(DATA_PATH, 'test_obs_list.txt')
outfile = os.path.join(DATA_PATH, 'test_obs_list_out.txt')
obslist_test = ['1061313008', '1061313128', '1061318864', '1061318984']
obslist = util.make_obslist(obsfile)
util.make_obsfile(obslist, outfile)
obslist_test_2 = util.make_obslist(outfile)
assert obslist_test == obslist, "The lists were not equal"
assert os.path.exists(outfile), "A file was not written"
assert obslist_test == obslist_test_2, "The new file did not read in properly"
os.remove(outfile)
import numpy as np
import argparse
from SSINS import util, INS, plot_lib
import os
import matplotlib.pyplot as plt
parser = argparse.ArgumentParser()
parser.add_argument('-o', '--obsfile')
parser.add_argument('-p', '--ppd_dir')
parser.add_argument('-i', '--ins_dir')
parser.add_argument('--outdir')
args = parser.parse_args()
obslist = util.make_obslist(args.obsfile)
missing_obs = util.make_obslist(
'/Users/mike_e_dubs/MWA/INS/Long_Run/Original_Jackknife_Revamp_Complete/missing_obs.txt'
)
maxlist = []
quadlist = np.zeros([1, 336])
linlist = np.zeros([1, 336])
for obs in obslist:
if obs not in missing_obs:
pow = np.load('%s/cfg-%s.npy' % (args.ppd_dir, obs))
pow_max = np.amax(pow)
if not os.path.exists('%s/%s_ms_poly_coeff_order_2_XX.npy' %
(args.outdir, obs)):
read_paths = util.read_paths_construct(args.ins_dir, 'original',
obs, 'INS')
from __future__ import division
from SSINS import INS, MF, util
import numpy as np
import glob
import pickle
import os
obsfile = '/Users/mike_e_dubs/Repositories/MJW-MWA/Obs_Lists/Long_Run_8s_Autos_OBSIDS.txt'
basedir = '/Users/mike_e_dubs/MWA/INS/Long_Run/Original'
outdir = '%s_Jackknife' % basedir
if not os.path.exists(outdir):
os.makedirs(outdir)
obslist = util.make_obslist(obsfile)
sig_list = [5, 10, 20, 40, 80]
shape_dict = {
'TV6': [1.74e8, 1.81e8],
'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8],
'broad6': [1.72e8, 1.83e8],
'broad7': [1.79e8, 1.9e8],
'broad8': [1.86e8, 1.97e8]
}
shapes = [
'TV6', 'TV7', 'TV8', 'broad6', 'broad7', 'broad8', 'streak', 'point',
'total'
]
occ_dict = {sig: {shape: {} for shape in shapes} for sig in sig_list}
from SSINS import util as u
import glob
LR = u.make_obslist('/Users/mike_e_dubs/Repositories/MJW-MWA/Obs_Lists/Long_Run_8s_Autos_OBSIDS.txt')
time_dir = '/Users/mike_e_dubs/MWA/INS/Long_Run/time_arrs'
t_list = glob.glob('%s/*lst_arr.npy' % time_dir)
t_list = [path[len(time_dir) + 1:len(time_dir) + 11] for path in t_list]
print(len(t_list))
arr_dir = '/Users/mike_e_dubs/MWA/INS/Long_Run/Original/metadata'
arr_list = glob.glob('%s/*pols.npy' % arr_dir)
arr_list = [path[len(arr_dir) + 1:len(arr_dir) + 11] for path in arr_list]
print(len(arr_list))
missing_obs = []
for obs in LR:
if (obs not in t_list) or (obs not in arr_list):
missing_obs.append(obs)
u.make_obsfile(missing_obs, '/Users/mike_e_dubs/MWA/INS/Long_Run/Original/missing_obs_AWS.txt')
parser.add_argument('-d', '--dlist', required=True,
help="Path to a text file where each line is a path to saved SSINS data")
parser.add_argument('-e', '--elist', required=True,
help="Path to a text file where each line is a path to saved match_events yaml")
parser.add_argument('-o', '--outdir', required=True,
help="The output directory. Puts out plots.")
parser.add_argument('-v', '--vmax', type=float, default=0,
help="The vmax for the color plots")
parser.add_argument('-f', '--freqs', required=True,
help="Path to text file of frequencies to tick on plots.")
parser.add_argument('-s', '--sig_fig', default=0,
help="Number of significant figures on frequency labels.")
parser.add_argument('-l', '--obslist', required=True,
help="Path to a text file of obsids")
args = parser.parse_args()
dlist = util.make_obslist(args.dlist)
elist = util.make_obslist(args.elist)
obslist = util.make_obslist(args.obslist)
freqs = np.array(util.make_obslist(args.freqs)).astype(float)
outdir = args.outdir
sig_fig = args.sig_fig
if args.vmax == 0:
vmax = None
vmin = None
else:
vmax = args.vmax
vmin = -args.vmax
make_sig_plots(outdir, dlist, elist, freqs, sig_fig, obslist)
help="Toggles creation of plots.")
parser.add_argument(
'-g',
'--gencsv',
help=
"If nonnull, creates an output CSV file for occ_csv.py with the given name (pass with extension)"
)
parser.add_argument(
'-n',
'--numthreads',
type=int,
default=4,
help="Sets the number of threads to use for evaluation (default: 4)")
args = parser.parse_args()
ins_file_list = util.make_obslist(args.ins_file_list)
if args.gencsv is not None:
f = open(args.gencsv, "w") #wipe old file
f.write("obsid,ins_file,mask_file,yml_file\n") #header
f.close()
#time the full length of the run if -v passed
if args.verbose:
start = time()
if args.numthreads == 1: #single thread fallback
print("using single thread execution path")
for ins_filepath in ins_file_list:
execbody(ins_filepath)
else: #multithreaded execution
from SSINS import util
from SSINS import INS
from SSINS import Catalog_Plot as cp
from SSINS import MF
import numpy as np
import os
obslist_path = '/Users/mike_e_dubs/HERA/Interesting_Obs_Master.txt'
obslist = util.make_obslist(obslist_path)
basedirs = ['/Users/mike_e_dubs/HERA/INS/IDR2_Prelim_Nocut/HERA_IDR2_Prelim_Set_nocut',
'/Users/mike_e_dubs/HERA/INS/IDR2_OR/HERA_IDR2_Prelim_Set_OR_original']
shape_dict = {'dig1': [1.125e8, 1.15625e8],
'dig2': [1.375e8, 1.40625e8],
'dig3': [1.625e8, 1.65625e8],
'dig4': [1.875e8, 1.90625e8],
'TV4': [1.74e8, 1.82e8],
'TV5': [1.82e8, 1.9e8],
'TV6': [1.9e8, 1.98e8],
'TV7': [1.98e8, 2.06e8]}
for i, flag_choice in enumerate(['None', 'original']):
for obsid in obslist:
for pol in ['xx', 'yy', 'xy', 'yx']:
obs = 'zen.%s.%s.HH' % (obsid, pol)
read_paths = util.read_paths_INS(basedirs[i], flag_choice, obs)
ins = INS(read_paths=read_paths, obs=obs, flag_choice=flag_choice,
outpath='/Users/mike_e_dubs/HERA/INS/Adam_Presentation/%s' % flag_choice)
cp.INS_plot(ins, vmax=0.05, ms_vmax=5, ms_vmin=-5)
if flag_choice is 'None':
from SSINS import util
import glob
GS_file = '/Users/mike_e_dubs/python_stuff/MJW-MWA/Obs_Lists/Golden_Set_OBSIDS.txt'
obslist = util.make_obslist(GS_file)
figpath = '/Users/mike_e_dubs/MWA/INS/Golden_Set/Golden_Set_COTTER_Filter_O3_S5/figs'
L = len(figpath)
dirty_obslist = glob.glob('%s/*data_match.png' % (figpath))
dirty_obslist = [obs[L + 1:L + 11] for obs in dirty_obslist]
for obs in dirty_obslist:
obslist.remove(obs)
outfile = '/Users/mike_e_dubs/python_stuff/MJW-MWA/Obs_Lists/GS_5sig_cubic_clean.txt'
util.make_obsfile(obslist, outfile)
import numpy as np
import yaml
import argparse
from matplotlib import use
use('Agg')
import matplotlib.pyplot as plt
from SSINS.util import make_obslist
parser = argparse.ArgumentParser()
parser.add_argument('--yml_file_list', help='The yml files to operate on')
parser.add_argument('--outdir', help="The output directory")
args = parser.parse_args()
yml_list = make_obslist(args.yml_file_list)
inner_keys = ['occ', 'autopow', 'crosspow']
chan_keys = ['TV%i' % chan for chan in [4, 5, 6]]
list_dict = {
chan_key: {key: []
for key in inner_keys}
for chan_key in chan_keys
}
for yml in yml_list:
with open(yml, 'r') as yml_file:
obsdict = yaml.load(yml_file, Loader=yaml.Loader)
for chan in chan_keys:
for key in inner_keys:
list_dict[chan][key].append(obsdict[chan][key])
Titles = [
from SSINS import util, INS, MF
from SSINS import Catalog_Plot as cp
import numpy as np
import glob
import pickle
obslist = util.make_obslist(
'/Users/mike_e_dubs/MWA/Obs_Lists/sidelobe_survey_obsIDs.txt')
indir = '/Users/mike_e_dubs/MWA/INS/Diffuse'
outpath = '%s_Filtered' % indir
sig_thresh = 5
shape_dict = {
'TV6': [1.74e8, 1.81e8],
'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8]
}
mf_kwargs = {'sig_thresh': sig_thresh, 'shape_dict': shape_dict}
occ_dict = {}
for obs in obslist:
if len(glob.glob('%s/figs/%s*' % (indir, obs))):
read_paths = util.read_paths_construct(indir, None, obs, 'INS')
ins = INS(obs=obs, outpath=outpath, read_paths=read_paths)
ins.data[-5:] = np.ma.masked
ins.data[0] = np.ma.masked
ins.data_ms = ins.mean_subtract()
mf = MF(ins, **mf_kwargs)
mf.apply_match_test()
cp.MF_plot(mf)
cp.INS_plot(ins, ms_vmin=-5, ms_vmax=5)
occ_dict['obs'] = (np.count_nonzero(ins.data_ms.mask[:-3]) /
ins.data[:-3].size)
parser = argparse.ArgumentParser()
parser.add_argument('-f', '--obsfile')
parser.add_argument('-i', '--indir')
parser.add_argument('-t', '--thresh', type=float)
parser.add_argument('-g', action='store_true')
parser.add_argument('--filter', action='store_true')
parser.add_argument('-w', type=int)
args = parser.parse_args()
if args.g:
prefix = 'cfg-'
else:
prefix = 'cf-'
obslist = u.make_obslist(args.obsfile)
sublist = []
for obs in obslist:
pow = np.load('%s/%s%s.npy' % (args.indir, prefix, obs))
if not args.filter:
cond = np.any(pow > args.thresh)
else:
pow_ds = np.zeros((2, 256 / args.w))
for i in range(2):
for k in range(256 / args.w):
pow_ds[i, k] = np.amax(pow[i, args.w * k:args.w * (k + 1)])
pow_ds = pow_ds[:, 10:20]
cond = np.any(
np.logical_and(pow_ds[:, 1:-1] > pow_ds[:, :-2] + args.thresh,
pow_ds[:, 1:-1] > pow_ds[:, 2:] + args.thresh))
'--verbose',
action='store_true',
help="Toggles verbose console output of file processing progress.")
parser.add_argument('-p',
'--plots',
action='store_true',
help="Toggles creation of plots.")
parser.add_argument(
'-g',
'--gencsv',
help=
"If nonnull, creates an output CSV file for occ_csv.py with the given name (pass with file extension & full path)"
)
inputargs = parser.parse_args()
ins_file_list = util.make_obslist(inputargs.ins_file_list)
if inputargs.gencsv is not None:
f = open(inputargs.gencsv, "w") #wipe old file
f.write("obsid,ins_file,mask_file,yml_file\n") #header
f.close()
#time the full length of the run if -v passed
if inputargs.verbose:
start = time()
print("using single thread execution path")
for ins_filepath in ins_file_list:
execbody(ins_filepath)
#print out full length of run
with open(path, 'rb') as file:
occ_dict = pickle.load(file)
subdict = occ_dict[5]['total']
x_data = np.zeros([len(subdict.keys()), 384])
y_data = np.copy(x_data)
x_bins = np.arange(384)
y_bins = np.linspace(0, 1, num=51)
edges = [16 * i for i in range(24)] + [15 + 16 * i for i in range(24)]
bool_ind = np.ones(384, dtype=bool)
bool_ind[edges] = 0
obs_ind = []
if args.obslist is not None:
obslist = make_obslist(args.obslist)
for i, obs in enumerate(subdict.keys()):
x_data[i] = np.arange(384)
y_data[i] = subdict[obs]
if args.obslist is not None:
if obs in obslist:
obs_ind.append(i)
if args.obslist is not None:
bool_obs_ind = np.zeros(len(subdict.keys()), dtype=bool)
bool_obs_ind[obs_ind] = 1
print(bool_obs_ind.shape)
plt.hist2d(x_data[:, bool_ind].flatten(), y_data[:, bool_ind].flatten(), bins=[x_bins, y_bins], norm=colors.LogNorm())
plt.title('2-d Long Run Occupation per Channel Histogram')
'--limit',
type=float,
default=0,
help="Sets the minimum total_occ level to write to outfile (default: 0)"
)
inputargs = parser.parse_args()
#start timer if verbose
if inputargs.verbose:
start = time()
dict_list = csv_to_dict_list(inputargs.csv)
#set up ignored channels if present
if inputargs.ch_ignore is not None:
fine_channels_ignore = util.make_obslist(inputargs.ch_ignore)
fine_channels_ignore = np.array(fine_channels_ignore).astype(int)
else:
fine_channels_ignore = None
if inputargs.time_ignore is not None:
time_ignore = util.make_obslist(inputargs.time_ignore)
time_ignore = np.array(time_ignore).astype(float)
else:
time_ignore = None
occ_dict_list = []
for ins_filepath in ins_file_list:
execbody(ins_filepath)
parser.add_argument('basedir')
parser.add_argument('outdir')
parser.add_argument('obsfile')
parser.add_argument('flag_choice')
parser.add_argument('order', type=int)
parser.add_argument('--labels', nargs='*')
parser.add_argument('--mins', nargs='*', type=float)
parser.add_argument('--maxs', nargs='*', type=float)
args = parser.parse_args()
if args.labels is not None:
shape_dict = {label: [min, max] for (label, min, max) in zip(args.labels, args.mins, args.maxs)}
else:
shape_dict = {}
obslist = util.make_obslist(args.obsfile)
edges = [0 + 16 * i for i in range(24)] + [15 + 16 * i for i in range(24)]
freqs = np.load('/Users/mike_e_dubs/python_stuff/MJW-MWA/Useful_Information/MWA_Highband_Freq_Array.npy')
for i, obs in enumerate(obslist):
read_paths = util.read_paths_INS(args.basedir, args.flag_choice, obs)
ins = INS(read_paths=read_paths, flag_choice=args.flag_choice, obs=obs,
outpath=args.outdir, order=args.order)
cp.INS_plot(ins)
mf = MF(ins, sig_thresh=5, shape_dict=shape_dict)
mf.apply_match_test(order=args.order)
cp.MF_plot(mf)
if not i:
occ_num = np.ma.masked_array(mf.INS.data.mask)
occ_den = np.ma.masked_array(np.ones(occ_num.shape))
occ_num[:, 0, edges] = np.ma.masked
from SSINS.util import make_obslist, make_obsfile
import os
master_list = []
rootdir = '/Users/mike_e_dubs/MWA/Obs_Lists'
for pointing in ['minus_two', 'minus_one', 'zenith', 'plus_one', 'plus_two']:
pointing_list = []
for page_ind in [1, 2, 3]:
path = '%s/2014_EoR_High_RA_350_pointing_%s_obs_page_%i.txt' % (
rootdir, pointing, page_ind)
if os.path.exists(path):
pointing_page_list = make_obslist(path)
pointing_list += pointing_page_list
print(pointing_list)
master_list += pointing_list
make_obsfile(
pointing_list,
'%s/2014_EoR_High_RA_350_Pointing_%s_obs.txt' % (rootdir, pointing))
make_obsfile(master_list, '%s/2014_EoR_High_RA_350_Master_obs.txt' % rootdir)
help="The output directory. Will be made if non-existant!")
parser.add_argument('-i',
'--ch_ignore',
help="Text file with channels to ignore")
parser.add_argument('-e',
'--red_events',
nargs='*',
action='append',
help="Redundant events for red_event_sort")
args = parser.parse_args()
times = []
lsts = []
occ_dict = {shape: [] for shape in args.shapes}
filelist = util.make_obslist(args.filelist)
if args.ch_ignore:
ch_ignore = util.make_obslist(args.ch_ignore)
ch_ignore = [int(ch) for ch in ch_ignore]
else:
ch_ignore = None
if args.red_events:
red_events = [tuple(lis) for lis in args.red_events]
for path in filelist:
prefix = path[:path.rfind('_')]
match_events_file = '%s_match_events.yml' % prefix
mask_file = '%s_mask.h5' % prefix
ins = INS(path,
mask_file=mask_file,
