def test_obslist(): obsfile = os.path.join(DATA_PATH, 'test_obs_list.txt') outfile = os.path.join(DATA_PATH, 'test_obs_list_out.txt') obslist_test = ['1061313008', '1061313128', '1061318864', '1061318984'] obslist = util.make_obslist(obsfile) util.make_obsfile(obslist, outfile) obslist_test_2 = util.make_obslist(outfile) assert obslist_test == obslist, "The lists were not equal" assert os.path.exists(outfile), "A file was not written" assert obslist_test == obslist_test_2, "The new file did not read in properly" os.remove(outfile)
import numpy as np import argparse from SSINS import util, INS, plot_lib import os import matplotlib.pyplot as plt parser = argparse.ArgumentParser() parser.add_argument('-o', '--obsfile') parser.add_argument('-p', '--ppd_dir') parser.add_argument('-i', '--ins_dir') parser.add_argument('--outdir') args = parser.parse_args() obslist = util.make_obslist(args.obsfile) missing_obs = util.make_obslist( '/Users/mike_e_dubs/MWA/INS/Long_Run/Original_Jackknife_Revamp_Complete/missing_obs.txt' ) maxlist = [] quadlist = np.zeros([1, 336]) linlist = np.zeros([1, 336]) for obs in obslist: if obs not in missing_obs: pow = np.load('%s/cfg-%s.npy' % (args.ppd_dir, obs)) pow_max = np.amax(pow) if not os.path.exists('%s/%s_ms_poly_coeff_order_2_XX.npy' % (args.outdir, obs)): read_paths = util.read_paths_construct(args.ins_dir, 'original', obs, 'INS')
from __future__ import division from SSINS import INS, MF, util import numpy as np import glob import pickle import os obsfile = '/Users/mike_e_dubs/Repositories/MJW-MWA/Obs_Lists/Long_Run_8s_Autos_OBSIDS.txt' basedir = '/Users/mike_e_dubs/MWA/INS/Long_Run/Original' outdir = '%s_Jackknife' % basedir if not os.path.exists(outdir): os.makedirs(outdir) obslist = util.make_obslist(obsfile) sig_list = [5, 10, 20, 40, 80] shape_dict = { 'TV6': [1.74e8, 1.81e8], 'TV7': [1.81e8, 1.88e8], 'TV8': [1.88e8, 1.95e8], 'broad6': [1.72e8, 1.83e8], 'broad7': [1.79e8, 1.9e8], 'broad8': [1.86e8, 1.97e8] } shapes = [ 'TV6', 'TV7', 'TV8', 'broad6', 'broad7', 'broad8', 'streak', 'point', 'total' ] occ_dict = {sig: {shape: {} for shape in shapes} for sig in sig_list}
from SSINS import util as u import glob LR = u.make_obslist('/Users/mike_e_dubs/Repositories/MJW-MWA/Obs_Lists/Long_Run_8s_Autos_OBSIDS.txt') time_dir = '/Users/mike_e_dubs/MWA/INS/Long_Run/time_arrs' t_list = glob.glob('%s/*lst_arr.npy' % time_dir) t_list = [path[len(time_dir) + 1:len(time_dir) + 11] for path in t_list] print(len(t_list)) arr_dir = '/Users/mike_e_dubs/MWA/INS/Long_Run/Original/metadata' arr_list = glob.glob('%s/*pols.npy' % arr_dir) arr_list = [path[len(arr_dir) + 1:len(arr_dir) + 11] for path in arr_list] print(len(arr_list)) missing_obs = [] for obs in LR: if (obs not in t_list) or (obs not in arr_list): missing_obs.append(obs) u.make_obsfile(missing_obs, '/Users/mike_e_dubs/MWA/INS/Long_Run/Original/missing_obs_AWS.txt')
parser.add_argument('-d', '--dlist', required=True, help="Path to a text file where each line is a path to saved SSINS data") parser.add_argument('-e', '--elist', required=True, help="Path to a text file where each line is a path to saved match_events yaml") parser.add_argument('-o', '--outdir', required=True, help="The output directory. Puts out plots.") parser.add_argument('-v', '--vmax', type=float, default=0, help="The vmax for the color plots") parser.add_argument('-f', '--freqs', required=True, help="Path to text file of frequencies to tick on plots.") parser.add_argument('-s', '--sig_fig', default=0, help="Number of significant figures on frequency labels.") parser.add_argument('-l', '--obslist', required=True, help="Path to a text file of obsids") args = parser.parse_args() dlist = util.make_obslist(args.dlist) elist = util.make_obslist(args.elist) obslist = util.make_obslist(args.obslist) freqs = np.array(util.make_obslist(args.freqs)).astype(float) outdir = args.outdir sig_fig = args.sig_fig if args.vmax == 0: vmax = None vmin = None else: vmax = args.vmax vmin = -args.vmax make_sig_plots(outdir, dlist, elist, freqs, sig_fig, obslist)
help="Toggles creation of plots.") parser.add_argument( '-g', '--gencsv', help= "If nonnull, creates an output CSV file for occ_csv.py with the given name (pass with extension)" ) parser.add_argument( '-n', '--numthreads', type=int, default=4, help="Sets the number of threads to use for evaluation (default: 4)") args = parser.parse_args() ins_file_list = util.make_obslist(args.ins_file_list) if args.gencsv is not None: f = open(args.gencsv, "w") #wipe old file f.write("obsid,ins_file,mask_file,yml_file\n") #header f.close() #time the full length of the run if -v passed if args.verbose: start = time() if args.numthreads == 1: #single thread fallback print("using single thread execution path") for ins_filepath in ins_file_list: execbody(ins_filepath) else: #multithreaded execution
from SSINS import util from SSINS import INS from SSINS import Catalog_Plot as cp from SSINS import MF import numpy as np import os obslist_path = '/Users/mike_e_dubs/HERA/Interesting_Obs_Master.txt' obslist = util.make_obslist(obslist_path) basedirs = ['/Users/mike_e_dubs/HERA/INS/IDR2_Prelim_Nocut/HERA_IDR2_Prelim_Set_nocut', '/Users/mike_e_dubs/HERA/INS/IDR2_OR/HERA_IDR2_Prelim_Set_OR_original'] shape_dict = {'dig1': [1.125e8, 1.15625e8], 'dig2': [1.375e8, 1.40625e8], 'dig3': [1.625e8, 1.65625e8], 'dig4': [1.875e8, 1.90625e8], 'TV4': [1.74e8, 1.82e8], 'TV5': [1.82e8, 1.9e8], 'TV6': [1.9e8, 1.98e8], 'TV7': [1.98e8, 2.06e8]} for i, flag_choice in enumerate(['None', 'original']): for obsid in obslist: for pol in ['xx', 'yy', 'xy', 'yx']: obs = 'zen.%s.%s.HH' % (obsid, pol) read_paths = util.read_paths_INS(basedirs[i], flag_choice, obs) ins = INS(read_paths=read_paths, obs=obs, flag_choice=flag_choice, outpath='/Users/mike_e_dubs/HERA/INS/Adam_Presentation/%s' % flag_choice) cp.INS_plot(ins, vmax=0.05, ms_vmax=5, ms_vmin=-5) if flag_choice is 'None':
from SSINS import util import glob GS_file = '/Users/mike_e_dubs/python_stuff/MJW-MWA/Obs_Lists/Golden_Set_OBSIDS.txt' obslist = util.make_obslist(GS_file) figpath = '/Users/mike_e_dubs/MWA/INS/Golden_Set/Golden_Set_COTTER_Filter_O3_S5/figs' L = len(figpath) dirty_obslist = glob.glob('%s/*data_match.png' % (figpath)) dirty_obslist = [obs[L + 1:L + 11] for obs in dirty_obslist] for obs in dirty_obslist: obslist.remove(obs) outfile = '/Users/mike_e_dubs/python_stuff/MJW-MWA/Obs_Lists/GS_5sig_cubic_clean.txt' util.make_obsfile(obslist, outfile)
import numpy as np import yaml import argparse from matplotlib import use use('Agg') import matplotlib.pyplot as plt from SSINS.util import make_obslist parser = argparse.ArgumentParser() parser.add_argument('--yml_file_list', help='The yml files to operate on') parser.add_argument('--outdir', help="The output directory") args = parser.parse_args() yml_list = make_obslist(args.yml_file_list) inner_keys = ['occ', 'autopow', 'crosspow'] chan_keys = ['TV%i' % chan for chan in [4, 5, 6]] list_dict = { chan_key: {key: [] for key in inner_keys} for chan_key in chan_keys } for yml in yml_list: with open(yml, 'r') as yml_file: obsdict = yaml.load(yml_file, Loader=yaml.Loader) for chan in chan_keys: for key in inner_keys: list_dict[chan][key].append(obsdict[chan][key]) Titles = [
from SSINS import util, INS, MF from SSINS import Catalog_Plot as cp import numpy as np import glob import pickle obslist = util.make_obslist( '/Users/mike_e_dubs/MWA/Obs_Lists/sidelobe_survey_obsIDs.txt') indir = '/Users/mike_e_dubs/MWA/INS/Diffuse' outpath = '%s_Filtered' % indir sig_thresh = 5 shape_dict = { 'TV6': [1.74e8, 1.81e8], 'TV7': [1.81e8, 1.88e8], 'TV8': [1.88e8, 1.95e8] } mf_kwargs = {'sig_thresh': sig_thresh, 'shape_dict': shape_dict} occ_dict = {} for obs in obslist: if len(glob.glob('%s/figs/%s*' % (indir, obs))): read_paths = util.read_paths_construct(indir, None, obs, 'INS') ins = INS(obs=obs, outpath=outpath, read_paths=read_paths) ins.data[-5:] = np.ma.masked ins.data[0] = np.ma.masked ins.data_ms = ins.mean_subtract() mf = MF(ins, **mf_kwargs) mf.apply_match_test() cp.MF_plot(mf) cp.INS_plot(ins, ms_vmin=-5, ms_vmax=5) occ_dict['obs'] = (np.count_nonzero(ins.data_ms.mask[:-3]) / ins.data[:-3].size)
parser = argparse.ArgumentParser() parser.add_argument('-f', '--obsfile') parser.add_argument('-i', '--indir') parser.add_argument('-t', '--thresh', type=float) parser.add_argument('-g', action='store_true') parser.add_argument('--filter', action='store_true') parser.add_argument('-w', type=int) args = parser.parse_args() if args.g: prefix = 'cfg-' else: prefix = 'cf-' obslist = u.make_obslist(args.obsfile) sublist = [] for obs in obslist: pow = np.load('%s/%s%s.npy' % (args.indir, prefix, obs)) if not args.filter: cond = np.any(pow > args.thresh) else: pow_ds = np.zeros((2, 256 / args.w)) for i in range(2): for k in range(256 / args.w): pow_ds[i, k] = np.amax(pow[i, args.w * k:args.w * (k + 1)]) pow_ds = pow_ds[:, 10:20] cond = np.any( np.logical_and(pow_ds[:, 1:-1] > pow_ds[:, :-2] + args.thresh, pow_ds[:, 1:-1] > pow_ds[:, 2:] + args.thresh))
'--verbose', action='store_true', help="Toggles verbose console output of file processing progress.") parser.add_argument('-p', '--plots', action='store_true', help="Toggles creation of plots.") parser.add_argument( '-g', '--gencsv', help= "If nonnull, creates an output CSV file for occ_csv.py with the given name (pass with file extension & full path)" ) inputargs = parser.parse_args() ins_file_list = util.make_obslist(inputargs.ins_file_list) if inputargs.gencsv is not None: f = open(inputargs.gencsv, "w") #wipe old file f.write("obsid,ins_file,mask_file,yml_file\n") #header f.close() #time the full length of the run if -v passed if inputargs.verbose: start = time() print("using single thread execution path") for ins_filepath in ins_file_list: execbody(ins_filepath) #print out full length of run
with open(path, 'rb') as file: occ_dict = pickle.load(file) subdict = occ_dict[5]['total'] x_data = np.zeros([len(subdict.keys()), 384]) y_data = np.copy(x_data) x_bins = np.arange(384) y_bins = np.linspace(0, 1, num=51) edges = [16 * i for i in range(24)] + [15 + 16 * i for i in range(24)] bool_ind = np.ones(384, dtype=bool) bool_ind[edges] = 0 obs_ind = [] if args.obslist is not None: obslist = make_obslist(args.obslist) for i, obs in enumerate(subdict.keys()): x_data[i] = np.arange(384) y_data[i] = subdict[obs] if args.obslist is not None: if obs in obslist: obs_ind.append(i) if args.obslist is not None: bool_obs_ind = np.zeros(len(subdict.keys()), dtype=bool) bool_obs_ind[obs_ind] = 1 print(bool_obs_ind.shape) plt.hist2d(x_data[:, bool_ind].flatten(), y_data[:, bool_ind].flatten(), bins=[x_bins, y_bins], norm=colors.LogNorm()) plt.title('2-d Long Run Occupation per Channel Histogram')
'--limit', type=float, default=0, help="Sets the minimum total_occ level to write to outfile (default: 0)" ) inputargs = parser.parse_args() #start timer if verbose if inputargs.verbose: start = time() dict_list = csv_to_dict_list(inputargs.csv) #set up ignored channels if present if inputargs.ch_ignore is not None: fine_channels_ignore = util.make_obslist(inputargs.ch_ignore) fine_channels_ignore = np.array(fine_channels_ignore).astype(int) else: fine_channels_ignore = None if inputargs.time_ignore is not None: time_ignore = util.make_obslist(inputargs.time_ignore) time_ignore = np.array(time_ignore).astype(float) else: time_ignore = None occ_dict_list = [] for ins_filepath in ins_file_list: execbody(ins_filepath)
parser.add_argument('basedir') parser.add_argument('outdir') parser.add_argument('obsfile') parser.add_argument('flag_choice') parser.add_argument('order', type=int) parser.add_argument('--labels', nargs='*') parser.add_argument('--mins', nargs='*', type=float) parser.add_argument('--maxs', nargs='*', type=float) args = parser.parse_args() if args.labels is not None: shape_dict = {label: [min, max] for (label, min, max) in zip(args.labels, args.mins, args.maxs)} else: shape_dict = {} obslist = util.make_obslist(args.obsfile) edges = [0 + 16 * i for i in range(24)] + [15 + 16 * i for i in range(24)] freqs = np.load('/Users/mike_e_dubs/python_stuff/MJW-MWA/Useful_Information/MWA_Highband_Freq_Array.npy') for i, obs in enumerate(obslist): read_paths = util.read_paths_INS(args.basedir, args.flag_choice, obs) ins = INS(read_paths=read_paths, flag_choice=args.flag_choice, obs=obs, outpath=args.outdir, order=args.order) cp.INS_plot(ins) mf = MF(ins, sig_thresh=5, shape_dict=shape_dict) mf.apply_match_test(order=args.order) cp.MF_plot(mf) if not i: occ_num = np.ma.masked_array(mf.INS.data.mask) occ_den = np.ma.masked_array(np.ones(occ_num.shape)) occ_num[:, 0, edges] = np.ma.masked
from SSINS.util import make_obslist, make_obsfile import os master_list = [] rootdir = '/Users/mike_e_dubs/MWA/Obs_Lists' for pointing in ['minus_two', 'minus_one', 'zenith', 'plus_one', 'plus_two']: pointing_list = [] for page_ind in [1, 2, 3]: path = '%s/2014_EoR_High_RA_350_pointing_%s_obs_page_%i.txt' % ( rootdir, pointing, page_ind) if os.path.exists(path): pointing_page_list = make_obslist(path) pointing_list += pointing_page_list print(pointing_list) master_list += pointing_list make_obsfile( pointing_list, '%s/2014_EoR_High_RA_350_Pointing_%s_obs.txt' % (rootdir, pointing)) make_obsfile(master_list, '%s/2014_EoR_High_RA_350_Master_obs.txt' % rootdir)
help="The output directory. Will be made if non-existant!") parser.add_argument('-i', '--ch_ignore', help="Text file with channels to ignore") parser.add_argument('-e', '--red_events', nargs='*', action='append', help="Redundant events for red_event_sort") args = parser.parse_args() times = [] lsts = [] occ_dict = {shape: [] for shape in args.shapes} filelist = util.make_obslist(args.filelist) if args.ch_ignore: ch_ignore = util.make_obslist(args.ch_ignore) ch_ignore = [int(ch) for ch in ch_ignore] else: ch_ignore = None if args.red_events: red_events = [tuple(lis) for lis in args.red_events] for path in filelist: prefix = path[:path.rfind('_')] match_events_file = '%s_match_events.yml' % prefix mask_file = '%s_mask.h5' % prefix ins = INS(path, mask_file=mask_file,