def test_freq_broadcast_subbands():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, f'{obs}_SSINS.h5')
out_prefix = os.path.join(DATA_PATH, f'{obs}_test')
match_outfile = f'{out_prefix}_SSINS_match_events.yml'
ins = INS(insfile)
# spoof the metric array
ins.metric_array[:] = 1
ins.metric_array[2, 10:20] = 10
ins.metric_array[4, 40:50] = 10
ins.metric_ms = ins.mean_subtract()
# Slice will go up to 21 since shapes are inclusive at the boundaries
# when boundary is on channel center
shape_dict = {
'shape1': [ins.freq_array[10], ins.freq_array[20]],
'shape2': [ins.freq_array[40], ins.freq_array[50]]
}
# boundaries are INCLUSIVE
broadcast_dict = {
'sb1': [ins.freq_array[0], ins.freq_array[29]],
'sb2': [ins.freq_array[30], ins.freq_array[59]]
}
mf = MF(ins.freq_array,
5,
shape_dict=shape_dict,
broadcast_streak=False,
broadcast_dict=broadcast_dict)
mf.apply_match_test(ins, event_record=True, freq_broadcast=True)
assert np.all(ins.metric_array.mask[2, :30])
assert not np.any(ins.metric_array.mask[2, 30:])
assert np.all(ins.metric_array.mask[4, 30:60])
assert not np.any(ins.metric_array.mask[4, :30])
assert not np.any(ins.metric_array.mask[4, 60:])
print(ins.match_events)
test_match_events = [(slice(2, 3), slice(10, 21), 'shape1'),
(slice(2, 3), slice(0, 30), 'freq_broadcast_sb1'),
(slice(4, 5), slice(40, 51), 'shape2'),
(slice(4, 5), slice(30, 60), 'freq_broadcast_sb2')]
for event, test_event in zip(ins.match_events, test_match_events):
assert event[:3] == test_event
def test_no_diff_start():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
# Don't diff - will fail to mask data array
ss = SS()
with pytest.warns(UserWarning, match="flag_choice will be ignored"):
ss.read(testfile, flag_choice='original', diff=False)
with pytest.warns(UserWarning, match="diff on read defaults to False"):
ss.read(testfile, flag_choice='original', diff=False)
ins = INS(ss)
assert ss.flag_choice is None
def test_select():
obs = '1061313128_99bl_1pol_half_time_SSINS'
testfile = os.path.join(DATA_PATH, '%s.h5' % obs)
ins = INS(testfile)
ins.metric_array.mask[7, :12] = True
Ntimes = len(ins.time_array)
ins.select(times=ins.time_array[3:-3], freq_chans=np.arange(24))
assert ins.metric_array.shape[0] == Ntimes - 6
assert ins.metric_array.shape[1] == 24
for param in ins._data_params:
assert getattr(ins, param).shape == ins.metric_array.shape
# Check that the mask is propagated
assert np.all(ins.metric_array.mask[4, :12])
assert np.count_nonzero(ins.metric_array.mask) == 12
def test_INS_plot():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
outdir = os.path.join(DATA_PATH, 'test_plots')
prefix = '%s/%s_raw' % (outdir, obs)
outfile = '%s_SSINS.pdf' % prefix
log_prefix = '%s/%s_log' % (outdir, obs)
log_outfile = '%s_SSINS.pdf' % log_prefix
symlog_prefix = '%s/%s_symlog' % (outdir, obs)
symlog_outfile = '%s_SSINS.pdf' % symlog_prefix
ins = INS(insfile)
xticks = np.arange(0, 384, 96)
xticklabels = ['%.1f' % (10**-6 * ins.freq_array[tick]) for tick in xticks]
yticks = np.arange(0, 50, 10)
yticklabels = ['%i' % (2 * tick) for tick in yticks]
cp.INS_plot(ins, prefix)
cp.INS_plot(ins,
log_prefix,
log=True,
xticks=xticks,
yticks=yticks,
xticklabels=xticklabels,
yticklabels=yticklabels,
title='Title')
cp.INS_plot(ins,
symlog_prefix,
symlog=True,
xticks=xticks,
yticks=yticks,
xticklabels=xticklabels,
yticklabels=yticklabels)
assert os.path.exists(outfile), "The first plot was not made"
assert os.path.exists(log_outfile), "The second plot was not made"
assert os.path.exists(symlog_outfile), "The third plot was not made"
os.remove(outfile)
os.remove(log_outfile)
os.remove(symlog_outfile)
os.rmdir(outdir)
def test_write_mwaf():
from astropy.io import fits
obs = '1061313128_99bl_1pol_half_time_SSINS'
testfile = os.path.join(DATA_PATH, '%s.h5' % obs)
prefix = os.path.join(DATA_PATH, '%s_test' % obs)
ins = INS(testfile)
mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')]
bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')]
# shape of that mwaf file
ins.metric_array = np.ma.ones([223, 768, 1])
ins.metric_array[100, 32 * 11:32 * (11 + 1)] = np.ma.masked
flags = ins.mask_to_flags()
new_flags = np.repeat(flags[:, np.newaxis, 32 * 11:32 * (11 + 1)],
8256,
axis=1).reshape((224 * 8256, 32))
# Test some defensive errors
with pytest.raises(IOError):
ins.write(prefix, output_type='mwaf', mwaf_files=bad_mwaf_files)
with pytest.raises(ValueError):
ins.write(prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
mwaf_method='bad_method')
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=None)
ins.write('%s_add' % prefix, output_type='mwaf', mwaf_files=mwaf_files)
ins.write('%s_replace' % prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
mwaf_method='replace')
with fits.open(mwaf_files[0]) as old_mwaf_hdu:
with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu:
assert np.all(add_mwaf_hdu[1].data['FLAGS'] ==
old_mwaf_hdu[1].data['FLAGS'] + new_flags)
with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu:
assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags)
for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]:
os.remove(path)
def test_write_meta():
obs = "1061313128_99bl_1pol_half_time"
testfile = os.path.join(DATA_PATH, f"{obs}.uvfits")
prefix = os.path.join(DATA_PATH, f"{obs}_test")
uvd = UVData()
uvd.read(testfile, freq_chans=np.arange(32))
ss = SS()
ss.read(testfile, freq_chans=np.arange(32), diff=True)
uvf = UVFlag(uvd, mode="flag", waterfall=True)
ins = INS(ss)
ins.metric_array[:] = 1
ins.weights_array[:] = 10
ins.weights_square_array[:] = 10
# Make some outliers
# Narrowband in 1th, 26th, and 31th frequency
ins.metric_array[1, 1, :] = 100
ins.metric_array[1, 30, :] = 100
ins.metric_array[3:14, 26, :] = 100
# Arbitrary shape in 2, 3, 4
ins.metric_array[3:14, 2:25, :] = 100
ins.metric_array[[0, -1], :, :] = np.ma.masked
ins.metric_array[:, [0, -1], :] = np.ma.masked
ins.metric_ms = ins.mean_subtract()
ch_wid = ins.freq_array[1] - ins.freq_array[0]
shape_dict = {
"shape":
[ins.freq_array[2] + 0.1 * ch_wid, ins.freq_array[24] - 0.1 * ch_wid]
}
mf = MF(ins.freq_array, 5, tb_aggro=0.5, shape_dict=shape_dict)
mf.apply_match_test(ins, time_broadcast=True)
util.write_meta(prefix, ins, uvf=uvf, mf=mf, clobber=True)
for data_type in ["data", "mask", "flags"]:
path = f"{prefix}_SSINS_{data_type}.h5"
assert os.path.exists(path)
os.remove(path)
for data_type in ["match_events", "matchfilter"]:
path = f"{prefix}_SSINS_{data_type}.yml"
assert os.path.exists(path)
os.remove(path)
def test_samp_thresh():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
ins = INS(insfile)
# Mock a simple metric_array and freq_array
ins.metric_array = np.ma.ones([10, 20, 1])
ins.weights_array = np.copy(ins.metric_array)
ins.freq_array = np.zeros([1, 20])
ins.freq_array = np.arange(20)
# Arbitrarily flag enough data in channel 10
mf = MF(ins.freq_array, 5, streak=False, N_samp_thresh=5)
ins.metric_array[3:, 10] = np.ma.masked
ins.metric_array[3:, 9] = np.ma.masked
# Put in an outlier so it gets to samp_thresh_test
ins.metric_array[0, 11] = 10
ins.metric_ms = ins.mean_subtract()
bool_ind = np.zeros(ins.metric_array.shape, dtype=bool)
bool_ind[:, 10] = 1
bool_ind[:, 9] = 1
bool_ind[0, 11] = 1
mf.apply_match_test(ins, event_record=True, apply_samp_thresh=True)
test_match_events = [(0, slice(11, 12), 'narrow'),
(0, slice(9, 10), 'samp_thresh'),
(0, slice(10, 11), 'samp_thresh')]
print(ins.match_events)
# Test stuff
assert np.all(
ins.metric_array.mask == bool_ind), "The right flags were not applied"
for i, event in enumerate(test_match_events):
assert ins.match_events[
i][:-1] == event, "The events weren't appended correctly"
# Test that exception is raised when N_samp_thresh is too high
with pytest.raises(ValueError):
mf = MF(ins.freq_array, 5, N_samp_thresh=100)
mf.apply_samp_thresh_test(ins)
def test_polyfit():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
ss = SS()
ss.read(testfile)
ins = INS(ss, order=1)
# Mock some data for which the polyfit is exact
x = np.arange(1, 11)
ins.metric_array = np.ma.masked_array([[3 * x + 5 for i in range(3)]])
ins.metric_array.mask = np.zeros(ins.metric_array.shape, dtype=bool)
ins.metric_array = np.swapaxes(ins.metric_array, 0, 2)
ins.weights_array = np.ones(ins.metric_array.shape)
ins.metric_ms = ins.mean_subtract()
assert np.all(ins.metric_ms == np.zeros(
ins.metric_ms.shape)), "The polyfit was not exact"
def test_mean_subtract():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
ss = SS()
ss.read(testfile, diff=True)
ins = INS(ss, order=0)
old_dat = np.copy(ins.metric_ms)
# Mask the first five frequencies and last two at the first and second times
ins.metric_array[0, :5] = np.ma.masked
# Calculate the new mean-subtracted spectrum only over the first few masked frequencies
ins.metric_ms[:, :5] = ins.mean_subtract(freq_slice=slice(0, 5))
# See if a new mean was calculated over the first five frequencies
assert not np.all(old_dat[1:, :5] == ins.metric_ms[1:, :5]), "All elements of the ms array are still equal"
def test_spectrum_type_file_init():
obs = "1061313128_99bl_1pol_half_time_SSINS"
auto_obs = "1061312640_mix_auto_SSINS_data"
cross_obs = "1061312640_mix_cross_SSINS_data"
testfile = os.path.join(DATA_PATH, f"{obs}.h5")
auto_testfile = os.path.join(DATA_PATH, f"{auto_obs}.h5")
cross_testfile = os.path.join(DATA_PATH, f"{cross_obs}.h5")
ins = INS(testfile)
assert ins.spectrum_type == "cross"
with pytest.raises(ValueError, match="Reading in a 'cross' spectrum as 'auto'."):
ins = INS(testfile, spectrum_type="auto")
with pytest.raises(ValueError, match="Requested spectrum type disagrees with saved spectrum. "):
ins = INS(auto_testfile, spectrum_type="cross")
with pytest.raises(ValueError, match="Requested spectrum type disagrees with saved spectrum. "):
ins = INS(cross_testfile, spectrum_type="auto")
del ins
ins = INS(cross_testfile)
del ins
ins = INS(auto_testfile, spectrum_type="auto")
'broad7': [1.79e8, 1.9e8],
'broad8': [1.86e8, 1.97e8]
}
shapes = [
'TV6', 'TV7', 'TV8', 'broad6', 'broad7', 'broad8', 'streak', 'point',
'total'
]
occ_dict = {sig: {shape: {} for shape in shapes} for sig in sig_list}
for obs in obslist:
flist = glob.glob('%s/metadata/%s*' % (basedir, obs))
if len(flist):
read_paths = util.read_paths_construct(basedir, 'original', obs, 'INS')
for sig_thresh in sig_list:
ins = INS(read_paths=read_paths,
obs=obs,
outpath=outdir,
flag_choice='original')
mf = MF(ins,
sig_thresh=sig_thresh,
N_thresh=15,
shape_dict=shape_dict)
mf.apply_match_test(apply_N_thresh=True)
occ_dict[sig_thresh]['total'][obs] = np.mean(ins.data.mask[:, 0, :,
0],
axis=0)
if len(ins.match_events):
event_frac = util.event_fraction(ins.match_events,
ins.data.shape[0], shapes,
384)
for shape in shapes[:-1]:
occ_dict[sig_thresh][shape][obs] = event_frac[shape]
uvd = UVData()
uvd.read(args.filename, read_data=False)
if xants == None or xants == []:
use_ants = uvd.get_ants()
else:
use_ants = [ant for ant in uvd.get_ants() if ant not in xants]
uvd.select(antenna_nums=use_ants)
bls = uvd.get_antpairs()
uvf = UVFlag(uvd, waterfall=True, mode='flag')
del uvd
# Make the SS object
ss = SS()
if args.num_baselines > 0:
ss.read(args.filename, bls=bls[:args.num_baselines], diff=args.no_diff)
ins = INS(ss)
Nbls = len(bls)
for slice_ind in range(args.num_baselines, Nbls, args.num_baselines):
ss = SS()
ss.read(args.filename,
bls=bls[slice_ind:slice_ind + args.num_baselines],
diff=args.no_diff)
new_ins = INS(ss)
ins = util.combine_ins(ins, new_ins)
else:
ss.read(args.filename, antenna_nums=use_ants, diff=args.no_diff)
ss.select(ant_str='cross')
ins = INS(ss)
# Clear some memory??
del ss
from __future__ import division
from SSINS import INS, plot_lib, util, MF
from matplotlib import cm
import matplotlib.pyplot as plt
import numpy as np
obs = 'zen.2458098.37904.xx.HH'
indir = '/Users/mike_e_dubs/HERA/INS/IDR2_Prelim_Nocut/HERA_IDR2_Prelim_Set_nocut'
outpath = '/Users/mike_e_dubs/General/%s' % obs
read_paths = util.read_paths_construct(indir, None, obs, 'INS')
indir2 = '/Users/mike_e_dubs/HERA/INS/IDR2_OR/HERA_IDR2_Prelim_Set_OR_original'
read_paths_orig = util.read_paths_construct(indir2, 'original', obs, 'INS')
ins2 = INS(read_paths=read_paths_orig,
obs=obs,
flag_choice='original',
outpath=outpath)
ins = INS(read_paths=read_paths, obs=obs, outpath=outpath)
aspect = ins.data.shape[2] / ins.data.shape[0]
fig, ax = plt.subplots(figsize=(16, 9), ncols=2)
#plot_lib.image_plot(fig, ax[0], ins.data[:, 0, :, 0], freq_array=ins.freq_array[0],
#cbar_label='Amplitude (UNCALIB)', aspect=aspect, vmax=0.03,
#ylabel='Time (10 s)')
plot_lib.image_plot(fig,
ax[0],
ins.data_ms[:, 0, :, 0],
freq_array=ins.freq_array[0],
cmap=cm.coolwarm,
def test_write_mwaf():
from astropy.io import fits
obs = '1061313128_99bl_1pol_half_time_SSINS'
testfile = os.path.join(DATA_PATH, '%s.h5' % obs)
prefix = os.path.join(DATA_PATH, '%s_test' % obs)
ins = INS(testfile)
mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')]
bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')]
# Compatible shape with mwaf file
ins.metric_array = np.ma.ones([55, 384, 1])
ins.metric_array[50, 16 * 11:16 * (11 + 1)] = np.ma.masked
# metadata from the input file, hardcoded for testing purposes
time_div = 4
freq_div = 2
NCHANS = 32
boxint = 11
Nbls = 8256
NSCANS = 224
flags = ins.mask_to_flags()
# Repeat in time
time_rep_flags = np.repeat(flags, time_div, axis=0)
# Repeat in freq
freq_time_rep_flags = np.repeat(time_rep_flags, freq_div, axis=1)
# Repeat in bls
freq_time_bls_rep_flags = np.repeat(
freq_time_rep_flags[:, np.newaxis,
NCHANS * boxint:NCHANS * (boxint + 1)],
Nbls,
axis=1)
# This shape is on MWA wiki. Reshape to this shape.
new_flags = freq_time_bls_rep_flags.reshape((NSCANS * Nbls, NCHANS))
# Test some defensive errors
with pytest.raises(IOError):
ins.write(prefix, output_type='mwaf', mwaf_files=bad_mwaf_files)
with pytest.raises(ValueError):
ins.write(prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
mwaf_method='bad_method')
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=None)
ins.write('%s_add' % prefix, output_type='mwaf', mwaf_files=mwaf_files)
ins.write('%s_replace' % prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
mwaf_method='replace')
with fits.open(mwaf_files[0]) as old_mwaf_hdu:
with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu:
assert np.all(add_mwaf_hdu[1].data['FLAGS'] ==
old_mwaf_hdu[1].data['FLAGS'] + new_flags)
with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu:
assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags)
for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]:
os.remove(path)
from SSINS import INS, plot_lib, util
import numpy as np
import matplotlib.pyplot as plt
inpath = '/Users/mike_e_dubs/General/1061318984'
outpath = '%s/figs' % inpath
obs = '1061318984'
read_paths = util.read_paths_construct(inpath, None, obs, 'INS')
ins = INS(read_paths=read_paths, obs=obs, outpath=outpath)
vmax = [None, 150]
fig, ax = plt.subplots(figsize=(14, 8), nrows=2)
fig.suptitle('Narrowband SSINS')
for i in range(2):
plot_lib.image_plot(fig,
ax[i],
ins.data[:, 0, :, 0],
cbar_label='Amplitude (UNCALIB)',
freq_array=ins.freq_array[0],
vmax=vmax[i])
fig.savefig('%s/1061318984_None_INS_data_XX.png' % outpath)
def execbody(ins_filepath):
slash_ind = ins_filepath.rfind('/')
obsid = ins_filepath[slash_ind + 1:slash_ind + 11]
ins = INS(ins_filepath)
ins.select(times=ins.time_array[3:-3])
ins.metric_ms = ins.mean_subtract()
shape_dict = {
'TV6': [1.74e8, 1.81e8],
'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8],
'TV9': [1.95e8, 2.02e8]
}
sig_thresh = {shape: 5 for shape in shape_dict}
sig_thresh['narrow'] = 5
sig_thresh['streak'] = 8
mf = MF(ins.freq_array,
sig_thresh,
shape_dict=shape_dict,
N_samp_thresh=len(ins.time_array) // 2)
ins.metric_array[ins.metric_array == 0] = np.ma.masked
ins.metric_ms = ins.mean_subtract()
ins.sig_array = np.ma.copy(ins.metric_ms)
#write plots if command flagged to do so
if args.plots:
prefix = '%s/%s_trimmed_zeromask' % (args.outdir, obsid)
freqs = np.arange(1.7e8, 2e8, 5e6)
xticks, xticklabels = util.make_ticks_labels(freqs,
ins.freq_array,
sig_fig=0)
yticks = [0, 20, 40]
yticklabels = []
for tick in yticks:
yticklabels.append(
Time(ins.time_array[tick], format='jd').iso[:-4])
Catalog_Plot.INS_plot(ins,
prefix,
xticks=xticks,
yticks=yticks,
xticklabels=xticklabels,
yticklabels=yticklabels,
data_cmap=cm.plasma,
ms_vmin=-5,
ms_vmax=5,
title=obsid,
xlabel='Frequency (Mhz)',
ylabel='Time (UTC)')
if args.verbose:
print("wrote trimmed zeromask plot for " + obsid)
mf.apply_match_test(ins, apply_samp_thresh=False)
mf.apply_samp_thresh_test(ins, event_record=True)
flagged_prefix = '%s/%s_trimmed_zeromask_MF_s8' % (args.outdir, obsid)
#write data/mask/match/ if command flagged to do so
if args.write:
ins.write(flagged_prefix, output_type='data', clobber=True)
ins.write(flagged_prefix, output_type='mask', clobber=True)
ins.write(flagged_prefix, output_type='match_events')
if args.verbose:
print("wrote data/mask/match files for " + obsid)
#write plots if command flagged to do so
if args.plots:
Catalog_Plot.INS_plot(ins,
flagged_prefix,
xticks=xticks,
yticks=yticks,
xticklabels=xticklabels,
yticklabels=yticklabels,
data_cmap=cm.plasma,
ms_vmin=-5,
ms_vmax=5,
title=obsid,
xlabel='Frequency (Mhz)',
ylabel='Time (UTC)')
if args.verbose:
print("wrote trimmed zeromask (w/ match filter) for " + obsid)
#a hardcoded csv generator for occ_csv
if args.gencsv is not None:
csv = "" + obsid + "," + flagged_prefix + "_SSINS_data.h5," + flagged_prefix + "_SSINS_mask.h5," + flagged_prefix + "_SSINS_match_events.yml\n"
with open(args.gencsv, "a") as csvfile:
csvfile.write(csv)
print("wrote entry for " + obsid)
from SSINS import Catalog_Plot as cp
from SSINS import MF
import numpy as np
import os
obslist_path = '/Users/mike_e_dubs/HERA/Interesting_Obs_Master.txt'
obslist = util.make_obslist(obslist_path)
basedirs = ['/Users/mike_e_dubs/HERA/INS/IDR2_Prelim_Nocut/HERA_IDR2_Prelim_Set_nocut',
'/Users/mike_e_dubs/HERA/INS/IDR2_OR/HERA_IDR2_Prelim_Set_OR_original']
shape_dict = {'dig1': [1.125e8, 1.15625e8],
'dig2': [1.375e8, 1.40625e8],
'dig3': [1.625e8, 1.65625e8],
'dig4': [1.875e8, 1.90625e8],
'TV4': [1.74e8, 1.82e8],
'TV5': [1.82e8, 1.9e8],
'TV6': [1.9e8, 1.98e8],
'TV7': [1.98e8, 2.06e8]}
for i, flag_choice in enumerate(['None', 'original']):
for obsid in obslist:
for pol in ['xx', 'yy', 'xy', 'yx']:
obs = 'zen.%s.%s.HH' % (obsid, pol)
read_paths = util.read_paths_INS(basedirs[i], flag_choice, obs)
ins = INS(read_paths=read_paths, obs=obs, flag_choice=flag_choice,
outpath='/Users/mike_e_dubs/HERA/INS/Adam_Presentation/%s' % flag_choice)
cp.INS_plot(ins, vmax=0.05, ms_vmax=5, ms_vmin=-5)
if flag_choice is 'None':
parser.add_argument('--mins', nargs='*', type=float)
parser.add_argument('--maxs', nargs='*', type=float)
args = parser.parse_args()
if args.labels is not None:
shape_dict = {label: [min, max] for (label, min, max) in zip(args.labels, args.mins, args.maxs)}
else:
shape_dict = {}
obslist = util.make_obslist(args.obsfile)
edges = [0 + 16 * i for i in range(24)] + [15 + 16 * i for i in range(24)]
freqs = np.load('/Users/mike_e_dubs/python_stuff/MJW-MWA/Useful_Information/MWA_Highband_Freq_Array.npy')
for i, obs in enumerate(obslist):
read_paths = util.read_paths_INS(args.basedir, args.flag_choice, obs)
ins = INS(read_paths=read_paths, flag_choice=args.flag_choice, obs=obs,
outpath=args.outdir, order=args.order)
cp.INS_plot(ins)
mf = MF(ins, sig_thresh=5, shape_dict=shape_dict)
mf.apply_match_test(order=args.order)
cp.MF_plot(mf)
if not i:
occ_num = np.ma.masked_array(mf.INS.data.mask)
occ_den = np.ma.masked_array(np.ones(occ_num.shape))
occ_num[:, 0, edges] = np.ma.masked
occ_den[:, 0, edges] = np.ma.masked
else:
occ_num = occ_num + mf.INS.data.mask
occ_den = occ_den + np.ones(occ_num.shape)
occ_freq = occ_num.sum(axis=(0, 1, 3)) / occ_den.sum(axis=(0, 1, 3)) * 100
occ_total = occ_num.sum() / occ_den.sum() * 100
import pickle
obslist = util.make_obslist(
'/Users/mike_e_dubs/MWA/Obs_Lists/sidelobe_survey_obsIDs.txt')
indir = '/Users/mike_e_dubs/MWA/INS/Diffuse'
outpath = '%s_Filtered' % indir
sig_thresh = 5
shape_dict = {
'TV6': [1.74e8, 1.81e8],
'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8]
}
mf_kwargs = {'sig_thresh': sig_thresh, 'shape_dict': shape_dict}
occ_dict = {}
for obs in obslist:
if len(glob.glob('%s/figs/%s*' % (indir, obs))):
read_paths = util.read_paths_construct(indir, None, obs, 'INS')
ins = INS(obs=obs, outpath=outpath, read_paths=read_paths)
ins.data[-5:] = np.ma.masked
ins.data[0] = np.ma.masked
ins.data_ms = ins.mean_subtract()
mf = MF(ins, **mf_kwargs)
mf.apply_match_test()
cp.MF_plot(mf)
cp.INS_plot(ins, ms_vmin=-5, ms_vmax=5)
occ_dict['obs'] = (np.count_nonzero(ins.data_ms.mask[:-3]) /
ins.data[:-3].size)
del ins
del mf
pickle.dump(occ_dict, open('%s/occ_dict_only_subflags' % outpath, 'wb'))
def test_data_params():
obs = '1061313128_99bl_1pol_half_time_SSINS'
testfile = os.path.join(DATA_PATH, '%s.h5' % obs)
ins = INS(testfile)
assert ins._data_params == ['metric_array', 'weights_array', 'metric_ms', 'sig_array']
from matplotlib import cm
import matplotlib.pyplot as plt
import numpy as np
basedir = '/Users/jonj/Test_folder/lwa'
obs = [
'LWA_50to51', 'LWA_51to52', 'LWA_52to53', 'LWA_53to54', 'LWA_54to55',
'LWA_55to56', 'LWA_56to57', 'LWA_57to58', 'LWA_58to59', 'LWA_59to60'
]
outpath = '%s/test_combines' % basedir
insarray = []
for x in obs:
read_paths = util.read_paths_construct(basedir, 'None', x, 'INS')
insarray.append(
INS(obs=x,
outpath=outpath,
read_paths=read_paths,
flag_choice='orginal'))
inscombined = INS_helpers.INS_concat(insarray, axis=0)
inscombined.obs = 'LWA_50to60'
inscombined.outpath = outpath
inscombined.vis_units = insarray[0].vis_units
inscombined.pols = insarray[0].pols
inscombined.save()
cp.INS_plot(inscombined, vmax=.05, ms_vmax=5, ms_vmin=-5)
shape_dict = {'44MHZ': [44e6, 45e6], '40.6MHZ': [40.4e6, 40.7e6]}
sig_thresh = 4.35
mf = MF(inscombined,
shape_dict=shape_dict,
L = len('%s/arrs/' % (args.inpath))
obs = arr[L:L + args.Lobs]
read_paths = {
'data':
arr,
'Nbls':
'%s/arrs/%s_%s_INS_Nbls.npym' % (args.inpath, obs, args.flag_choice),
'freq_array':
'%s/metadata/%s_freq_array.npy' % (args.inpath, obs),
'pols':
'%s/metadata/%s_pols.npy' % (args.inpath, obs),
'vis_units':
'%s/metadata/%s_vis_units.npy' % (args.inpath, obs)
}
ins = INS(obs=obs,
outpath=args.outpath,
flag_choice=args.flag_choice,
read_paths=read_paths)
# ins.data[:, :, :82] = np.ma.masked
# ins.data[:, :, -21:] = np.ma.masked
ins.data_ms = ins.mean_subtract()
ins.counts, ins.bins, ins.sig_thresh = ins.hist_make()
cp.INS_plot(ins, **ms_plot_kwargs)
mf = MF(ins,
shape_dict=shape_dict,
point=args.point,
streak=args.streak,
**mf_kwargs)
for test in args.tests:
getattr(mf, 'apply_%s_test' % test)(args.order)
ins.save()
cp.MF_plot(mf, **ms_plot_kwargs)
raise ValueError("indir and outdir are the same")
if args.rfi_flag:
ss = SS()
if args.correct:
ss.read(args.uvd, phase_to_pointing_center=True,
correct_cable_len=True, flag_choice='original', diff=True,
remove_dig_gains=True, remove_coarse_band=True,
data_array_dtype=np.complex64)
else:
ss.read(args.uvd, phase_to_pointing_center=True,
correct_cable_len=True, flag_choice='original', diff=True,
remove_dig_gains=False, remove_coarse_band=False,
data_array_dtype=np.complex64)
ins = INS(ss)
prefix = f'{args.outdir}/{args.obsid}'
Catalog_Plot.INS_plot(ins, prefix, data_cmap=cm.plasma, ms_vmin=-5, ms_vmax=5,
title=args.obsid, xlabel='Frequency (Mhz)',
ylabel='Time (UTC)')
# Try to save memory - hope for garbage collector
del ss
# Set up MF flagging for routine shapes
shape_dict = {'TV6': [1.74e8, 1.81e8], 'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8], 'TV9': [1.95e8, 2.02e8]}
sig_thresh = {shape: 5 for shape in shape_dict}
sig_thresh['narrow'] = 5
sig_thresh['streak'] = 10
mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict,
def test_apply_match_test():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
ins = INS(insfile)
# Mock a simple metric_array and freq_array
ins.metric_array[:] = np.ones_like(ins.metric_array)
ins.weights_array = np.copy(ins.metric_array)
ins.weights_square_array = np.copy(ins.weights_array)
# Make a shape dictionary for a shape that will be injected later
ch_wid = ins.freq_array[1] - ins.freq_array[0]
shape = [
ins.freq_array[7] - 0.2 * ch_wid, ins.freq_array[12] + 0.2 * ch_wid
]
shape_dict = {'shape': shape}
sig_thresh = {'shape': 5, 'narrow': 5, 'streak': 5}
mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict)
# Inject a shape, narrow, and streak event
ins.metric_array[3, 5] = 10
ins.metric_array[5] = 10
ins.metric_array[7, 7:13] = 10
ins.metric_ms = ins.mean_subtract()
ins.sig_array = np.ma.copy(ins.metric_ms)
mf.apply_match_test(ins, event_record=True)
# Check that the right events are flagged
test_mask = np.zeros(ins.metric_array.shape, dtype=bool)
test_mask[3, 5] = 1
test_mask[5] = 1
test_mask[7, 7:13] = 1
assert np.all(test_mask == ins.metric_array.mask), "Flags are incorrect"
test_match_events_slc = [
(slice(5, 6), slice(0, ins.Nfreqs), 'streak'),
(slice(7, 8), slice(7, 13), 'shape'),
(slice(3,
4), slice(5,
6), 'narrow_%.3fMHz' % (ins.freq_array[5] * 10**(-6)))
]
for i, event in enumerate(test_match_events_slc):
assert ins.match_events[i][:-1] == test_match_events_slc[
i], f"{i}th event is wrong"
assert not np.any([ins.match_events[i][-1] < 5 for i in range(3)
]), "Some significances were less than 5"
# Test a funny if block that is required when the last time in a shape is flagged
ins.metric_array[1:, 7:13] = np.ma.masked
ins.metric_ms[0, 7:13] = 10
mf.apply_match_test(ins, event_record=True)
assert np.all(ins.metric_ms.mask[:, 7:13]
), "All the times were not flagged for the shape"
maxlist = []
quadlist = np.zeros([1, 336])
linlist = np.zeros([1, 336])
for obs in obslist:
if obs not in missing_obs:
pow = np.load('%s/cfg-%s.npy' % (args.ppd_dir, obs))
pow_max = np.amax(pow)
if not os.path.exists('%s/%s_ms_poly_coeff_order_2_XX.npy' %
(args.outdir, obs)):
read_paths = util.read_paths_construct(args.ins_dir, 'original',
obs, 'INS')
ins = INS(read_paths=read_paths,
order=2,
coeff_write=True,
outpath=args.outdir,
obs=obs)
coeff = np.load('%s/%s_ms_poly_coeff_order_2_XX.npy' %
(args.outdir, obs))
med = np.absolute(coeff[:-1])
maxlist.append(pow_max)
quadlist = np.vstack([quadlist, med[:1]])
linlist = np.vstack([linlist, med[1:2]])
print('The minimum peak is %f' % min(maxlist))
print('The maximum peak is %f' % max(maxlist))
print('The minimum quad coeff is %f' % np.amin(quadlist))
print('The maximum quad coeff is %f' % np.amax(quadlist))
print('The minimum lin coeff is %f' % np.amin(linlist))
def test_mask_to_flags():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
prefix = os.path.join(DATA_PATH, '%s_test' % obs)
flags_outfile = '%s_SSINS_flags.h5' % prefix
ss = SS()
ss.read(testfile, diff=True)
uvd = UVData()
uvd.read(testfile)
uvf = UVFlag(uvd, mode='flag', waterfall=True)
# start with some flags so that we can test the intended OR operation
uvf.flag_array[6, :] = True
ins = INS(ss)
# Check error handling
with pytest.raises(ValueError):
bad_uvf = UVFlag(uvd, mode='metric', waterfall=True)
err_uvf = ins.flag_uvf(uvf=bad_uvf)
with pytest.raises(ValueError):
bad_uvf = UVFlag(uvd, mode='flag', waterfall=False)
err_uvf = ins.flag_uvf(uvf=bad_uvf)
with pytest.raises(ValueError):
bad_uvf = UVFlag(uvd, mode='flag', waterfall=True)
# Pretend the data is off by 1 day
bad_uvf.time_array += 1
err_uvf = ins.flag_uvf(uvf=bad_uvf)
# Pretend we flagged the INS object
freq_inds_1 = np.arange(0, len(ins.freq_array), 2)
freq_inds_2 = np.arange(1, len(ins.freq_array), 2)
ins.metric_array[1, freq_inds_1] = np.ma.masked
ins.metric_array[3, freq_inds_1] = np.ma.masked
ins.metric_array[7, freq_inds_2] = np.ma.masked
ins.metric_array[-2, freq_inds_2] = np.ma.masked
# Make a NEW uvflag object
new_uvf = ins.flag_uvf(uvf=uvf, inplace=False)
# Construct the expected flags by hand
test_flags = np.zeros_like(new_uvf.flag_array)
test_flags[1:5, freq_inds_1] = True
test_flags[6, :] = True
test_flags[7, freq_inds_2] = True
test_flags[8, freq_inds_2] = True
test_flags[-3:-1, freq_inds_2] = True
# Check that new flags are correct
assert np.all(new_uvf.flag_array ==
test_flags), "Test flags were not equal to calculated flags."
# Check that the input uvf was not edited in place
assert new_uvf != uvf, "The UVflag object was edited inplace and should not have been."
# Edit the uvf inplace
inplace_uvf = ins.flag_uvf(uvf=uvf, inplace=True)
# Check that new flags are correct
assert np.all(inplace_uvf.flag_array ==
test_flags), "Test flags were not equal to calculated flags."
# Check that the input uvf was not edited in place
assert inplace_uvf == uvf, "The UVflag object was not edited inplace and should have been."
# Test write/read
ins.write(prefix, output_type='flags', uvf=uvf)
read_uvf = UVFlag(flags_outfile, mode='flag', waterfall=True)
# Check equality
assert read_uvf == uvf, "UVFlag objsect differs after read"
os.remove(flags_outfile)
def test_calc_occ():
obs = "1061313128_99bl_1pol_half_time_SSINS"
testfile = os.path.join(DATA_PATH, f"{obs}.h5")
ins = INS(testfile)
# Mock some flaggable data
ins.select(freq_chans=np.arange(32), times=ins.time_array[:22])
ins.metric_array[:] = 1
ins.weights_array[:] = 10
ins.weights_square_array[:] = 10
# Make some outliers
# Narrowband in 1th, 26th, and 31th frequency
ins.metric_array[1, 1, :] = 100
ins.metric_array[1, 30, :] = 100
ins.metric_array[3:14, 26, :] = 100
# Arbitrary shape in 2, 3, 4
ins.metric_array[3:14, 2:25, :] = 100
ins.metric_array[[0, -1], :, :] = np.ma.masked
ins.metric_array[:, [0, -1], :] = np.ma.masked
ins.metric_ms = ins.mean_subtract()
num_int_flag = 2
num_chan_flag = 2
num_init_flag = np.sum(ins.metric_array.mask)
ch_wid = ins.freq_array[1] - ins.freq_array[0]
shape_dict = {
"shape":
[ins.freq_array[2] + 0.1 * ch_wid, ins.freq_array[24] - 0.1 * ch_wid]
}
mf = MF(ins.freq_array, 5, tb_aggro=0.5, shape_dict=shape_dict)
mf.apply_match_test(ins, time_broadcast=True)
occ_dict = util.calc_occ(ins,
mf,
num_init_flag,
num_int_flag=2,
lump_narrowband=False)
assert occ_dict["streak"] == 0
assert occ_dict["narrow_%.3fMHz" % (ins.freq_array[1] * 10**(-6))] == 0.05
assert occ_dict["narrow_%.3fMHz" % (ins.freq_array[26] * 10**(-6))] == 1
assert occ_dict["narrow_%.3fMHz" % (ins.freq_array[30] * 10**(-6))] == 0.05
assert occ_dict["shape"] == 1
occ_dict = util.calc_occ(ins,
mf,
num_init_flag,
num_int_flag=2,
lump_narrowband=True)
# total narrow over total valid
assert occ_dict["narrow"] == 24 / 600
assert occ_dict["streak"] == 0
assert occ_dict["shape"] == 1
assert "narrow_%.3fMHz" % (ins.freq_array[1] *
10**(-6)) not in occ_dict.keys()
assert "narrow_%.3fMHz" % (ins.freq_array[30] *
10**(-6)) not in occ_dict.keys()
yml_outpath = os.path.join(DATA_PATH, "test_occ_.yml")
with open(yml_outpath, "w") as occ_file:
yaml.safe_dump(occ_dict, occ_file)
os.remove(yml_outpath)
def test_write_mwaf():
from astropy.io import fits
obs = '1061313128_99bl_1pol_half_time_SSINS'
testfile = os.path.join(DATA_PATH, '%s.h5' % obs)
prefix = os.path.join(DATA_PATH, '%s_test' % obs)
ins = INS(testfile)
mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')]
bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')]
metafits_file = os.path.join(DATA_PATH, '1061313128.metafits')
# Compatible shape with mwaf file
ins.metric_array = np.ma.ones([55, 384, 1])
ins.metric_array[50, 16 * 12:int(16 * (12 + 0.5))] = np.ma.masked
# metadata from the input file
NCHANS = 32
Nbls = 8256
NSCANS = 224
# hard code the answer
new_flags = np.zeros((NSCANS * Nbls, NCHANS), dtype=bool)
new_flags[Nbls * 200:Nbls * 208, :16] = 1
# Test some defensive errors
with pytest.raises(IOError):
ins.write(prefix,
output_type='mwaf',
mwaf_files=bad_mwaf_files,
metafits_file=metafits_file)
with pytest.raises(ValueError):
ins.write(prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
mwaf_method='bad_method',
metafits_file=metafits_file)
with pytest.raises(ValueError):
ins.write(prefix,
output_type='mwaf',
mwaf_files=None,
metafits_file=metafits_file)
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=mwaf_files)
ins.write('%s_add' % prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
metafits_file=metafits_file)
ins.write('%s_replace' % prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
mwaf_method='replace',
metafits_file=metafits_file)
with fits.open(mwaf_files[0]) as old_mwaf_hdu:
with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu:
assert np.all(add_mwaf_hdu[1].data['FLAGS'] ==
old_mwaf_hdu[1].data['FLAGS'] + new_flags)
with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu:
assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags)
for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]:
os.remove(path)
parser.add_argument('-u', '--uvd', nargs='*', help='The path to the uvdata files')
parser.add_argument('-n', '--nsample_default', default=1, type=float, help='The default nsample to use.')
parser.add_argument('-f', '--rfi_flag', action='store_true', help="Whether or not to do rfi flagging with SSINS")
args = parser.parse_args()
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
print("The filelist is %s" % args.uvd)
indir = args.uvd[0][:args.uvd[0].rfind('/')]
if indir == args.outdir:
raise ValueError("indir and outdir are the same")
if args.rfi_flag:
if args.insfile is not None:
ins = INS(args.insfile, mask_file=args.maskfile)
else:
ss = SS()
ss.read(args.uvd, phase_to_pointing_center=True, correct_cable_len=True,
flag_choice='original', diff=True)
ins = INS(ss)
prefix = '%s/%s' % (args.outdir, args.obsid)
ins.write(prefix)
freqs = np.arange(1.7e8, 2e8, 5e6)
xticks, xticklabels = util.make_ticks_labels(freqs, ins.freq_array, sig_fig=0)
yticks = [0, 20, 40]
yticklabels = []
for tick in yticks:
yticklabels.append(Time(ins.time_array[tick], format='jd').iso[:-4])
def test_spectrum_type_bad_input():
obs = "1061313128_99bl_1pol_half_time_SSINS"
testfile = os.path.join(DATA_PATH, f"{obs}.h5")
with pytest.raises(ValueError,
match="Requested spectrum_type is invalid."):
ins = INS(testfile, spectrum_type="foo")
