def test_time_broadcast():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, f'{obs}_SSINS.h5')
out_prefix = os.path.join(DATA_PATH, f'{obs}_test')
match_outfile = f'{out_prefix}_SSINS_match_events.yml'
ins = INS(insfile)
# Mock a simple metric_array and freq_array
ins.metric_array[:] = 1
ins.weights_array = np.copy(ins.metric_array)
ins.weights_square_array = np.copy(ins.weights_array)
ins.metric_ms = ins.mean_subtract()
ins.sig_array = np.ma.copy(ins.metric_ms)
# Arbitrarily flag enough data in channel 10
sig_thresh = {'narrow': 5}
mf = MF(ins.freq_array, sig_thresh, streak=False, tb_aggro=0.5)
ins.metric_array[4:, 9] = np.ma.masked
ins.metric_array[4:, 10] = np.ma.masked
# Put in an outlier so it gets to samp_thresh_test
ins.metric_array[2, 9] = 100
ins.metric_array[1, 10] = 100
ins.metric_ms = ins.mean_subtract()
bool_ind = np.zeros(ins.metric_array.shape, dtype=bool)
bool_ind[:, 10] = 1
bool_ind[:, 9] = 1
mf.apply_match_test(ins, event_record=True, time_broadcast=True)
print(ins.match_events)
test_match_events = [
(slice(1, 2), slice(10, 11),
'narrow_%.3fMHz' % (ins.freq_array[10] * 10**(-6))),
(slice(0, ins.Ntimes), slice(10, 11),
'time_broadcast_narrow_%.3fMHz' % (ins.freq_array[10] * 10**(-6))),
(slice(2, 3), slice(9, 10),
'narrow_%.3fMHz' % (ins.freq_array[9] * 10**(-6))),
(slice(0, ins.Ntimes), slice(9, 10),
'time_broadcast_narrow_%.3fMHz' % (ins.freq_array[9] * 10**(-6)))
]
# Test stuff
assert np.all(
ins.metric_array.mask == bool_ind), "The right flags were not applied"
for i, event in enumerate(test_match_events):
assert ins.match_events[
i][:-1] == event, "The events weren't appended correctly"
# Test that writing with samp_thresh flags is OK
ins.write(out_prefix, output_type='match_events')
test_match_events_read = ins.match_events_read(match_outfile)
os.remove(match_outfile)
assert ins.match_events == test_match_events_read
# Test that exception is raised when tb_aggro is too high
with pytest.raises(ValueError):
mf = MF(ins.freq_array, {'narrow': 5, 'streak': 5}, tb_aggro=100)
mf.apply_samp_thresh_test(ins, (slice(1, 2), slice(10, 11), 'narrow'))
def test_samp_thresh():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
out_prefix = os.path.join(DATA_PATH, '%s_test' % obs)
match_outfile = '%s_SSINS_match_events.yml' % out_prefix
ins = INS(insfile)
# Mock a simple metric_array and freq_array
ins.metric_array = np.ma.ones([10, 20, 1])
ins.weights_array = np.copy(ins.metric_array)
ins.metric_ms = ins.mean_subtract()
ins.sig_array = np.ma.copy(ins.metric_ms)
ins.freq_array = np.zeros([1, 20])
ins.freq_array = np.arange(20)
# Arbitrarily flag enough data in channel 10
sig_thresh = {'narrow': 5}
mf = MF(ins.freq_array, sig_thresh, streak=False, N_samp_thresh=5)
ins.metric_array[3:, 10] = np.ma.masked
ins.metric_array[3:, 9] = np.ma.masked
# Put in an outlier so it gets to samp_thresh_test
ins.metric_array[0, 11] = 10
ins.metric_ms = ins.mean_subtract()
bool_ind = np.zeros(ins.metric_array.shape, dtype=bool)
bool_ind[:, 10] = 1
bool_ind[:, 9] = 1
bool_ind[0, 11] = 1
mf.apply_match_test(ins, event_record=True, apply_samp_thresh=True)
test_match_events = [(0, slice(11, 12), 'narrow')]
test_match_events += [(ind, slice(9, 10), 'samp_thresh')
for ind in range(3)]
test_match_events += [(ind, slice(10, 11), 'samp_thresh')
for ind in range(3)]
# Test stuff
assert np.all(
ins.metric_array.mask == bool_ind), "The right flags were not applied"
for i, event in enumerate(test_match_events):
assert ins.match_events[
i][:-1] == event, "The events weren't appended correctly"
# Test that writing with samp_thresh flags is OK
ins.write(out_prefix, output_type='match_events')
test_match_events_read = ins.match_events_read(match_outfile)
os.remove(match_outfile)
assert ins.match_events == test_match_events_read
# Test that exception is raised when N_samp_thresh is too high
with pytest.raises(ValueError):
mf = MF(ins.freq_array, {'narrow': 5, 'streak': 5}, N_samp_thresh=100)
mf.apply_samp_thresh_test(ins)
xticks=xticks, yticks=yticks, xticklabels=xticklabels,
yticklabels=yticklabels, data_cmap=cm.plasma,
ms_vmin=-5, ms_vmax=5, title=args.obsid,
xlabel='Frequency (Mhz)', ylabel='Time (UTC)')
# Try to save memory - hope for garbage collector
del ss
# Set up MF flagging for routine shapes
shape_dict = {'TV6': [1.74e8, 1.81e8], 'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8], 'TV9': [1.95e8, 2.02e8]}
sig_thresh = {shape: 5 for shape in shape_dict}
sig_thresh['narrow'] = 5
sig_thresh['streak'] = 8
mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict,
N_samp_thresh=len(ins.time_array) // 2)
mf.apply_match_test(ins, apply_samp_thresh=False)
mf.apply_samp_thresh_test(ins, event_record=True)
Catalog_Plot.INS_plot(ins, '%s_flagged' % prefix,
xticks=xticks, yticks=yticks, xticklabels=xticklabels,
yticklabels=yticklabels, data_cmap=cm.plasma,
ms_vmin=-5, ms_vmax=5, title=args.obsid,
xlabel='Frequency (Mhz)', ylabel='Time (UTC)')
ins.write(prefix, output_type='mask')
uvd = UVData()
uvd.read(args.uvd, phase_to_pointing_center=True, correct_cable_len=True)
uvf = UVFlag(uvd, mode='flag', waterfall=True)
uvf.flag_array = ins.mask_to_flags()
utils.apply_uvflag(uvd, uvf, inplace=True)
if np.any(uvd.nsample_array == 0):
uvd.nsample_array[uvd.nsample_array == 0] = args.nsample_default
def execbody(ins_filepath):
slash_ind = ins_filepath.rfind('/')
obsid = ins_filepath[slash_ind + 1:slash_ind + 11]
ins = INS(ins_filepath)
ins.select(times=ins.time_array[3:-3])
ins.metric_ms = ins.mean_subtract()
shape_dict = {
'TV6': [1.74e8, 1.81e8],
'TV7': [1.81e8, 1.88e8],
'TV8': [1.88e8, 1.95e8],
'TV9': [1.95e8, 2.02e8]
}
sig_thresh = {shape: 5 for shape in shape_dict}
sig_thresh['narrow'] = 5
sig_thresh['streak'] = 8
mf = MF(ins.freq_array,
sig_thresh,
shape_dict=shape_dict,
N_samp_thresh=len(ins.time_array) // 2)
ins.metric_array[ins.metric_array == 0] = np.ma.masked
ins.metric_ms = ins.mean_subtract()
ins.sig_array = np.ma.copy(ins.metric_ms)
#write plots if command flagged to do so
if args.plots:
prefix = '%s/%s_trimmed_zeromask' % (args.outdir, obsid)
freqs = np.arange(1.7e8, 2e8, 5e6)
xticks, xticklabels = util.make_ticks_labels(freqs,
ins.freq_array,
sig_fig=0)
yticks = [0, 20, 40]
yticklabels = []
for tick in yticks:
yticklabels.append(
Time(ins.time_array[tick], format='jd').iso[:-4])
Catalog_Plot.INS_plot(ins,
prefix,
xticks=xticks,
yticks=yticks,
xticklabels=xticklabels,
yticklabels=yticklabels,
data_cmap=cm.plasma,
ms_vmin=-5,
ms_vmax=5,
title=obsid,
xlabel='Frequency (Mhz)',
ylabel='Time (UTC)')
if args.verbose:
print("wrote trimmed zeromask plot for " + obsid)
mf.apply_match_test(ins, apply_samp_thresh=False)
mf.apply_samp_thresh_test(ins, event_record=True)
flagged_prefix = '%s/%s_trimmed_zeromask_MF_s8' % (args.outdir, obsid)
#write data/mask/match/ if command flagged to do so
if args.write:
ins.write(flagged_prefix, output_type='data', clobber=True)
ins.write(flagged_prefix, output_type='mask', clobber=True)
ins.write(flagged_prefix, output_type='match_events')
if args.verbose:
print("wrote data/mask/match files for " + obsid)
#write plots if command flagged to do so
if args.plots:
Catalog_Plot.INS_plot(ins,
flagged_prefix,
xticks=xticks,
yticks=yticks,
xticklabels=xticklabels,
yticklabels=yticklabels,
data_cmap=cm.plasma,
ms_vmin=-5,
ms_vmax=5,
title=obsid,
xlabel='Frequency (Mhz)',
ylabel='Time (UTC)')
if args.verbose:
print("wrote trimmed zeromask (w/ match filter) for " + obsid)
#a hardcoded csv generator for occ_csv
if args.gencsv is not None:
csv = "" + obsid + "," + flagged_prefix + "_SSINS_data.h5," + flagged_prefix + "_SSINS_mask.h5," + flagged_prefix + "_SSINS_match_events.yml\n"
with open(args.gencsv, "a") as csvfile:
csvfile.write(csv)
print("wrote entry for " + obsid)