def test_polyfit():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
ss = SS()
ss.read(testfile, diff=True)
ins = INS(ss, order=1)
# Mock some data for which the polyfit is exact
x = np.arange(1, 11)
ins.metric_array = np.ma.masked_array([[3 * x + 5 for i in range(3)]])
ins.metric_array.mask = np.zeros(ins.metric_array.shape, dtype=bool)
ins.metric_array = np.swapaxes(ins.metric_array, 0, 2)
ins.weights_array = np.ones(ins.metric_array.shape)
ins.metric_ms, coeffs = ins.mean_subtract(return_coeffs=True)
test_coeffs = np.zeros((ins.order + 1, ) + ins.metric_ms.shape[1:])
test_coeffs[0, :] = 3
test_coeffs[1, :] = 5
assert np.all(ins.metric_ms == np.zeros(ins.metric_ms.shape)), "The polyfit was not exact"
assert np.all(np.allclose(coeffs, test_coeffs)), "The polyfit got the wrong coefficients"
ins.metric_array[:] = np.ma.masked
ins.metric_ms = ins.mean_subtract()
assert np.all(ins.metric_ms.mask), "The metric_ms array was not all masked"
def test_match_test():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
ins = INS(insfile)
# Mock a simple metric_array and freq_array
ins.metric_array = np.ma.ones([10, 20, 1])
ins.weights_array = np.copy(ins.metric_array)
ins.freq_array = np.zeros([1, 20])
ins.freq_array = np.arange(20)
# Make a shape dictionary for a shape that will be injected later
shape = [7.9, 12.1]
shape_dict = {'shape': shape}
sig_thresh = {'shape': 5, 'narrow': 5, 'streak': 5}
mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict)
# Inject a shape, narrow, and streak event
ins.metric_array[3, 5] = 10
ins.metric_array[5] = 10
ins.metric_array[7, 7:13] = 10
ins.metric_ms = ins.mean_subtract()
t_max, f_max, R_max, shape_max = mf.match_test(ins)
print(shape_max)
assert t_max == 5, "Wrong time"
assert f_max == slice(0, 20), "Wrong freq"
assert shape_max == 'streak', "Wrong shape"
def test_apply_match_test():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
ins = INS(insfile)
# Mock a simple metric_array and freq_array
ins.metric_array = np.ma.ones([10, 20, 1])
ins.weights_array = np.copy(ins.metric_array)
ins.freq_array = np.zeros([1, 20])
ins.freq_array = np.arange(20)
# Make a shape dictionary for a shape that will be injected later
shape = [7.9, 12.1]
shape_dict = {'shape': shape}
sig_thresh = {'shape': 5, 'narrow': 5, 'streak': 5}
mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict)
# Inject a shape, narrow, and streak event
ins.metric_array[3, 5] = 10
ins.metric_array[5] = 10
ins.metric_array[7, 7:13] = 10
ins.metric_ms = ins.mean_subtract()
ins.sig_array = np.ma.copy(ins.metric_ms)
mf.apply_match_test(ins, event_record=True)
# Check that the right events are flagged
test_mask = np.zeros(ins.metric_array.shape, dtype=bool)
test_mask[3, 5] = 1
test_mask[5] = 1
test_mask[7, 7:13] = 1
assert np.all(test_mask == ins.metric_array.mask), "Flags are incorrect"
test_match_events_slc = [(5, slice(0, 20), 'streak'),
(7, slice(7, 13), 'shape'),
(3, slice(5, 6), 'narrow')]
for i, event in enumerate(test_match_events_slc):
assert ins.match_events[i][:-1] == test_match_events_slc[
i], "%ith event is wrong" % i
assert not np.any([ins.match_events[i][-1] < 5 for i in range(3)
]), "Some significances were less than 5"
# Test a funny if block that is required when the last time in a shape is flagged
ins.metric_array[1:, 7:13] = np.ma.masked
ins.metric_ms[0, 7:13] = 10
mf.apply_match_test(ins, event_record=True)
assert np.all(ins.metric_ms.mask[:, 7:13]
), "All the times were not flagged for the shape"
def test_polyfit():
obs = '1061313128_99bl_1pol_half_time'
testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs)
file_type = 'uvfits'
ss = SS()
ss.read(testfile)
ins = INS(ss, order=1)
# Mock some data for which the polyfit is exact
x = np.arange(1, 11)
ins.metric_array = np.ma.masked_array([[3 * x + 5 for i in range(3)]])
ins.metric_array.mask = np.zeros(ins.metric_array.shape, dtype=bool)
ins.metric_array = np.swapaxes(ins.metric_array, 0, 2)
ins.weights_array = np.ones(ins.metric_array.shape)
ins.metric_ms = ins.mean_subtract()
assert np.all(ins.metric_ms == np.zeros(
ins.metric_ms.shape)), "The polyfit was not exact"
def test_samp_thresh():
obs = '1061313128_99bl_1pol_half_time'
insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs)
out_prefix = os.path.join(DATA_PATH, '%s_test' % obs)
match_outfile = '%s_SSINS_match_events.yml' % out_prefix
ins = INS(insfile)
# Mock a simple metric_array and freq_array
ins.metric_array = np.ma.ones([10, 20, 1])
ins.weights_array = np.copy(ins.metric_array)
ins.metric_ms = ins.mean_subtract()
ins.sig_array = np.ma.copy(ins.metric_ms)
ins.freq_array = np.zeros([1, 20])
ins.freq_array = np.arange(20)
# Arbitrarily flag enough data in channel 10
sig_thresh = {'narrow': 5}
mf = MF(ins.freq_array, sig_thresh, streak=False, N_samp_thresh=5)
ins.metric_array[3:, 10] = np.ma.masked
ins.metric_array[3:, 9] = np.ma.masked
# Put in an outlier so it gets to samp_thresh_test
ins.metric_array[0, 11] = 10
ins.metric_ms = ins.mean_subtract()
bool_ind = np.zeros(ins.metric_array.shape, dtype=bool)
bool_ind[:, 10] = 1
bool_ind[:, 9] = 1
bool_ind[0, 11] = 1
mf.apply_match_test(ins, event_record=True, apply_samp_thresh=True)
test_match_events = [(0, slice(11, 12), 'narrow')]
test_match_events += [(ind, slice(9, 10), 'samp_thresh')
for ind in range(3)]
test_match_events += [(ind, slice(10, 11), 'samp_thresh')
for ind in range(3)]
# Test stuff
assert np.all(
ins.metric_array.mask == bool_ind), "The right flags were not applied"
for i, event in enumerate(test_match_events):
assert ins.match_events[
i][:-1] == event, "The events weren't appended correctly"
# Test that writing with samp_thresh flags is OK
ins.write(out_prefix, output_type='match_events')
test_match_events_read = ins.match_events_read(match_outfile)
os.remove(match_outfile)
assert ins.match_events == test_match_events_read
# Test that exception is raised when N_samp_thresh is too high
with pytest.raises(ValueError):
mf = MF(ins.freq_array, {'narrow': 5, 'streak': 5}, N_samp_thresh=100)
mf.apply_samp_thresh_test(ins)
def test_write_mwaf():
from astropy.io import fits
obs = '1061313128_99bl_1pol_half_time_SSINS'
testfile = os.path.join(DATA_PATH, '%s.h5' % obs)
prefix = os.path.join(DATA_PATH, '%s_test' % obs)
ins = INS(testfile)
mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')]
bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')]
# Compatible shape with mwaf file
ins.metric_array = np.ma.ones([55, 384, 1])
ins.metric_array[50, 16 * 11: 16 * (11 + 1)] = np.ma.masked
# metadata from the input file, hardcoded for testing purposes
time_div = 4
freq_div = 2
NCHANS = 32
boxint = 11
Nbls = 8256
NSCANS = 224
flags = ins.mask_to_flags()
# Repeat in time
time_rep_flags = np.repeat(flags, time_div, axis=0)
# Repeat in freq
freq_time_rep_flags = np.repeat(time_rep_flags, freq_div, axis=1)
# Repeat in bls
freq_time_bls_rep_flags = np.repeat(freq_time_rep_flags[:, np.newaxis, NCHANS * boxint: NCHANS * (boxint + 1)], Nbls, axis=1)
# This shape is on MWA wiki. Reshape to this shape.
new_flags = freq_time_bls_rep_flags.reshape((NSCANS * Nbls, NCHANS))
# Test some defensive errors
with pytest.raises(IOError):
ins.write(prefix, output_type='mwaf', mwaf_files=bad_mwaf_files)
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=mwaf_files,
mwaf_method='bad_method')
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=None)
ins.write('%s_add' % prefix, output_type='mwaf', mwaf_files=mwaf_files)
ins.write('%s_replace' % prefix, output_type='mwaf', mwaf_files=mwaf_files,
mwaf_method='replace')
with fits.open(mwaf_files[0]) as old_mwaf_hdu:
with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu:
assert np.all(add_mwaf_hdu[1].data['FLAGS'] == old_mwaf_hdu[1].data['FLAGS'] + new_flags)
with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu:
assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags)
for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]:
os.remove(path)
def test_write_mwaf():
from astropy.io import fits
obs = '1061313128_99bl_1pol_half_time_SSINS'
testfile = os.path.join(DATA_PATH, '%s.h5' % obs)
prefix = os.path.join(DATA_PATH, '%s_test' % obs)
ins = INS(testfile)
mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')]
bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')]
metafits_file = os.path.join(DATA_PATH, '1061313128.metafits')
# Compatible shape with mwaf file
ins.metric_array = np.ma.ones([55, 384, 1])
ins.metric_array[50, 16 * 12: int(16 * (12 + 0.5))] = np.ma.masked
# metadata from the input file
NCHANS = 32
Nbls = 8256
NSCANS = 224
# hard code the answer
new_flags = np.zeros((NSCANS * Nbls, NCHANS), dtype=bool)
new_flags[Nbls * 200:Nbls * 208, :16] = 1
# Test some defensive errors
with pytest.raises(IOError):
ins.write(prefix, output_type='mwaf', mwaf_files=bad_mwaf_files,
metafits_file=metafits_file)
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=mwaf_files,
mwaf_method='bad_method', metafits_file=metafits_file)
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=None,
metafits_file=metafits_file)
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=mwaf_files)
ins.write('%s_add' % prefix, output_type='mwaf', mwaf_files=mwaf_files,
metafits_file=metafits_file)
ins.write('%s_replace' % prefix, output_type='mwaf', mwaf_files=mwaf_files,
mwaf_method='replace', metafits_file=metafits_file)
with fits.open(mwaf_files[0]) as old_mwaf_hdu:
with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu:
assert np.all(add_mwaf_hdu[1].data['FLAGS'] == old_mwaf_hdu[1].data['FLAGS'] + new_flags)
with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu:
assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags)
for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]:
os.remove(path)
def test_write_mwaf():
from astropy.io import fits
obs = '1061313128_99bl_1pol_half_time_SSINS'
testfile = os.path.join(DATA_PATH, '%s.h5' % obs)
prefix = os.path.join(DATA_PATH, '%s_test' % obs)
ins = INS(testfile)
mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')]
bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')]
# shape of that mwaf file
ins.metric_array = np.ma.ones([223, 768, 1])
ins.metric_array[100, 32 * 11:32 * (11 + 1)] = np.ma.masked
flags = ins.mask_to_flags()
new_flags = np.repeat(flags[:, np.newaxis, 32 * 11:32 * (11 + 1)],
8256,
axis=1).reshape((224 * 8256, 32))
# Test some defensive errors
with pytest.raises(IOError):
ins.write(prefix, output_type='mwaf', mwaf_files=bad_mwaf_files)
with pytest.raises(ValueError):
ins.write(prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
mwaf_method='bad_method')
with pytest.raises(ValueError):
ins.write(prefix, output_type='mwaf', mwaf_files=None)
ins.write('%s_add' % prefix, output_type='mwaf', mwaf_files=mwaf_files)
ins.write('%s_replace' % prefix,
output_type='mwaf',
mwaf_files=mwaf_files,
mwaf_method='replace')
with fits.open(mwaf_files[0]) as old_mwaf_hdu:
with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu:
assert np.all(add_mwaf_hdu[1].data['FLAGS'] ==
old_mwaf_hdu[1].data['FLAGS'] + new_flags)
with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu:
assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags)
for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]:
os.remove(path)
