def test_polyfit(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) file_type = 'uvfits' ss = SS() ss.read(testfile, diff=True) ins = INS(ss, order=1) # Mock some data for which the polyfit is exact x = np.arange(1, 11) ins.metric_array = np.ma.masked_array([[3 * x + 5 for i in range(3)]]) ins.metric_array.mask = np.zeros(ins.metric_array.shape, dtype=bool) ins.metric_array = np.swapaxes(ins.metric_array, 0, 2) ins.weights_array = np.ones(ins.metric_array.shape) ins.metric_ms, coeffs = ins.mean_subtract(return_coeffs=True) test_coeffs = np.zeros((ins.order + 1, ) + ins.metric_ms.shape[1:]) test_coeffs[0, :] = 3 test_coeffs[1, :] = 5 assert np.all(ins.metric_ms == np.zeros(ins.metric_ms.shape)), "The polyfit was not exact" assert np.all(np.allclose(coeffs, test_coeffs)), "The polyfit got the wrong coefficients" ins.metric_array[:] = np.ma.masked ins.metric_ms = ins.mean_subtract() assert np.all(ins.metric_ms.mask), "The metric_ms array was not all masked"
def test_match_test(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs) ins = INS(insfile) # Mock a simple metric_array and freq_array ins.metric_array = np.ma.ones([10, 20, 1]) ins.weights_array = np.copy(ins.metric_array) ins.freq_array = np.zeros([1, 20]) ins.freq_array = np.arange(20) # Make a shape dictionary for a shape that will be injected later shape = [7.9, 12.1] shape_dict = {'shape': shape} sig_thresh = {'shape': 5, 'narrow': 5, 'streak': 5} mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict) # Inject a shape, narrow, and streak event ins.metric_array[3, 5] = 10 ins.metric_array[5] = 10 ins.metric_array[7, 7:13] = 10 ins.metric_ms = ins.mean_subtract() t_max, f_max, R_max, shape_max = mf.match_test(ins) print(shape_max) assert t_max == 5, "Wrong time" assert f_max == slice(0, 20), "Wrong freq" assert shape_max == 'streak', "Wrong shape"
def test_apply_match_test(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs) ins = INS(insfile) # Mock a simple metric_array and freq_array ins.metric_array = np.ma.ones([10, 20, 1]) ins.weights_array = np.copy(ins.metric_array) ins.freq_array = np.zeros([1, 20]) ins.freq_array = np.arange(20) # Make a shape dictionary for a shape that will be injected later shape = [7.9, 12.1] shape_dict = {'shape': shape} sig_thresh = {'shape': 5, 'narrow': 5, 'streak': 5} mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict) # Inject a shape, narrow, and streak event ins.metric_array[3, 5] = 10 ins.metric_array[5] = 10 ins.metric_array[7, 7:13] = 10 ins.metric_ms = ins.mean_subtract() ins.sig_array = np.ma.copy(ins.metric_ms) mf.apply_match_test(ins, event_record=True) # Check that the right events are flagged test_mask = np.zeros(ins.metric_array.shape, dtype=bool) test_mask[3, 5] = 1 test_mask[5] = 1 test_mask[7, 7:13] = 1 assert np.all(test_mask == ins.metric_array.mask), "Flags are incorrect" test_match_events_slc = [(5, slice(0, 20), 'streak'), (7, slice(7, 13), 'shape'), (3, slice(5, 6), 'narrow')] for i, event in enumerate(test_match_events_slc): assert ins.match_events[i][:-1] == test_match_events_slc[ i], "%ith event is wrong" % i assert not np.any([ins.match_events[i][-1] < 5 for i in range(3) ]), "Some significances were less than 5" # Test a funny if block that is required when the last time in a shape is flagged ins.metric_array[1:, 7:13] = np.ma.masked ins.metric_ms[0, 7:13] = 10 mf.apply_match_test(ins, event_record=True) assert np.all(ins.metric_ms.mask[:, 7:13] ), "All the times were not flagged for the shape"
def test_polyfit(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) file_type = 'uvfits' ss = SS() ss.read(testfile) ins = INS(ss, order=1) # Mock some data for which the polyfit is exact x = np.arange(1, 11) ins.metric_array = np.ma.masked_array([[3 * x + 5 for i in range(3)]]) ins.metric_array.mask = np.zeros(ins.metric_array.shape, dtype=bool) ins.metric_array = np.swapaxes(ins.metric_array, 0, 2) ins.weights_array = np.ones(ins.metric_array.shape) ins.metric_ms = ins.mean_subtract() assert np.all(ins.metric_ms == np.zeros( ins.metric_ms.shape)), "The polyfit was not exact"
def test_samp_thresh(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs) out_prefix = os.path.join(DATA_PATH, '%s_test' % obs) match_outfile = '%s_SSINS_match_events.yml' % out_prefix ins = INS(insfile) # Mock a simple metric_array and freq_array ins.metric_array = np.ma.ones([10, 20, 1]) ins.weights_array = np.copy(ins.metric_array) ins.metric_ms = ins.mean_subtract() ins.sig_array = np.ma.copy(ins.metric_ms) ins.freq_array = np.zeros([1, 20]) ins.freq_array = np.arange(20) # Arbitrarily flag enough data in channel 10 sig_thresh = {'narrow': 5} mf = MF(ins.freq_array, sig_thresh, streak=False, N_samp_thresh=5) ins.metric_array[3:, 10] = np.ma.masked ins.metric_array[3:, 9] = np.ma.masked # Put in an outlier so it gets to samp_thresh_test ins.metric_array[0, 11] = 10 ins.metric_ms = ins.mean_subtract() bool_ind = np.zeros(ins.metric_array.shape, dtype=bool) bool_ind[:, 10] = 1 bool_ind[:, 9] = 1 bool_ind[0, 11] = 1 mf.apply_match_test(ins, event_record=True, apply_samp_thresh=True) test_match_events = [(0, slice(11, 12), 'narrow')] test_match_events += [(ind, slice(9, 10), 'samp_thresh') for ind in range(3)] test_match_events += [(ind, slice(10, 11), 'samp_thresh') for ind in range(3)] # Test stuff assert np.all( ins.metric_array.mask == bool_ind), "The right flags were not applied" for i, event in enumerate(test_match_events): assert ins.match_events[ i][:-1] == event, "The events weren't appended correctly" # Test that writing with samp_thresh flags is OK ins.write(out_prefix, output_type='match_events') test_match_events_read = ins.match_events_read(match_outfile) os.remove(match_outfile) assert ins.match_events == test_match_events_read # Test that exception is raised when N_samp_thresh is too high with pytest.raises(ValueError): mf = MF(ins.freq_array, {'narrow': 5, 'streak': 5}, N_samp_thresh=100) mf.apply_samp_thresh_test(ins)
def test_write_mwaf(): from astropy.io import fits obs = '1061313128_99bl_1pol_half_time_SSINS' testfile = os.path.join(DATA_PATH, '%s.h5' % obs) prefix = os.path.join(DATA_PATH, '%s_test' % obs) ins = INS(testfile) mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')] bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')] # Compatible shape with mwaf file ins.metric_array = np.ma.ones([55, 384, 1]) ins.metric_array[50, 16 * 11: 16 * (11 + 1)] = np.ma.masked # metadata from the input file, hardcoded for testing purposes time_div = 4 freq_div = 2 NCHANS = 32 boxint = 11 Nbls = 8256 NSCANS = 224 flags = ins.mask_to_flags() # Repeat in time time_rep_flags = np.repeat(flags, time_div, axis=0) # Repeat in freq freq_time_rep_flags = np.repeat(time_rep_flags, freq_div, axis=1) # Repeat in bls freq_time_bls_rep_flags = np.repeat(freq_time_rep_flags[:, np.newaxis, NCHANS * boxint: NCHANS * (boxint + 1)], Nbls, axis=1) # This shape is on MWA wiki. Reshape to this shape. new_flags = freq_time_bls_rep_flags.reshape((NSCANS * Nbls, NCHANS)) # Test some defensive errors with pytest.raises(IOError): ins.write(prefix, output_type='mwaf', mwaf_files=bad_mwaf_files) with pytest.raises(ValueError): ins.write(prefix, output_type='mwaf', mwaf_files=mwaf_files, mwaf_method='bad_method') with pytest.raises(ValueError): ins.write(prefix, output_type='mwaf', mwaf_files=None) ins.write('%s_add' % prefix, output_type='mwaf', mwaf_files=mwaf_files) ins.write('%s_replace' % prefix, output_type='mwaf', mwaf_files=mwaf_files, mwaf_method='replace') with fits.open(mwaf_files[0]) as old_mwaf_hdu: with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu: assert np.all(add_mwaf_hdu[1].data['FLAGS'] == old_mwaf_hdu[1].data['FLAGS'] + new_flags) with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu: assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags) for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]: os.remove(path)
def test_write_mwaf(): from astropy.io import fits obs = '1061313128_99bl_1pol_half_time_SSINS' testfile = os.path.join(DATA_PATH, '%s.h5' % obs) prefix = os.path.join(DATA_PATH, '%s_test' % obs) ins = INS(testfile) mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')] bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')] metafits_file = os.path.join(DATA_PATH, '1061313128.metafits') # Compatible shape with mwaf file ins.metric_array = np.ma.ones([55, 384, 1]) ins.metric_array[50, 16 * 12: int(16 * (12 + 0.5))] = np.ma.masked # metadata from the input file NCHANS = 32 Nbls = 8256 NSCANS = 224 # hard code the answer new_flags = np.zeros((NSCANS * Nbls, NCHANS), dtype=bool) new_flags[Nbls * 200:Nbls * 208, :16] = 1 # Test some defensive errors with pytest.raises(IOError): ins.write(prefix, output_type='mwaf', mwaf_files=bad_mwaf_files, metafits_file=metafits_file) with pytest.raises(ValueError): ins.write(prefix, output_type='mwaf', mwaf_files=mwaf_files, mwaf_method='bad_method', metafits_file=metafits_file) with pytest.raises(ValueError): ins.write(prefix, output_type='mwaf', mwaf_files=None, metafits_file=metafits_file) with pytest.raises(ValueError): ins.write(prefix, output_type='mwaf', mwaf_files=mwaf_files) ins.write('%s_add' % prefix, output_type='mwaf', mwaf_files=mwaf_files, metafits_file=metafits_file) ins.write('%s_replace' % prefix, output_type='mwaf', mwaf_files=mwaf_files, mwaf_method='replace', metafits_file=metafits_file) with fits.open(mwaf_files[0]) as old_mwaf_hdu: with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu: assert np.all(add_mwaf_hdu[1].data['FLAGS'] == old_mwaf_hdu[1].data['FLAGS'] + new_flags) with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu: assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags) for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]: os.remove(path)
def test_write_mwaf(): from astropy.io import fits obs = '1061313128_99bl_1pol_half_time_SSINS' testfile = os.path.join(DATA_PATH, '%s.h5' % obs) prefix = os.path.join(DATA_PATH, '%s_test' % obs) ins = INS(testfile) mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')] bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')] # shape of that mwaf file ins.metric_array = np.ma.ones([223, 768, 1]) ins.metric_array[100, 32 * 11:32 * (11 + 1)] = np.ma.masked flags = ins.mask_to_flags() new_flags = np.repeat(flags[:, np.newaxis, 32 * 11:32 * (11 + 1)], 8256, axis=1).reshape((224 * 8256, 32)) # Test some defensive errors with pytest.raises(IOError): ins.write(prefix, output_type='mwaf', mwaf_files=bad_mwaf_files) with pytest.raises(ValueError): ins.write(prefix, output_type='mwaf', mwaf_files=mwaf_files, mwaf_method='bad_method') with pytest.raises(ValueError): ins.write(prefix, output_type='mwaf', mwaf_files=None) ins.write('%s_add' % prefix, output_type='mwaf', mwaf_files=mwaf_files) ins.write('%s_replace' % prefix, output_type='mwaf', mwaf_files=mwaf_files, mwaf_method='replace') with fits.open(mwaf_files[0]) as old_mwaf_hdu: with fits.open('%s_add_12.mwaf' % prefix) as add_mwaf_hdu: assert np.all(add_mwaf_hdu[1].data['FLAGS'] == old_mwaf_hdu[1].data['FLAGS'] + new_flags) with fits.open('%s_replace_12.mwaf' % prefix) as replace_mwaf_hdu: assert np.all(replace_mwaf_hdu[1].data['FLAGS'] == new_flags) for path in ['%s_add_12.mwaf' % prefix, '%s_replace_12.mwaf' % prefix]: os.remove(path)