def test_time_broadcast_no_new_event(): """ Tests that a new event is not added because the agression threshold is not exceeded. """ obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, f'{obs}_SSINS.h5') out_prefix = os.path.join(DATA_PATH, f'{obs}_test') match_outfile = f'{out_prefix}_SSINS_match_events.yml' ins = INS(insfile) # Mock a simple metric_array and freq_array (and weights...) ins.metric_array[:] = 1 ins.weights_array = np.copy(ins.metric_array) ins.weights_square_array = np.copy(ins.weights_array) ins.metric_ms = ins.mean_subtract() ins.sig_array = np.ma.copy(ins.metric_ms) sig_thresh = {'narrow': 5} mf = MF(ins.freq_array, sig_thresh, streak=False, tb_aggro=0.5) # Put in an outlier so it gets to samp_thresh_test ins.metric_array[1, 10] = 100 ins.metric_ms = ins.mean_subtract() mf.apply_match_test(ins, event_record=True, time_broadcast=True) event = mf.time_broadcast(ins, ins.match_events[0], event_record=True) assert event == ins.match_events[0]
def test_polyfit(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) file_type = 'uvfits' ss = SS() ss.read(testfile, diff=True) ins = INS(ss, order=1) # Mock some data for which the polyfit is exact x = np.arange(1, ins.Ntimes + 1) for ind in range(ins.Nfreqs): ins.metric_array[:, ind, 0] = 3 * x + 5 ins.metric_array.mask = np.zeros(ins.metric_array.shape, dtype=bool) ins.weights_array = np.ones(ins.metric_array.shape) ins.weights_square_array = np.copy(ins.weights_array) ins.metric_ms, coeffs = ins.mean_subtract(return_coeffs=True) test_coeffs = np.zeros((ins.order + 1, ) + ins.metric_ms.shape[1:]) test_coeffs[0, :] = 3 test_coeffs[1, :] = 5 assert np.all(np.allclose(ins.metric_ms, np.zeros( ins.metric_ms.shape))), "The polyfit was not exact" assert np.all(np.allclose( coeffs, test_coeffs)), "The polyfit got the wrong coefficients" ins.metric_array[:] = np.ma.masked ins.metric_ms = ins.mean_subtract() assert np.all(ins.metric_ms.mask), "The metric_ms array was not all masked"
def test_time_broadcast(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, f'{obs}_SSINS.h5') out_prefix = os.path.join(DATA_PATH, f'{obs}_test') match_outfile = f'{out_prefix}_SSINS_match_events.yml' ins = INS(insfile) # Mock a simple metric_array and freq_array ins.metric_array[:] = 1 ins.weights_array = np.copy(ins.metric_array) ins.weights_square_array = np.copy(ins.weights_array) ins.metric_ms = ins.mean_subtract() ins.sig_array = np.ma.copy(ins.metric_ms) # Arbitrarily flag enough data in channel 10 sig_thresh = {'narrow': 5} mf = MF(ins.freq_array, sig_thresh, streak=False, tb_aggro=0.5) ins.metric_array[4:, 9] = np.ma.masked ins.metric_array[4:, 10] = np.ma.masked # Put in an outlier so it gets to samp_thresh_test ins.metric_array[2, 9] = 100 ins.metric_array[1, 10] = 100 ins.metric_ms = ins.mean_subtract() bool_ind = np.zeros(ins.metric_array.shape, dtype=bool) bool_ind[:, 10] = 1 bool_ind[:, 9] = 1 mf.apply_match_test(ins, event_record=True, time_broadcast=True) print(ins.match_events) test_match_events = [ (slice(1, 2), slice(10, 11), 'narrow_%.3fMHz' % (ins.freq_array[10] * 10**(-6))), (slice(0, ins.Ntimes), slice(10, 11), 'time_broadcast_narrow_%.3fMHz' % (ins.freq_array[10] * 10**(-6))), (slice(2, 3), slice(9, 10), 'narrow_%.3fMHz' % (ins.freq_array[9] * 10**(-6))), (slice(0, ins.Ntimes), slice(9, 10), 'time_broadcast_narrow_%.3fMHz' % (ins.freq_array[9] * 10**(-6))) ] # Test stuff assert np.all( ins.metric_array.mask == bool_ind), "The right flags were not applied" for i, event in enumerate(test_match_events): assert ins.match_events[ i][:-1] == event, "The events weren't appended correctly" # Test that writing with samp_thresh flags is OK ins.write(out_prefix, output_type='match_events') test_match_events_read = ins.match_events_read(match_outfile) os.remove(match_outfile) assert ins.match_events == test_match_events_read # Test that exception is raised when tb_aggro is too high with pytest.raises(ValueError): mf = MF(ins.freq_array, {'narrow': 5, 'streak': 5}, tb_aggro=100) mf.apply_samp_thresh_test(ins, (slice(1, 2), slice(10, 11), 'narrow'))
def test_samp_thresh(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs) out_prefix = os.path.join(DATA_PATH, '%s_test' % obs) match_outfile = '%s_SSINS_match_events.yml' % out_prefix ins = INS(insfile) # Mock a simple metric_array and freq_array ins.metric_array = np.ma.ones([10, 20, 1]) ins.weights_array = np.copy(ins.metric_array) ins.metric_ms = ins.mean_subtract() ins.sig_array = np.ma.copy(ins.metric_ms) ins.freq_array = np.zeros([1, 20]) ins.freq_array = np.arange(20) # Arbitrarily flag enough data in channel 10 sig_thresh = {'narrow': 5} mf = MF(ins.freq_array, sig_thresh, streak=False, N_samp_thresh=5) ins.metric_array[3:, 10] = np.ma.masked ins.metric_array[3:, 9] = np.ma.masked # Put in an outlier so it gets to samp_thresh_test ins.metric_array[0, 11] = 10 ins.metric_ms = ins.mean_subtract() bool_ind = np.zeros(ins.metric_array.shape, dtype=bool) bool_ind[:, 10] = 1 bool_ind[:, 9] = 1 bool_ind[0, 11] = 1 mf.apply_match_test(ins, event_record=True, apply_samp_thresh=True) test_match_events = [(0, slice(11, 12), 'narrow')] test_match_events += [(ind, slice(9, 10), 'samp_thresh') for ind in range(3)] test_match_events += [(ind, slice(10, 11), 'samp_thresh') for ind in range(3)] # Test stuff assert np.all( ins.metric_array.mask == bool_ind), "The right flags were not applied" for i, event in enumerate(test_match_events): assert ins.match_events[ i][:-1] == event, "The events weren't appended correctly" # Test that writing with samp_thresh flags is OK ins.write(out_prefix, output_type='match_events') test_match_events_read = ins.match_events_read(match_outfile) os.remove(match_outfile) assert ins.match_events == test_match_events_read # Test that exception is raised when N_samp_thresh is too high with pytest.raises(ValueError): mf = MF(ins.freq_array, {'narrow': 5, 'streak': 5}, N_samp_thresh=100) mf.apply_samp_thresh_test(ins)
def test_match_test(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs) ins = INS(insfile) # Mock a simple metric_array and freq_array ins.metric_array[:] = np.ones_like(ins.metric_array) ins.weights_array = np.copy(ins.metric_array) ins.weights_square_array = np.copy(ins.weights_array) # Make a shape dictionary for a shape that will be injected later ch_wid = ins.freq_array[1] - ins.freq_array[0] shape = [ ins.freq_array[8] - 0.2 * ch_wid, ins.freq_array[12] + 0.2 * ch_wid ] shape_dict = {'shape': shape} sig_thresh = {'shape': 5, 'narrow': 5, 'streak': 5} mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict) # Inject a shape, narrow, and streak event ins.metric_array[3, 5] = 10 ins.metric_array[5] = 10 ins.metric_array[7, 7:13] = 10 ins.metric_ms = ins.mean_subtract() t_max, f_max, shape_max, sig_max = mf.match_test(ins) assert t_max == slice(5, 6), "Wrong time" assert f_max == slice(0, ins.Nfreqs), "Wrong freq" assert shape_max == 'streak', "Wrong shape"
def test_freq_broadcast_whole_band(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, f'{obs}_SSINS.h5') out_prefix = os.path.join(DATA_PATH, f'{obs}_test') match_outfile = f'{out_prefix}_SSINS_match_events.yml' ins = INS(insfile) # spoof the metric array ins.metric_array[:] = 1 ins.metric_array[2, 10:20] = 10 ins.metric_array[4, 40:50] = 10 ins.metric_ms = ins.mean_subtract() shape_dict = { 'shape1': [ins.freq_array[10], ins.freq_array[20]], 'shape2': [ins.freq_array[40], ins.freq_array[50]] } mf = MF(ins.freq_array, 5, shape_dict=shape_dict, broadcast_streak=True) mf.apply_match_test(ins, event_record=True, freq_broadcast=True) assert np.all(ins.metric_array.mask[2]) assert np.all(ins.metric_array.mask[4])
def test_match_test(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs) ins = INS(insfile) # Mock a simple metric_array and freq_array ins.metric_array = np.ma.ones([10, 20, 1]) ins.weights_array = np.copy(ins.metric_array) ins.freq_array = np.zeros([1, 20]) ins.freq_array = np.arange(20) # Make a shape dictionary for a shape that will be injected later shape = [7.9, 12.1] shape_dict = {'shape': shape} sig_thresh = {'shape': 5, 'narrow': 5, 'streak': 5} mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict) # Inject a shape, narrow, and streak event ins.metric_array[3, 5] = 10 ins.metric_array[5] = 10 ins.metric_array[7, 7:13] = 10 ins.metric_ms = ins.mean_subtract() t_max, f_max, R_max, shape_max = mf.match_test(ins) print(shape_max) assert t_max == 5, "Wrong time" assert f_max == slice(0, 20), "Wrong freq" assert shape_max == 'streak', "Wrong shape"
def test_freq_broadcast_no_new_event(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, f'{obs}_SSINS.h5') out_prefix = os.path.join(DATA_PATH, f'{obs}_test') match_outfile = f'{out_prefix}_SSINS_match_events.yml' ins = INS(insfile) # spoof the metric array ins.metric_array[:] = 1 ins.metric_array[2, 10:20] = 10 ins.metric_ms = ins.mean_subtract() # Slice will go up to 21 since shapes are inclusive at the boundaries # when boundary is on channel center shape_dict = {'shape1': [ins.freq_array[10], ins.freq_array[20]]} broadcast_dict = {'sb2': [ins.freq_array[30], ins.freq_array[59]]} test_event = (slice(2, 3), slice(10, 21), 'shape1', 10) mf = MF(ins.freq_array, 5, shape_dict=shape_dict, broadcast_streak=False, broadcast_dict=broadcast_dict) mf.apply_match_test(ins, event_record=True, freq_broadcast=True) event = mf.freq_broadcast(ins, event=test_event, event_record=True) assert event == test_event assert ins.match_events[0][:-1] == test_event[:-1]
def test_calc_occ(): obs = "1061313128_99bl_1pol_half_time_SSINS" testfile = os.path.join(DATA_PATH, f"{obs}.h5") ins = INS(testfile) # Mock some flaggable data ins.select(freq_chans=np.arange(32), times=ins.time_array[:22]) ins.metric_array[:] = 1 ins.weights_array[:] = 10 ins.weights_square_array[:] = 10 # Make some outliers # Narrowband in 1th, 26th, and 31th frequency ins.metric_array[1, 1, :] = 100 ins.metric_array[1, 30, :] = 100 ins.metric_array[3:14, 26, :] = 100 # Arbitrary shape in 2, 3, 4 ins.metric_array[3:14, 2:25, :] = 100 ins.metric_array[[0, -1], :, :] = np.ma.masked ins.metric_array[:, [0, -1], :] = np.ma.masked ins.metric_ms = ins.mean_subtract() num_int_flag = 2 num_chan_flag = 2 num_init_flag = np.sum(ins.metric_array.mask) ch_wid = ins.freq_array[1] - ins.freq_array[0] shape_dict = {"shape": [ins.freq_array[2] + 0.1 * ch_wid, ins.freq_array[24] - 0.1 * ch_wid]} mf = MF(ins.freq_array, 5, tb_aggro=0.5, shape_dict=shape_dict) mf.apply_match_test(ins, time_broadcast=True) occ_dict = util.calc_occ(ins, mf, num_init_flag, num_int_flag=2, lump_narrowband=False) assert occ_dict["streak"] == 0 assert occ_dict["narrow_%.3fMHz" % (ins.freq_array[1] * 10**(-6))] == 0.05 assert occ_dict["narrow_%.3fMHz" % (ins.freq_array[26] * 10**(-6))] == 1 assert occ_dict["narrow_%.3fMHz" % (ins.freq_array[30] * 10**(-6))] == 0.05 assert occ_dict["shape"] == 1 occ_dict = util.calc_occ(ins, mf, num_init_flag, num_int_flag=2, lump_narrowband=True) # total narrow over total valid assert occ_dict["narrow"] == 24 / 600 assert occ_dict["streak"] == 0 assert occ_dict["shape"] == 1 assert "narrow_%.3fMHz" % (ins.freq_array[1] * 10**(-6)) not in occ_dict.keys() assert "narrow_%.3fMHz" % (ins.freq_array[30] * 10**(-6)) not in occ_dict.keys() yml_outpath = os.path.join(DATA_PATH, "test_occ_.yml") with open(yml_outpath, "w") as occ_file: yaml.safe_dump(occ_dict, occ_file) os.remove(yml_outpath)
def test_apply_match_test(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, '%s_SSINS.h5' % obs) ins = INS(insfile) # Mock a simple metric_array and freq_array ins.metric_array = np.ma.ones([10, 20, 1]) ins.weights_array = np.copy(ins.metric_array) ins.freq_array = np.zeros([1, 20]) ins.freq_array = np.arange(20) # Make a shape dictionary for a shape that will be injected later shape = [7.9, 12.1] shape_dict = {'shape': shape} sig_thresh = {'shape': 5, 'narrow': 5, 'streak': 5} mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict) # Inject a shape, narrow, and streak event ins.metric_array[3, 5] = 10 ins.metric_array[5] = 10 ins.metric_array[7, 7:13] = 10 ins.metric_ms = ins.mean_subtract() ins.sig_array = np.ma.copy(ins.metric_ms) mf.apply_match_test(ins, event_record=True) # Check that the right events are flagged test_mask = np.zeros(ins.metric_array.shape, dtype=bool) test_mask[3, 5] = 1 test_mask[5] = 1 test_mask[7, 7:13] = 1 assert np.all(test_mask == ins.metric_array.mask), "Flags are incorrect" test_match_events_slc = [(5, slice(0, 20), 'streak'), (7, slice(7, 13), 'shape'), (3, slice(5, 6), 'narrow')] for i, event in enumerate(test_match_events_slc): assert ins.match_events[i][:-1] == test_match_events_slc[ i], "%ith event is wrong" % i assert not np.any([ins.match_events[i][-1] < 5 for i in range(3) ]), "Some significances were less than 5" # Test a funny if block that is required when the last time in a shape is flagged ins.metric_array[1:, 7:13] = np.ma.masked ins.metric_ms[0, 7:13] = 10 mf.apply_match_test(ins, event_record=True) assert np.all(ins.metric_ms.mask[:, 7:13] ), "All the times were not flagged for the shape"
def test_write(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) file_type = 'uvfits' prefix = os.path.join(DATA_PATH, '%s_test' % obs) data_outfile = '%s_SSINS_data.h5' % prefix z_score_outfile = '%s_SSINS_z_score.h5' % prefix mask_outfile = '%s_SSINS_mask.h5' % prefix match_outfile = '%s_SSINS_match_events.yml' % prefix sep_data_outfile = '%s.SSINS.data.h5' % prefix ss = SS() ss.read(testfile, flag_choice='original', diff=True) ins = INS(ss) # Mock some events ins.match_events.append((slice(0, 1), slice(1, 3), 'shape', 5)) ins.match_events.append((slice(1, 2), slice(1, 3), 'shape', 5)) ins.metric_array[:2, 1:3] = np.ma.masked ins.metric_ms = ins.mean_subtract() ins.write(prefix, output_type='data', clobber=True) ins.write(prefix, output_type='z_score', clobber=True) ins.write(prefix, output_type='mask', clobber=True) ins.write(prefix, output_type='match_events', clobber=True) ins.write(prefix, output_type='data', sep='.', clobber=True) with pytest.raises(ValueError): ins.write(prefix, output_type='bad_label') with pytest.raises(ValueError): ins.write(prefix, output_type='flags') new_ins = INS(data_outfile, mask_file=mask_outfile, match_events_file=match_outfile) assert np.all(ins.metric_array == new_ins.metric_array ), "Elements of the metric array were not equal" assert np.all(ins.weights_array == new_ins.weights_array ), "Elements of the weights array were not equal" assert np.all(ins.metric_array.mask == new_ins.metric_array.mask ), "Elements of the mask were not equal" assert np.all(ins.metric_ms == new_ins.metric_ms ), "Elements of the metric_ms were not equal" assert np.all(ins.match_events == new_ins.match_events ), "Elements of the match_events were not equal" assert os.path.exists( sep_data_outfile), "sep_data_outfile was note written" for path in [ data_outfile, z_score_outfile, mask_outfile, match_outfile, sep_data_outfile ]: os.remove(path)
def test_freq_broadcast_subbands(): obs = '1061313128_99bl_1pol_half_time' insfile = os.path.join(DATA_PATH, f'{obs}_SSINS.h5') out_prefix = os.path.join(DATA_PATH, f'{obs}_test') match_outfile = f'{out_prefix}_SSINS_match_events.yml' ins = INS(insfile) # spoof the metric array ins.metric_array[:] = 1 ins.metric_array[2, 10:20] = 10 ins.metric_array[4, 40:50] = 10 ins.metric_ms = ins.mean_subtract() # Slice will go up to 21 since shapes are inclusive at the boundaries # when boundary is on channel center shape_dict = { 'shape1': [ins.freq_array[10], ins.freq_array[20]], 'shape2': [ins.freq_array[40], ins.freq_array[50]] } # boundaries are INCLUSIVE broadcast_dict = { 'sb1': [ins.freq_array[0], ins.freq_array[29]], 'sb2': [ins.freq_array[30], ins.freq_array[59]] } mf = MF(ins.freq_array, 5, shape_dict=shape_dict, broadcast_streak=False, broadcast_dict=broadcast_dict) mf.apply_match_test(ins, event_record=True, freq_broadcast=True) assert np.all(ins.metric_array.mask[2, :30]) assert not np.any(ins.metric_array.mask[2, 30:]) assert np.all(ins.metric_array.mask[4, 30:60]) assert not np.any(ins.metric_array.mask[4, :30]) assert not np.any(ins.metric_array.mask[4, 60:]) print(ins.match_events) test_match_events = [(slice(2, 3), slice(10, 21), 'shape1'), (slice(2, 3), slice(0, 30), 'freq_broadcast_sb1'), (slice(4, 5), slice(40, 51), 'shape2'), (slice(4, 5), slice(30, 60), 'freq_broadcast_sb2')] for event, test_event in zip(ins.match_events, test_match_events): assert event[:3] == test_event
def test_write_meta(): obs = "1061313128_99bl_1pol_half_time" testfile = os.path.join(DATA_PATH, f"{obs}.uvfits") prefix = os.path.join(DATA_PATH, f"{obs}_test") uvd = UVData() uvd.read(testfile, freq_chans=np.arange(32)) ss = SS() ss.read(testfile, freq_chans=np.arange(32), diff=True) uvf = UVFlag(uvd, mode="flag", waterfall=True) ins = INS(ss) ins.metric_array[:] = 1 ins.weights_array[:] = 10 ins.weights_square_array[:] = 10 # Make some outliers # Narrowband in 1th, 26th, and 31th frequency ins.metric_array[1, 1, :] = 100 ins.metric_array[1, 30, :] = 100 ins.metric_array[3:14, 26, :] = 100 # Arbitrary shape in 2, 3, 4 ins.metric_array[3:14, 2:25, :] = 100 ins.metric_array[[0, -1], :, :] = np.ma.masked ins.metric_array[:, [0, -1], :] = np.ma.masked ins.metric_ms = ins.mean_subtract() ch_wid = ins.freq_array[1] - ins.freq_array[0] shape_dict = { "shape": [ins.freq_array[2] + 0.1 * ch_wid, ins.freq_array[24] - 0.1 * ch_wid] } mf = MF(ins.freq_array, 5, tb_aggro=0.5, shape_dict=shape_dict) mf.apply_match_test(ins, time_broadcast=True) util.write_meta(prefix, ins, uvf=uvf, mf=mf, clobber=True) for data_type in ["data", "mask", "flags"]: path = f"{prefix}_SSINS_{data_type}.h5" assert os.path.exists(path) os.remove(path) for data_type in ["match_events", "matchfilter"]: path = f"{prefix}_SSINS_{data_type}.yml" assert os.path.exists(path) os.remove(path)
def test_polyfit(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) file_type = 'uvfits' ss = SS() ss.read(testfile) ins = INS(ss, order=1) # Mock some data for which the polyfit is exact x = np.arange(1, 11) ins.metric_array = np.ma.masked_array([[3 * x + 5 for i in range(3)]]) ins.metric_array.mask = np.zeros(ins.metric_array.shape, dtype=bool) ins.metric_array = np.swapaxes(ins.metric_array, 0, 2) ins.weights_array = np.ones(ins.metric_array.shape) ins.metric_ms = ins.mean_subtract() assert np.all(ins.metric_ms == np.zeros( ins.metric_ms.shape)), "The polyfit was not exact"
def test_mean_subtract(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) file_type = 'uvfits' ss = SS() ss.read(testfile, diff=True) ins = INS(ss, order=0) old_dat = np.copy(ins.metric_ms) # Mask the first five frequencies and last two at the first and second times ins.metric_array[0, :5] = np.ma.masked # Calculate the new mean-subtracted spectrum only over the first few masked frequencies ins.metric_ms[:, :5] = ins.mean_subtract(freq_slice=slice(0, 5)) # See if a new mean was calculated over the first five frequencies assert not np.all(old_dat[1:, :5] == ins.metric_ms[1:, :5]), "All elements of the ms array are still equal"
'vmin': -5, 'vmax': 5, 'mask_color': 'black' }] for i in range(2): for k, obj in zip([0, 2], [raw_ins, OR_ins]): pl.image_plot(fig, ax[k, i], getattr(obj, attr[i])[:, 0, :, 0], cbar_label=cbar_label[i], freq_array=obj.freq_array[0], **kwargs[i]) raw_ins.data[:, 0, :82] = np.ma.masked raw_ins.data[:, 0, -21:] = np.ma.masked mf = MF(raw_ins, sig_thresh=5, shape_dict=shape_dict, N_thresh=20) raw_ins.data_ms = raw_ins.mean_subtract(order=1) mf.apply_match_test(apply_N_thresh=True, order=1) raw_ins.data_ms = raw_ins.mean_subtract() for i in range(2): pl.image_plot(fig, ax[1, i], getattr(raw_ins, attr[i])[:, 0, :, 0], cbar_label=cbar_label[i], freq_array=raw_ins.freq_array[0], **kwargs[i]) fig.savefig('%s/%s_flag_comp.png' % (figpath, obs)) raw_ins.save() plt.close(fig)
read_paths = { 'data': arr, 'Nbls': '%s/arrs/%s_%s_INS_Nbls.npym' % (args.inpath, obs, args.flag_choice), 'freq_array': '%s/metadata/%s_freq_array.npy' % (args.inpath, obs), 'pols': '%s/metadata/%s_pols.npy' % (args.inpath, obs), 'vis_units': '%s/metadata/%s_vis_units.npy' % (args.inpath, obs) } ins = INS(obs=obs, outpath=args.outpath, flag_choice=args.flag_choice, read_paths=read_paths) # ins.data[:, :, :82] = np.ma.masked # ins.data[:, :, -21:] = np.ma.masked ins.data_ms = ins.mean_subtract() ins.counts, ins.bins, ins.sig_thresh = ins.hist_make() cp.INS_plot(ins, **ms_plot_kwargs) mf = MF(ins, shape_dict=shape_dict, point=args.point, streak=args.streak, **mf_kwargs) for test in args.tests: getattr(mf, 'apply_%s_test' % test)(args.order) ins.save() cp.MF_plot(mf, **ms_plot_kwargs)
parser = argparse.ArgumentParser() parser.add_argument('obs', action='store', help='The observation ID') parser.add_argument('inpath', action='store', help='The path to the data file') parser.add_argument('outpath', action='store', help='The base directory for saving all outputs') args = parser.parse_args() ss = SS() ss.read(args.inpath, ant_str='cross') ins = INS(ss) ins.metric_array[:, :82] = np.ma.masked ins.metric_array[:, :-21] = np.ma.masked ins.order = 1 ins.metric_ms = ins.mean_subtract() shape_dict = { 'TV4': [1.74e8, 1.82e8], 'TV5': [1.82e8, 1.9e8], 'TV6': [1.9e8, 1.98e8], 'dig1': [1.125e8, 1.15625e8], 'dig2': [1.375e8, 1.40625e8], 'dig3': [1.625e8, 1.65625e8], 'dig4': [1.875e8, 1.90625e8] } mf = MF(ins.freq_array, 5, shape_dict=shape_dict, N_samp_thresh=15) mf.apply_match_test(ins, apply_samp_thresh=True) prefix = '%s/%s' % (args.outpath, args.obs)
def execbody(ins_filepath): slash_ind = ins_filepath.rfind('/') obsid = ins_filepath[slash_ind + 1:slash_ind + 11] ins = INS(ins_filepath) ins.select(times=ins.time_array[3:-3]) ins.metric_ms = ins.mean_subtract() shape_dict = { 'TV6': [1.74e8, 1.81e8], 'TV7': [1.81e8, 1.88e8], 'TV8': [1.88e8, 1.95e8], 'TV9': [1.95e8, 2.02e8] } sig_thresh = {shape: 5 for shape in shape_dict} sig_thresh['narrow'] = 5 sig_thresh['streak'] = 8 mf = MF(ins.freq_array, sig_thresh, shape_dict=shape_dict, N_samp_thresh=len(ins.time_array) // 2) ins.metric_array[ins.metric_array == 0] = np.ma.masked ins.metric_ms = ins.mean_subtract() ins.sig_array = np.ma.copy(ins.metric_ms) #write plots if command flagged to do so if args.plots: prefix = '%s/%s_trimmed_zeromask' % (args.outdir, obsid) freqs = np.arange(1.7e8, 2e8, 5e6) xticks, xticklabels = util.make_ticks_labels(freqs, ins.freq_array, sig_fig=0) yticks = [0, 20, 40] yticklabels = [] for tick in yticks: yticklabels.append( Time(ins.time_array[tick], format='jd').iso[:-4]) Catalog_Plot.INS_plot(ins, prefix, xticks=xticks, yticks=yticks, xticklabels=xticklabels, yticklabels=yticklabels, data_cmap=cm.plasma, ms_vmin=-5, ms_vmax=5, title=obsid, xlabel='Frequency (Mhz)', ylabel='Time (UTC)') if args.verbose: print("wrote trimmed zeromask plot for " + obsid) mf.apply_match_test(ins, apply_samp_thresh=False) mf.apply_samp_thresh_test(ins, event_record=True) flagged_prefix = '%s/%s_trimmed_zeromask_MF_s8' % (args.outdir, obsid) #write data/mask/match/ if command flagged to do so if args.write: ins.write(flagged_prefix, output_type='data', clobber=True) ins.write(flagged_prefix, output_type='mask', clobber=True) ins.write(flagged_prefix, output_type='match_events') if args.verbose: print("wrote data/mask/match files for " + obsid) #write plots if command flagged to do so if args.plots: Catalog_Plot.INS_plot(ins, flagged_prefix, xticks=xticks, yticks=yticks, xticklabels=xticklabels, yticklabels=yticklabels, data_cmap=cm.plasma, ms_vmin=-5, ms_vmax=5, title=obsid, xlabel='Frequency (Mhz)', ylabel='Time (UTC)') if args.verbose: print("wrote trimmed zeromask (w/ match filter) for " + obsid) #a hardcoded csv generator for occ_csv if args.gencsv is not None: csv = "" + obsid + "," + flagged_prefix + "_SSINS_data.h5," + flagged_prefix + "_SSINS_mask.h5," + flagged_prefix + "_SSINS_match_events.yml\n" with open(args.gencsv, "a") as csvfile: csvfile.write(csv) print("wrote entry for " + obsid)
#ylabel='Time (10 s)') plot_lib.image_plot(fig, ax[0], ins.data_ms[:, 0, :, 0], freq_array=ins.freq_array[0], cmap=cm.coolwarm, aspect=aspect, cbar_label='Deviation ($\hat{\sigma}$)', vmin=-5, vmax=5, ylabel='Time (10 s)') fig_mf, ax_mf = plt.subplots(figsize=(16, 9), ncols=2) ins.data.mask[:, 0, :82, 0] = True ins.data.mask[:, 0, -21:, 0] = True ins.data_ms = ins.mean_subtract(order=1) mf = MF(ins, sig_thresh=5, N_thresh=0, shape_dict={ 'TV4': [1.74e8, 1.82e8], 'TV5': [1.82e8, 1.9e8], 'TV6': [1.9e8, 1.98e8] }) mf.apply_match_test(apply_N_thresh=False, order=1) plot_lib.image_plot(fig_mf, ax_mf[0], ins.data[:, 0, :, 0], freq_array=ins.freq_array[0], cbar_label='Amplitude (UNCALIB)',