def test_sky_noise_jy(self): fqs = np.linspace(0.1, 0.2, 100) lsts = np.linspace(0, 2 * np.pi, 500) omp = noise.bm_poly_to_omega_p(fqs) tsky = noise.resample_Tsky(fqs, lsts) jy2T = noise.jy2T(fqs, omega_p=omp) / 1e3 jy2T.shape = (1, -1) nos_jy = noise.sky_noise_jy(tsky, fqs, lsts, inttime=10.7, omega_p=omp) self.assertEqual(nos_jy.shape, (500, 100)) np.testing.assert_allclose(np.average(nos_jy, axis=0), 0, atol=0.7) scaling = np.average(tsky, axis=0) / jy2T np.testing.assert_allclose(np.std(nos_jy, axis=0) / scaling * np.sqrt(1e6 * 10.7), 1.0, atol=0.1) np.random.seed(0) nos_jy = noise.sky_noise_jy(tsky, fqs, lsts, inttime=None, omega_p=omp) np.testing.assert_allclose( np.std(nos_jy, axis=0) / scaling * np.sqrt(1e6 * aipy.const.sidereal_day / 500), 1.0, atol=0.1) np.random.seed(0) nos_jy = noise.sky_noise_jy(tsky, fqs, lsts, B=.1, inttime=10.7, omega_p=omp) np.testing.assert_allclose(np.std(nos_jy, axis=0) / scaling * np.sqrt(1e8 * 10.7), 1.0, atol=0.1)
def generate_noise(self): omega_p = noise.bm_poly_to_omega_p(self.fqs) tsky = noise.resample_Tsky(self.fqs, self.lsts, Tsky_mdl=noise.HERA_Tsky_mdl['xx']) t_rx = 150. return noise.sky_noise_jy(tsky + t_rx, self.fqs, self.lsts, omega_p)
def test_trim_model(self): # load data V = VisClean(os.path.join(DATA_PATH, "PyGSM_Jy_downselect.uvh5")) V.read(bls=[(23, 23, 'ee'), (23, 24, 'ee')]) # interpolate to 768 frequencies freqs = np.linspace(120e6, 180e6, 768) for k in V.data: V.data[k] = interpolate.interp1d(V.freqs, V.data[k], axis=1, fill_value='extrapolate', kind='cubic')(freqs) V.flags[k] = np.zeros_like(V.data[k], dtype=np.bool) V.freqs = freqs V.Nfreqs = len(V.freqs) # add noise np.random.seed(0) k = (23, 24, 'ee') Op = noise.bm_poly_to_omega_p(V.freqs / 1e9) V.data[k] += noise.sky_noise_jy(V.data[(23, 23, 'ee')], V.freqs / 1e9, V.lsts, Op, inttime=50) # add lots of random flags f = np.zeros(V.Nfreqs, dtype=np.bool)[None, :] f[:, 127:156] = True f[:, 300:303] = True f[:, 450:455] = True f[:, 625:630] = True V.flags[k] += f # vis clean V.vis_clean(data=V.data, flags=V.flags, keys=[k], tol=1e-6, min_dly=300, ax='freq', overwrite=True, window='tukey', alpha=0.2) V.fft_data(V.data, window='bh', overwrite=True, assign='dfft1') V.fft_data(V.clean_data, window='bh', overwrite=True, assign='dfft2') # trim model mdl, n = vis_clean.trim_model(V.clean_model, V.clean_resid, V.dnu, noise_thresh=3.0, delay_cut=500, kernel_size=21, polyfit_deg=None) clean_data2 = deepcopy(V.clean_data) clean_data2[k][V.flags[k]] = mdl[k][V.flags[k]] V.fft_data(clean_data2, window='bh', overwrite=True, assign='dfft3') # get averaged spectra n1 = vis_clean.noise_eq_bandwidth(dspec.gen_window('bh', V.Nfreqs)) n2 = vis_clean.noise_eq_bandwidth(dspec.gen_window('bh', V.Nfreqs) * ~V.flags[k][0]) d1 = np.mean(np.abs(V.dfft1[k]), axis=0) * n1 d2 = np.mean(np.abs(V.dfft2[k]), axis=0) * n2 d3 = np.mean(np.abs(V.dfft3[k]), axis=0) * n2 # confirm that dfft3 and dfft1 match while dfft2 and dfft1 do not near CLEAN boundary select = (np.abs(V.delays) < 300) & (np.abs(V.delays) > 100) assert np.isclose(np.mean(np.abs(d1)[select]), np.mean(np.abs(d3)[select]), atol=10) assert not np.isclose(np.mean(np.abs(d1)[select]), np.mean(np.abs(d2)[select]), atol=10) # test that polynomial fitting is a good fit _, n1 = vis_clean.trim_model(V.clean_model, V.clean_resid, V.dnu, noise_thresh=3.0, delay_cut=500, kernel_size=None, polyfit_deg=None) _, n2 = vis_clean.trim_model(V.clean_model, V.clean_resid, V.dnu, noise_thresh=3.0, delay_cut=500, kernel_size=None, polyfit_deg=5) assert (np.std(n1[k] - n2[k]) / np.mean(n2[k])) < 0.1 # assert residual is below 10% of fit # test well-conditioned check takes effect V2 = deepcopy(V) V2.clean_resid[k][:-2] = 0.0 # zero all the data except last two integrations _, n2 = vis_clean.trim_model(V2.clean_model, V2.clean_resid, V2.dnu, noise_thresh=3.0, delay_cut=500, kernel_size=None, polyfit_deg=5) assert np.all(np.isclose(n2[k][-1], n1[k][-1])) # assert non-zeroed output are same as n1 (no polyfit)
def gen_HERA_vis(self, tsamples, fsamples, bl_len_ns=400., add_rfi=False, inject_frb=False): #### Convert time samples to appropriate LST hours where 60 time samples = 10 min #### !!!! LST is in rads not hrs sph = 60. / .1667 lst_add = tsamples / sph fqs = np.linspace(.1, .2, fsamples, endpoint=False) lsts = np.linspace(np.pi / 2., np.pi / 2. + lst_add, tsamples) times = lsts / (2. * np.pi) * a.const.sidereal_day #### FOREGROUNDS #### # Diffuse Tsky_mdl = noise.HERA_Tsky_mdl['xx'] vis_fg_diffuse = foregrounds.diffuse_foreground( Tsky_mdl, lsts, fqs, bl_len_ns) # Point Sources vis_fg_pntsrc = foregrounds.pntsrc_foreground(lsts, fqs, bl_len_ns, nsrcs=1000) # FRBs vis_fg_frb = np.asarray(gen_simulated_frb(NFREQ=1024, NTIME=61, width=1., sim=True, delta_t=10., freq=(200, 100), FREQ_REF=150., fluence=(60., 600.), scintillate=True, dm=(300., 1800.))[0].T, dtype=np.complex128) vis_fg_frb *= np.exp( 1j * (lsts[30] + .1 * np.pi * np.random.randn(61, 1024))) # Combined if inject_frb: vis_fg = vis_fg_diffuse + vis_fg_pntsrc + vis_fg_frb else: vis_fg = vis_fg_diffuse + vis_fg_pntsrc #### Noise #### tsky = noise.resample_Tsky(fqs, lsts, Tsky_mdl=noise.HERA_Tsky_mdl['xx']) t_rx = 150. nos_jy = noise.sky_noise_jy(tsky + t_rx, fqs, lsts) # Add Noise vis_fg_nos = vis_fg + nos_jy #### RFI #### if add_rfi: g = sigchain.gen_gains(fqs, [1, 2, 3]) with open(add_rfi, 'r') as infile: RFIdict = yaml.load(infile) rfi = RFI_Sim(RFIdict) rfi.applyRFI() self.rfi_true = rfi.getFlags() vis_fg_nos_rfi = np.copy(vis_fg_nos) + rfi.getRFI() vis_total_rfi = sigchain.apply_gains(vis_fg_nos_rfi, g, (1, 2)) # add cross-talk xtalk = sigchain.gen_xtalk(np.linspace(.1, .2, 1024), amplitude=.001) vis_total_rfi = sigchain.apply_xtalk(vis_total_rfi, xtalk) self.data_rfi = vis_total_rfi else: g = sigchain.gen_gains(fqs, [1, 2, 3]) vis_total_norfi = sigchain.apply_gains(vis_fg_nos, g, (1, 2)) self.data = vis_total_norfi
fp.create_dataset('lsts', data=lsts) for a in ants: fp.create_dataset('g%d' % a, data=gains[a]) for bls in reds: for (i, j) in bls: data[(i, j)] = fp.create_dataset('%d-%d' % (i, j), (len(lsts), len(fqs)), chunks=True, dtype='complex64') for bls in reds: bl_len = get_distance(bls[0], antpos) bl_len_ns = bl_len / aipy.const.c * 1e11 vis_fg_pntsrc = foregrounds.pntsrc_foreground(lsts, fqs, bl_len_ns, nsrcs=200) vis_fg_diffuse = foregrounds.diffuse_foreground(Tsky_mdl, lsts, fqs, bl_len_ns) true_vis[bls[0]] = vis_fg_pntsrc + vis_fg_diffuse for (i, j) in bls: print(i, j) nos_jy = noise.sky_noise_jy(tsky + 150., fqs, lsts) vis_tot = nos_jy + true_vis[bls[0]] data[(i, j)][:, :] = sigchain.apply_gains(vis_tot, gains, (i, j)) fp.close()