def test_sky_noise_jy(self):
fqs = np.linspace(0.1, 0.2, 100)
lsts = np.linspace(0, 2 * np.pi, 500)
omp = noise.bm_poly_to_omega_p(fqs)
tsky = noise.resample_Tsky(fqs, lsts)
jy2T = noise.jy2T(fqs, omega_p=omp) / 1e3
jy2T.shape = (1, -1)
nos_jy = noise.sky_noise_jy(tsky, fqs, lsts, inttime=10.7, omega_p=omp)
self.assertEqual(nos_jy.shape, (500, 100))
np.testing.assert_allclose(np.average(nos_jy, axis=0), 0, atol=0.7)
scaling = np.average(tsky, axis=0) / jy2T
np.testing.assert_allclose(np.std(nos_jy, axis=0) / scaling *
np.sqrt(1e6 * 10.7),
1.0,
atol=0.1)
np.random.seed(0)
nos_jy = noise.sky_noise_jy(tsky, fqs, lsts, inttime=None, omega_p=omp)
np.testing.assert_allclose(
np.std(nos_jy, axis=0) / scaling *
np.sqrt(1e6 * aipy.const.sidereal_day / 500),
1.0,
atol=0.1)
np.random.seed(0)
nos_jy = noise.sky_noise_jy(tsky,
fqs,
lsts,
B=.1,
inttime=10.7,
omega_p=omp)
np.testing.assert_allclose(np.std(nos_jy, axis=0) / scaling *
np.sqrt(1e8 * 10.7),
1.0,
atol=0.1)
def generate_noise(self):
omega_p = noise.bm_poly_to_omega_p(self.fqs)
tsky = noise.resample_Tsky(self.fqs,
self.lsts,
Tsky_mdl=noise.HERA_Tsky_mdl['xx'])
t_rx = 150.
return noise.sky_noise_jy(tsky + t_rx, self.fqs, self.lsts, omega_p)
def test_trim_model(self):
# load data
V = VisClean(os.path.join(DATA_PATH, "PyGSM_Jy_downselect.uvh5"))
V.read(bls=[(23, 23, 'ee'), (23, 24, 'ee')])
# interpolate to 768 frequencies
freqs = np.linspace(120e6, 180e6, 768)
for k in V.data:
V.data[k] = interpolate.interp1d(V.freqs, V.data[k], axis=1, fill_value='extrapolate', kind='cubic')(freqs)
V.flags[k] = np.zeros_like(V.data[k], dtype=np.bool)
V.freqs = freqs
V.Nfreqs = len(V.freqs)
# add noise
np.random.seed(0)
k = (23, 24, 'ee')
Op = noise.bm_poly_to_omega_p(V.freqs / 1e9)
V.data[k] += noise.sky_noise_jy(V.data[(23, 23, 'ee')], V.freqs / 1e9, V.lsts, Op, inttime=50)
# add lots of random flags
f = np.zeros(V.Nfreqs, dtype=np.bool)[None, :]
f[:, 127:156] = True
f[:, 300:303] = True
f[:, 450:455] = True
f[:, 625:630] = True
V.flags[k] += f
# vis clean
V.vis_clean(data=V.data, flags=V.flags, keys=[k], tol=1e-6, min_dly=300, ax='freq', overwrite=True, window='tukey', alpha=0.2)
V.fft_data(V.data, window='bh', overwrite=True, assign='dfft1')
V.fft_data(V.clean_data, window='bh', overwrite=True, assign='dfft2')
# trim model
mdl, n = vis_clean.trim_model(V.clean_model, V.clean_resid, V.dnu, noise_thresh=3.0, delay_cut=500,
kernel_size=21, polyfit_deg=None)
clean_data2 = deepcopy(V.clean_data)
clean_data2[k][V.flags[k]] = mdl[k][V.flags[k]]
V.fft_data(clean_data2, window='bh', overwrite=True, assign='dfft3')
# get averaged spectra
n1 = vis_clean.noise_eq_bandwidth(dspec.gen_window('bh', V.Nfreqs))
n2 = vis_clean.noise_eq_bandwidth(dspec.gen_window('bh', V.Nfreqs) * ~V.flags[k][0])
d1 = np.mean(np.abs(V.dfft1[k]), axis=0) * n1
d2 = np.mean(np.abs(V.dfft2[k]), axis=0) * n2
d3 = np.mean(np.abs(V.dfft3[k]), axis=0) * n2
# confirm that dfft3 and dfft1 match while dfft2 and dfft1 do not near CLEAN boundary
select = (np.abs(V.delays) < 300) & (np.abs(V.delays) > 100)
assert np.isclose(np.mean(np.abs(d1)[select]), np.mean(np.abs(d3)[select]), atol=10)
assert not np.isclose(np.mean(np.abs(d1)[select]), np.mean(np.abs(d2)[select]), atol=10)
# test that polynomial fitting is a good fit
_, n1 = vis_clean.trim_model(V.clean_model, V.clean_resid, V.dnu, noise_thresh=3.0, delay_cut=500,
kernel_size=None, polyfit_deg=None)
_, n2 = vis_clean.trim_model(V.clean_model, V.clean_resid, V.dnu, noise_thresh=3.0, delay_cut=500,
kernel_size=None, polyfit_deg=5)
assert (np.std(n1[k] - n2[k]) / np.mean(n2[k])) < 0.1 # assert residual is below 10% of fit
# test well-conditioned check takes effect
V2 = deepcopy(V)
V2.clean_resid[k][:-2] = 0.0 # zero all the data except last two integrations
_, n2 = vis_clean.trim_model(V2.clean_model, V2.clean_resid, V2.dnu, noise_thresh=3.0, delay_cut=500,
kernel_size=None, polyfit_deg=5)
assert np.all(np.isclose(n2[k][-1], n1[k][-1])) # assert non-zeroed output are same as n1 (no polyfit)
def gen_HERA_vis(self,
tsamples,
fsamples,
bl_len_ns=400.,
add_rfi=False,
inject_frb=False):
#### Convert time samples to appropriate LST hours where 60 time samples = 10 min
#### !!!! LST is in rads not hrs
sph = 60. / .1667
lst_add = tsamples / sph
fqs = np.linspace(.1, .2, fsamples, endpoint=False)
lsts = np.linspace(np.pi / 2., np.pi / 2. + lst_add, tsamples)
times = lsts / (2. * np.pi) * a.const.sidereal_day
#### FOREGROUNDS ####
# Diffuse
Tsky_mdl = noise.HERA_Tsky_mdl['xx']
vis_fg_diffuse = foregrounds.diffuse_foreground(
Tsky_mdl, lsts, fqs, bl_len_ns)
# Point Sources
vis_fg_pntsrc = foregrounds.pntsrc_foreground(lsts,
fqs,
bl_len_ns,
nsrcs=1000)
# FRBs
vis_fg_frb = np.asarray(gen_simulated_frb(NFREQ=1024,
NTIME=61,
width=1.,
sim=True,
delta_t=10.,
freq=(200, 100),
FREQ_REF=150.,
fluence=(60., 600.),
scintillate=True,
dm=(300., 1800.))[0].T,
dtype=np.complex128)
vis_fg_frb *= np.exp(
1j * (lsts[30] + .1 * np.pi * np.random.randn(61, 1024)))
# Combined
if inject_frb:
vis_fg = vis_fg_diffuse + vis_fg_pntsrc + vis_fg_frb
else:
vis_fg = vis_fg_diffuse + vis_fg_pntsrc
#### Noise ####
tsky = noise.resample_Tsky(fqs,
lsts,
Tsky_mdl=noise.HERA_Tsky_mdl['xx'])
t_rx = 150.
nos_jy = noise.sky_noise_jy(tsky + t_rx, fqs, lsts)
# Add Noise
vis_fg_nos = vis_fg + nos_jy
#### RFI ####
if add_rfi:
g = sigchain.gen_gains(fqs, [1, 2, 3])
with open(add_rfi, 'r') as infile:
RFIdict = yaml.load(infile)
rfi = RFI_Sim(RFIdict)
rfi.applyRFI()
self.rfi_true = rfi.getFlags()
vis_fg_nos_rfi = np.copy(vis_fg_nos) + rfi.getRFI()
vis_total_rfi = sigchain.apply_gains(vis_fg_nos_rfi, g, (1, 2))
# add cross-talk
xtalk = sigchain.gen_xtalk(np.linspace(.1, .2, 1024),
amplitude=.001)
vis_total_rfi = sigchain.apply_xtalk(vis_total_rfi, xtalk)
self.data_rfi = vis_total_rfi
else:
g = sigchain.gen_gains(fqs, [1, 2, 3])
vis_total_norfi = sigchain.apply_gains(vis_fg_nos, g, (1, 2))
self.data = vis_total_norfi
fp.create_dataset('lsts', data=lsts)
for a in ants:
fp.create_dataset('g%d' % a, data=gains[a])
for bls in reds:
for (i, j) in bls:
data[(i, j)] = fp.create_dataset('%d-%d' % (i, j),
(len(lsts), len(fqs)),
chunks=True,
dtype='complex64')
for bls in reds:
bl_len = get_distance(bls[0], antpos)
bl_len_ns = bl_len / aipy.const.c * 1e11
vis_fg_pntsrc = foregrounds.pntsrc_foreground(lsts,
fqs,
bl_len_ns,
nsrcs=200)
vis_fg_diffuse = foregrounds.diffuse_foreground(Tsky_mdl, lsts, fqs,
bl_len_ns)
true_vis[bls[0]] = vis_fg_pntsrc + vis_fg_diffuse
for (i, j) in bls:
print(i, j)
nos_jy = noise.sky_noise_jy(tsky + 150., fqs, lsts)
vis_tot = nos_jy + true_vis[bls[0]]
data[(i, j)][:, :] = sigchain.apply_gains(vis_tot, gains, (i, j))
fp.close()