def flag(self, vis, vis_mask, li, gi, tf, ts, **kwargs):
"""Function that does the actual flag."""
# if all have been masked, no need to flag again
if vis_mask.all():
return
first_threshold = self.params['first_threshold']
exp_factor = self.params['exp_factor']
distribution = self.params['distribution']
max_threshold_len = self.params['max_threshold_len']
sensitivity = self.params['sensitivity']
min_connected = self.params['min_connected']
flag_direction = self.params['flag_direction']
tk_size = self.params['tk_size']
fk_size = self.params['fk_size']
threshold_num = max(0, int(self.params['threshold_num']))
vis_abs = np.abs(vis) # operate only on the amplitude
# first round
# first complete masked vals due to ns by interpolate
itp = interpolate.Interpolate(vis_abs, vis_mask)
background = itp.fit()
# Gaussian fileter
gf = gaussian_filter.GaussianFilter(background, time_kernal_size=tk_size, freq_kernal_size=fk_size)
background = gf.fit()
# sum-threshold
vis_diff = vis_abs - background
# an initial run of N = 1 only to remove extremely high amplitude RFI
st = sum_threshold.SumThreshold(vis_diff, vis_mask, first_threshold, exp_factor, distribution, 1, min_connected)
st.execute(sensitivity, flag_direction)
# if all have been masked, no need to flag again
if st.vis_mask.all():
vis_mask[:] = st.vis_mask
return
# next rounds
for i in xrange(threshold_num):
# Gaussian fileter
gf = gaussian_filter.GaussianFilter(vis_diff, st.vis_mask, time_kernal_size=tk_size, freq_kernal_size=fk_size)
background = gf.fit()
# sum-threshold
vis_diff = vis_diff - background
st = sum_threshold.SumThreshold(vis_diff, st.vis_mask, first_threshold, exp_factor, distribution, max_threshold_len, min_connected)
st.execute(sensitivity, flag_direction)
# if all have been masked, no need to flag again
if st.vis_mask.all():
break
# replace vis_mask with the flagged mask
vis_mask[:] = st.vis_mask
def flag(self, vis, vis_mask, li, gi, fb, ts, **kwargs):
"""Function that does the actual flag."""
# if all have been masked, no need to flag again
if vis_mask.all():
return
first_threshold = self.params['first_threshold']
exp_factor = self.params['exp_factor']
distribution = self.params['distribution']
max_threshold_len = self.params['max_threshold_len']
sensitivity = self.params['sensitivity']
min_connected = self.params['min_connected']
flag_direction = self.params['flag_direction']
vis_abs = np.abs(vis) # operate only on the amplitude
# first complete masked vals due to ns by interpolate
itp = interpolate.Interpolate(vis_abs, vis_mask)
background = itp.fit()
background1 = background.copy()
# nt, nf = background.shape
# for fi in range(nf):
# background[:, fi] = multiscale.median_wavelet_smooth(background[:, fi], level=4)
# for ti in range(nt):
# if fb[0] == fb[1]:
# background[ti, :] = multiscale.median_wavelet_smooth(background[ti, :], level=2)
# else:
# background[ti, :] = multiscale.median_wavelet_smooth(background[ti, :], level=3)
background[:] = multiscale.median_wavelet_smooth(background, level=2)
# sum-threshold
# vis_diff = vis_abs - background
vis_diff = background1 - background
# an initial run of N = 1 only to remove extremely high amplitude RFI
st = sum_threshold.SumThreshold(vis_diff, vis_mask, first_threshold,
exp_factor, distribution, 1,
min_connected)
st.execute(sensitivity, flag_direction)
# if all have been masked, no need to flag again
if st.vis_mask.all():
vis_mask[:] = st.vis_mask
return
vis_diff = np.where(st.vis_mask, 0, vis_diff)
st = sum_threshold.SumThreshold(vis_diff, st.vis_mask, first_threshold,
exp_factor, distribution,
max_threshold_len, min_connected)
st.execute(sensitivity, flag_direction)
vis_mask[:] = st.vis_mask
def init_vis(self, vis, vis_mask, li, gi, bl, ts, **kwargs):
time = ts['sec1970'][:]
n_time = time.shape[0]
tblock_len = self.params['tblock_len']
if tblock_len is None:
tblock_len = n_time
if self.params['deweight_time_slope']:
n_poly = 2
else:
n_poly = 1
vis_var = ts['vis_var'].local_data
logger.debug('est. var %d %d'%(n_time, tblock_len))
_vis = np.ma.array(vis.copy())
_vis.mask = vis_mask
median = np.ma.median(_vis, axis=0)
vis -= median[None, ...]
#vis_fit = sub_ortho_poly(vis, time, ~vis_mask.astype('bool') , n_poly)
#for i in range(vis_fit.shape[-1]):
# good = vis_fit[:, 0, i] != 0
# plt.plot(np.arange(vis_fit.shape[0])[good], vis_fit[good, 0, i], '.')
# plt.show()
if self.params['noise_weight']:
for st in range(0, n_time, tblock_len):
et = st + tblock_len
_time = time[st:et] #ts['sec1970'][st:et]
_vis_mask = (vis_mask[st:et,...]).astype('bool')
_vis = vis[st:et,...]
_vis[_vis_mask] = 0.
# rm bright sources for var est.
_vis.shape = _time.shape + (-1, )
_vis_mask.shape = _vis.shape
bg = gaussian_filter.GaussianFilter(
interpolate.Interpolate(_vis, _vis_mask).fit(),
time_kernal_size=0.5, freq_kernal_size=1,
filter_direction = ('time', )).fit()
_vis = _vis - bg
_vis.shape = (-1, ) + vis.shape[1:]
_vis_mask.shape = _vis.shape
_vis[_vis_mask] = 0.
#for i in range(5):
_vars = sp.sum(_vis ** 2., axis=0)
_cont = sp.sum(~_vis_mask, axis=0) * 1.
_bad = _cont == 0
_cont[_bad] = np.inf
_vars /= _cont
#_vis_mask += (_vis - 3 * np.sqrt(_vars[None, ...])) > 0.
#_vis[_vis_mask] = 0.
msg = 'min vars = %f, max vars = %f'%(_vars.min(), _vars.max())
#logger.debug(msg)
logger.info(msg)
#_vars[_bad] = T_infinity ** 2.
#_bad = _vars < T_small ** 2
#_vars[_bad] = T_small ** 2
#vis_var[st:et, li, ...] += _vars[None, :] * vis_fit[st:et, ...]
vis_var[st:et, li, ...] += _vars[None, :] * median[None, ...]
#vis_var[st:et, li, ...][_vis_mask, ...] = T_infinity ** 2.
else:
vis_var[:] = 1.
def _plot(self, vis1, x_axis, xmin, xmax, gi, bl, ts):
axhh = self.axhh
axvv = self.axvv
label = 'M%03d'%(bl[0] - 1)
#axhh.plot(x_axis, np.ma.mean(vis1[:,:,0], axis=1), '.', label = label)
#axvv.plot(x_axis, np.ma.mean(vis1[:,:,1], axis=1), '.')
if len(vis1.shape) == 3:
vis1 = np.ma.mean(vis1, axis=1)
time = np.arange(x_axis.shape[0])
_n = 0
t_block = self.params['t_block']
if self.params['rm_slop']: _n = 2
elif self.params['rm_mean']: _n = 1
if t_block is None: t_block = len(time)
for ii in range(0, len(time), t_block):
st = ii
ed = ii + t_block
_vis1 = vis1[st:ed, ...]
_time = time[st:ed]
bg = gaussian_filter.GaussianFilter(
interpolate.Interpolate(_vis1, _vis1.mask).fit(),
time_kernal_size=0.5, freq_kernal_size=1,
filter_direction = ('time', )).fit()
if _n != 0:
_vis1 -= sub_ortho_poly(bg, _time, ~_vis1.mask, _n)
if self.params['rm_point_sources']:
bg = gaussian_filter.GaussianFilter(
interpolate.Interpolate(_vis1, _vis1.mask).fit(),
time_kernal_size=0.5, freq_kernal_size=1,
filter_direction = ('time', )).fit()
_vis1 -= bg
vis1[st:ed, ...] = _vis1
_l = axhh.plot(x_axis, vis1[:,0], '-', drawstyle='steps-mid',
label = label)
#_l = axhh.plot(x_axis, vis1[:,0], '.', c=_l[0].get_color())
_l = axvv.plot(x_axis, vis1[:,1], '-', drawstyle='steps-mid')
#_l = axvv.plot(x_axis, vis1[:,1], '.', c=_l[0].get_color())
if xmin is None: xmin = x_axis[0]
if xmax is None: xmax = x_axis[-1]
self.xmin = min([xmin, self.xmin])
self.xmax = max([xmax, self.xmax])
if self.params['nvss_cat'] is not None:
dec_min = np.min(ts['dec'][:, gi])
dec_max = np.max(ts['dec'][:, gi])
#dec_min = 25.65294 #15.15294
#dec_max = 25.65294 #15.15294
#if dec_min == dec_max:
dec_min -= 2./60.
dec_max += 2./60.
#dec_min = 25.541339365641274
#dec_max = 25.61497357686361
#dec_min = 25.458541361490884
#dec_max = 25.532173665364585
#dec_min = 25.789624659220376
#dec_max = 25.863256963094077
self.nvss_range.append([xmin, xmax, dec_min, dec_max])