def surgical_scrub(ar, chanthresh=None, subintthresh=None, binthresh=None):
"""Surgically scrub RFI from the data.
Input:
ar: The archive to be cleaned.
Outputs:
None - The archive is cleaned in place.
"""
import psrchive # Temporarily, because python bindings
# are not available on all computers
patient = ar.clone()
patient.pscrunch()
patient.remove_baseline()
# Remove profile from dedispersed data
patient.dedisperse()
data = patient.get_data().squeeze()
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
clean_utils.remove_profile_inplace(patient, template)
# re-set DM to 0
patient.dededisperse()
# Get weights
weights = patient.get_weights()
# Get data (select first polarization - recall we already P-scrunched)
data = patient.get_data()[:, 0, :, :]
data = clean_utils.apply_weights(data, weights)
# Mask profiles where weight is 0
mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool))
mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2)
data = np.ma.masked_array(data, mask=mask_3d)
# RFI-ectomy must be recommended by average of tests
avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \
chanthresh=chanthresh, \
subintthresh=subintthresh, \
binthresh=binthresh)
for (isub, ichan) in np.argwhere(avg_test_results >= 1):
# Be sure to set weights on the original archive, and
# not the clone we've been working with.
integ = ar.get_Integration(int(isub))
integ.set_weight(int(ichan), 0.0)
def _clean(self, ar):
plot = False
#if self.configs.plot is None:
# plot = False
#else:
# plot = self.configs.plot
#if plot:
# import matplotlib.pyplot as plt
patient = ar.clone()
patient.pscrunch()
patient.remove_baseline()
# Remove profile from dedispersed data
patient.dedisperse()
print('Loading template')
data = patient.get_data().squeeze()
if self.configs.template is None:
# Sum over all axes except last, which is phase bins
template = np.apply_over_axes(np.sum, data,
tuple(range(data.ndim -
1))).squeeze()
# smooth data
template = savgol_filter(template, 5, 1)
else:
template_ar = psrchive.Archive_load(self.configs.template)
template_ar.pscrunch()
template_ar.remove_baseline()
template_ar.dedisperse()
if len(template_ar.get_frequencies()) > 1 and len(
template_ar.get_frequencies()) < len(
patient.get_frequencies()):
print(
"Template channel number doesn't match data... f-scrunching!"
)
template_ar.fscrunch()
template_data = template_ar.get_data().squeeze()
template = np.apply_over_axes(np.sum, template_data,
tuple(range(template_data.ndim -
1))).squeeze()
# make sure template is 1D
if len(np.shape(template)) > 1: # sum over frequencies too
template_ar.fscrunch()
print(
"2D template found. Assuming it has same frequency coverage and channels as data!"
)
template_phs = np.apply_over_axes(
np.sum, template_data,
tuple(range(template_data.ndim - 1))).squeeze()
else:
template_phs = template
print('Estimating template and profile phase offset')
if self.configs.template is None:
phs = 0
else:
# Calculate phase offset of template in number of bins, using full obs
# Get profile data of full obs
profile = np.apply_over_axes(np.sum, data,
tuple(range(data.ndim -
1))).squeeze()
if np.shape(template_phs) != np.shape(profile):
print(
'template and profile have different numbers of phase bins'
)
#err = (lambda (amp, phs, base): amp*clean_utils.fft_rotate(template_phs, phs) + base - profile)
#err = lambda amp, phs: amp*clean_utils.fft_rotate(template_phs, phs) - profile
err = lambda x: x[0] * clean_utils.fft_rotate(template_phs, x[1]
) - profile
amp_guess = np.median(profile) / np.median(template_phs)
phase_guess = -(np.argmax(profile) - np.argmax(template_phs))
#params, status = leastsq(err, [amp_guess, phase_guess, np.min(profile) - np.min(template_phs)])
params, status = leastsq(err, [amp_guess, phase_guess])
phs = params[1]
print('Template phase offset = {0}'.format(round(phs, 3)))
print('Removing profile from patient')
if plot:
preop_patient = patient.clone()
preop_weights = preop_patient.get_weights()
clean_utils.remove_profile_inplace(patient, template, phs)
print('Accessing weights and applying to patient')
# re-set DM to 0
# patient.dededisperse()
# Get weights
weights = patient.get_weights()
# Get data (select first polarization - recall we already P-scrunched)
data = patient.get_data()[:, 0, :, :]
data = clean_utils.apply_weights(data, weights)
if plot:
preop_data = preop_patient.get_data()[:, 0, :, :]
preop_patient = [] # clear for the sake of memory
preop_data = clean_utils.apply_weights(preop_data, weights)
# Mask profiles where weight is 0
mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool))
mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2)
data = np.ma.masked_array(data, mask=mask_3d)
if plot:
preop_data = np.ma.masked_array(preop_data, mask=mask_3d)
print('Masking on-pulse region as determined from template')
# consider residual only in off-pulse region
if len(np.shape(template)) > 1: # sum over frequencies
print('Estimating on-pulse region by f-scrunching 2D template')
template_ar.fscrunch()
template_1D = np.apply_over_axes(np.sum, template_ar.get_data(),
(0, 1)).squeeze()
else:
template_1D = template
# Rotate template by apropriate amount
template_rot = clean_utils.fft_rotate(template_1D, phs).squeeze()
# masked_template = np.ma.masked_greater(template_rot, np.min(template_rot) + 0.01*np.ptp(template_rot))
masked_template = np.ma.masked_greater(template_rot,
np.median(template_rot))
masked_std = np.ma.std(masked_template)
# use this std of masked data as cutoff
masked_template = np.ma.masked_greater(
template_rot,
np.median(template_rot) + masked_std)
if plot:
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
plt.plot(np.apply_over_axes(np.sum, preop_data,
tuple(range(data.ndim - 1))).squeeze(),
alpha=1)
# Do fit again to scale template
subchan, err, params = clean_utils.remove_profile1d(
np.apply_over_axes(np.sum, preop_data, (0, 1)).squeeze(),
0,
0,
template_rot,
0,
return_params=True)
# plt.plot(params[0]*template_rot + params[1], alpha=0.5)
# plt.plot(params[0]*masked_template + params[1], 'k')
plt.plot(params[0] * template_rot, alpha=0.5)
plt.plot(params[0] * masked_template, 'k')
plt.legend(('Pre-op data', 'Scaled and rotated template',
'Masked template'))
# Loop through chans and subints to mask on-pulse phase bins
for ii in range(0, np.shape(data)[0]):
for jj in range(0, np.shape(data)[1]):
data.mask[ii, jj, :] = masked_template.mask
data = np.ma.masked_array(data, mask=data.mask)
if plot:
plt.subplot(1, 2, 2)
plt.plot(
np.apply_over_axes(np.ma.sum, data,
tuple(range(data.ndim - 1))).squeeze())
plt.title("Residual data")
plt.savefig('data_and_template.png')
print(
'Calculating robust statistics to determine where RFI removal is required'
)
# RFI-ectomy must be recommended by average of tests
# BWM: Ok, so this is where the magical stuff actually happens - need to know actually WHAT are the comprehensive stats
# DJR: At this stage the stats are; (found to work well experimentally)
# geometric mean, peak-to-peak, standard deviation, normaltest.
# In original coast_guard they were;
# mean, peak-to-peak, standard deviation, and max value of FFT
avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \
chanthresh=self.configs.chanthresh, \
subintthresh=self.configs.subintthresh, \
chan_order=self.configs.chan_order, \
chan_breakpoints=self.configs.chan_breakpoints, \
chan_numpieces=self.configs.chan_numpieces, \
subint_order=self.configs.subint_order, \
subint_breakpoints=self.configs.subint_breakpoints, \
subint_numpieces=self.configs.subint_numpieces, \
cut_edge=False, \
)
print('Applying RFI masking weights to archive')
for (isub, ichan) in np.argwhere(avg_test_results >= 1):
# Be sure to set weights on the original archive, and
# not the clone we've been working with.
integ = ar.get_Integration(int(isub))
integ.set_weight(int(ichan), 0.0)
freq_fraczap = clean_utils.freq_fraczap(ar)
def _clean(self, ar):
patient = ar.clone()
patient.pscrunch()
patient.remove_baseline()
# Remove profile from dedispersed data
patient.dedisperse()
data = patient.get_data().squeeze()
if self.configs.template is None:
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
else:
template_ar = psrchive.Archive_load(self.configs.template)
template_ar.pscrunch()
template_ar.remove_baseline()
template = np.apply_over_axes(np.sum, template_ar.get_data(),
(0, 1)).squeeze()
# make sure template is 1D
if len(np.shape(template)) > 1: # sum over frequencies too
template_phs = np.apply_over_axes(np.sum, template.squeeze(),
0).squeeze()
else:
template_phs = template
if self.configs.template is None:
phs = 0
else:
# Calculate phase offset of template in number of bins, using full obs
profile = patient.clone()
profile.tscrunch()
profile.fscrunch()
# Get profile data of full obs
profile = profile.get_data()[0, 0, 0, :]
if np.shape(template_phs) != np.shape(profile):
print(
'template and profile have different numbers of phase bins'
)
err = lambda (amp, phs): amp * clean_utils.fft_rotate(
template_phs, phs) - profile
params, status = leastsq(err, [1, 0])
phs = params[1]
print('Found template phase offset = ', round(phs, 3))
clean_utils.remove_profile_inplace(patient, template, phs)
# re-set DM to 0
patient.dededisperse()
# Get weights
weights = patient.get_weights()
# Get data (select first polarization - recall we already P-scrunched)
data = patient.get_data()[:, 0, :, :]
data = clean_utils.apply_weights(data, weights)
# Mask profiles where weight is 0
mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool))
mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2)
data = np.ma.masked_array(data, mask=mask_3d)
# RFI-ectomy must be recommended by average of tests
avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \
chanthresh=self.configs.chanthresh, \
subintthresh=self.configs.subintthresh, \
chan_order=self.configs.chan_order, \
chan_breakpoints=self.configs.chan_breakpoints, \
chan_numpieces=self.configs.chan_numpieces, \
subint_order=self.configs.subint_order, \
subint_breakpoints=self.configs.subint_breakpoints, \
subint_numpieces=self.configs.subint_numpieces, \
)
for (isub, ichan) in np.argwhere(avg_test_results >= 1):
# Be sure to set weights on the original archive, and
# not the clone we've been working with.
integ = ar.get_Integration(int(isub))
integ.set_weight(int(ichan), 0.0)
def _clean(self, ar):
patient = ar.clone()
patient.pscrunch()
patient.remove_baseline()
# Shi Dai, 2019/01/02/, apply weights before forming the template
# Get weights
weights = patient.get_weights()
# Remove profile from dedispersed data
patient.dedisperse()
data = patient.get_data().squeeze()
# apply weights
data = clean_utils.apply_weights(data, weights)
#template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
# Shi Dai, 2019/05/16, using 2D template
nsub, nchan, nbin = data.shape
temp_T = np.sum(data, 0)
temp_reshape = temp_T.reshape(
(26, nchan / 26, nbin)) # hard coded to use 26 sub-bands
template = np.sum(temp_reshape, axis=1)
print("Using 2D template with %d channels." % (template.shape[0]))
clean_utils.remove_profile_inplace(patient, template)
#np.save('data', data)
#np.save('template', template)
# re-set DM to 0
patient.dededisperse()
# Get data (select first polarization - recall we already P-scrunched)
data = patient.get_data()[:, 0, :, :]
data = clean_utils.apply_weights(data, weights)
# # Remove profile from dedispersed data
# patient.dedisperse()
# data = patient.get_data().squeeze()
# template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
# clean_utils.remove_profile_inplace(patient, template)
# # re-set DM to 0
# patient.dededisperse()
#
# # Get weights
# weights = patient.get_weights()
# # Get data (select first polarization - recall we already P-scrunched)
# data = patient.get_data()[:,0,:,:]
# data = clean_utils.apply_weights(data, weights)
# Mask profiles where weight is 0
mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool))
mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2)
data = np.ma.masked_array(data, mask=mask_3d)
# RFI-ectomy must be recommended by average of tests
avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \
chanthresh=self.configs.chanthresh, \
subintthresh=self.configs.subintthresh, \
chan_order=self.configs.chan_order, \
chan_breakpoints=self.configs.chan_breakpoints, \
chan_numpieces=self.configs.chan_numpieces, \
subint_order=self.configs.subint_order, \
subint_breakpoints=self.configs.subint_breakpoints, \
subint_numpieces=self.configs.subint_numpieces, \
)
for (isub, ichan) in np.argwhere(avg_test_results >= 1):
# Be sure to set weights on the original archive, and
# not the clone we've been working with.
integ = ar.get_Integration(int(isub))
integ.set_weight(int(ichan), 0.0)