def power_wash(ar):
"""Power wash RFI out of the data.
Input:
ar: The archive to be cleaned.
Outputs:
None - The archive is cleaned in place.
"""
ar.pscrunch()
ar.remove_baseline()
ar.dedisperse()
# Remove profile
data = ar.get_data().squeeze()
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
clean_utils.remove_profile_inplace(ar, template, None)
ar.dededisperse()
bad_chans = []
bad_subints = []
bad_pairs = []
std_sub_vs_chan = np.std(data, axis=2)
print std_sub_vs_chan.shape
#mean_sub_vs_chan = np.mean(data, axis=2)
# Identify bad sub-int/channel pairs
subintweights = clean_utils.get_subint_weights(ar).astype(bool)
chanweights = clean_utils.get_chan_weights(ar).astype(bool)
for isub in range(ar.get_nsubint()):
for ichan in range(ar.get_nchan()):
plt.figure()
plt.subplot(2, 1, 1)
plt.plot(std_sub_vs_chan[isub, :], 'k-')
subint = clean_utils.scale_chans(std_sub_vs_chan[isub, :], \
chanweights=chanweights)
print clean_utils.get_hot_bins(subint)
plt.subplot(2, 1, 2)
plt.plot(subint, 'r-')
plt.title("Subint #%d" % isub)
plt.figure()
plt.subplot(2, 1, 1)
plt.plot(std_sub_vs_chan[:, ichan], 'k-')
chan = clean_utils.scale_subints(std_sub_vs_chan[:, ichan], \
subintweights=subintweights)
print clean_utils.get_hot_bins(chan)
plt.subplot(2, 1, 2)
plt.plot(chan, 'r-')
plt.title("Chan #%d" % ichan)
plt.show()
chanstds = np.sum(std_sub_vs_chan, axis=0)
plt.subplot(2, 1, 1)
plt.plot(chanstds)
chanstds = clean_utils.scale_chans(chanstds, chanweights=chanweights)
plt.subplot(2, 1, 2)
plt.plot(chanstds)
bad_chans.extend(np.argwhere(chanstds > 1).squeeze())
plt.show()
def surgical_scrub(ar, chanthresh=None, subintthresh=None, binthresh=None):
"""Surgically scrub RFI from the data.
Input:
ar: The archive to be cleaned.
Outputs:
None - The archive is cleaned in place.
"""
import psrchive # Temporarily, because python bindings
# are not available on all computers
patient = ar.clone()
patient.pscrunch()
patient.remove_baseline()
# Remove profile from dedispersed data
patient.dedisperse()
data = patient.get_data().squeeze()
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
clean_utils.remove_profile_inplace(patient, template)
# re-set DM to 0
patient.dededisperse()
# Get weights
weights = patient.get_weights()
# Get data (select first polarization - recall we already P-scrunched)
data = patient.get_data()[:, 0, :, :]
data = clean_utils.apply_weights(data, weights)
# Mask profiles where weight is 0
mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool))
mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2)
data = np.ma.masked_array(data, mask=mask_3d)
# RFI-ectomy must be recommended by average of tests
avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \
chanthresh=chanthresh, \
subintthresh=subintthresh, \
binthresh=binthresh)
for (isub, ichan) in np.argwhere(avg_test_results >= 1):
# Be sure to set weights on the original archive, and
# not the clone we've been working with.
integ = ar.get_Integration(int(isub))
integ.set_weight(int(ichan), 0.0)
def deep_clean(toclean, chanthresh=None, subintthresh=None, binthresh=None):
import psrchive # Temporarily, because python bindings
# are not available on all computers
if chanthresh is None:
chanthresh = config.cfg.clean_chanthresh
if subintthresh is None:
subintthresh = config.cfg.clean_subintthresh
if binthresh is None:
binthresh = config.cfg.clean_binthresh
ar = toclean.clone()
ar.pscrunch()
ar.remove_baseline()
ar.dedisperse()
# Remove profile
data = ar.get_data().squeeze()
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
clean_utils.remove_profile_inplace(ar, template, None)
ar.dededisperse()
# First clean channels
chandata = clean_utils.get_chans(ar, remove_prof=True)
chanweights = clean_utils.get_chan_weights(ar).astype(bool)
chanmeans = clean_utils.scale_chans(chandata.mean(axis=1),
chanweights=chanweights)
chanmeans /= clean_utils.get_robust_std(chanmeans, chanweights)
chanstds = clean_utils.scale_chans(chandata.std(axis=1),
chanweights=chanweights)
chanstds /= clean_utils.get_robust_std(chanstds, chanweights)
badchans = np.concatenate((np.argwhere(np.abs(chanmeans) >= chanthresh), \
np.argwhere(np.abs(chanstds) >= chanthresh)))
badchans = np.unique(badchans)
utils.print_info(
"Number of channels to be de-weighted: %d" % len(badchans), 2)
for ichan in badchans:
utils.print_info("De-weighting chan# %d" % ichan, 3)
clean_utils.zero_weight_chan(ar, ichan)
clean_utils.zero_weight_chan(toclean, ichan)
# Next clean subints
subintdata = clean_utils.get_subints(ar, remove_prof=True)
subintweights = clean_utils.get_subint_weights(ar).astype(bool)
subintmeans = clean_utils.scale_subints(subintdata.mean(axis=1), \
subintweights=subintweights)
subintmeans /= clean_utils.get_robust_std(subintmeans, subintweights)
subintstds = clean_utils.scale_subints(subintdata.std(axis=1), \
subintweights=subintweights)
subintstds /= clean_utils.get_robust_std(subintstds, subintweights)
badsubints = np.concatenate((np.argwhere(np.abs(subintmeans) >= subintthresh), \
np.argwhere(np.abs(subintstds) >= subintthresh)))
if config.debug.CLEAN:
plt.subplots_adjust(hspace=0.4)
chanax = plt.subplot(4, 1, 1)
plt.plot(np.arange(len(chanmeans)), chanmeans, 'k-')
plt.axhline(chanthresh, c='k', ls='--')
plt.axhline(-chanthresh, c='k', ls='--')
plt.xlabel('Channel Number', size='x-small')
plt.ylabel('Average', size='x-small')
plt.subplot(4, 1, 2, sharex=chanax)
plt.plot(np.arange(len(chanstds)), chanstds, 'k-')
plt.axhline(chanthresh, c='k', ls='--')
plt.axhline(-chanthresh, c='k', ls='--')
plt.xlabel('Channel Number', size='x-small')
plt.ylabel('Standard Deviation', size='x-small')
subintax = plt.subplot(4, 1, 3)
plt.plot(np.arange(len(subintmeans)), subintmeans, 'k-')
plt.axhline(subintthresh, c='k', ls='--')
plt.axhline(-subintthresh, c='k', ls='--')
plt.xlabel('Sub-int Number', size='x-small')
plt.ylabel('Average', size='x-small')
plt.subplot(4, 1, 4, sharex=subintax)
plt.plot(np.arange(len(subintstds)), subintstds, 'k-')
plt.axhline(subintthresh, c='k', ls='--')
plt.axhline(-subintthresh, c='k', ls='--')
plt.xlabel('Sub-int Number', size='x-small')
plt.ylabel('Standard Deviation', size='x-small')
plt.show()
badsubints = np.unique(badsubints)
utils.print_info(
"Number of sub-ints to be de-weighted: %d" % len(badsubints), 2)
for isub in badsubints:
utils.print_info("De-weighting subint# %d" % isub, 3)
clean_utils.zero_weight_subint(ar, isub)
clean_utils.zero_weight_subint(toclean, isub)
# Re-dedisperse the data
ar.dedisperse()
# Now replace hot bins
utils.print_info("Will find and clean 'hot' bins", 2)
clean_utils.clean_hot_bins(toclean, thresh=binthresh)
def _clean(self, ar):
patient = ar.clone()
patient.pscrunch()
patient.remove_baseline()
# Remove profile from dedispersed data
patient.dedisperse()
data = patient.get_data().squeeze()
if self.configs.template is None:
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
else:
template_ar = psrchive.Archive_load(self.configs.template)
template_ar.pscrunch()
template_ar.remove_baseline()
template = np.apply_over_axes(np.sum, template_ar.get_data(),
(0, 1)).squeeze()
# make sure template is 1D
if len(np.shape(template)) > 1: # sum over frequencies too
template_phs = np.apply_over_axes(np.sum, template.squeeze(),
0).squeeze()
else:
template_phs = template
if self.configs.template is None:
phs = 0
else:
# Calculate phase offset of template in number of bins, using full obs
profile = patient.clone()
profile.tscrunch()
profile.fscrunch()
# Get profile data of full obs
profile = profile.get_data()[0, 0, 0, :]
if np.shape(template_phs) != np.shape(profile):
print(
'template and profile have different numbers of phase bins'
)
err = lambda (amp, phs): amp * clean_utils.fft_rotate(
template_phs, phs) - profile
params, status = leastsq(err, [1, 0])
phs = params[1]
print('Found template phase offset = ', round(phs, 3))
clean_utils.remove_profile_inplace(patient, template, phs)
# re-set DM to 0
patient.dededisperse()
# Get weights
weights = patient.get_weights()
# Get data (select first polarization - recall we already P-scrunched)
data = patient.get_data()[:, 0, :, :]
data = clean_utils.apply_weights(data, weights)
# Mask profiles where weight is 0
mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool))
mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2)
data = np.ma.masked_array(data, mask=mask_3d)
# RFI-ectomy must be recommended by average of tests
avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \
chanthresh=self.configs.chanthresh, \
subintthresh=self.configs.subintthresh, \
chan_order=self.configs.chan_order, \
chan_breakpoints=self.configs.chan_breakpoints, \
chan_numpieces=self.configs.chan_numpieces, \
subint_order=self.configs.subint_order, \
subint_breakpoints=self.configs.subint_breakpoints, \
subint_numpieces=self.configs.subint_numpieces, \
)
for (isub, ichan) in np.argwhere(avg_test_results >= 1):
# Be sure to set weights on the original archive, and
# not the clone we've been working with.
integ = ar.get_Integration(int(isub))
integ.set_weight(int(ichan), 0.0)
def _clean(self, ar):
plot = False
#if self.configs.plot is None:
# plot = False
#else:
# plot = self.configs.plot
#if plot:
# import matplotlib.pyplot as plt
patient = ar.clone()
patient.pscrunch()
patient.remove_baseline()
# Remove profile from dedispersed data
patient.dedisperse()
print('Loading template')
data = patient.get_data().squeeze()
if self.configs.template is None:
# Sum over all axes except last, which is phase bins
template = np.apply_over_axes(np.sum, data,
tuple(range(data.ndim -
1))).squeeze()
# smooth data
template = savgol_filter(template, 5, 1)
else:
template_ar = psrchive.Archive_load(self.configs.template)
template_ar.pscrunch()
template_ar.remove_baseline()
template_ar.dedisperse()
if len(template_ar.get_frequencies()) > 1 and len(
template_ar.get_frequencies()) < len(
patient.get_frequencies()):
print(
"Template channel number doesn't match data... f-scrunching!"
)
template_ar.fscrunch()
template_data = template_ar.get_data().squeeze()
template = np.apply_over_axes(np.sum, template_data,
tuple(range(template_data.ndim -
1))).squeeze()
# make sure template is 1D
if len(np.shape(template)) > 1: # sum over frequencies too
template_ar.fscrunch()
print(
"2D template found. Assuming it has same frequency coverage and channels as data!"
)
template_phs = np.apply_over_axes(
np.sum, template_data,
tuple(range(template_data.ndim - 1))).squeeze()
else:
template_phs = template
print('Estimating template and profile phase offset')
if self.configs.template is None:
phs = 0
else:
# Calculate phase offset of template in number of bins, using full obs
# Get profile data of full obs
profile = np.apply_over_axes(np.sum, data,
tuple(range(data.ndim -
1))).squeeze()
if np.shape(template_phs) != np.shape(profile):
print(
'template and profile have different numbers of phase bins'
)
#err = (lambda (amp, phs, base): amp*clean_utils.fft_rotate(template_phs, phs) + base - profile)
#err = lambda amp, phs: amp*clean_utils.fft_rotate(template_phs, phs) - profile
err = lambda x: x[0] * clean_utils.fft_rotate(template_phs, x[1]
) - profile
amp_guess = np.median(profile) / np.median(template_phs)
phase_guess = -(np.argmax(profile) - np.argmax(template_phs))
#params, status = leastsq(err, [amp_guess, phase_guess, np.min(profile) - np.min(template_phs)])
params, status = leastsq(err, [amp_guess, phase_guess])
phs = params[1]
print('Template phase offset = {0}'.format(round(phs, 3)))
print('Removing profile from patient')
if plot:
preop_patient = patient.clone()
preop_weights = preop_patient.get_weights()
clean_utils.remove_profile_inplace(patient, template, phs)
print('Accessing weights and applying to patient')
# re-set DM to 0
# patient.dededisperse()
# Get weights
weights = patient.get_weights()
# Get data (select first polarization - recall we already P-scrunched)
data = patient.get_data()[:, 0, :, :]
data = clean_utils.apply_weights(data, weights)
if plot:
preop_data = preop_patient.get_data()[:, 0, :, :]
preop_patient = [] # clear for the sake of memory
preop_data = clean_utils.apply_weights(preop_data, weights)
# Mask profiles where weight is 0
mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool))
mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2)
data = np.ma.masked_array(data, mask=mask_3d)
if plot:
preop_data = np.ma.masked_array(preop_data, mask=mask_3d)
print('Masking on-pulse region as determined from template')
# consider residual only in off-pulse region
if len(np.shape(template)) > 1: # sum over frequencies
print('Estimating on-pulse region by f-scrunching 2D template')
template_ar.fscrunch()
template_1D = np.apply_over_axes(np.sum, template_ar.get_data(),
(0, 1)).squeeze()
else:
template_1D = template
# Rotate template by apropriate amount
template_rot = clean_utils.fft_rotate(template_1D, phs).squeeze()
# masked_template = np.ma.masked_greater(template_rot, np.min(template_rot) + 0.01*np.ptp(template_rot))
masked_template = np.ma.masked_greater(template_rot,
np.median(template_rot))
masked_std = np.ma.std(masked_template)
# use this std of masked data as cutoff
masked_template = np.ma.masked_greater(
template_rot,
np.median(template_rot) + masked_std)
if plot:
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
plt.plot(np.apply_over_axes(np.sum, preop_data,
tuple(range(data.ndim - 1))).squeeze(),
alpha=1)
# Do fit again to scale template
subchan, err, params = clean_utils.remove_profile1d(
np.apply_over_axes(np.sum, preop_data, (0, 1)).squeeze(),
0,
0,
template_rot,
0,
return_params=True)
# plt.plot(params[0]*template_rot + params[1], alpha=0.5)
# plt.plot(params[0]*masked_template + params[1], 'k')
plt.plot(params[0] * template_rot, alpha=0.5)
plt.plot(params[0] * masked_template, 'k')
plt.legend(('Pre-op data', 'Scaled and rotated template',
'Masked template'))
# Loop through chans and subints to mask on-pulse phase bins
for ii in range(0, np.shape(data)[0]):
for jj in range(0, np.shape(data)[1]):
data.mask[ii, jj, :] = masked_template.mask
data = np.ma.masked_array(data, mask=data.mask)
if plot:
plt.subplot(1, 2, 2)
plt.plot(
np.apply_over_axes(np.ma.sum, data,
tuple(range(data.ndim - 1))).squeeze())
plt.title("Residual data")
plt.savefig('data_and_template.png')
print(
'Calculating robust statistics to determine where RFI removal is required'
)
# RFI-ectomy must be recommended by average of tests
# BWM: Ok, so this is where the magical stuff actually happens - need to know actually WHAT are the comprehensive stats
# DJR: At this stage the stats are; (found to work well experimentally)
# geometric mean, peak-to-peak, standard deviation, normaltest.
# In original coast_guard they were;
# mean, peak-to-peak, standard deviation, and max value of FFT
avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \
chanthresh=self.configs.chanthresh, \
subintthresh=self.configs.subintthresh, \
chan_order=self.configs.chan_order, \
chan_breakpoints=self.configs.chan_breakpoints, \
chan_numpieces=self.configs.chan_numpieces, \
subint_order=self.configs.subint_order, \
subint_breakpoints=self.configs.subint_breakpoints, \
subint_numpieces=self.configs.subint_numpieces, \
cut_edge=False, \
)
print('Applying RFI masking weights to archive')
for (isub, ichan) in np.argwhere(avg_test_results >= 1):
# Be sure to set weights on the original archive, and
# not the clone we've been working with.
integ = ar.get_Integration(int(isub))
integ.set_weight(int(ichan), 0.0)
freq_fraczap = clean_utils.freq_fraczap(ar)
def _clean(self, ar):
patient = ar.clone()
patient.pscrunch()
patient.remove_baseline()
# Shi Dai, 2019/01/02/, apply weights before forming the template
# Get weights
weights = patient.get_weights()
# Remove profile from dedispersed data
patient.dedisperse()
data = patient.get_data().squeeze()
# apply weights
data = clean_utils.apply_weights(data, weights)
#template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
# Shi Dai, 2019/05/16, using 2D template
nsub, nchan, nbin = data.shape
temp_T = np.sum(data, 0)
temp_reshape = temp_T.reshape(
(26, nchan / 26, nbin)) # hard coded to use 26 sub-bands
template = np.sum(temp_reshape, axis=1)
print("Using 2D template with %d channels." % (template.shape[0]))
clean_utils.remove_profile_inplace(patient, template)
#np.save('data', data)
#np.save('template', template)
# re-set DM to 0
patient.dededisperse()
# Get data (select first polarization - recall we already P-scrunched)
data = patient.get_data()[:, 0, :, :]
data = clean_utils.apply_weights(data, weights)
# # Remove profile from dedispersed data
# patient.dedisperse()
# data = patient.get_data().squeeze()
# template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
# clean_utils.remove_profile_inplace(patient, template)
# # re-set DM to 0
# patient.dededisperse()
#
# # Get weights
# weights = patient.get_weights()
# # Get data (select first polarization - recall we already P-scrunched)
# data = patient.get_data()[:,0,:,:]
# data = clean_utils.apply_weights(data, weights)
# Mask profiles where weight is 0
mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool))
mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2)
data = np.ma.masked_array(data, mask=mask_3d)
# RFI-ectomy must be recommended by average of tests
avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \
chanthresh=self.configs.chanthresh, \
subintthresh=self.configs.subintthresh, \
chan_order=self.configs.chan_order, \
chan_breakpoints=self.configs.chan_breakpoints, \
chan_numpieces=self.configs.chan_numpieces, \
subint_order=self.configs.subint_order, \
subint_breakpoints=self.configs.subint_breakpoints, \
subint_numpieces=self.configs.subint_numpieces, \
)
for (isub, ichan) in np.argwhere(avg_test_results >= 1):
# Be sure to set weights on the original archive, and
# not the clone we've been working with.
integ = ar.get_Integration(int(isub))
integ.set_weight(int(ichan), 0.0)