def power_wash(ar):
"""Power wash RFI out of the data.
Input:
ar: The archive to be cleaned.
Outputs:
None - The archive is cleaned in place.
"""
ar.pscrunch()
ar.remove_baseline()
ar.dedisperse()
# Remove profile
data = ar.get_data().squeeze()
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
clean_utils.remove_profile_inplace(ar, template, None)
ar.dededisperse()
bad_chans = []
bad_subints = []
bad_pairs = []
std_sub_vs_chan = np.std(data, axis=2)
print std_sub_vs_chan.shape
#mean_sub_vs_chan = np.mean(data, axis=2)
# Identify bad sub-int/channel pairs
subintweights = clean_utils.get_subint_weights(ar).astype(bool)
chanweights = clean_utils.get_chan_weights(ar).astype(bool)
for isub in range(ar.get_nsubint()):
for ichan in range(ar.get_nchan()):
plt.figure()
plt.subplot(2, 1, 1)
plt.plot(std_sub_vs_chan[isub, :], 'k-')
subint = clean_utils.scale_chans(std_sub_vs_chan[isub, :], \
chanweights=chanweights)
print clean_utils.get_hot_bins(subint)
plt.subplot(2, 1, 2)
plt.plot(subint, 'r-')
plt.title("Subint #%d" % isub)
plt.figure()
plt.subplot(2, 1, 1)
plt.plot(std_sub_vs_chan[:, ichan], 'k-')
chan = clean_utils.scale_subints(std_sub_vs_chan[:, ichan], \
subintweights=subintweights)
print clean_utils.get_hot_bins(chan)
plt.subplot(2, 1, 2)
plt.plot(chan, 'r-')
plt.title("Chan #%d" % ichan)
plt.show()
chanstds = np.sum(std_sub_vs_chan, axis=0)
plt.subplot(2, 1, 1)
plt.plot(chanstds)
chanstds = clean_utils.scale_chans(chanstds, chanweights=chanweights)
plt.subplot(2, 1, 2)
plt.plot(chanstds)
bad_chans.extend(np.argwhere(chanstds > 1).squeeze())
plt.show()
def deep_clean(toclean, chanthresh=None, subintthresh=None, binthresh=None):
import psrchive # Temporarily, because python bindings
# are not available on all computers
if chanthresh is None:
chanthresh = config.cfg.clean_chanthresh
if subintthresh is None:
subintthresh = config.cfg.clean_subintthresh
if binthresh is None:
binthresh = config.cfg.clean_binthresh
ar = toclean.clone()
ar.pscrunch()
ar.remove_baseline()
ar.dedisperse()
# Remove profile
data = ar.get_data().squeeze()
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
clean_utils.remove_profile_inplace(ar, template, None)
ar.dededisperse()
# First clean channels
chandata = clean_utils.get_chans(ar, remove_prof=True)
chanweights = clean_utils.get_chan_weights(ar).astype(bool)
chanmeans = clean_utils.scale_chans(chandata.mean(axis=1),
chanweights=chanweights)
chanmeans /= clean_utils.get_robust_std(chanmeans, chanweights)
chanstds = clean_utils.scale_chans(chandata.std(axis=1),
chanweights=chanweights)
chanstds /= clean_utils.get_robust_std(chanstds, chanweights)
badchans = np.concatenate((np.argwhere(np.abs(chanmeans) >= chanthresh), \
np.argwhere(np.abs(chanstds) >= chanthresh)))
badchans = np.unique(badchans)
utils.print_info(
"Number of channels to be de-weighted: %d" % len(badchans), 2)
for ichan in badchans:
utils.print_info("De-weighting chan# %d" % ichan, 3)
clean_utils.zero_weight_chan(ar, ichan)
clean_utils.zero_weight_chan(toclean, ichan)
# Next clean subints
subintdata = clean_utils.get_subints(ar, remove_prof=True)
subintweights = clean_utils.get_subint_weights(ar).astype(bool)
subintmeans = clean_utils.scale_subints(subintdata.mean(axis=1), \
subintweights=subintweights)
subintmeans /= clean_utils.get_robust_std(subintmeans, subintweights)
subintstds = clean_utils.scale_subints(subintdata.std(axis=1), \
subintweights=subintweights)
subintstds /= clean_utils.get_robust_std(subintstds, subintweights)
badsubints = np.concatenate((np.argwhere(np.abs(subintmeans) >= subintthresh), \
np.argwhere(np.abs(subintstds) >= subintthresh)))
if config.debug.CLEAN:
plt.subplots_adjust(hspace=0.4)
chanax = plt.subplot(4, 1, 1)
plt.plot(np.arange(len(chanmeans)), chanmeans, 'k-')
plt.axhline(chanthresh, c='k', ls='--')
plt.axhline(-chanthresh, c='k', ls='--')
plt.xlabel('Channel Number', size='x-small')
plt.ylabel('Average', size='x-small')
plt.subplot(4, 1, 2, sharex=chanax)
plt.plot(np.arange(len(chanstds)), chanstds, 'k-')
plt.axhline(chanthresh, c='k', ls='--')
plt.axhline(-chanthresh, c='k', ls='--')
plt.xlabel('Channel Number', size='x-small')
plt.ylabel('Standard Deviation', size='x-small')
subintax = plt.subplot(4, 1, 3)
plt.plot(np.arange(len(subintmeans)), subintmeans, 'k-')
plt.axhline(subintthresh, c='k', ls='--')
plt.axhline(-subintthresh, c='k', ls='--')
plt.xlabel('Sub-int Number', size='x-small')
plt.ylabel('Average', size='x-small')
plt.subplot(4, 1, 4, sharex=subintax)
plt.plot(np.arange(len(subintstds)), subintstds, 'k-')
plt.axhline(subintthresh, c='k', ls='--')
plt.axhline(-subintthresh, c='k', ls='--')
plt.xlabel('Sub-int Number', size='x-small')
plt.ylabel('Standard Deviation', size='x-small')
plt.show()
badsubints = np.unique(badsubints)
utils.print_info(
"Number of sub-ints to be de-weighted: %d" % len(badsubints), 2)
for isub in badsubints:
utils.print_info("De-weighting subint# %d" % isub, 3)
clean_utils.zero_weight_subint(ar, isub)
clean_utils.zero_weight_subint(toclean, isub)
# Re-dedisperse the data
ar.dedisperse()
# Now replace hot bins
utils.print_info("Will find and clean 'hot' bins", 2)
clean_utils.clean_hot_bins(toclean, thresh=binthresh)