def _clean(self, ar):
nchan = ar.get_nchan()
nsub = ar.get_nsubint()
weights = (ar.get_weights() > 0)
nchan_masked = np.sum(weights.sum(axis=0) == 0)
nsub_masked = np.sum(weights.sum(axis=1) == 0)
sub_badfrac = 1 - weights.sum(axis=1) / float(nchan - nchan_masked)
chan_badfrac = 1 - weights.sum(axis=0) / float(nsub - nsub_masked)
sub_is_bad = np.argwhere(sub_badfrac > self.configs.badchantol)
utils.print_debug(
'Number of subints to mask because too many '
'channels are already masked: %d (%.1f %%)' %
(sub_is_bad.size, 100.0 * sub_is_bad.size / nsub), 'clean')
for isub in sub_is_bad:
clean_utils.zero_weight_subint(ar, isub)
chan_is_bad = np.argwhere(chan_badfrac > self.configs.badsubtol)
utils.print_debug(
'Number of channels to mask because too many '
'subints are already masked: %d (%.1f %%)' %
(chan_is_bad.size, 100.0 * chan_is_bad.size / nchan), 'clean')
for ichan in chan_is_bad:
clean_utils.zero_weight_chan(ar, ichan)
def _clean(self, ar):
nchan = ar.get_nchan()
nsub = ar.get_nsubint()
weights = (ar.get_weights() > 0)
nchan_masked = np.sum(weights.sum(axis=0) == 0)
nsub_masked = np.sum(weights.sum(axis=1) == 0)
sub_badfrac = 1 - weights.sum(axis=1) / float(nchan - nchan_masked)
chan_badfrac = 1 - weights.sum(axis=0) / float(nsub - nsub_masked)
sub_is_bad = np.argwhere(sub_badfrac > self.configs.badchantol)
for isub in sub_is_bad:
clean_utils.zero_weight_subint(ar, isub)
chan_is_bad = np.argwhere(chan_badfrac > self.configs.badsubtol)
for ichan in chan_is_bad:
clean_utils.zero_weight_chan(ar, ichan)
def __remove_bad_subints(self, ar):
"""Zero-weights bad subints.
The file is modified in-place. However, zero-weighting
is used for trimming, so the process is reversible.
Inputs:
ar: The psrchive archive object to clean.
Outputs:
None
"""
if self.configs.badsubints:
for tozap in self.configs.badsubints:
if type(tozap) is int:
clean_utils.zero_weight_subint(ar, tozap)
else:
losubint, hisubint = tozap
for xx in range(losubint, hisubint + 1):
clean_utils.zero_weight_subint(ar, xx)
def deep_clean(toclean, chanthresh=None, subintthresh=None, binthresh=None):
import psrchive # Temporarily, because python bindings
# are not available on all computers
if chanthresh is None:
chanthresh = config.cfg.clean_chanthresh
if subintthresh is None:
subintthresh = config.cfg.clean_subintthresh
if binthresh is None:
binthresh = config.cfg.clean_binthresh
ar = toclean.clone()
ar.pscrunch()
ar.remove_baseline()
ar.dedisperse()
# Remove profile
data = ar.get_data().squeeze()
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
clean_utils.remove_profile_inplace(ar, template, None)
ar.dededisperse()
# First clean channels
chandata = clean_utils.get_chans(ar, remove_prof=True)
chanweights = clean_utils.get_chan_weights(ar).astype(bool)
chanmeans = clean_utils.scale_chans(chandata.mean(axis=1),
chanweights=chanweights)
chanmeans /= clean_utils.get_robust_std(chanmeans, chanweights)
chanstds = clean_utils.scale_chans(chandata.std(axis=1),
chanweights=chanweights)
chanstds /= clean_utils.get_robust_std(chanstds, chanweights)
badchans = np.concatenate((np.argwhere(np.abs(chanmeans) >= chanthresh), \
np.argwhere(np.abs(chanstds) >= chanthresh)))
badchans = np.unique(badchans)
utils.print_info(
"Number of channels to be de-weighted: %d" % len(badchans), 2)
for ichan in badchans:
utils.print_info("De-weighting chan# %d" % ichan, 3)
clean_utils.zero_weight_chan(ar, ichan)
clean_utils.zero_weight_chan(toclean, ichan)
# Next clean subints
subintdata = clean_utils.get_subints(ar, remove_prof=True)
subintweights = clean_utils.get_subint_weights(ar).astype(bool)
subintmeans = clean_utils.scale_subints(subintdata.mean(axis=1), \
subintweights=subintweights)
subintmeans /= clean_utils.get_robust_std(subintmeans, subintweights)
subintstds = clean_utils.scale_subints(subintdata.std(axis=1), \
subintweights=subintweights)
subintstds /= clean_utils.get_robust_std(subintstds, subintweights)
badsubints = np.concatenate((np.argwhere(np.abs(subintmeans) >= subintthresh), \
np.argwhere(np.abs(subintstds) >= subintthresh)))
if config.debug.CLEAN:
plt.subplots_adjust(hspace=0.4)
chanax = plt.subplot(4, 1, 1)
plt.plot(np.arange(len(chanmeans)), chanmeans, 'k-')
plt.axhline(chanthresh, c='k', ls='--')
plt.axhline(-chanthresh, c='k', ls='--')
plt.xlabel('Channel Number', size='x-small')
plt.ylabel('Average', size='x-small')
plt.subplot(4, 1, 2, sharex=chanax)
plt.plot(np.arange(len(chanstds)), chanstds, 'k-')
plt.axhline(chanthresh, c='k', ls='--')
plt.axhline(-chanthresh, c='k', ls='--')
plt.xlabel('Channel Number', size='x-small')
plt.ylabel('Standard Deviation', size='x-small')
subintax = plt.subplot(4, 1, 3)
plt.plot(np.arange(len(subintmeans)), subintmeans, 'k-')
plt.axhline(subintthresh, c='k', ls='--')
plt.axhline(-subintthresh, c='k', ls='--')
plt.xlabel('Sub-int Number', size='x-small')
plt.ylabel('Average', size='x-small')
plt.subplot(4, 1, 4, sharex=subintax)
plt.plot(np.arange(len(subintstds)), subintstds, 'k-')
plt.axhline(subintthresh, c='k', ls='--')
plt.axhline(-subintthresh, c='k', ls='--')
plt.xlabel('Sub-int Number', size='x-small')
plt.ylabel('Standard Deviation', size='x-small')
plt.show()
badsubints = np.unique(badsubints)
utils.print_info(
"Number of sub-ints to be de-weighted: %d" % len(badsubints), 2)
for isub in badsubints:
utils.print_info("De-weighting subint# %d" % isub, 3)
clean_utils.zero_weight_subint(ar, isub)
clean_utils.zero_weight_subint(toclean, isub)
# Re-dedisperse the data
ar.dedisperse()
# Now replace hot bins
utils.print_info("Will find and clean 'hot' bins", 2)
clean_utils.clean_hot_bins(toclean, thresh=binthresh)