def __prune_band_edges(self, ar):
"""Prune the edges of the band. This is useful for
removing channels where there is no response.
The file is modified in-place. However, zero-weighting
is used for pruning, so the process is reversible.
Inputs:
ar: The psrchive archive object to clean.
Outputs:
None
"""
if self.configs.response is None:
utils.print_info('No freq range specified for band pruning. Skipping...', 2)
else:
lofreq, hifreq = self.configs.response
# Use absolute value in case band is flipped (BW<0)
# bw = ar.get_bandwidth() # assigned but never used
nchan = ar.get_nchan()
# chanbw = bw/nchan # assigned but never used
utils.print_info('Pruning frequency band to (%g-%g MHz)' % (lofreq, hifreq), 2)
# Loop over channels
for ichan in range(nchan):
# Get profile for subint=0, pol=0
prof = ar.get_Profile(0, 0, ichan)
freq = prof.get_centre_frequency()
if (freq < lofreq) or (freq > hifreq):
clean_utils.zero_weight_chan(ar, ichan)
def _clean(self, ar):
nchan = ar.get_nchan()
nsub = ar.get_nsubint()
weights = (ar.get_weights() > 0)
nchan_masked = np.sum(weights.sum(axis=0) == 0)
nsub_masked = np.sum(weights.sum(axis=1) == 0)
sub_badfrac = 1 - weights.sum(axis=1) / float(nchan - nchan_masked)
chan_badfrac = 1 - weights.sum(axis=0) / float(nsub - nsub_masked)
sub_is_bad = np.argwhere(sub_badfrac > self.configs.badchantol)
utils.print_debug(
'Number of subints to mask because too many '
'channels are already masked: %d (%.1f %%)' %
(sub_is_bad.size, 100.0 * sub_is_bad.size / nsub), 'clean')
for isub in sub_is_bad:
clean_utils.zero_weight_subint(ar, isub)
chan_is_bad = np.argwhere(chan_badfrac > self.configs.badsubtol)
utils.print_debug(
'Number of channels to mask because too many '
'subints are already masked: %d (%.1f %%)' %
(chan_is_bad.size, 100.0 * chan_is_bad.size / nchan), 'clean')
for ichan in chan_is_bad:
clean_utils.zero_weight_chan(ar, ichan)
def __remove_bad_channels(self, ar):
"""Zero-weight bad channels and channels containing bad
frequencies. However, zero-weighting
is used for trimming, so the process is reversible.
Inputs:
ar: The psrchive archive object to clean.
Outputs:
None
"""
if self.configs.badchans:
nremoved = 0
for tozap in self.configs.badchans:
if type(tozap) is types.IntType:
# A single bad channel to zap
clean_utils.zero_weight_chan(ar, tozap)
nremoved += 1
else:
# An (inclusive) interval of bad channels to zap
lochan, hichan = tozap
for xx in xrange(lochan, hichan):
clean_utils.zero_weight_chan(ar, tozap)
nremoved += 1
utils.print_debug("Removed %d channels due to bad chans " \
"(%s) in %s" % (nremoved, self.configs.badfreqs, \
ar.get_filename()), 'clean')
if self.configs.badfreqs:
nremoved = 0
# Get a list of frequencies
nchan = ar.get_nchan()
lofreqs = np.empty(nchan)
hifreqs = np.empty(nchan)
chanbw = ar.get_bandwidth()/nchan
for ichan in xrange(nchan):
prof = ar.get_Profile(0, 0, ichan)
ctr = prof.get_centre_frequency()
lofreqs[ichan] = ctr - chanbw/2.0
hifreqs[ichan] = ctr + chanbw/2.0
for tozap in self.configs.badfreqs:
if type(tozap) is types.FloatType:
# A single bad freq to zap
for ichan in np.argwhere((lofreqs<=tozap) & (hifreqs>tozap)):
ichan = ichan.squeeze()
clean_utils.zero_weight_chan(ar, ichan)
nremoved += 1
else:
# An (inclusive) interval of bad freqs to zap
flo, fhi = tozap
for ichan in np.argwhere((hifreqs>=flo) & (lofreqs<=fhi)):
ichan = ichan.squeeze()
clean_utils.zero_weight_chan(ar, ichan)
nremoved += 1
utils.print_debug("Removed %d channels due to bad freqs " \
"(%s) in %s" % (nremoved, self.configs.badfreqs, \
ar.get_filename()), 'clean')
def _clean(self, ar):
nchan = ar.get_nchan()
nsub = ar.get_nsubint()
weights = (ar.get_weights() > 0)
nchan_masked = np.sum(weights.sum(axis=0) == 0)
nsub_masked = np.sum(weights.sum(axis=1) == 0)
sub_badfrac = 1 - weights.sum(axis=1) / float(nchan - nchan_masked)
chan_badfrac = 1 - weights.sum(axis=0) / float(nsub - nsub_masked)
sub_is_bad = np.argwhere(sub_badfrac > self.configs.badchantol)
for isub in sub_is_bad:
clean_utils.zero_weight_subint(ar, isub)
chan_is_bad = np.argwhere(chan_badfrac > self.configs.badsubtol)
for ichan in chan_is_bad:
clean_utils.zero_weight_chan(ar, ichan)
def __trim_edge_channels(self, ar):
"""Trim the edge channels of an input file to remove
band-pass roll-off and the effect of aliasing.
The file is modified in-place. However, zero-weighting
is used for trimming, so the process is reversible.
Inputs:
ar: The psrchive archive object to clean.
Outputs:
None
"""
nchan = ar.get_nchan()
bw = float(ar.get_bandwidth())
num_to_trim = max(self.configs.trimnum,
int(self.configs.trimfrac * nchan + 0.5),
int(self.configs.trimbw / bw * nchan + 0.5))
if num_to_trim > 0:
utils.print_info('Trimming %d channels from each band-edge.' % num_to_trim, 2)
for ichan in range(num_to_trim):
clean_utils.zero_weight_chan(ar, ichan) # trim at beginning
clean_utils.zero_weight_chan(ar, nchan - ichan - 1) # trim at end
def deep_clean(toclean, chanthresh=None, subintthresh=None, binthresh=None):
import psrchive # Temporarily, because python bindings
# are not available on all computers
if chanthresh is None:
chanthresh = config.cfg.clean_chanthresh
if subintthresh is None:
subintthresh = config.cfg.clean_subintthresh
if binthresh is None:
binthresh = config.cfg.clean_binthresh
ar = toclean.clone()
ar.pscrunch()
ar.remove_baseline()
ar.dedisperse()
# Remove profile
data = ar.get_data().squeeze()
template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze()
clean_utils.remove_profile_inplace(ar, template, None)
ar.dededisperse()
# First clean channels
chandata = clean_utils.get_chans(ar, remove_prof=True)
chanweights = clean_utils.get_chan_weights(ar).astype(bool)
chanmeans = clean_utils.scale_chans(chandata.mean(axis=1),
chanweights=chanweights)
chanmeans /= clean_utils.get_robust_std(chanmeans, chanweights)
chanstds = clean_utils.scale_chans(chandata.std(axis=1),
chanweights=chanweights)
chanstds /= clean_utils.get_robust_std(chanstds, chanweights)
badchans = np.concatenate((np.argwhere(np.abs(chanmeans) >= chanthresh), \
np.argwhere(np.abs(chanstds) >= chanthresh)))
badchans = np.unique(badchans)
utils.print_info(
"Number of channels to be de-weighted: %d" % len(badchans), 2)
for ichan in badchans:
utils.print_info("De-weighting chan# %d" % ichan, 3)
clean_utils.zero_weight_chan(ar, ichan)
clean_utils.zero_weight_chan(toclean, ichan)
# Next clean subints
subintdata = clean_utils.get_subints(ar, remove_prof=True)
subintweights = clean_utils.get_subint_weights(ar).astype(bool)
subintmeans = clean_utils.scale_subints(subintdata.mean(axis=1), \
subintweights=subintweights)
subintmeans /= clean_utils.get_robust_std(subintmeans, subintweights)
subintstds = clean_utils.scale_subints(subintdata.std(axis=1), \
subintweights=subintweights)
subintstds /= clean_utils.get_robust_std(subintstds, subintweights)
badsubints = np.concatenate((np.argwhere(np.abs(subintmeans) >= subintthresh), \
np.argwhere(np.abs(subintstds) >= subintthresh)))
if config.debug.CLEAN:
plt.subplots_adjust(hspace=0.4)
chanax = plt.subplot(4, 1, 1)
plt.plot(np.arange(len(chanmeans)), chanmeans, 'k-')
plt.axhline(chanthresh, c='k', ls='--')
plt.axhline(-chanthresh, c='k', ls='--')
plt.xlabel('Channel Number', size='x-small')
plt.ylabel('Average', size='x-small')
plt.subplot(4, 1, 2, sharex=chanax)
plt.plot(np.arange(len(chanstds)), chanstds, 'k-')
plt.axhline(chanthresh, c='k', ls='--')
plt.axhline(-chanthresh, c='k', ls='--')
plt.xlabel('Channel Number', size='x-small')
plt.ylabel('Standard Deviation', size='x-small')
subintax = plt.subplot(4, 1, 3)
plt.plot(np.arange(len(subintmeans)), subintmeans, 'k-')
plt.axhline(subintthresh, c='k', ls='--')
plt.axhline(-subintthresh, c='k', ls='--')
plt.xlabel('Sub-int Number', size='x-small')
plt.ylabel('Average', size='x-small')
plt.subplot(4, 1, 4, sharex=subintax)
plt.plot(np.arange(len(subintstds)), subintstds, 'k-')
plt.axhline(subintthresh, c='k', ls='--')
plt.axhline(-subintthresh, c='k', ls='--')
plt.xlabel('Sub-int Number', size='x-small')
plt.ylabel('Standard Deviation', size='x-small')
plt.show()
badsubints = np.unique(badsubints)
utils.print_info(
"Number of sub-ints to be de-weighted: %d" % len(badsubints), 2)
for isub in badsubints:
utils.print_info("De-weighting subint# %d" % isub, 3)
clean_utils.zero_weight_subint(ar, isub)
clean_utils.zero_weight_subint(toclean, isub)
# Re-dedisperse the data
ar.dedisperse()
# Now replace hot bins
utils.print_info("Will find and clean 'hot' bins", 2)
clean_utils.clean_hot_bins(toclean, thresh=binthresh)