def power_wash(ar): """Power wash RFI out of the data. Input: ar: The archive to be cleaned. Outputs: None - The archive is cleaned in place. """ ar.pscrunch() ar.remove_baseline() ar.dedisperse() # Remove profile data = ar.get_data().squeeze() template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze() clean_utils.remove_profile_inplace(ar, template, None) ar.dededisperse() bad_chans = [] bad_subints = [] bad_pairs = [] std_sub_vs_chan = np.std(data, axis=2) print std_sub_vs_chan.shape #mean_sub_vs_chan = np.mean(data, axis=2) # Identify bad sub-int/channel pairs subintweights = clean_utils.get_subint_weights(ar).astype(bool) chanweights = clean_utils.get_chan_weights(ar).astype(bool) for isub in range(ar.get_nsubint()): for ichan in range(ar.get_nchan()): plt.figure() plt.subplot(2, 1, 1) plt.plot(std_sub_vs_chan[isub, :], 'k-') subint = clean_utils.scale_chans(std_sub_vs_chan[isub, :], \ chanweights=chanweights) print clean_utils.get_hot_bins(subint) plt.subplot(2, 1, 2) plt.plot(subint, 'r-') plt.title("Subint #%d" % isub) plt.figure() plt.subplot(2, 1, 1) plt.plot(std_sub_vs_chan[:, ichan], 'k-') chan = clean_utils.scale_subints(std_sub_vs_chan[:, ichan], \ subintweights=subintweights) print clean_utils.get_hot_bins(chan) plt.subplot(2, 1, 2) plt.plot(chan, 'r-') plt.title("Chan #%d" % ichan) plt.show() chanstds = np.sum(std_sub_vs_chan, axis=0) plt.subplot(2, 1, 1) plt.plot(chanstds) chanstds = clean_utils.scale_chans(chanstds, chanweights=chanweights) plt.subplot(2, 1, 2) plt.plot(chanstds) bad_chans.extend(np.argwhere(chanstds > 1).squeeze()) plt.show()
def surgical_scrub(ar, chanthresh=None, subintthresh=None, binthresh=None): """Surgically scrub RFI from the data. Input: ar: The archive to be cleaned. Outputs: None - The archive is cleaned in place. """ import psrchive # Temporarily, because python bindings # are not available on all computers patient = ar.clone() patient.pscrunch() patient.remove_baseline() # Remove profile from dedispersed data patient.dedisperse() data = patient.get_data().squeeze() template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze() clean_utils.remove_profile_inplace(patient, template) # re-set DM to 0 patient.dededisperse() # Get weights weights = patient.get_weights() # Get data (select first polarization - recall we already P-scrunched) data = patient.get_data()[:, 0, :, :] data = clean_utils.apply_weights(data, weights) # Mask profiles where weight is 0 mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool)) mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2) data = np.ma.masked_array(data, mask=mask_3d) # RFI-ectomy must be recommended by average of tests avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \ chanthresh=chanthresh, \ subintthresh=subintthresh, \ binthresh=binthresh) for (isub, ichan) in np.argwhere(avg_test_results >= 1): # Be sure to set weights on the original archive, and # not the clone we've been working with. integ = ar.get_Integration(int(isub)) integ.set_weight(int(ichan), 0.0)
def deep_clean(toclean, chanthresh=None, subintthresh=None, binthresh=None): import psrchive # Temporarily, because python bindings # are not available on all computers if chanthresh is None: chanthresh = config.cfg.clean_chanthresh if subintthresh is None: subintthresh = config.cfg.clean_subintthresh if binthresh is None: binthresh = config.cfg.clean_binthresh ar = toclean.clone() ar.pscrunch() ar.remove_baseline() ar.dedisperse() # Remove profile data = ar.get_data().squeeze() template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze() clean_utils.remove_profile_inplace(ar, template, None) ar.dededisperse() # First clean channels chandata = clean_utils.get_chans(ar, remove_prof=True) chanweights = clean_utils.get_chan_weights(ar).astype(bool) chanmeans = clean_utils.scale_chans(chandata.mean(axis=1), chanweights=chanweights) chanmeans /= clean_utils.get_robust_std(chanmeans, chanweights) chanstds = clean_utils.scale_chans(chandata.std(axis=1), chanweights=chanweights) chanstds /= clean_utils.get_robust_std(chanstds, chanweights) badchans = np.concatenate((np.argwhere(np.abs(chanmeans) >= chanthresh), \ np.argwhere(np.abs(chanstds) >= chanthresh))) badchans = np.unique(badchans) utils.print_info( "Number of channels to be de-weighted: %d" % len(badchans), 2) for ichan in badchans: utils.print_info("De-weighting chan# %d" % ichan, 3) clean_utils.zero_weight_chan(ar, ichan) clean_utils.zero_weight_chan(toclean, ichan) # Next clean subints subintdata = clean_utils.get_subints(ar, remove_prof=True) subintweights = clean_utils.get_subint_weights(ar).astype(bool) subintmeans = clean_utils.scale_subints(subintdata.mean(axis=1), \ subintweights=subintweights) subintmeans /= clean_utils.get_robust_std(subintmeans, subintweights) subintstds = clean_utils.scale_subints(subintdata.std(axis=1), \ subintweights=subintweights) subintstds /= clean_utils.get_robust_std(subintstds, subintweights) badsubints = np.concatenate((np.argwhere(np.abs(subintmeans) >= subintthresh), \ np.argwhere(np.abs(subintstds) >= subintthresh))) if config.debug.CLEAN: plt.subplots_adjust(hspace=0.4) chanax = plt.subplot(4, 1, 1) plt.plot(np.arange(len(chanmeans)), chanmeans, 'k-') plt.axhline(chanthresh, c='k', ls='--') plt.axhline(-chanthresh, c='k', ls='--') plt.xlabel('Channel Number', size='x-small') plt.ylabel('Average', size='x-small') plt.subplot(4, 1, 2, sharex=chanax) plt.plot(np.arange(len(chanstds)), chanstds, 'k-') plt.axhline(chanthresh, c='k', ls='--') plt.axhline(-chanthresh, c='k', ls='--') plt.xlabel('Channel Number', size='x-small') plt.ylabel('Standard Deviation', size='x-small') subintax = plt.subplot(4, 1, 3) plt.plot(np.arange(len(subintmeans)), subintmeans, 'k-') plt.axhline(subintthresh, c='k', ls='--') plt.axhline(-subintthresh, c='k', ls='--') plt.xlabel('Sub-int Number', size='x-small') plt.ylabel('Average', size='x-small') plt.subplot(4, 1, 4, sharex=subintax) plt.plot(np.arange(len(subintstds)), subintstds, 'k-') plt.axhline(subintthresh, c='k', ls='--') plt.axhline(-subintthresh, c='k', ls='--') plt.xlabel('Sub-int Number', size='x-small') plt.ylabel('Standard Deviation', size='x-small') plt.show() badsubints = np.unique(badsubints) utils.print_info( "Number of sub-ints to be de-weighted: %d" % len(badsubints), 2) for isub in badsubints: utils.print_info("De-weighting subint# %d" % isub, 3) clean_utils.zero_weight_subint(ar, isub) clean_utils.zero_weight_subint(toclean, isub) # Re-dedisperse the data ar.dedisperse() # Now replace hot bins utils.print_info("Will find and clean 'hot' bins", 2) clean_utils.clean_hot_bins(toclean, thresh=binthresh)
def _clean(self, ar): patient = ar.clone() patient.pscrunch() patient.remove_baseline() # Remove profile from dedispersed data patient.dedisperse() data = patient.get_data().squeeze() if self.configs.template is None: template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze() else: template_ar = psrchive.Archive_load(self.configs.template) template_ar.pscrunch() template_ar.remove_baseline() template = np.apply_over_axes(np.sum, template_ar.get_data(), (0, 1)).squeeze() # make sure template is 1D if len(np.shape(template)) > 1: # sum over frequencies too template_phs = np.apply_over_axes(np.sum, template.squeeze(), 0).squeeze() else: template_phs = template if self.configs.template is None: phs = 0 else: # Calculate phase offset of template in number of bins, using full obs profile = patient.clone() profile.tscrunch() profile.fscrunch() # Get profile data of full obs profile = profile.get_data()[0, 0, 0, :] if np.shape(template_phs) != np.shape(profile): print( 'template and profile have different numbers of phase bins' ) err = lambda (amp, phs): amp * clean_utils.fft_rotate( template_phs, phs) - profile params, status = leastsq(err, [1, 0]) phs = params[1] print('Found template phase offset = ', round(phs, 3)) clean_utils.remove_profile_inplace(patient, template, phs) # re-set DM to 0 patient.dededisperse() # Get weights weights = patient.get_weights() # Get data (select first polarization - recall we already P-scrunched) data = patient.get_data()[:, 0, :, :] data = clean_utils.apply_weights(data, weights) # Mask profiles where weight is 0 mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool)) mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2) data = np.ma.masked_array(data, mask=mask_3d) # RFI-ectomy must be recommended by average of tests avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \ chanthresh=self.configs.chanthresh, \ subintthresh=self.configs.subintthresh, \ chan_order=self.configs.chan_order, \ chan_breakpoints=self.configs.chan_breakpoints, \ chan_numpieces=self.configs.chan_numpieces, \ subint_order=self.configs.subint_order, \ subint_breakpoints=self.configs.subint_breakpoints, \ subint_numpieces=self.configs.subint_numpieces, \ ) for (isub, ichan) in np.argwhere(avg_test_results >= 1): # Be sure to set weights on the original archive, and # not the clone we've been working with. integ = ar.get_Integration(int(isub)) integ.set_weight(int(ichan), 0.0)
def _clean(self, ar): plot = False #if self.configs.plot is None: # plot = False #else: # plot = self.configs.plot #if plot: # import matplotlib.pyplot as plt patient = ar.clone() patient.pscrunch() patient.remove_baseline() # Remove profile from dedispersed data patient.dedisperse() print('Loading template') data = patient.get_data().squeeze() if self.configs.template is None: # Sum over all axes except last, which is phase bins template = np.apply_over_axes(np.sum, data, tuple(range(data.ndim - 1))).squeeze() # smooth data template = savgol_filter(template, 5, 1) else: template_ar = psrchive.Archive_load(self.configs.template) template_ar.pscrunch() template_ar.remove_baseline() template_ar.dedisperse() if len(template_ar.get_frequencies()) > 1 and len( template_ar.get_frequencies()) < len( patient.get_frequencies()): print( "Template channel number doesn't match data... f-scrunching!" ) template_ar.fscrunch() template_data = template_ar.get_data().squeeze() template = np.apply_over_axes(np.sum, template_data, tuple(range(template_data.ndim - 1))).squeeze() # make sure template is 1D if len(np.shape(template)) > 1: # sum over frequencies too template_ar.fscrunch() print( "2D template found. Assuming it has same frequency coverage and channels as data!" ) template_phs = np.apply_over_axes( np.sum, template_data, tuple(range(template_data.ndim - 1))).squeeze() else: template_phs = template print('Estimating template and profile phase offset') if self.configs.template is None: phs = 0 else: # Calculate phase offset of template in number of bins, using full obs # Get profile data of full obs profile = np.apply_over_axes(np.sum, data, tuple(range(data.ndim - 1))).squeeze() if np.shape(template_phs) != np.shape(profile): print( 'template and profile have different numbers of phase bins' ) #err = (lambda (amp, phs, base): amp*clean_utils.fft_rotate(template_phs, phs) + base - profile) #err = lambda amp, phs: amp*clean_utils.fft_rotate(template_phs, phs) - profile err = lambda x: x[0] * clean_utils.fft_rotate(template_phs, x[1] ) - profile amp_guess = np.median(profile) / np.median(template_phs) phase_guess = -(np.argmax(profile) - np.argmax(template_phs)) #params, status = leastsq(err, [amp_guess, phase_guess, np.min(profile) - np.min(template_phs)]) params, status = leastsq(err, [amp_guess, phase_guess]) phs = params[1] print('Template phase offset = {0}'.format(round(phs, 3))) print('Removing profile from patient') if plot: preop_patient = patient.clone() preop_weights = preop_patient.get_weights() clean_utils.remove_profile_inplace(patient, template, phs) print('Accessing weights and applying to patient') # re-set DM to 0 # patient.dededisperse() # Get weights weights = patient.get_weights() # Get data (select first polarization - recall we already P-scrunched) data = patient.get_data()[:, 0, :, :] data = clean_utils.apply_weights(data, weights) if plot: preop_data = preop_patient.get_data()[:, 0, :, :] preop_patient = [] # clear for the sake of memory preop_data = clean_utils.apply_weights(preop_data, weights) # Mask profiles where weight is 0 mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool)) mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2) data = np.ma.masked_array(data, mask=mask_3d) if plot: preop_data = np.ma.masked_array(preop_data, mask=mask_3d) print('Masking on-pulse region as determined from template') # consider residual only in off-pulse region if len(np.shape(template)) > 1: # sum over frequencies print('Estimating on-pulse region by f-scrunching 2D template') template_ar.fscrunch() template_1D = np.apply_over_axes(np.sum, template_ar.get_data(), (0, 1)).squeeze() else: template_1D = template # Rotate template by apropriate amount template_rot = clean_utils.fft_rotate(template_1D, phs).squeeze() # masked_template = np.ma.masked_greater(template_rot, np.min(template_rot) + 0.01*np.ptp(template_rot)) masked_template = np.ma.masked_greater(template_rot, np.median(template_rot)) masked_std = np.ma.std(masked_template) # use this std of masked data as cutoff masked_template = np.ma.masked_greater( template_rot, np.median(template_rot) + masked_std) if plot: plt.figure(figsize=(10, 5)) plt.subplot(1, 2, 1) plt.plot(np.apply_over_axes(np.sum, preop_data, tuple(range(data.ndim - 1))).squeeze(), alpha=1) # Do fit again to scale template subchan, err, params = clean_utils.remove_profile1d( np.apply_over_axes(np.sum, preop_data, (0, 1)).squeeze(), 0, 0, template_rot, 0, return_params=True) # plt.plot(params[0]*template_rot + params[1], alpha=0.5) # plt.plot(params[0]*masked_template + params[1], 'k') plt.plot(params[0] * template_rot, alpha=0.5) plt.plot(params[0] * masked_template, 'k') plt.legend(('Pre-op data', 'Scaled and rotated template', 'Masked template')) # Loop through chans and subints to mask on-pulse phase bins for ii in range(0, np.shape(data)[0]): for jj in range(0, np.shape(data)[1]): data.mask[ii, jj, :] = masked_template.mask data = np.ma.masked_array(data, mask=data.mask) if plot: plt.subplot(1, 2, 2) plt.plot( np.apply_over_axes(np.ma.sum, data, tuple(range(data.ndim - 1))).squeeze()) plt.title("Residual data") plt.savefig('data_and_template.png') print( 'Calculating robust statistics to determine where RFI removal is required' ) # RFI-ectomy must be recommended by average of tests # BWM: Ok, so this is where the magical stuff actually happens - need to know actually WHAT are the comprehensive stats # DJR: At this stage the stats are; (found to work well experimentally) # geometric mean, peak-to-peak, standard deviation, normaltest. # In original coast_guard they were; # mean, peak-to-peak, standard deviation, and max value of FFT avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \ chanthresh=self.configs.chanthresh, \ subintthresh=self.configs.subintthresh, \ chan_order=self.configs.chan_order, \ chan_breakpoints=self.configs.chan_breakpoints, \ chan_numpieces=self.configs.chan_numpieces, \ subint_order=self.configs.subint_order, \ subint_breakpoints=self.configs.subint_breakpoints, \ subint_numpieces=self.configs.subint_numpieces, \ cut_edge=False, \ ) print('Applying RFI masking weights to archive') for (isub, ichan) in np.argwhere(avg_test_results >= 1): # Be sure to set weights on the original archive, and # not the clone we've been working with. integ = ar.get_Integration(int(isub)) integ.set_weight(int(ichan), 0.0) freq_fraczap = clean_utils.freq_fraczap(ar)
def _clean(self, ar): patient = ar.clone() patient.pscrunch() patient.remove_baseline() # Shi Dai, 2019/01/02/, apply weights before forming the template # Get weights weights = patient.get_weights() # Remove profile from dedispersed data patient.dedisperse() data = patient.get_data().squeeze() # apply weights data = clean_utils.apply_weights(data, weights) #template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze() # Shi Dai, 2019/05/16, using 2D template nsub, nchan, nbin = data.shape temp_T = np.sum(data, 0) temp_reshape = temp_T.reshape( (26, nchan / 26, nbin)) # hard coded to use 26 sub-bands template = np.sum(temp_reshape, axis=1) print("Using 2D template with %d channels." % (template.shape[0])) clean_utils.remove_profile_inplace(patient, template) #np.save('data', data) #np.save('template', template) # re-set DM to 0 patient.dededisperse() # Get data (select first polarization - recall we already P-scrunched) data = patient.get_data()[:, 0, :, :] data = clean_utils.apply_weights(data, weights) # # Remove profile from dedispersed data # patient.dedisperse() # data = patient.get_data().squeeze() # template = np.apply_over_axes(np.sum, data, (0, 1)).squeeze() # clean_utils.remove_profile_inplace(patient, template) # # re-set DM to 0 # patient.dededisperse() # # # Get weights # weights = patient.get_weights() # # Get data (select first polarization - recall we already P-scrunched) # data = patient.get_data()[:,0,:,:] # data = clean_utils.apply_weights(data, weights) # Mask profiles where weight is 0 mask_2d = np.bitwise_not(np.expand_dims(weights, 2).astype(bool)) mask_3d = mask_2d.repeat(ar.get_nbin(), axis=2) data = np.ma.masked_array(data, mask=mask_3d) # RFI-ectomy must be recommended by average of tests avg_test_results = clean_utils.comprehensive_stats(data, axis=2, \ chanthresh=self.configs.chanthresh, \ subintthresh=self.configs.subintthresh, \ chan_order=self.configs.chan_order, \ chan_breakpoints=self.configs.chan_breakpoints, \ chan_numpieces=self.configs.chan_numpieces, \ subint_order=self.configs.subint_order, \ subint_breakpoints=self.configs.subint_breakpoints, \ subint_numpieces=self.configs.subint_numpieces, \ ) for (isub, ichan) in np.argwhere(avg_test_results >= 1): # Be sure to set weights on the original archive, and # not the clone we've been working with. integ = ar.get_Integration(int(isub)) integ.set_weight(int(ichan), 0.0)