def full_matched_filter_and_cluster(self, template, template_norm, stilde, window): """ Return the complex snr and normalization. Calculated the matched filter, threshold, and cluster. Parameters ---------- htilde : FrequencySeries The template waveform. Must come from the FilterBank class. template_norm : float The htilde, template normalization factor. stilde : FrequencySeries The strain data to be filtered. window : int The size of the cluster window in samples. Returns ------- snr : TimeSeries A time series containing the complex snr. norm : float The normalization of the complex snr. corrrelation: FrequencySeries A frequency series containing the correlation vector. idx : Array List of indices of the triggers. snrv : Array The snr values at the trigger locations. """ snr, corr, norm = matched_filter_core(template, stilde, low_frequency_cutoff=self.flow, high_frequency_cutoff=self.fhigh, h_norm=template_norm, out=self.snr_mem, corr_out=self.corr_mem) idx, snrv = events.threshold(snr[stilde.analyze], self.snr_threshold / norm) if len(idx) == 0: return [], [], [], [], [] logging.info("%s points above threshold" % str(len(idx))) idx, snrv = events.cluster_reduce(idx, snrv, window) logging.info("%s clustered points" % str(len(idx))) return snr, norm, corr, idx, snrv
def full_matched_filter_and_cluster_fc(self, segnum, template_norm, window): """ Return the complex snr and normalization. Calculated the matched filter, threshold, and cluster. Parameters ---------- segnum : int Index into the list of segments at MatchedFilterControl construction against which to filter. template_norm : float The htilde, template normalization factor. window : int Size of the window over which to cluster triggers, in samples Returns ------- snr : TimeSeries A time series containing the complex snr. norm : float The normalization of the complex snr. corrrelation: FrequencySeries A frequency series containing the correlation vector. idx : Array List of indices of the triggers. snrv : Array The snr values at the trigger locations. """ norm = (4.0 * self.delta_f) / sqrt(template_norm) self.correlators[segnum].correlate() self.ifft.execute() idx, snrv = events.threshold( self.snr_mem[self.segments[segnum].analyze], self.snr_threshold / norm) idx, snrv = events.cluster_reduce(idx, snrv, window) if len(idx) == 0: return [], [], [], [], [] logging.info("%s points above threshold" % str(len(idx))) return self.snr_mem, norm, self.corr_mem, idx, snrv
def full_matched_filter_and_cluster_fc(self, segnum, template_norm, window): """ Return the complex snr and normalization. Calculated the matched filter, threshold, and cluster. Parameters ---------- segnum : int Index into the list of segments at MatchedFilterControl construction against which to filter. template_norm : float The htilde, template normalization factor. window : int Size of the window over which to cluster triggers, in samples Returns ------- snr : TimeSeries A time series containing the complex snr. norm : float The normalization of the complex snr. corrrelation: FrequencySeries A frequency series containing the correlation vector. idx : Array List of indices of the triggers. snrv : Array The snr values at the trigger locations. """ norm = (4.0 * self.delta_f) / sqrt(template_norm) self.correlators[segnum].correlate() self.ifft.execute() idx, snrv = events.threshold(self.snr_mem[self.segments[segnum].analyze], self.snr_threshold / norm) idx, snrv = events.cluster_reduce(idx, snrv, window) if len(idx) == 0: return [], [], [], [], [] logging.info("%s points above threshold" % str(len(idx))) return self.snr_mem, norm, self.corr_mem, idx, snrv
def heirarchical_matched_filter_and_cluster(self, htilde, template_norm, stilde, window): """ Return the complex snr and normalization. Calculated the matched filter, threshold, and cluster. Parameters ---------- htilde : FrequencySeries The template waveform. Must come from the FilterBank class. template_norm : float The htilde, template normalization factor. stilde : FrequencySeries The strain data to be filtered. window : int The size of the cluster window in samples. Returns ------- snr : TimeSeries A time series containing the complex snr at the reduced sample rate. norm : float The normalization of the complex snr. corrrelation: FrequencySeries A frequency series containing the correlation vector. idx : Array List of indices of the triggers. snrv : Array The snr values at the trigger locations. """ from pycbc.fft.fftw_pruned import pruned_c2cifft, fft_transpose norm = (4.0 * stilde.delta_f) / sqrt(template_norm) correlate(htilde[self.kmin_red:self.kmax_red], stilde[self.kmin_red:self.kmax_red], self.corr_mem[self.kmin_red:self.kmax_red]) ifft(self.corr_mem, self.snr_mem) if not hasattr(stilde, 'red_analyze'): stilde.red_analyze = \ slice(stilde.analyze.start/self.downsample_factor, stilde.analyze.stop/self.downsample_factor) idx_red, snrv_red = events.threshold( self.snr_mem[stilde.red_analyze], self.snr_threshold / norm * self.upsample_threshold) if len(idx_red) == 0: return [], None, [], [], [] idx_red, _ = events.cluster_reduce(idx_red, snrv_red, window / self.downsample_factor) logging.info("%s points above threshold at reduced resolution"\ %(str(len(idx_red)),)) # The fancy upsampling is here if self.upsample_method == 'pruned_fft': idx = (idx_red + stilde.analyze.start/self.downsample_factor)\ * self.downsample_factor idx = smear(idx, self.downsample_factor) # cache transposed versions of htilde and stilde if not hasattr(self.corr_mem_full, 'transposed'): self.corr_mem_full.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype) if not hasattr(htilde, 'transposed'): htilde.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype) htilde.transposed[self.kmin_full:self.kmax_full] = htilde[ self.kmin_full:self.kmax_full] htilde.transposed = fft_transpose(htilde.transposed) if not hasattr(stilde, 'transposed'): stilde.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype) stilde.transposed[self.kmin_full:self.kmax_full] = stilde[ self.kmin_full:self.kmax_full] stilde.transposed = fft_transpose(stilde.transposed) correlate(htilde.transposed, stilde.transposed, self.corr_mem_full.transposed) snrv = pruned_c2cifft(self.corr_mem_full.transposed, self.inter_vec, idx, pretransposed=True) idx = idx - stilde.analyze.start idx2, snrv = events.threshold(Array(snrv, copy=False), self.snr_threshold / norm) if len(idx2) > 0: correlate(htilde[self.kmax_red:self.kmax_full], stilde[self.kmax_red:self.kmax_full], self.corr_mem_full[self.kmax_red:self.kmax_full]) idx, snrv = events.cluster_reduce(idx[idx2], snrv, window) else: idx, snrv = [], [] logging.info("%s points at full rate and clustering" % len(idx)) return self.snr_mem, norm, self.corr_mem_full, idx, snrv else: raise ValueError("Invalid upsample method")
def compute_max_snr_over_sky_loc_stat(hplus, hcross, hphccorr, hpnorm=None, hcnorm=None, out=None, thresh=0, analyse_slice=None): """ Compute the maximized over sky location statistic. Parameters ----------- hplus : TimeSeries This is the IFFTed complex SNR time series of (h+, data). If not normalized, supply the normalization factor so this can be done! It is recommended to normalize this before sending through this function hcross : TimeSeries This is the IFFTed complex SNR time series of (hx, data). If not normalized, supply the normalization factor so this can be done! hphccorr : float The real component of the overlap between the two polarizations Re[(h+, hx)]. Note that the imaginary component does not enter the detection statistic. This must be normalized and is sign-sensitive. thresh : float Used for optimization. If we do not care about the value of SNR values below thresh we can calculate a quick statistic that will always overestimate SNR and then only calculate the proper, more expensive, statistic at points where the quick SNR is above thresh. hpsigmasq : float The normalization factor (h+, h+). Default = None (=1, already normalized) hcsigmasq : float The normalization factor (hx, hx). Default = None (=1, already normalized) out : TimeSeries (optional, default=None) If given, use this array to store the output. Returns -------- det_stat : TimeSeries The SNR maximized over sky location """ # NOTE: Not much optimization has been done here! This may need to be # C-ified using scipy.weave. if out is None: out = zeros(len(hplus)) out.non_zero_locs = numpy.array([], dtype=out.dtype) else: if not hasattr(out, 'non_zero_locs'): # Doing this every time is not a zero-cost operation out.data[:] = 0 out.non_zero_locs = numpy.array([], dtype=out.dtype) else: # Only set non zero locations to zero out.data[out.non_zero_locs] = 0 # If threshold is given we can limit the points at which to compute the # full statistic if thresh: # This is the statistic that always overestimates the SNR... # It allows some unphysical freedom that the full statistic does not idx_p, _ = events.threshold(hplus[analyse_slice], thresh / (2**0.5 * hpnorm)) idx_c, _ = events.threshold(hcross[analyse_slice], thresh / (2**0.5 * hcnorm)) idx_p = idx_p + analyse_slice.start idx_c = idx_c + analyse_slice.start hp_red = hplus[idx_p] * hpnorm hc_red = hcross[idx_p] * hcnorm stat_p = hp_red.real**2 + hp_red.imag**2 + \ hc_red.real**2 + hc_red.imag**2 locs_p = idx_p[stat_p > (thresh * thresh)] hp_red = hplus[idx_c] * hpnorm hc_red = hcross[idx_c] * hcnorm stat_c = hp_red.real**2 + hp_red.imag**2 + \ hc_red.real**2 + hc_red.imag**2 locs_c = idx_c[stat_c > (thresh * thresh)] locs = numpy.unique(numpy.concatenate((locs_p, locs_c))) hplus = hplus[locs] hcross = hcross[locs] hplus = hplus * hpnorm hcross = hcross * hcnorm # Calculate and sanity check the denominator denom = 1 - hphccorr * hphccorr if denom < 0: if hphccorr > 1: err_msg = "Overlap between hp and hc is given as %f. " % (hphccorr) err_msg += "How can an overlap be bigger than 1?" raise ValueError(err_msg) else: err_msg = "There really is no way to raise this error!?! " err_msg += "If you're seeing this, it is bad." raise ValueError(err_msg) if denom == 0: # This case, of hphccorr==1, makes the statistic degenerate # This case should not physically be possible luckily. err_msg = "You have supplied a real overlap between hp and hc of 1. " err_msg += "Ian is reasonably certain this is physically impossible " err_msg += "so why are you seeing this?" raise ValueError(err_msg) assert (len(hplus) == len(hcross)) # Now the stuff where comp. cost may be a problem hplus_magsq = numpy.real(hplus) * numpy.real(hplus) + \ numpy.imag(hplus) * numpy.imag(hplus) hcross_magsq = numpy.real(hcross) * numpy.real(hcross) + \ numpy.imag(hcross) * numpy.imag(hcross) rho_pluscross = numpy.real(hplus) * numpy.real(hcross) + numpy.imag( hplus) * numpy.imag(hcross) sqroot = (hplus_magsq - hcross_magsq)**2 sqroot += 4 * (hphccorr * hplus_magsq - rho_pluscross) * \ (hphccorr * hcross_magsq - rho_pluscross) # Sometimes this can be less than 0 due to numeric imprecision, catch this. if (sqroot < 0).any(): indices = numpy.arange(len(sqroot))[sqroot < 0] # This should not be *much* smaller than 0 due to numeric imprecision if (sqroot[indices] < -0.0001).any(): err_msg = "Square root has become negative. Something wrong here!" raise ValueError(err_msg) sqroot[indices] = 0 sqroot = numpy.sqrt(sqroot) det_stat_sq = 0.5 * (hplus_magsq + hcross_magsq - \ 2 * rho_pluscross*hphccorr + sqroot) det_stat = numpy.sqrt(det_stat_sq) if thresh: out.data[locs] = det_stat out.non_zero_locs = locs return out else: return Array(det_stat, copy=False)
def compute_max_snr_over_sky_loc_stat(hplus, hcross, hphccorr, hpnorm=None, hcnorm=None, out=None, thresh=0, analyse_slice=None): """ Compute the maximized over sky location statistic. Parameters ----------- hplus : TimeSeries This is the IFFTed complex SNR time series of (h+, data). If not normalized, supply the normalization factor so this can be done! It is recommended to normalize this before sending through this function hcross : TimeSeries This is the IFFTed complex SNR time series of (hx, data). If not normalized, supply the normalization factor so this can be done! hphccorr : float The real component of the overlap between the two polarizations Re[(h+, hx)]. Note that the imaginary component does not enter the detection statistic. This must be normalized and is sign-sensitive. thresh : float Used for optimization. If we do not care about the value of SNR values below thresh we can calculate a quick statistic that will always overestimate SNR and then only calculate the proper, more expensive, statistic at points where the quick SNR is above thresh. hpsigmasq : float The normalization factor (h+, h+). Default = None (=1, already normalized) hcsigmasq : float The normalization factor (hx, hx). Default = None (=1, already normalized) out : TimeSeries (optional, default=None) If given, use this array to store the output. Returns -------- det_stat : TimeSeries The SNR maximized over sky location """ # NOTE: Not much optimization has been done here! This may need to be # C-ified using scipy.weave. if out is None: out = zeros(len(hplus)) out.non_zero_locs = numpy.array([], dtype=out.dtype) else: if not hasattr(out, 'non_zero_locs'): # Doing this every time is not a zero-cost operation out.data[:] = 0 out.non_zero_locs = numpy.array([], dtype=out.dtype) else: # Only set non zero locations to zero out.data[out.non_zero_locs] = 0 # If threshold is given we can limit the points at which to compute the # full statistic if thresh: # This is the statistic that always overestimates the SNR... # It allows some unphysical freedom that the full statistic does not idx_p, _ = events.threshold(hplus[analyse_slice], thresh / (2**0.5 * hpnorm)) idx_c, _ = events.threshold(hcross[analyse_slice], thresh / (2**0.5 * hcnorm)) idx_p = idx_p + analyse_slice.start idx_c = idx_c + analyse_slice.start hp_red = hplus[idx_p] * hpnorm hc_red = hcross[idx_p] * hcnorm stat_p = hp_red.real**2 + hp_red.imag**2 + \ hc_red.real**2 + hc_red.imag**2 locs_p = idx_p[stat_p > (thresh*thresh)] hp_red = hplus[idx_c] * hpnorm hc_red = hcross[idx_c] * hcnorm stat_c = hp_red.real**2 + hp_red.imag**2 + \ hc_red.real**2 + hc_red.imag**2 locs_c = idx_c[stat_c > (thresh*thresh)] locs = numpy.unique(numpy.concatenate((locs_p, locs_c))) hplus = hplus[locs] hcross = hcross[locs] hplus = hplus * hpnorm hcross = hcross * hcnorm # Calculate and sanity check the denominator denom = 1 - hphccorr*hphccorr if denom < 0: if hphccorr > 1: err_msg = "Overlap between hp and hc is given as %f. " %(hphccorr) err_msg += "How can an overlap be bigger than 1?" raise ValueError(err_msg) else: err_msg = "There really is no way to raise this error!?! " err_msg += "If you're seeing this, it is bad." raise ValueError(err_msg) if denom == 0: # This case, of hphccorr==1, makes the statistic degenerate # This case should not physically be possible luckily. err_msg = "You have supplied a real overlap between hp and hc of 1. " err_msg += "Ian is reasonably certain this is physically impossible " err_msg += "so why are you seeing this?" raise ValueError(err_msg) assert(len(hplus) == len(hcross)) # Now the stuff where comp. cost may be a problem hplus_magsq = numpy.real(hplus) * numpy.real(hplus) + \ numpy.imag(hplus) * numpy.imag(hplus) hcross_magsq = numpy.real(hcross) * numpy.real(hcross) + \ numpy.imag(hcross) * numpy.imag(hcross) rho_pluscross = numpy.real(hplus) * numpy.real(hcross) + numpy.imag(hplus)*numpy.imag(hcross) sqroot = (hplus_magsq - hcross_magsq)**2 sqroot += 4 * (hphccorr * hplus_magsq - rho_pluscross) * \ (hphccorr * hcross_magsq - rho_pluscross) # Sometimes this can be less than 0 due to numeric imprecision, catch this. if (sqroot < 0).any(): indices = numpy.arange(len(sqroot))[sqroot < 0] # This should not be *much* smaller than 0 due to numeric imprecision if (sqroot[indices] < -0.0001).any(): err_msg = "Square root has become negative. Something wrong here!" raise ValueError(err_msg) sqroot[indices] = 0 sqroot = numpy.sqrt(sqroot) det_stat_sq = 0.5 * (hplus_magsq + hcross_magsq - \ 2 * rho_pluscross*hphccorr + sqroot) det_stat = numpy.sqrt(det_stat_sq) if thresh: out.data[locs] = det_stat out.non_zero_locs = locs return out else: return Array(det_stat, copy=False)
def heirarchical_matched_filter_and_cluster(self, segnum, template_norm, window): """ Return the complex snr and normalization. Calculated the matched filter, threshold, and cluster. Parameters ---------- segnum : int Index into the list of segments at MatchedFilterControl construction template_norm : float The htilde, template normalization factor. window : int Size of the window over which to cluster triggers, in samples Returns ------- snr : TimeSeries A time series containing the complex snr at the reduced sample rate. norm : float The normalization of the complex snr. corrrelation: FrequencySeries A frequency series containing the correlation vector. idx : Array List of indices of the triggers. snrv : Array The snr values at the trigger locations. """ from pycbc.fft.fftw_pruned import pruned_c2cifft, fft_transpose htilde = self.htilde stilde = self.segments[segnum] norm = (4.0 * stilde.delta_f) / sqrt(template_norm) correlate(htilde[self.kmin_red:self.kmax_red], stilde[self.kmin_red:self.kmax_red], self.corr_mem[self.kmin_red:self.kmax_red]) ifft(self.corr_mem, self.snr_mem) if not hasattr(stilde, 'red_analyze'): stilde.red_analyze = \ slice(stilde.analyze.start/self.downsample_factor, stilde.analyze.stop/self.downsample_factor) idx_red, snrv_red = events.threshold(self.snr_mem[stilde.red_analyze], self.snr_threshold / norm * self.upsample_threshold) if len(idx_red) == 0: return [], None, [], [], [] idx_red, _ = events.cluster_reduce(idx_red, snrv_red, window / self.downsample_factor) logging.info("%s points above threshold at reduced resolution"\ %(str(len(idx_red)),)) # The fancy upsampling is here if self.upsample_method=='pruned_fft': idx = (idx_red + stilde.analyze.start/self.downsample_factor)\ * self.downsample_factor idx = smear(idx, self.downsample_factor) # cache transposed versions of htilde and stilde if not hasattr(self.corr_mem_full, 'transposed'): self.corr_mem_full.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype) if not hasattr(htilde, 'transposed'): htilde.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype) htilde.transposed[self.kmin_full:self.kmax_full] = htilde[self.kmin_full:self.kmax_full] htilde.transposed = fft_transpose(htilde.transposed) if not hasattr(stilde, 'transposed'): stilde.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype) stilde.transposed[self.kmin_full:self.kmax_full] = stilde[self.kmin_full:self.kmax_full] stilde.transposed = fft_transpose(stilde.transposed) correlate(htilde.transposed, stilde.transposed, self.corr_mem_full.transposed) snrv = pruned_c2cifft(self.corr_mem_full.transposed, self.inter_vec, idx, pretransposed=True) idx = idx - stilde.analyze.start idx2, snrv = events.threshold(Array(snrv, copy=False), self.snr_threshold / norm) if len(idx2) > 0: correlate(htilde[self.kmax_red:self.kmax_full], stilde[self.kmax_red:self.kmax_full], self.corr_mem_full[self.kmax_red:self.kmax_full]) idx, snrv = events.cluster_reduce(idx[idx2], snrv, window) else: idx, snrv = [], [] logging.info("%s points at full rate and clustering" % len(idx)) return self.snr_mem, norm, self.corr_mem_full, idx, snrv else: raise ValueError("Invalid upsample method")