Esempio n. 1
0
def spectralWhitening(data, sr=None, smoothi=None, freq_domain=False, apply_filter=None):
    """
    Apply spectral whitening to data.
    
    sr: sampling rate (only needed for smoothing)
    smoothi: None or int
    Data is divided by its smoothed (Default: None) amplitude spectrum.
    """
    if freq_domain:
        mask = False
        spec = data
    else:
        mask = np.ma.getmask(data)
        N = len(data)
        nfft = nextpow2(N)
        spec = fft(data, nfft)
    #df = sr/N
    spec_ampl = np.sqrt(np.abs(np.multiply(spec, np.conjugate(spec))))
    if isinstance(smoothi, basestring) and isnumber(smoothi) and smoothi > 0:
        smoothi = int(smoothi * N / sr)
        spec /= ifftshift(smooth(fftshift(spec_ampl), smoothi))
    else:
        spec /= spec_ampl
    if apply_filter is not None:
        spec *= filterResp(*apply_filter, sr=sr, N=len(spec), whole=True)[1]
    if freq_domain:
        return spec
    else:
        ret = np.real(ifft(spec, nfft)[:N])
        if USE_FFTW3:
            ret = ret.copy()
        return fillArray(ret, mask=mask, fill_value=0.)
Esempio n. 2
0
 def filter2(self, freqmin=None, freqmax=None, corners=2, zerophase=False):
     """
     Wrapper for Trace.filter, make entry in self.stats.filter.
     """
     if self.stats.is_fft:
         self.data *= filterResp(freqmin, freqmax, corners=corners, zerophase=zerophase,
                                 sr=self.stats.sampling_rate, N=self.stats.nfft,
                                  whole=True)[1]
         self.stats.filter += 'BP%4.2f,%4.2f,%d,%d' % (freqmin, freqmax, corners, zerophase)
     else:
         mask = np.ma.getmask(self.data)
         if freqmin and freqmax:
             self.filter("bandpass", freqmin=freqmin, freqmax=freqmax, corners=corners, zerophase=zerophase)
             self.stats.filter += 'BP%4.2f,%4.2f,%d,%d' % (freqmin, freqmax, corners, zerophase)
         elif not freqmin and freqmax:
             self.filter("lowpass", freq=freqmax, corners=corners, zerophase=zerophase)
             self.stats.filter += 'LP%4.2f,%d,%d' % (freqmax, corners, zerophase)
         elif freqmin and not freqmax:
             self.filter("highpass", freq=freqmin, corners=corners, zerophase=zerophase)
             self.stats.filter += 'HP%4.2f,%d,%d' % (freqmin, corners, zerophase)
         self.data = fillArray(self.data, mask=mask, fill_value=0.)
Esempio n. 3
0
 def filter2(self, freqmin=None, freqmax=None, corners=2, zerophase=False):
     """
     Wrapper for Trace.filter, make entry in self.stats.filter.
     """
     if self.stats.is_fft:
         self.data *= filterResp(freqmin,
                                 freqmax,
                                 corners=corners,
                                 zerophase=zerophase,
                                 sr=self.stats.sampling_rate,
                                 N=self.stats.nfft,
                                 whole=True)[1]
         self.stats.filter += 'BP%4.2f,%4.2f,%d,%d' % (freqmin, freqmax,
                                                       corners, zerophase)
     else:
         mask = np.ma.getmask(self.data)
         if freqmin and freqmax:
             self.filter("bandpass",
                         freqmin=freqmin,
                         freqmax=freqmax,
                         corners=corners,
                         zerophase=zerophase)
             self.stats.filter += 'BP%4.2f,%4.2f,%d,%d' % (
                 freqmin, freqmax, corners, zerophase)
         elif not freqmin and freqmax:
             self.filter("lowpass",
                         freq=freqmax,
                         corners=corners,
                         zerophase=zerophase)
             self.stats.filter += 'LP%4.2f,%d,%d' % (freqmax, corners,
                                                     zerophase)
         elif freqmin and not freqmax:
             self.filter("highpass",
                         freq=freqmin,
                         corners=corners,
                         zerophase=zerophase)
             self.stats.filter += 'HP%4.2f,%d,%d' % (freqmin, corners,
                                                     zerophase)
         self.data = fillArray(self.data, mask=mask, fill_value=0.)
Esempio n. 4
0
def spectralWhitening(data,
                      sr=None,
                      smoothi=None,
                      freq_domain=False,
                      apply_filter=None):
    """
    Apply spectral whitening to data.
    
    sr: sampling rate (only needed for smoothing)
    smoothi: None or int
    Data is divided by its smoothed (Default: None) amplitude spectrum.
    """
    if freq_domain:
        mask = False
        spec = data
    else:
        mask = np.ma.getmask(data)
        N = len(data)
        nfft = nextpow2(N)
        spec = fft(data, nfft)
    #df = sr/N
    spec_ampl = np.sqrt(np.abs(np.multiply(spec, np.conjugate(spec))))
    if isinstance(smoothi, basestring) and isnumber(smoothi) and smoothi > 0:
        smoothi = int(smoothi * N / sr)
        spec /= ifftshift(smooth(fftshift(spec_ampl), smoothi))
    else:
        spec /= spec_ampl
    if apply_filter is not None:
        spec *= filterResp(*apply_filter, sr=sr, N=len(spec), whole=True)[1]
    if freq_domain:
        return spec
    else:
        ret = np.real(ifft(spec, nfft)[:N])
        if USE_FFTW3:
            ret = ret.copy()
        return fillArray(ret, mask=mask, fill_value=0.)
Esempio n. 5
0
def timeNorm(data, method=None, param=None, recursive=0):
    """
    Calculates normalized data. See Bensen et al.(2007)

    Method is a string. There are the following methods:

    1bit: reduce data to +1 if >0 and -1 if <0
    clip: clip data to the root mean square (rms)
    eventremoval: automatic event detection and removal - if an value is bigger
        than the threshold, the following values are set to zero.
        param: (threshold, number of samples (eg. 30min) to set to zero)
    stalta: automatic event removing with recursive sta/lta trigger
    runningmean: the data is normalized with the running average
        The width of the normalization window determines how much amplitude
        information is retained (N=1 -> 1bit normalization, N very big ->
        rescaled data). Half of the maximum period of the passband filter_ works
        well.
        param: width of window (should be odd)
    runningmean_over_filtered: the data is normalized with the running average
        over the filtered data.
        A band pass filter_ between 20s and 100s period can remove local
        seismicity.
        param: (width of window in seconds, sampling rate, filter_, freq1, freq2)
            filter_: in ('band', 'low', high')
            if filter_ in ('low', 'high') only on frequency is needed
    waterlevel: any amplitude above the waterlevel (multiple of rms) is
        down-weighted by a factor. This procedure is repeated all of the
        waveform data is under the water-level
        param: (water-level factor, reducing factor)

    """
    mask = np.ma.getmask(data)
    if method == '1bit':
        data = np.sign(data)
    elif method == 'clip':
        std = np.std(data)
        data[data > std] = std
        data[data < -std] = -std
    elif method == 'eventremoval':
        if param == None:
            # remove 30 min (at 10Hz) after events if data is bigger than 2000
            param = (2000, 30 * 60 * 10)
        clip = np.nonzero(abs(data) >= param[0])[0]
        if len(clip) > 0:
            clip = clip[0]
            index = min(clip + param[1], len(data))
            data[clip:index] = 0
            if index < len(data):
                data[index:] = timeNorm(data[index:],
                                        method=method,
                                        param=param)
    elif method == 'stalta':
        if param is None:
            # STA: 3s at 100Hz, LTA: 10s, trigger on: 1.2, trigger off:1.0
            param = (100 * 3, 100 * 10, 1.2, 1.0)
        cft = obspy.signal.trigger.recSTALTA(data, param[0], param[1])
        trg = obspy.signal.trigger.triggerOnset(cft, param[2], param[3])
        for on, off in trg:
            data[on:off] = 0
    elif method == 'runningmean':
        if param == None:
            # smooth over 20s at 10Hz
            param = 10 * 10
        smoothed = smooth(np.abs(data), param)
        data /= smoothed
    elif method == 'runningmean_over_filtered':
        if param is None:
            # smooth over 20s at 10Hz over bandpassed data
            param = (10, 10, 'band', 1 / 50., 1 / 15.)
        sr = param[1]
        over = int(param[0] * sr)
        filter_ = param[2]
        if filter_ == 'band':
            data2 = obspy.signal.bandpass(data, param[3], param[4], sr)
        elif filter_ == 'low':
            data2 = obspy.signal.lowpass(data, param[3], sr)
        elif filter_ == 'high':
            data2 = obspy.signal.highpass(data, param[3], sr)
        else:
            raise ValueError("filter_ should be in ('band', 'high', 'low')")
        data /= smooth(np.abs(data2), over)
    elif method == 'waterlevel':
        if param == None:
            # data above 6*rms is recursively reduced by a factor of 10
            param = (6., 10.)
        waterlevel = param[0] * np.std(data)
        indices = np.abs(data) > waterlevel
        if np.any(indices):
            if param[1] == 0:
                data[indices] = 0
            else:
                data[indices] /= param[1]
            data = timeNorm(data,
                            method=method,
                            param=param,
                            recursive=recursive + 1)
    elif method == 'waterlevel_rm':
        if param == None:
            # running mean over 5s at 10Hz data
            # data above 4*rms is recursively reduced by a factor of 10
            param = (5 * 10, 4., 10.)
        running_mean = smooth(np.abs(data), param[0])
        waterlevel = param[1] * np.std(running_mean)
        indices = (running_mean > waterlevel) + (np.abs(data) > waterlevel)
        if np.any(indices):
            param = list(param)
            frac_zeros = 1. * np.count_nonzero(indices) / len(data)
            if param[2] == 0:
                data[indices] = 0
                param[1] *= (1 + frac_zeros)
            else:
                data[indices] /= param[2]
                param[1] *= (1 + frac_zeros * (1 - 1 / param[2]))
            print recursive, frac_zeros, waterlevel
            data = timeNorm(data,
                            method=method,
                            param=param,
                            recursive=recursive + 1)
    elif method == 'waterlevel_env':
        if param == None:
            # data above 4*rms is recursively reduced by a factor of 10
            param = (4., 10.)
        param = list(param)
        if len(param) == 2:
            param.append(0)
            param.append(0)
        env = obspy.signal.cpxtrace.envelope(data)[1][:len(data)]

        # correct std because of zeros
        waterlevel = param[0] * np.std(env) / (1 - param[2])
        #        import pylab as plt
        #        from imaging import plotTrace
        #        from sito import Trace
        #        trace = Trace(data=data)
        #        trace2 = Trace(data=env)
        #        plotTrace(trace)
        #        plotTrace(trace2)
        #        plt.figure()
        #        plt.plot(data)
        #        plt.plot(env)
        #        plt.hlines(waterlevel, 0, len(data))
        #        plt.show()
        indices = env > waterlevel
        frac_zeros = 1. * np.count_nonzero(indices) / len(data)
        if np.any(indices) and frac_zeros > 0.0005 and param[3] < 20:

            if param[1] == 0:
                data[indices] = 0
                #param[0] *= (1 + frac_zeros)
            else:
                data[indices] /= param[2]
                #param[0] *= (1 + frac_zeros * (1 - 1 / param[1]))
            print param[3], frac_zeros, param[2], waterlevel
            param[2] += frac_zeros
            param[3] += 1
            data = timeNorm(data, method=method, param=param)
    elif method == 'waterlevel_env2':
        if param == None:
            # data above 4*rms is recursively reduced by a factor of 10
            param = (4., 10.)
        N = len(data)
        env = obspy.signal.cpxtrace.envelope(data)[1][:N]
        if mask is not False:
            env[mask] = 0.
        num_stds = 96  # 24*4 =^ every 15min
        if N < 86400:  # 24*3600
            num_stds = N // 900
        len_parts = N // num_stds  # N//96 = N//24//4 =^ 15min
        len_stds = len_parts // 15  # len_parts//15 =^ 1min
        stds = np.array([
            np.std(env[i:i + len_stds])
            for i in np.arange(num_stds) * len_parts
        ])
        if np.min(stds) == 0:
            stds = stds[stds != 0.]
            num_stds = len(stds)
        stds = np.sort(stds)[num_stds // 15:-num_stds // 15]
        stds = stds[stds < np.min(stds) * 2.]
        waterlevel = param[0] * np.mean(stds)
        #        import pylab as plt
        #        from imaging import plotTrace
        #        from sito import Trace
        #        trace = Trace(data=data)
        #        trace2 = Trace(data=env)
        #        plotTrace(trace)
        #        plotTrace(trace2)
        #        plt.figure()
        #        plt.plot(data)
        #        plt.plot(env)
        #        plt.hlines(waterlevel, 0, len(data))
        #        plt.show()
        indices = env > waterlevel
        #frac_zeros = 1. * np.count_nonzero(indices) / len(data)
        if np.any(indices):
            if param[1] == 0:
                # not setting values to zero but masking them
                # -> they will stay zero after spectral whitening
                # and 1bit normalization
                mask = np.ma.mask_or(mask, indices)
                #data[indices] = 0
            else:
                data[indices] /= param[2]
    elif method is not None:
        raise ValueError('The method passed to timeNorm() is not known.')
    return fillArray(data, mask=mask, fill_value=0.)
Esempio n. 6
0
def _prepare_stream(stream, output_file,
            freq_domain=True, filter=(None, None), downsample=None,  #@ReservedAssignment
            simulate=None,
            eventremoval=None, param_removal=None,
            normalize=None, param_norm=None,
            whitening=None, filter_before_whitening=True,
            use_this_filter_after_whitening=None,
            discard=0.8 * 24 * 3600,
#            edge_smooth=(0, 0),
            trim=None):
    for tr in stream:
        tr.data = tr.data.astype('float64')
    # use special parameters for merging, because sometimes there is a
    # gap of one sample in IPOC data
    # method = 1: overlapping data: take the one of second trace
    # fill_value=0 fill gaps with zeros
    stream.merge(method=1, interpolation_samples=10)  #, fill_value=0)
    for tr in stream:
        tr.data = util.fillArray(tr.data, fill_value=0.)
        tr.stats.filter = ''
    stream.demean()
    if simulate and tr.stats.station in simulate:
        paz = simulate[tr.stats.station]
        print 'Simulate STS2 for station %s' % tr.stats.station
        stream.simulate(paz_remove=paz['remove'], paz_simulate=paz['simulate'],
                        taper_fraction=0.0003)
    if filter is not None and filter_before_whitening:
        stream.filter2(*filter)
    if downsample:
        stream.downsample2(downsample)
    if eventremoval:
        stream.timeNorm(method=eventremoval, param=param_removal)
    if (freq_domain and discard and
        1. * stream[0].stats.npts / stream[0].stats.sampling_rate < discard):
        return 'Coverage %f station %s day %s'
#    if edge_smooth != (0, 0):
#        for tr in stream:
#            tzeros1 = tr.stats.starttime + edge_smooth[0]
#            tzeros2 = tr.stats.endtime - edge_smooth[0]
#            tsmooth1 = tzeros1 + edge_smooth[1]
#            tsmooth2 = tzeros2 - edge_smooth[1]
#            tr.select(tr.stats.starttime, tzeros1).data[:] = 0
#            tr.select(tzeros2, tr.stats.endtime).data[:] = 0
#            tr_sm = tr.select(tzeros1, tsmooth1)
#            tr_sm.data *= 1 + np.cos(np.linspace(0, np.pi, len(tr_sm)))
#            tr_sm = tr.select(tsmooth2, tzeros2)
#            tr_sm.data *= 1 + np.cos(np.linspace(np.pi, 2 * np.pi, len(tr_sm)))
    if freq_domain and normalize is None and not trim:
        stream.fft()
    if whitening:
        if use_this_filter_after_whitening:
            filter = use_this_filter_after_whitening  #@ReservedAssignment
        stream.spectralWhitening(smoothi=whitening, apply_filter=filter)
    if normalize:
        stream.timeNorm(method=normalize, param=param_norm)
    if trim and trim == 'day':
        for tr in stream:
            t = tr.stats.starttime + 3600
            date = t.__class__(t.date)
            tr.trim(date, date + 24 * 3600)
    if freq_domain and (normalize is not None or trim != (0, 0)):
            stream.fft()
    if output_file:
        stream.write(output_file, 'Q')
    else:
        return stream
Esempio n. 7
0
def timeNorm(data, method=None, param=None, recursive=0):
    """
    Calculates normalized data. See Bensen et al.(2007)

    Method is a string. There are the following methods:

    1bit: reduce data to +1 if >0 and -1 if <0
    clip: clip data to the root mean square (rms)
    eventremoval: automatic event detection and removal - if an value is bigger
        than the threshold, the following values are set to zero.
        param: (threshold, number of samples (eg. 30min) to set to zero)
    stalta: automatic event removing with recursive sta/lta trigger
    runningmean: the data is normalized with the running average
        The width of the normalization window determines how much amplitude
        information is retained (N=1 -> 1bit normalization, N very big ->
        rescaled data). Half of the maximum period of the passband filter_ works
        well.
        param: width of window (should be odd)
    runningmean_over_filtered: the data is normalized with the running average
        over the filtered data.
        A band pass filter_ between 20s and 100s period can remove local
        seismicity.
        param: (width of window in seconds, sampling rate, filter_, freq1, freq2)
            filter_: in ('band', 'low', high')
            if filter_ in ('low', 'high') only on frequency is needed
    waterlevel: any amplitude above the waterlevel (multiple of rms) is
        down-weighted by a factor. This procedure is repeated all of the
        waveform data is under the water-level
        param: (water-level factor, reducing factor)

    """
    mask = np.ma.getmask(data)
    if method == '1bit':
        data = np.sign(data)
    elif method == 'clip':
        std = np.std(data)
        data[data > std] = std
        data[data < -std] = -std
    elif method == 'eventremoval':
        if param == None:
            # remove 30 min (at 10Hz) after events if data is bigger than 2000 
            param = (2000, 30 * 60 * 10)
        clip = np.nonzero(abs(data) >= param[0])[0]
        if len(clip) > 0:
            clip = clip[0]
            index = min(clip + param[1], len(data))
            data[clip:index] = 0
            if index < len(data):
                data[index:] = timeNorm(data[index:], method=method, param=param)
    elif method == 'stalta':
        if param is None:
            # STA: 3s at 100Hz, LTA: 10s, trigger on: 1.2, trigger off:1.0 
            param = (100 * 3, 100 * 10, 1.2, 1.0)
        cft = obspy.signal.trigger.recSTALTA(data, param[0], param[1])
        trg = obspy.signal.trigger.triggerOnset(cft, param[2], param[3])
        for on, off in trg:
            data[on:off] = 0
    elif method == 'runningmean':
        if param == None:
            # smooth over 20s at 10Hz
            param = 10 * 10
        smoothed = smooth(np.abs(data), param)
        data /= smoothed
    elif method == 'runningmean_over_filtered':
        if param is None:
            # smooth over 20s at 10Hz over bandpassed data
            param = (10, 10, 'band', 1 / 50., 1 / 15.)
        sr = param[1]
        over = int(param[0] * sr)
        filter_ = param[2]
        if filter_ == 'band':
            data2 = obspy.signal.bandpass(data, param[3], param[4], sr)
        elif filter_ == 'low':
            data2 = obspy.signal.lowpass(data, param[3], sr)
        elif filter_ == 'high':
            data2 = obspy.signal.highpass(data, param[3], sr)
        else:
            raise ValueError("filter_ should be in ('band', 'high', 'low')")
        data /= smooth(np.abs(data2), over)
    elif method == 'waterlevel':
        if param == None:
            # data above 6*rms is recursively reduced by a factor of 10
            param = (6., 10.)
        waterlevel = param[0] * np.std(data)
        indices = np.abs(data) > waterlevel
        if np.any(indices):
            if param[1] == 0:
                data[indices] = 0
            else:
                data[indices] /= param[1]
            data = timeNorm(data, method=method, param=param, recursive=recursive + 1)
    elif method == 'waterlevel_rm':
        if param == None:
            # running mean over 5s at 10Hz data
            # data above 4*rms is recursively reduced by a factor of 10
            param = (5 * 10, 4., 10.)
        running_mean = smooth(np.abs(data), param[0])
        waterlevel = param[1] * np.std(running_mean)
        indices = (running_mean > waterlevel) + (np.abs(data) > waterlevel)
        if np.any(indices):
            param = list(param)
            frac_zeros = 1. * np.count_nonzero(indices) / len(data)
            if param[2] == 0:
                data[indices] = 0
                param[1] *= (1 + frac_zeros)
            else:
                data[indices] /= param[2]
                param[1] *= (1 + frac_zeros * (1 - 1 / param[2]))
            print recursive, frac_zeros, waterlevel
            data = timeNorm(data, method=method, param=param, recursive=recursive + 1)
    elif method == 'waterlevel_env':
        if param == None:
            # data above 4*rms is recursively reduced by a factor of 10
            param = (4., 10.)
        param = list(param)
        if len(param) == 2:
            param.append(0)
            param.append(0)
        env = obspy.signal.cpxtrace.envelope(data)[1][:len(data)]

        # correct std because of zeros
        waterlevel = param[0] * np.std(env) / (1 - param[2])
#        import pylab as plt
#        from imaging import plotTrace
#        from sito import Trace
#        trace = Trace(data=data)
#        trace2 = Trace(data=env)
#        plotTrace(trace)
#        plotTrace(trace2)
#        plt.figure()
#        plt.plot(data)
#        plt.plot(env)
#        plt.hlines(waterlevel, 0, len(data))
#        plt.show()
        indices = env > waterlevel
        frac_zeros = 1. * np.count_nonzero(indices) / len(data)
        if np.any(indices) and frac_zeros > 0.0005 and param[3] < 20:

            if param[1] == 0:
                data[indices] = 0
                #param[0] *= (1 + frac_zeros)
            else:
                data[indices] /= param[2]
                #param[0] *= (1 + frac_zeros * (1 - 1 / param[1]))
            print param[3], frac_zeros, param[2], waterlevel
            param[2] += frac_zeros
            param[3] += 1
            data = timeNorm(data, method=method, param=param)
    elif method == 'waterlevel_env2':
        if param == None:
            # data above 4*rms is recursively reduced by a factor of 10
            param = (4., 10.)
        N = len(data)
        env = obspy.signal.cpxtrace.envelope(data)[1][:N]
        if mask is not False:
            env[mask] = 0.
        num_stds = 96 # 24*4 =^ every 15min
        if N < 86400: # 24*3600
            num_stds = N // 900
        len_parts = N // num_stds # N//96 = N//24//4 =^ 15min
        len_stds = len_parts // 15 # len_parts//15 =^ 1min
        stds = np.array([np.std(env[i:i + len_stds]) for i in np.arange(num_stds) * len_parts])
        if np.min(stds) == 0:
            stds = stds[stds != 0.]
            num_stds = len(stds)
        stds = np.sort(stds)[num_stds // 15:-num_stds // 15]
        stds = stds[stds < np.min(stds) * 2.]
        waterlevel = param[0] * np.mean(stds)
#        import pylab as plt
#        from imaging import plotTrace
#        from sito import Trace
#        trace = Trace(data=data)
#        trace2 = Trace(data=env)
#        plotTrace(trace)
#        plotTrace(trace2)
#        plt.figure()
#        plt.plot(data)
#        plt.plot(env)
#        plt.hlines(waterlevel, 0, len(data))
#        plt.show()
        indices = env > waterlevel
        #frac_zeros = 1. * np.count_nonzero(indices) / len(data)
        if np.any(indices):
            if param[1] == 0:
                # not setting values to zero but masking them
                # -> they will stay zero after spectral whitening
                # and 1bit normalization
                mask = np.ma.mask_or(mask, indices)
                #data[indices] = 0
            else:
                data[indices] /= param[2]
    elif method is not None:
        raise ValueError('The method passed to timeNorm() is not known.')
    return fillArray(data, mask=mask, fill_value=0.)