def spectralWhitening(data, sr=None, smoothi=None, freq_domain=False, apply_filter=None):
"""
Apply spectral whitening to data.
sr: sampling rate (only needed for smoothing)
smoothi: None or int
Data is divided by its smoothed (Default: None) amplitude spectrum.
"""
if freq_domain:
mask = False
spec = data
else:
mask = np.ma.getmask(data)
N = len(data)
nfft = nextpow2(N)
spec = fft(data, nfft)
#df = sr/N
spec_ampl = np.sqrt(np.abs(np.multiply(spec, np.conjugate(spec))))
if isinstance(smoothi, basestring) and isnumber(smoothi) and smoothi > 0:
smoothi = int(smoothi * N / sr)
spec /= ifftshift(smooth(fftshift(spec_ampl), smoothi))
else:
spec /= spec_ampl
if apply_filter is not None:
spec *= filterResp(*apply_filter, sr=sr, N=len(spec), whole=True)[1]
if freq_domain:
return spec
else:
ret = np.real(ifft(spec, nfft)[:N])
if USE_FFTW3:
ret = ret.copy()
return fillArray(ret, mask=mask, fill_value=0.)
def filter2(self, freqmin=None, freqmax=None, corners=2, zerophase=False):
"""
Wrapper for Trace.filter, make entry in self.stats.filter.
"""
if self.stats.is_fft:
self.data *= filterResp(freqmin, freqmax, corners=corners, zerophase=zerophase,
sr=self.stats.sampling_rate, N=self.stats.nfft,
whole=True)[1]
self.stats.filter += 'BP%4.2f,%4.2f,%d,%d' % (freqmin, freqmax, corners, zerophase)
else:
mask = np.ma.getmask(self.data)
if freqmin and freqmax:
self.filter("bandpass", freqmin=freqmin, freqmax=freqmax, corners=corners, zerophase=zerophase)
self.stats.filter += 'BP%4.2f,%4.2f,%d,%d' % (freqmin, freqmax, corners, zerophase)
elif not freqmin and freqmax:
self.filter("lowpass", freq=freqmax, corners=corners, zerophase=zerophase)
self.stats.filter += 'LP%4.2f,%d,%d' % (freqmax, corners, zerophase)
elif freqmin and not freqmax:
self.filter("highpass", freq=freqmin, corners=corners, zerophase=zerophase)
self.stats.filter += 'HP%4.2f,%d,%d' % (freqmin, corners, zerophase)
self.data = fillArray(self.data, mask=mask, fill_value=0.)
def filter2(self, freqmin=None, freqmax=None, corners=2, zerophase=False):
"""
Wrapper for Trace.filter, make entry in self.stats.filter.
"""
if self.stats.is_fft:
self.data *= filterResp(freqmin,
freqmax,
corners=corners,
zerophase=zerophase,
sr=self.stats.sampling_rate,
N=self.stats.nfft,
whole=True)[1]
self.stats.filter += 'BP%4.2f,%4.2f,%d,%d' % (freqmin, freqmax,
corners, zerophase)
else:
mask = np.ma.getmask(self.data)
if freqmin and freqmax:
self.filter("bandpass",
freqmin=freqmin,
freqmax=freqmax,
corners=corners,
zerophase=zerophase)
self.stats.filter += 'BP%4.2f,%4.2f,%d,%d' % (
freqmin, freqmax, corners, zerophase)
elif not freqmin and freqmax:
self.filter("lowpass",
freq=freqmax,
corners=corners,
zerophase=zerophase)
self.stats.filter += 'LP%4.2f,%d,%d' % (freqmax, corners,
zerophase)
elif freqmin and not freqmax:
self.filter("highpass",
freq=freqmin,
corners=corners,
zerophase=zerophase)
self.stats.filter += 'HP%4.2f,%d,%d' % (freqmin, corners,
zerophase)
self.data = fillArray(self.data, mask=mask, fill_value=0.)
def spectralWhitening(data,
sr=None,
smoothi=None,
freq_domain=False,
apply_filter=None):
"""
Apply spectral whitening to data.
sr: sampling rate (only needed for smoothing)
smoothi: None or int
Data is divided by its smoothed (Default: None) amplitude spectrum.
"""
if freq_domain:
mask = False
spec = data
else:
mask = np.ma.getmask(data)
N = len(data)
nfft = nextpow2(N)
spec = fft(data, nfft)
#df = sr/N
spec_ampl = np.sqrt(np.abs(np.multiply(spec, np.conjugate(spec))))
if isinstance(smoothi, basestring) and isnumber(smoothi) and smoothi > 0:
smoothi = int(smoothi * N / sr)
spec /= ifftshift(smooth(fftshift(spec_ampl), smoothi))
else:
spec /= spec_ampl
if apply_filter is not None:
spec *= filterResp(*apply_filter, sr=sr, N=len(spec), whole=True)[1]
if freq_domain:
return spec
else:
ret = np.real(ifft(spec, nfft)[:N])
if USE_FFTW3:
ret = ret.copy()
return fillArray(ret, mask=mask, fill_value=0.)
def timeNorm(data, method=None, param=None, recursive=0):
"""
Calculates normalized data. See Bensen et al.(2007)
Method is a string. There are the following methods:
1bit: reduce data to +1 if >0 and -1 if <0
clip: clip data to the root mean square (rms)
eventremoval: automatic event detection and removal - if an value is bigger
than the threshold, the following values are set to zero.
param: (threshold, number of samples (eg. 30min) to set to zero)
stalta: automatic event removing with recursive sta/lta trigger
runningmean: the data is normalized with the running average
The width of the normalization window determines how much amplitude
information is retained (N=1 -> 1bit normalization, N very big ->
rescaled data). Half of the maximum period of the passband filter_ works
well.
param: width of window (should be odd)
runningmean_over_filtered: the data is normalized with the running average
over the filtered data.
A band pass filter_ between 20s and 100s period can remove local
seismicity.
param: (width of window in seconds, sampling rate, filter_, freq1, freq2)
filter_: in ('band', 'low', high')
if filter_ in ('low', 'high') only on frequency is needed
waterlevel: any amplitude above the waterlevel (multiple of rms) is
down-weighted by a factor. This procedure is repeated all of the
waveform data is under the water-level
param: (water-level factor, reducing factor)
"""
mask = np.ma.getmask(data)
if method == '1bit':
data = np.sign(data)
elif method == 'clip':
std = np.std(data)
data[data > std] = std
data[data < -std] = -std
elif method == 'eventremoval':
if param == None:
# remove 30 min (at 10Hz) after events if data is bigger than 2000
param = (2000, 30 * 60 * 10)
clip = np.nonzero(abs(data) >= param[0])[0]
if len(clip) > 0:
clip = clip[0]
index = min(clip + param[1], len(data))
data[clip:index] = 0
if index < len(data):
data[index:] = timeNorm(data[index:],
method=method,
param=param)
elif method == 'stalta':
if param is None:
# STA: 3s at 100Hz, LTA: 10s, trigger on: 1.2, trigger off:1.0
param = (100 * 3, 100 * 10, 1.2, 1.0)
cft = obspy.signal.trigger.recSTALTA(data, param[0], param[1])
trg = obspy.signal.trigger.triggerOnset(cft, param[2], param[3])
for on, off in trg:
data[on:off] = 0
elif method == 'runningmean':
if param == None:
# smooth over 20s at 10Hz
param = 10 * 10
smoothed = smooth(np.abs(data), param)
data /= smoothed
elif method == 'runningmean_over_filtered':
if param is None:
# smooth over 20s at 10Hz over bandpassed data
param = (10, 10, 'band', 1 / 50., 1 / 15.)
sr = param[1]
over = int(param[0] * sr)
filter_ = param[2]
if filter_ == 'band':
data2 = obspy.signal.bandpass(data, param[3], param[4], sr)
elif filter_ == 'low':
data2 = obspy.signal.lowpass(data, param[3], sr)
elif filter_ == 'high':
data2 = obspy.signal.highpass(data, param[3], sr)
else:
raise ValueError("filter_ should be in ('band', 'high', 'low')")
data /= smooth(np.abs(data2), over)
elif method == 'waterlevel':
if param == None:
# data above 6*rms is recursively reduced by a factor of 10
param = (6., 10.)
waterlevel = param[0] * np.std(data)
indices = np.abs(data) > waterlevel
if np.any(indices):
if param[1] == 0:
data[indices] = 0
else:
data[indices] /= param[1]
data = timeNorm(data,
method=method,
param=param,
recursive=recursive + 1)
elif method == 'waterlevel_rm':
if param == None:
# running mean over 5s at 10Hz data
# data above 4*rms is recursively reduced by a factor of 10
param = (5 * 10, 4., 10.)
running_mean = smooth(np.abs(data), param[0])
waterlevel = param[1] * np.std(running_mean)
indices = (running_mean > waterlevel) + (np.abs(data) > waterlevel)
if np.any(indices):
param = list(param)
frac_zeros = 1. * np.count_nonzero(indices) / len(data)
if param[2] == 0:
data[indices] = 0
param[1] *= (1 + frac_zeros)
else:
data[indices] /= param[2]
param[1] *= (1 + frac_zeros * (1 - 1 / param[2]))
print recursive, frac_zeros, waterlevel
data = timeNorm(data,
method=method,
param=param,
recursive=recursive + 1)
elif method == 'waterlevel_env':
if param == None:
# data above 4*rms is recursively reduced by a factor of 10
param = (4., 10.)
param = list(param)
if len(param) == 2:
param.append(0)
param.append(0)
env = obspy.signal.cpxtrace.envelope(data)[1][:len(data)]
# correct std because of zeros
waterlevel = param[0] * np.std(env) / (1 - param[2])
# import pylab as plt
# from imaging import plotTrace
# from sito import Trace
# trace = Trace(data=data)
# trace2 = Trace(data=env)
# plotTrace(trace)
# plotTrace(trace2)
# plt.figure()
# plt.plot(data)
# plt.plot(env)
# plt.hlines(waterlevel, 0, len(data))
# plt.show()
indices = env > waterlevel
frac_zeros = 1. * np.count_nonzero(indices) / len(data)
if np.any(indices) and frac_zeros > 0.0005 and param[3] < 20:
if param[1] == 0:
data[indices] = 0
#param[0] *= (1 + frac_zeros)
else:
data[indices] /= param[2]
#param[0] *= (1 + frac_zeros * (1 - 1 / param[1]))
print param[3], frac_zeros, param[2], waterlevel
param[2] += frac_zeros
param[3] += 1
data = timeNorm(data, method=method, param=param)
elif method == 'waterlevel_env2':
if param == None:
# data above 4*rms is recursively reduced by a factor of 10
param = (4., 10.)
N = len(data)
env = obspy.signal.cpxtrace.envelope(data)[1][:N]
if mask is not False:
env[mask] = 0.
num_stds = 96 # 24*4 =^ every 15min
if N < 86400: # 24*3600
num_stds = N // 900
len_parts = N // num_stds # N//96 = N//24//4 =^ 15min
len_stds = len_parts // 15 # len_parts//15 =^ 1min
stds = np.array([
np.std(env[i:i + len_stds])
for i in np.arange(num_stds) * len_parts
])
if np.min(stds) == 0:
stds = stds[stds != 0.]
num_stds = len(stds)
stds = np.sort(stds)[num_stds // 15:-num_stds // 15]
stds = stds[stds < np.min(stds) * 2.]
waterlevel = param[0] * np.mean(stds)
# import pylab as plt
# from imaging import plotTrace
# from sito import Trace
# trace = Trace(data=data)
# trace2 = Trace(data=env)
# plotTrace(trace)
# plotTrace(trace2)
# plt.figure()
# plt.plot(data)
# plt.plot(env)
# plt.hlines(waterlevel, 0, len(data))
# plt.show()
indices = env > waterlevel
#frac_zeros = 1. * np.count_nonzero(indices) / len(data)
if np.any(indices):
if param[1] == 0:
# not setting values to zero but masking them
# -> they will stay zero after spectral whitening
# and 1bit normalization
mask = np.ma.mask_or(mask, indices)
#data[indices] = 0
else:
data[indices] /= param[2]
elif method is not None:
raise ValueError('The method passed to timeNorm() is not known.')
return fillArray(data, mask=mask, fill_value=0.)
def _prepare_stream(stream, output_file,
freq_domain=True, filter=(None, None), downsample=None, #@ReservedAssignment
simulate=None,
eventremoval=None, param_removal=None,
normalize=None, param_norm=None,
whitening=None, filter_before_whitening=True,
use_this_filter_after_whitening=None,
discard=0.8 * 24 * 3600,
# edge_smooth=(0, 0),
trim=None):
for tr in stream:
tr.data = tr.data.astype('float64')
# use special parameters for merging, because sometimes there is a
# gap of one sample in IPOC data
# method = 1: overlapping data: take the one of second trace
# fill_value=0 fill gaps with zeros
stream.merge(method=1, interpolation_samples=10) #, fill_value=0)
for tr in stream:
tr.data = util.fillArray(tr.data, fill_value=0.)
tr.stats.filter = ''
stream.demean()
if simulate and tr.stats.station in simulate:
paz = simulate[tr.stats.station]
print 'Simulate STS2 for station %s' % tr.stats.station
stream.simulate(paz_remove=paz['remove'], paz_simulate=paz['simulate'],
taper_fraction=0.0003)
if filter is not None and filter_before_whitening:
stream.filter2(*filter)
if downsample:
stream.downsample2(downsample)
if eventremoval:
stream.timeNorm(method=eventremoval, param=param_removal)
if (freq_domain and discard and
1. * stream[0].stats.npts / stream[0].stats.sampling_rate < discard):
return 'Coverage %f station %s day %s'
# if edge_smooth != (0, 0):
# for tr in stream:
# tzeros1 = tr.stats.starttime + edge_smooth[0]
# tzeros2 = tr.stats.endtime - edge_smooth[0]
# tsmooth1 = tzeros1 + edge_smooth[1]
# tsmooth2 = tzeros2 - edge_smooth[1]
# tr.select(tr.stats.starttime, tzeros1).data[:] = 0
# tr.select(tzeros2, tr.stats.endtime).data[:] = 0
# tr_sm = tr.select(tzeros1, tsmooth1)
# tr_sm.data *= 1 + np.cos(np.linspace(0, np.pi, len(tr_sm)))
# tr_sm = tr.select(tsmooth2, tzeros2)
# tr_sm.data *= 1 + np.cos(np.linspace(np.pi, 2 * np.pi, len(tr_sm)))
if freq_domain and normalize is None and not trim:
stream.fft()
if whitening:
if use_this_filter_after_whitening:
filter = use_this_filter_after_whitening #@ReservedAssignment
stream.spectralWhitening(smoothi=whitening, apply_filter=filter)
if normalize:
stream.timeNorm(method=normalize, param=param_norm)
if trim and trim == 'day':
for tr in stream:
t = tr.stats.starttime + 3600
date = t.__class__(t.date)
tr.trim(date, date + 24 * 3600)
if freq_domain and (normalize is not None or trim != (0, 0)):
stream.fft()
if output_file:
stream.write(output_file, 'Q')
else:
return stream
def timeNorm(data, method=None, param=None, recursive=0):
"""
Calculates normalized data. See Bensen et al.(2007)
Method is a string. There are the following methods:
1bit: reduce data to +1 if >0 and -1 if <0
clip: clip data to the root mean square (rms)
eventremoval: automatic event detection and removal - if an value is bigger
than the threshold, the following values are set to zero.
param: (threshold, number of samples (eg. 30min) to set to zero)
stalta: automatic event removing with recursive sta/lta trigger
runningmean: the data is normalized with the running average
The width of the normalization window determines how much amplitude
information is retained (N=1 -> 1bit normalization, N very big ->
rescaled data). Half of the maximum period of the passband filter_ works
well.
param: width of window (should be odd)
runningmean_over_filtered: the data is normalized with the running average
over the filtered data.
A band pass filter_ between 20s and 100s period can remove local
seismicity.
param: (width of window in seconds, sampling rate, filter_, freq1, freq2)
filter_: in ('band', 'low', high')
if filter_ in ('low', 'high') only on frequency is needed
waterlevel: any amplitude above the waterlevel (multiple of rms) is
down-weighted by a factor. This procedure is repeated all of the
waveform data is under the water-level
param: (water-level factor, reducing factor)
"""
mask = np.ma.getmask(data)
if method == '1bit':
data = np.sign(data)
elif method == 'clip':
std = np.std(data)
data[data > std] = std
data[data < -std] = -std
elif method == 'eventremoval':
if param == None:
# remove 30 min (at 10Hz) after events if data is bigger than 2000
param = (2000, 30 * 60 * 10)
clip = np.nonzero(abs(data) >= param[0])[0]
if len(clip) > 0:
clip = clip[0]
index = min(clip + param[1], len(data))
data[clip:index] = 0
if index < len(data):
data[index:] = timeNorm(data[index:], method=method, param=param)
elif method == 'stalta':
if param is None:
# STA: 3s at 100Hz, LTA: 10s, trigger on: 1.2, trigger off:1.0
param = (100 * 3, 100 * 10, 1.2, 1.0)
cft = obspy.signal.trigger.recSTALTA(data, param[0], param[1])
trg = obspy.signal.trigger.triggerOnset(cft, param[2], param[3])
for on, off in trg:
data[on:off] = 0
elif method == 'runningmean':
if param == None:
# smooth over 20s at 10Hz
param = 10 * 10
smoothed = smooth(np.abs(data), param)
data /= smoothed
elif method == 'runningmean_over_filtered':
if param is None:
# smooth over 20s at 10Hz over bandpassed data
param = (10, 10, 'band', 1 / 50., 1 / 15.)
sr = param[1]
over = int(param[0] * sr)
filter_ = param[2]
if filter_ == 'band':
data2 = obspy.signal.bandpass(data, param[3], param[4], sr)
elif filter_ == 'low':
data2 = obspy.signal.lowpass(data, param[3], sr)
elif filter_ == 'high':
data2 = obspy.signal.highpass(data, param[3], sr)
else:
raise ValueError("filter_ should be in ('band', 'high', 'low')")
data /= smooth(np.abs(data2), over)
elif method == 'waterlevel':
if param == None:
# data above 6*rms is recursively reduced by a factor of 10
param = (6., 10.)
waterlevel = param[0] * np.std(data)
indices = np.abs(data) > waterlevel
if np.any(indices):
if param[1] == 0:
data[indices] = 0
else:
data[indices] /= param[1]
data = timeNorm(data, method=method, param=param, recursive=recursive + 1)
elif method == 'waterlevel_rm':
if param == None:
# running mean over 5s at 10Hz data
# data above 4*rms is recursively reduced by a factor of 10
param = (5 * 10, 4., 10.)
running_mean = smooth(np.abs(data), param[0])
waterlevel = param[1] * np.std(running_mean)
indices = (running_mean > waterlevel) + (np.abs(data) > waterlevel)
if np.any(indices):
param = list(param)
frac_zeros = 1. * np.count_nonzero(indices) / len(data)
if param[2] == 0:
data[indices] = 0
param[1] *= (1 + frac_zeros)
else:
data[indices] /= param[2]
param[1] *= (1 + frac_zeros * (1 - 1 / param[2]))
print recursive, frac_zeros, waterlevel
data = timeNorm(data, method=method, param=param, recursive=recursive + 1)
elif method == 'waterlevel_env':
if param == None:
# data above 4*rms is recursively reduced by a factor of 10
param = (4., 10.)
param = list(param)
if len(param) == 2:
param.append(0)
param.append(0)
env = obspy.signal.cpxtrace.envelope(data)[1][:len(data)]
# correct std because of zeros
waterlevel = param[0] * np.std(env) / (1 - param[2])
# import pylab as plt
# from imaging import plotTrace
# from sito import Trace
# trace = Trace(data=data)
# trace2 = Trace(data=env)
# plotTrace(trace)
# plotTrace(trace2)
# plt.figure()
# plt.plot(data)
# plt.plot(env)
# plt.hlines(waterlevel, 0, len(data))
# plt.show()
indices = env > waterlevel
frac_zeros = 1. * np.count_nonzero(indices) / len(data)
if np.any(indices) and frac_zeros > 0.0005 and param[3] < 20:
if param[1] == 0:
data[indices] = 0
#param[0] *= (1 + frac_zeros)
else:
data[indices] /= param[2]
#param[0] *= (1 + frac_zeros * (1 - 1 / param[1]))
print param[3], frac_zeros, param[2], waterlevel
param[2] += frac_zeros
param[3] += 1
data = timeNorm(data, method=method, param=param)
elif method == 'waterlevel_env2':
if param == None:
# data above 4*rms is recursively reduced by a factor of 10
param = (4., 10.)
N = len(data)
env = obspy.signal.cpxtrace.envelope(data)[1][:N]
if mask is not False:
env[mask] = 0.
num_stds = 96 # 24*4 =^ every 15min
if N < 86400: # 24*3600
num_stds = N // 900
len_parts = N // num_stds # N//96 = N//24//4 =^ 15min
len_stds = len_parts // 15 # len_parts//15 =^ 1min
stds = np.array([np.std(env[i:i + len_stds]) for i in np.arange(num_stds) * len_parts])
if np.min(stds) == 0:
stds = stds[stds != 0.]
num_stds = len(stds)
stds = np.sort(stds)[num_stds // 15:-num_stds // 15]
stds = stds[stds < np.min(stds) * 2.]
waterlevel = param[0] * np.mean(stds)
# import pylab as plt
# from imaging import plotTrace
# from sito import Trace
# trace = Trace(data=data)
# trace2 = Trace(data=env)
# plotTrace(trace)
# plotTrace(trace2)
# plt.figure()
# plt.plot(data)
# plt.plot(env)
# plt.hlines(waterlevel, 0, len(data))
# plt.show()
indices = env > waterlevel
#frac_zeros = 1. * np.count_nonzero(indices) / len(data)
if np.any(indices):
if param[1] == 0:
# not setting values to zero but masking them
# -> they will stay zero after spectral whitening
# and 1bit normalization
mask = np.ma.mask_or(mask, indices)
#data[indices] = 0
else:
data[indices] /= param[2]
elif method is not None:
raise ValueError('The method passed to timeNorm() is not known.')
return fillArray(data, mask=mask, fill_value=0.)