def fillRMS(strSta, dUTC):
rmsData['rsam_estacion'] = strSta[0].stats['station']
rmsData['rsam_canal'] = strSta[0].stats['channel']
rmsData['rsam_fecha_proceso'] = str(dUTC.datetime)
rmsData['rms'] = getRMS(strSta[0].data)
dataFilt1 = filter.bandpass(strSta[0].data, 0.05, 0.125,
strSta[0].stats['sampling_rate'])
rmsData['rsam_banda1'] = getRMS(dataFilt1)
dataFilt2 = filter.bandpass(strSta[0].data, 2, 8,
strSta[0].stats['sampling_rate'])
rmsData['rsam_banda2'] = getRMS(dataFilt2)
dataFilt3 = filter.bandpass(strSta[0].data, 0.25, 2,
strSta[0].stats['sampling_rate'])
rmsData['rsam_banda3'] = getRMS(dataFilt3)
dataFilt4 = filter.highpass(strSta[0].data,
10.0,
strSta[0].stats['sampling_rate'],
corners=1,
zerophase=True)
rmsData['rsam_banda4'] = getRMS(dataFilt4)
def bandpass(self,fmin,fmax):
"""
bandpass(fmin,fmax)
Apply a zero-phase bandpass to all traces.
"""
self.trace_x=flt.bandpass(self.trace_x,fmin,fmax,1.0/self.dt,corners=2,zerophase=True)
self.trace_y=flt.bandpass(self.trace_y,fmin,fmax,1.0/self.dt,corners=2,zerophase=True)
self.trace_z=flt.bandpass(self.trace_z,fmin,fmax,1.0/self.dt,corners=2,zerophase=True)
def test_bandpass_high_corner_at_nyquist(self):
"""
Check that using exactly Nyquist for high corner gives correct results.
See #1451.
"""
tr = read()[0]
data = tr.data[:1000]
df = tr.stats.sampling_rate
nyquist = df / 2.0
for low_corner in (6.0, 8.55, 8.59):
for corners in (3, 4, 5, 6):
# this is filtering with high corner slightly below what we
# catch and change into highpass
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
expected = bandpass(data,
low_corner,
nyquist * (1 - 1.1e-6),
df=df,
corners=corners)
self.assertEqual(len(w), 0)
# all of these should be changed into a highpass
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
got1 = bandpass(data,
low_corner,
nyquist * (1 - 0.9e-6),
df=df,
corners=corners)
got2 = bandpass(data,
low_corner,
nyquist,
df=df,
corners=corners)
got3 = bandpass(data,
low_corner,
nyquist + 1.78,
df=df,
corners=corners)
self.assertEqual(len(w), 3)
for w_ in w:
self.assertTrue('Selected high corner frequency ' in
str(w[0].message))
self.assertTrue('Applying a high-pass instead.' in str(
w[0].message))
for got in (got1, got2, got3):
np.testing.assert_allclose(got,
expected,
rtol=1e-3,
atol=0.9)
def test_bandpass_vs_pitsa(self):
"""
Test Butterworth bandpass filter against Butterworth bandpass filter
of PITSA. Note that the corners value is twice the value of the filter
sections in PITSA. The rms of the difference between ObsPy and PITSA
tends to get bigger with higher order filtering.
"""
# load test file
file = os.path.join(self.path, 'rjob_20051006.gz')
with gzip.open(file) as f:
data = np.loadtxt(f)
# parameters for the test
samp_rate = 200.0
freq1 = 5
freq2 = 10
corners = 4
# filter trace
datcorr = bandpass(data, freq1, freq2, df=samp_rate, corners=corners)
# load pitsa file
filename = os.path.join(self.path, 'rjob_20051006_bandpass.gz')
with gzip.open(filename) as f:
data_pitsa = np.loadtxt(f)
# calculate normalized rms
rms = np.sqrt(
np.sum((datcorr - data_pitsa)**2) / np.sum(data_pitsa**2))
self.assertEqual(rms < 1.0e-05, True)
def plot_cc_withtime_stack(ccffile,freqmin,freqmax,ccomp,maxlag=None):
'''
plot the filtered cross-correlation functions between station-pair sta1-sta2
for all of the available days stored in ccfdir
example: plot_cc_withtime('/Users/chengxin/Documents/Harvard/Kanto_basin/Mesonet_BW/STACK/E.AYHM/E.AYHM_E.ENZM.h5',1,5,'ZZ',50)
'''
#---basic parameters----
if not maxlag:
maxlag = 100
#-----check whether file exists------
if os.path.isfile(ccffile):
with pyasdf.ASDFDataSet(ccffile,mode='r') as ds:
rlist = ds.auxiliary_data['Allstacked'].list()
if not rlist:
raise ValueError('no data stacked for %s'%ccffile)
dt = ds.auxiliary_data['Allstacked'][rlist[0]].parameters['dt']
tt = np.arange(-maxlag/dt, maxlag/dt+1)*dt
dist = ds.auxiliary_data['Allstacked'][rlist[0]].parameters['dist']
#-----loop through each day-----
slist = ds.auxiliary_data.list()
for ii in range(len(slist)-1):
if ii%60==0:
plt.figure(figsize=(9,6))
iday = slist[ii]
#-----in case there is no data on that day-----
try:
data = ds.auxiliary_data[iday][ccomp].data[:]
data = bandpass(data,freqmin,freqmax,int(1/dt),corners=4, zerophase=True)
#--normalize the data----
data = data/max(data)
except Exception:
data = np.zeros(tt.shape,dtype=np.float32)
npts = len(data)
#----make index----
indx0 = npts//2
tindx = int(maxlag/dt)
if ii==0:
color = 'b-'
else:
color = 'k-'
plt.plot(tt,data[indx0-tindx:indx0+tindx+1]+ii*2,color,linewidth=0.5)
plt.text(maxlag*0.7,ii*2,iday,fontsize=6)
plt.grid(True)
#---------highlight zero time------------
plt.plot([0,0],[0,ii*2],'r--',linewidth=1.5)
plt.title('%s %s, dist:%6.1fkm @%4.1f-%4.1f Hz' % (ccffile.split('/')[-1],ccomp,dist,freqmin,freqmax))
plt.xlabel('time [s]')
plt.ylabel('days')
plt.show()
def data_preprocess(data, filter_model='bandpass'):
if filter_model == 'bandpass':
data = bandpass(data, freqmin=4, freqmax=15, df=100)
data = (data - np.mean(data)) / (np.max(np.absolute(data)) + 1)
data = np.absolute(data)
return data
def preprocessing(data, path, muteP, fs, fmin=5, fmax=30):
"""
Preprocess data. only used for SES3D waveforms.
:param data: data to preprocess
:param path: path where data is stored
:param muteP: if muteP, time domain taper is applied to mute the p-wave arrival
:param fs: sampling frequency
:param fmin: lower bound for bandpass filtering
:param fmax: higher bound for bandpass filtering
:return: preprocessed data
"""
# mute unphysical reflections and p-wave
rylgh_int = np.loadtxt(path + "rayleigh_interval.txt")
pwave = rylgh_int[0]
refl = rylgh_int[1]
argmax = np.argmax(data)
if muteP:
data[:argmax - pwave] = 0
data[argmax + refl:] = 0
# filter
data = bandpass(data, fmin, fmax, fs, zerophase=True)
# demean
data = signal.detrend(data, type="constant")
return data
def test_bandpassVsPitsa(self):
"""
Test Butterworth bandpass filter against Butterworth bandpass filter
of PITSA. Note that the corners value is twice the value of the filter
sections in PITSA. The rms of the difference between ObsPy and PITSA
tends to get bigger with higher order filtering.
"""
# load test file
file = os.path.join(self.path, 'rjob_20051006.gz')
with gzip.open(file) as f:
data = np.loadtxt(f)
# parameters for the test
samp_rate = 200.0
freq1 = 5
freq2 = 10
corners = 4
# filter trace
datcorr = bandpass(data, freq1, freq2, df=samp_rate, corners=corners)
# load pitsa file
filename = os.path.join(self.path, 'rjob_20051006_bandpass.gz')
with gzip.open(filename) as f:
data_pitsa = np.loadtxt(f)
# calculate normalized rms
rms = np.sqrt(np.sum((datcorr - data_pitsa) ** 2) /
np.sum(data_pitsa ** 2))
self.assertEqual(rms < 1.0e-05, True)
def filtering(st,ty,args):
hip = args.highpass
lop = args.lowpass
bdp = args.bandpass
if hip != "0":
elements = hip.split()
cors = int(elements[0])
freq = eval(elements[1])
for i in range(len(st)):
st[i].data = highpass(st[i].data, freq, df=st[i].stats.sampling_rate,
corners=cors, zerophase=args.zeroph)
if lop != "0":
elements = lop.split()
cors = int(elements[0])
freq = eval(elements[1])
for i in range(len(st)):
st[i].data = lowpass(st[i].data, freq, df=st[i].stats.sampling_rate,
corners=cors, zerophase=args.zeroph)
if bdp != "0":
elements = bdp.split()
cors = int(elements[0])
freq_min = eval(elements[1])
freq_max = eval(elements[2])
for i in range(len(st)):
st[i].data = bandpass(st[i].data, freq_min, freq_max,
df=st[i].stats.sampling_rate, corners=cors,
zerophase=args.zeroph)
return st
def signal_filter(signal_raw, mvmt_level, criteria, f_lo, f_hi, samp_rate, filt, t_start, signal_min, filt_type='fft', hilb_sig=None):
#signal = signal-np.mean(signal)
filt_signal = []
if filt_type == 'fft':
if filt == True:
signal = bandpass(signal_raw, 0.05, 100, samp_rate)
else: signal = signal_raw
for t in range(0,signal.shape[0]-samp_rate,samp_rate/30):
if not(np.where(signal[t:t+samp_rate]<=signal_min)[0].shape[0] > 0):
if mvmt_level[ int((t + t_start)/float(samp_rate)*30)] == criteria:
## plt.plot(signal[t:t+512])
## plt.show()
## plt.plot((np.abs(np.fft.fft(signal[t:t+512]))**2)[f_lo:f_hi])
## plt.show()
## pdb.set_trace()
filt_signal.append((np.abs(np.fft.fft(signal[t:t+samp_rate]))**2)[f_lo:f_hi])
#pdb.set_trace()
elif filt_type == 'hilbert':
for t in range(0,hilb_sig.shape[0]-samp_rate,samp_rate/30):
if not(np.where(signal_raw[t:t+samp_rate]<=signal_min)[0].shape[0] > 0):
if mvmt_level[ int((t + t_start)/float(samp_rate)*30)] == criteria:
filt_signal.append(np.sum(hilb_sig[t:t+samp_rate,:],axis=0))
return filt_signal
def filt_ratio(trace,plot=False):
"""
Compute the amplitude ratio between filtered and unfiltered traces
from Langer et al. 2006
"""
from obspy.signal import filter
x = trace.tr
x_filt = filter.bandpass(trace.tr,0.7,1.5,1./trace.dt)
x_filt = x_filt
r = x_filt/x
r = r/np.max(r)
t = np.arange(len(trace.tr))*trace.dt
val, tval = window_p(r,t,0,opt='max')
if plot:
import matplotlib.pyplot as plt
fig = plt.figure()
fig.set_facecolor('white')
ax1 = fig.add_subplot(311)
ax1.plot(t,x,'k')
ax1.set_xticklabels('')
ax2 = fig.add_subplot(312)
ax2.plot(t,x_filt,'k')
ax2.set_xticklabels('')
ax3 = fig.add_subplot(313)
ax3.plot(t,r,'k')
ax3.plot(tval,val,'r')
ax3.set_xlabel('Time (s)')
plt.show()
return val
def plot_spectrum2(sfile,iday,icomp,freqmin,freqmax):
'''
this script plots the noise spectrum for the idayth on icomp (results from step1)
and compare it with the waveforms in time-domain
'''
dt = 0.05
sta = sfile.split('/')[-1].split('.')[1]
ds = pyasdf.ASDFDataSet(sfile,mode='r')
comp = ds.auxiliary_data.list()
#--check whether it exists----
if icomp in comp:
tlist = ds.auxiliary_data[icomp].list()
if iday in tlist:
spect = ds.auxiliary_data[icomp][iday].data[:]
#---look at hourly----
if spect.ndim==2:
nfft = spect.shape[1]
plt.title('station %s %s @ %s' % (sta,icomp,iday))
for ii in range(spect.shape[0]):
waveform = np.real(np.fft.irfft(spect[ii])[0:nfft])
waveform = bandpass(waveform,freqmin,freqmax,int(1/dt),corners=4, zerophase=True)
waveform = waveform*0.5/max(waveform)
plt.plot(np.arange(0,nfft)*dt,waveform+ii,'k-')
#plt.plot([0,nfft*dt],[ii,ii],'r--',linewidth=0.2)
plt.show()
def test_bandpass_zphsh_vs_pitsa(self):
"""
Test Butterworth zero-phase bandpass filter against Butterworth
zero-phase bandpass filter of PITSA. Note that the corners value is
twice the value of the filter sections in PITSA. The rms of the
difference between ObsPy and PITSA tends to get bigger with higher
order filtering.
Note: The Zero-Phase filters deviate from PITSA's zero-phase filters
at the end of the trace! The rms for the test is calculated omitting
the last 200 samples, as this part of the trace is assumed to
generally be of low interest/importance.
"""
# load test file
filename = os.path.join(self.path, 'rjob_20051006.gz')
with gzip.open(filename) as f:
data = np.loadtxt(f)
# parameters for the test
samp_rate = 200.0
freq1 = 5
freq2 = 10
corners = 2
# filter trace
datcorr = bandpass(data, freq1, freq2, df=samp_rate,
corners=corners, zerophase=True)
# load pitsa file
filename = os.path.join(self.path, 'rjob_20051006_bandpassZPHSH.gz')
with gzip.open(filename) as f:
data_pitsa = np.loadtxt(f)
# calculate normalized rms
rms = np.sqrt(np.sum((datcorr[:-200] - data_pitsa[:-200]) ** 2) /
np.sum(data_pitsa[:-200] ** 2))
self.assertEqual(rms < 1.0e-05, True)
def envsac(tr, low, high, delta, debug=False):
"""
Function to generate an envelope from given seismic data - requires obspy
:type tr: obspy.Trace
:type low: float
:param low: Lowcut in Hz
:type high: float
:param high: Highcut in Hz
:type delta: float
:param delta: Sampling interval desired
:return env: Envelope as an obspy.Trace object
"""
from obspy.signal.filter import bandpass, envelope
if debug:
print 'Detrend'
env=tr.detrend('simple') # Demean the data
del tr
if debug:
print 'Resample'
env.resample(40) # Downsample to 40Hz
if debug:
print 'Bandpass'
env.data=bandpass(env.data, low, high, env.stats.sampling_rate,\
3, True) # Filter
if debug:
print 'Envelope'
env.data=envelope(env.data) # Generate envelope
if debug:
print 'Resample'
env.resample(1) # Deimate to 1 Hz
return env
def running_absolute_mean(self, trace):
"""
Running-aboulute-mean normalization.
"""
# if the temporal normalization weight is not set, return the original trace
if self.temp_width == None: return
# compute normalizing weight (filter if specified) [Bensen et al., 2007]
delta = trace.stats.delta
filtered = trace.data.copy()
if self.ram_fband != None:
# filter the original trace to compute weight
filtered = bandpass(filtered, df=1.0/delta, zerophase=True, \
freqmin=self.ram_fband[0], freqmax=self.ram_fband[1])
# smoothen by convolving with an average mask
winlen = 2 * int(0.5 * self.temp_width / delta) + 1
avg_mask = np.ones(winlen) / (1.0 * winlen)
# filtered = fftconvolve(np.abs(filtered), avg_mask, 'same')
filtered = np.convolve(np.abs(filtered), avg_mask, 'same')
# except there is near-zero weight
MAX_AMP = np.max(np.abs(filtered))
for n in range(trace.stats.npts):
trace.data[n] = 0 if (filtered[n]<=1e-8*MAX_AMP) else (trace.data[n]/filtered[n])
trace.taper(type='cosine', max_percentage=0.005)
def snr_cc(symmetric_cc,df,distance,cmin,cmax,intervals,plotting=False):
snr_filt = np.zeros(len(intervals)-1)
if plotting:
plt.figure()
plt.subplot(len(intervals)+1,1,1)
plt.plot(symmetric_cc)
ax = plt.gca()
ax.set_yticklabels([])
for i in range(len(intervals)-1):
lim1 = intervals[i]
lim2 = intervals[i+1]
signal = bandpass(symmetric_cc,lim1,lim2,df)
idx1 = int(dist/cmax*df)
idx2 = int(dist/cmin*df)
snr_filt[i] = np.mean(np.abs(signal[idx1:idx2])) / np.mean(np.abs(np.append(signal[0:idx1],signal[idx2:int(len(signal)/2)])))
if plotting:
plt.subplot(len(intervals)+1,1,i+2)
plt.plot(signal)
plt.plot([idx1,idx1],[np.min(signal),np.max(signal)],'r')
plt.plot([idx2,idx2],[np.min(signal),np.max(signal)],'r')
if lim1==0.:
lim1 = 1./999
plt.text(idx2,np.max(signal)/2.,"%d - %ds SNR: %.1f" %(1./lim2,1./lim1,snr_filt[i]))
ax = plt.gca()
ax.set_yticklabels([])
if plotting:
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=None)
return snr_filt
def test_bandpassZPHSHVsPitsa(self):
"""
Test Butterworth zero-phase bandpass filter against Butterworth
zero-phase bandpass filter of PITSA. Note that the corners value is
twice the value of the filter sections in PITSA. The rms of the
difference between ObsPy and PITSA tends to get bigger with higher
order filtering.
Note: The Zero-Phase filters deviate from PITSA's zero-phase filters
at the end of the trace! The rms for the test is calculated omitting
the last 200 samples, as this part of the trace is assumed to
generally be of low interest/importance.
"""
# load test file
filename = os.path.join(self.path, 'rjob_20051006.gz')
with gzip.open(filename) as f:
data = np.loadtxt(f)
# parameters for the test
samp_rate = 200.0
freq1 = 5
freq2 = 10
corners = 2
# filter trace
datcorr = bandpass(data, freq1, freq2, df=samp_rate,
corners=corners, zerophase=True)
# load pitsa file
filename = os.path.join(self.path, 'rjob_20051006_bandpassZPHSH.gz')
with gzip.open(filename) as f:
data_pitsa = np.loadtxt(f)
# calculate normalized rms
rms = np.sqrt(np.sum((datcorr[:-200] - data_pitsa[:-200]) ** 2) /
np.sum(data_pitsa[:-200] ** 2))
self.assertEqual(rms < 1.0e-05, True)
def filt_ratio(trace, plot=False):
"""
Compute the amplitude ratio between filtered and unfiltered traces
from Langer et al. 2006
"""
from obspy.signal import filter
x = trace.tr
x_filt = filter.bandpass(trace.tr, 0.7, 1.5, 1. / trace.dt)
x_filt = x_filt
r = x_filt / x
r = r / np.max(r)
t = np.arange(len(trace.tr)) * trace.dt
val, tval = window_p(r, t, 0, opt='max')
if plot:
import matplotlib.pyplot as plt
fig = plt.figure()
fig.set_facecolor('white')
ax1 = fig.add_subplot(311)
ax1.plot(t, x, 'k')
ax1.set_xticklabels('')
ax2 = fig.add_subplot(312)
ax2.plot(t, x_filt, 'k')
ax2.set_xticklabels('')
ax3 = fig.add_subplot(313)
ax3.plot(t, r, 'k')
ax3.plot(tval, val, 'r')
ax3.set_xlabel('Time (s)')
plt.show()
return val
def plot_multi_freq_stack(sfile,freqmin,freqmax,nfreq,ccomp,tags=None):
'''
plot the stacked ccfs for sta1-sta2 at multi-frequency bands between freqmin-freqmax.
this may be useful to show the dispersive property of the surface waves, which could
be used together with plot_ZH_pmotion to identify surface wave components
example: plot_multi_freq_stack('/Users/chengxin/Documents/Harvard/Kanto_basin/Mesonet_BW/STACK/E.AYHM/E.AYHM_E.BKKM.h5',0.08,6,15,'RR')
'''
#---basic parameters----
maxlag = 100
#------set path and type-------------
ds = pyasdf.ASDFDataSet(sfile,mode='r')
slist = ds.auxiliary_data.list()
#-----check tags information------
if not tags:
tags = "Allstacked"
if tags not in slist:
raise ValueError('tags %s not in the file %s' % (tags,sfile))
#--------read the data and parameters------------
parameters = ds.auxiliary_data[tags][ccomp].parameters
corr = ds.auxiliary_data[tags][ccomp].data[:]
sampling_rate = int(1/parameters['dt'])
npts = int(2*sampling_rate*parameters['lag'])
indx = npts//2
tt = np.arange(-maxlag*sampling_rate, maxlag*sampling_rate+1)/sampling_rate
#------make frequency information-----
freq = np.zeros(nfreq,dtype=np.float32)
step = (np.log(freqmin)-np.log(freqmax))/(nfreq-1)
for ii in range(nfreq):
freq[ii]=np.exp(np.log(freqmin)-ii*step)
indx0 = maxlag*sampling_rate
#----loop through each freq-----
for ii in range(1,nfreq-1):
if ii==1:
plt.figure(figsize=(9,6))
f1 = freq[ii-1]
f2 = freq[ii+1]
ncorr = bandpass(corr,f1,f2,sampling_rate,corners=4,zerophase=True)
ncorr = ncorr/max(ncorr)
#------plot the signals-------
plt.plot(tt,ncorr[indx-indx0:indx+indx0+1]+ii,'k-',linewidth=0.6)
ttext = '{0:4.2f}-{1:4.2f} Hz'.format(f1,f2)
plt.text(maxlag*0.9,ii+0.3,ttext,fontsize=6)
if ii==1:
plt.title('%s at %s component' % (sfile.split('/')[-1],ccomp))
plt.xlabel('time [s]')
plt.ylabel('waveform #')
plt.grid(True)
plt.show()
del ds
def my_hilb_transform(signal, f_lo, f_hi, win_size, samp_rate):
hilb_signals = np.zeros(shape = (signal.shape[0], f_hi-f_lo))
cnt = 0
for f in range(f_lo,f_hi):
hilb_signals[:,cnt] = \
savgol_filter(np.abs(hilbert(bandpass(signal, f, f+1, samp_rate))**2), win_size, 2)
cnt += 1
return hilb_signals
def kurto_bandpass(trace, plot=False):
"""
Returns the frequency bandpass which maximizes the kurtosis
from Antoni 2007
"""
from waveloc.kurtogram import Fast_Kurtogram
import matplotlib.pyplot as plt
data = trace.tr[trace.i1:trace.i2]
N = len(data)
N2 = np.log2(N) - 7
nlevel = int(np.fix(N2))
dt = trace.dt
if plot:
import matplotlib.gridspec as gridspec
G = gridspec.GridSpec(3, 2)
fig = plt.figure(figsize=(15, 6))
fig.set_facecolor('white')
plt.subplot(G[:, 0])
Kwav, Level_w, freq_w, c, f_lower, f_upper = Fast_Kurtogram(np.array(
data, dtype=float),
nlevel,
Fs=1. / dt,
opt2=1,
verbose=plot)
if plot:
from obspy.signal.filter import bandpass
data_filt = bandpass(trace.tr, f_lower, f_upper, 1. / dt)
t = np.linspace(0, len(trace.tr) * dt, len(trace.tr))
plt.subplot(G[0, 1])
plt.plot(t, trace.tr, 'k')
plt.plot(t[trace.i1:trace.i2], trace.tr[trace.i1:trace.i2], 'r')
k = kurtosis(trace.tr[trace.i1:trace.i2],
axis=0,
fisher=False,
bias=True)
plt.xlim([t[0], t[-1]])
plt.figtext(.84, .9, 'K=%.1f' % k)
plt.subplot(G[1, 1])
plt.plot(t, data_filt, 'k')
plt.plot(t[trace.i1:trace.i2], data_filt[trace.i1:trace.i2], 'r')
k = kurtosis(data_filt[trace.i1:trace.i2],
axis=0,
fisher=False,
bias=True)
plt.xlim([t[0], t[-1]])
plt.figtext(.84, .57, 'K=%.1f' % k)
plt.xlabel('Time (s)')
plt.show()
return f_lower, f_upper
def bandpass(self, low, high):
"""
Perform a frequency bandpass in-place on a trace to reduce noise
Args:
low: Remove frequencies below this value
high: Remove frequencies above this value
"""
self.trace = bandpass(self.trace, low, high, self.freq)
def bp_filter(self,
freqmin,
freqmax,
zerophase=False,
rmean=False,
taper=False):
"""
Apply a band-pass filter to the data. If data are segmented into
multiple traces, apply the same filter to all traces. Calls
:func:`obspy.signal.filter` to do the filtering.
:param freqmin: Low frequency corner
:param freqmax: High frequency corner
:param zerophase: If ``True`` applies a non-causal bandpass filter. If
``False`` applies a causal bandpass filter.
:param rmean: If ``True`` remove mean before filtering.
:param taper: If ``True`` apply taper before filtering.
:type freqmin: float
:type freqmax: float
:type zerophase: bool
:type rmean: bool
:type taper: bool
:raises UserWarning: if no data in stream.
"""
if zerophase:
logging.info("Non-causal band-pass filtering single traces : \
%.2fHz - %.2fHz\n" % (freqmin, freqmax))
else:
logging.info("Causal band-pass filtering single traces : \
%.2fHz - %.2fHz\n" % (freqmin, freqmax))
if rmean:
self.rmean()
if taper:
self.taper()
for itr in range(self.stream.count()):
tr = self.stream.traces[itr]
xs = filter.bandpass(tr.data,
freqmin,
freqmax,
tr.stats.sampling_rate,
zerophase=zerophase)
tr.data = xs
self.stream.traces[itr] = tr
try:
self.trace = self.stream.traces[0]
except IndexError:
raise UserWarning('No data in stream at bp_filter.')
def shortproc(st, lowcut, highcut, filt_order, samp_rate, debug=0):
"""
Basic function to bandpass, downsample. Works in place
on data. This is employed to ensure all parts of the data are processed
in the same way.
:type st: obspy.Stream
:param st: Stream to process
:type highcut: float
:param highcut: High cut for bandpass in Hz
:type lowcut: float
:param lowcut: Low cut for bandpass in Hz
:type filt_order: int
:param filt_order: Number of corners for bandpass filter
:type samp_rate: float
:param samp_rate: Sampling rate desired in Hz
:type debug: int
:param debug: Debug flag from 0-5, higher numbers = more output
:return: obspy.Stream
"""
for tr in st:
if debug > 4:
tr.plot()
# Check data quality first
qual=_check_daylong(tr)
if not qual:
msg="Data have more zeros than actual data, please check the raw" +\
"data set-up and manually sort it"
raise ValueError(msg)
# Check sampling rate and resample
if tr.stats.sampling_rate != samp_rate:
tr.resample(samp_rate)
# Filtering section
tr=tr.detrend('simple') # Detrend data before filtering
tr.data=bandpass(tr.data, lowcut, highcut,
tr.stats.sampling_rate, filt_order, True)
# Correct FRAN N,E channels to 1,2 as this is correct
# if tr.stats.station=='FRAN' and tr.stats.channel=='SHN':
# print 'Correcting FRAN SHN to SH1'
# tr.stats.channel='SH1'
# if tr.stats.station=='FRAN' and tr.stats.channel=='SHE':
# print 'Correcting FRAN SHE to SH2'
# tr.stats.channel='SH2'
# Account for two letter channel names in s-files and therefore templates
tr.stats.channel=tr.stats.channel[0]+tr.stats.channel[2]
# Final visual check for debug
if debug > 4:
tr.plot()
return st
def clean_up(corr, sampling_rate, freqmin, freqmax):
if corr.ndim == 2:
axis = 1
else:
axis = 0
corr = scipy.signal.detrend(corr, axis=axis, type='constant')
corr = scipy.signal.detrend(corr, axis=axis, type='linear')
percent = np.min([sampling_rate * 20 / corr.shape[axis],0.05])
taper = scipy.signal.tukey(corr.shape[axis], percent)
corr *= taper
corr = bandpass(corr, freqmin, freqmax, sampling_rate, zerophase=True)
return corr
def test_bandpass_high_corner_at_nyquist(self):
"""
Check that using exactly Nyquist for high corner gives correct results.
See #1451.
"""
tr = read()[0]
data = tr.data[:1000]
df = tr.stats.sampling_rate
nyquist = df / 2.0
for low_corner in (6.0, 8.55, 8.59):
for corners in (3, 4, 5, 6):
# this is filtering with high corner slightly below what we
# catch and change into highpass
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
expected = bandpass(
data, low_corner, nyquist * (1 - 1.1e-6), df=df,
corners=corners)
self.assertEqual(len(w), 0)
# all of these should be changed into a highpass
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
got1 = bandpass(data, low_corner, nyquist * (1 - 0.9e-6),
df=df, corners=corners)
got2 = bandpass(data, low_corner, nyquist,
df=df, corners=corners)
got3 = bandpass(data, low_corner, nyquist + 1.78,
df=df, corners=corners)
self.assertEqual(len(w), 3)
for w_ in w:
self.assertTrue('Selected high corner frequency ' in
str(w[0].message))
self.assertTrue('Applying a high-pass instead.' in
str(w[0].message))
for got in (got1, got2, got3):
np.testing.assert_allclose(got, expected, rtol=1e-3,
atol=0.9)
def getMetrics(trace):
data = trace.data
mean = data.mean()
median = np.median(data)
stdv = data.std()
maximum = np.amax(data)
trace.taper(type='hamming', max_percentage=0.05, max_length=5)
data = trace.data
repFreq = central_frequency_unwindowed(data, df)
filtered = bandpass(data, 0.01, 1.5625, df)
sumEnergy = np.sum(
welch(filtered, np.hamming(len(data)), next_pow_2(len(data))))
return [mean, median, stdv, maximum, repFreq, sumEnergy]
def shortproc(st, lowcut, highcut, filt_order, samp_rate, debug=0):
"""
Basic function to bandpass, downsample. Works in place
on data. This is employed to ensure all parts of the data are processed
in the same way.
:type st: obspy.Stream
:param st: Stream to process
:type highcut: float
:param highcut: High cut for bandpass in Hz
:type lowcut: float
:param lowcut: Low cut for bandpass in Hz
:type filt_order: int
:param filt_order: Number of corners for bandpass filter
:type samp_rate: float
:param samp_rate: Sampling rate desired in Hz
:type debug: int
:param debug: Debug flag from 0-5, higher numbers = more output
:return: obspy.Stream
..rubric:: Note
Will convert channel names to two charectars long
"""
# Add sanity check for filter
if highcut >= 0.5*samp_rate:
raise IOError('Highcut must be lower than the nyquist')
for tr in st:
if debug > 4:
tr.plot()
# Check data quality first
qual=_check_daylong(tr)
if not qual:
msg="Data have more zeros than actual data, please check the raw" +\
"data set-up and manually sort it"
raise ValueError(msg)
# Check sampling rate and resample
if tr.stats.sampling_rate != samp_rate:
tr.resample(samp_rate)
# Filtering section
tr=tr.detrend('simple') # Detrend data before filtering
tr.data=bandpass(tr.data, lowcut, highcut,
tr.stats.sampling_rate, filt_order, True)
# Convert to two charectar channel names
tr.stats.channel=tr.stats.channel[0]+tr.stats.channel[-1]
# Final visual check for debug
if debug > 4:
tr.plot()
return st
def bp_filter(self, freqmin, freqmax, zerophase=False,
rmean=False, taper=False):
"""
Apply a band-pass filter to the data. If data are segmented into
multiple traces, apply the same filter to all traces. Calls
:func:`obspy.signal.filter` to do the filtering.
:param freqmin: Low frequency corner
:param freqmax: High frequency corner
:param zerophase: If ``True`` applies a non-causal bandpass filter. If
``False`` applies a causal bandpass filter.
:param rmean: If ``True`` remove mean before filtering.
:param taper: If ``True`` apply taper before filtering.
:type freqmin: float
:type freqmax: float
:type zerophase: bool
:type rmean: bool
:type taper: bool
:raises UserWarning: if no data in stream.
"""
if zerophase:
logging.info("Non-causal band-pass filtering single traces : \
%.2fHz - %.2fHz\n" % (freqmin, freqmax))
else:
logging.info("Causal band-pass filtering single traces : \
%.2fHz - %.2fHz\n" % (freqmin, freqmax))
if rmean:
self.rmean()
if taper:
self.taper()
for itr in range(self.stream.count()):
tr = self.stream.traces[itr]
xs = filter.bandpass(tr.data, freqmin, freqmax,
tr.stats.sampling_rate, zerophase=zerophase)
tr.data = xs
self.stream.traces[itr] = tr
try:
self.trace = self.stream.traces[0]
except IndexError:
raise UserWarning('No data in stream at bp_filter.')
def callback(self, fdb):
filter = [(fdb.data[i] / 1023.0 - 0.5) * 10
for j in range(len(fdb.data))] # -5dB to +5dB
ori = y[(self.currblock + 1) * self.n:(self.currblock + 2) * self.n]
arr = [ori] * len(filter)
for j in range(len(filter)):
if filter[j] < 0:
arr[j] = bandstop(
ori, self.xf[j * (self.n / 2 / len(filter)),
(j + 1) * (self.n / 2 / len(filter))], fs)
elif filter[j] > 0:
arr[j] = bandpass(
ori, self.xf[j * (self.n / 20), (j + 1) * (self.n / 20)],
fs)
arr[j] = (0.5 * 10 - filter[j]) * ori + filter[j] * arr[
j] #apply dB accordingly . e.g. ....*ori + ....*arr[j]
def kurto_bandpass(trace,plot=False):
"""
Returns the frequency bandpass which maximizes the kurtosis
from Antoni 2007
"""
from waveloc.kurtogram import Fast_Kurtogram
import matplotlib.pyplot as plt
data = trace.tr[trace.i1:trace.i2]
N = len(data)
N2 = np.log2(N)-7
nlevel = int(np.fix(N2))
dt = trace.dt
if plot:
import matplotlib.gridspec as gridspec
G = gridspec.GridSpec(3, 2)
fig = plt.figure(figsize=(15, 6))
fig.set_facecolor('white')
plt.subplot(G[:,0])
Kwav, Level_w, freq_w, c, f_lower, f_upper = Fast_Kurtogram(np.array(data,dtype=float), nlevel,Fs=1./dt,opt2=1,verbose=plot)
if plot:
from obspy.signal.filter import bandpass
data_filt = bandpass(trace.tr,f_lower,f_upper,1./dt)
t = np.linspace(0,len(trace.tr)*dt,len(trace.tr))
plt.subplot(G[0,1])
plt.plot(t,trace.tr,'k')
plt.plot(t[trace.i1:trace.i2],trace.tr[trace.i1:trace.i2],'r')
k = kurtosis(trace.tr[trace.i1:trace.i2], axis=0, fisher=False, bias=True)
plt.xlim([t[0],t[-1]])
plt.figtext(.84,.9,'K=%.1f'%k)
plt.subplot(G[1,1])
plt.plot(t,data_filt,'k')
plt.plot(t[trace.i1:trace.i2],data_filt[trace.i1:trace.i2],'r')
k = kurtosis(data_filt[trace.i1:trace.i2], axis=0, fisher=False, bias=True)
plt.xlim([t[0],t[-1]])
plt.figtext(.84,.57,'K=%.1f'%k)
plt.xlabel('Time (s)')
plt.show()
return f_lower, f_upper
def filter(self):
""" Filter data for each band.
"""
n_bands = self._N_bands()
LEN = self.tr.stats.npts
df = self.tr.stats.sampling_rate
# create zeros 2D array for BF
BF = np.zeros(shape=(n_bands, LEN))
for j in range(n_bands):
octave_high = (self.freqmin + self.freqmin * 2.0) / 2.0 * (2 ** j)
octave_low = octave_high / 2.0
BF[j] = bandpass(self.tr.data, octave_low, octave_high, df, corners=self.cnr, zerophase=False)
BF[j] = cosine_taper(LEN, self.perc_taper) * BF[j]
return BF
def bp(ar, header, freqmin, freqmax, zerophase=True, verbose=False):
"""
Vertical butterworth bandpass. This filter is not as effective as :py:func:`triangular` and thus is not available through the command line interface or through :py:func:`readgssi.readgssi.readgssi`.
Filter design and implementation are dictated by :py:func:`obspy.signal.filter.bandpass`.
:param np.ndarray ar: The radar array
:param dict header: The file header dictionary
:param int freqmin: The lower corner of the bandpass
:param int freqmax: The upper corner of the bandpass
:param bool zerophase: Whether to run the filter forwards and backwards in order to counteract the phase shift
:param bool verbose: Verbose, defaults to False
:rtype: :py:class:`numpy.ndarray`
"""
from obspy.signal.filter import bandpass
if verbose:
fx.printmsg('vertical butterworth bandpass filter')
fx.printmsg(
'NOTE: better results are achieved with readgssi.filtering.triangular()'
)
#samp_freq = 1 / ((header['rhf_depth'] * 2) / header['cr'] / header['rh_nsamp'])
samp_freq = header['samp_freq']
freqmin = freqmin * 10**6
freqmax = freqmax * 10**6
corners = 1
if verbose:
fx.printmsg('sampling frequency: %.2E Hz' % samp_freq)
fx.printmsg('minimum filter frequency: %.2E Hz' % freqmin)
fx.printmsg('maximum filter frequency: %.2E Hz' % freqmax)
fx.printmsg('corners: %s, zerophase: %s' % (corners, zerophase))
i = 0
for t in ar.T:
f = bandpass(data=t,
freqmin=freqmin,
freqmax=freqmax,
df=samp_freq,
corners=corners,
zerophase=zerophase)
ar[:, i] = f
i += 1
return ar
def band_pass(sig, min_freq, max_freq):
"""Plain bandpass
Parameters
----------
sig: np.array
whole EEG signal (single channel)
min_freq, max_freq: (int, int)
lower and upper bound of bandpass filter.
Returns
-------
filtered: nparray
filtered signal between min_freq, max_freq, default sampling rate is 500 hz
"""
filtered = ob.bandpass(sig, min_freq, max_freq, df=500)
return filtered
def band_pass(sig, min_freq, max_freq):
"""Plain bandpass
Parameters
----------
sig: np.array
whole EEG signal (single channel)
min_freq, max_freq: (int, int)
lower and upper bound of bandpass filter.
Returns
-------
filtered: nparray
filtered signal between min_freq, max_freq, default sampling rate is 500 hz
"""
filtered = ob.bandpass(sig,min_freq, max_freq, df = 500)
return filtered
def bandpass(self, freqmin, freqmax, corners=4, zerophase=False, traces=None):
"""
Butterworth-Bandpass Filter of the data.
Filter data from ``freqmin`` to ``freqmax`` using ``corners`` corners.
:param freqmin: Pass band low corner frequency in Hz.
:param freqmax: Pass band high corner frequency in Hz.
:param corners: Filter corners.
:param zerophase: If True, apply filter once forwards and once
backwards. This results in twice the number of corners but
zero phase shift in the resulting filtered trace.
:param traces: List of ``SEGYTrace`` objects with data to operate on.
Default is to operate on all traces.
"""
if not traces:
traces = self.traces
for i, tr in enumerate(traces):
df = 1.0 / (tr.header.sample_interval_in_ms_for_this_trace / 1.0e6)
tr.data = filter.bandpass(tr.data, freqmin, freqmax, df, corners=corners, zerophase=zerophase)
def NormalizeFilter(array2d, hist_bin, delta_sec, pmin=15., pmax=50.):
'''
Normalize correlogram and bandpass filter.
:param array2d: 2D correlogram to be normalized.
:param hist_bin: number of correlation in each inter-receiver distance bin
:param delta_sec: time step in seconds
:param pmin: lower bandpass period (in seconds)
:param pmmax: upper bandpass period (in seconds)
'''
## Getting the dimensions of the correlogram
nbins, npts = array2d.shape
## For each correlogram bin
for nb in range(nbins):
if hist_bin[nb] > 0:
## Normalize the correlation stack by number correlation pair
data = array2d[nb, :] / hist_bin[nb]
## Bandpass filtering
array2d[nb, :] = bandpass(data, df=1./delta_sec, freqmin=1./pmax,
freqmax=1./pmin, corners=4, zerophase=True)
return array2d
def bp(ar, header, freqmin, freqmax, verbose=False):
"""
Vertical frequency domain bandpass
"""
if verbose:
fx.printmsg('vertical frequency filtering...')
samp_freq = 1 / (header['rhf_depth'] / header['cr'] / header['rh_nsamp'])
freqmin = freqmin * 10 ** 6
freqmax = freqmax * 10 ** 6
if verbose:
fx.printmsg('Sampling frequency: %.2E Hz' % samp_freq)
fx.printmsg('Minimum filter frequency: %.2E Hz' % freqmin)
fx.printmsg('Maximum filter frequency: %.2E Hz' % freqmax)
i = 0
for t in ar.T:
f = bandpass(data=t, freqmin=freqmin, freqmax=freqmax, df=samp_freq, corners=2, zerophase=False)
ar[:,i] = f
i += 1
return ar
def test_cwt_class(self):
# file_path = os.path.join(ROOT_DIR, "260", "RAW", "WM.OBS01..SHZ.D.2015.260")
file_path = os.path.join("/media/junqueira/DATA/Eva_geysir",
"VI.G1..HHZ.2017.349.mseed")
st = read(file_path)
tr = st[0]
tr.taper(max_percentage=0.01)
wf = tr.data
wf = bandpass(wf,
freqmin=4,
freqmax=12,
df=200,
corners=3,
zerophase=True)
wf /= np.amax(wf)
cw = ConvolveWavelet(file_path, fmin=4., fmax=40., nf=50)
cw.setup_wavelet()
print(cw)
t0 = time.time()
data = cw.cf_lowpass()
print(time.time() - t0)
data /= np.amax(data)
n_0 = (15 * 3600 + 10 * 60) * 50
n_f = (15 * 3600 + 10 * 60 + 240) * 50
print("Max value at index: ", np.argmax(data[n_0:n_f]))
# print("Max value at time: ", get_pick_time(data, 50., "2015-09-17T00:00:27.840000Z"))
print("Max value at time: ", cw.detect_max_pick_in_time(data))
print("Max values at time: ", cw.detect_picks_in_time(data, sigmas=5))
sigma = np.sqrt(np.var(data))
s_p = np.zeros(shape=len(data))
s_p[:] = 5 * sigma
plt.plot(data)
plt.plot(wf)
plt.plot(s_p)
plt.plot(-1 * s_p)
for indx in cw.detect_picks(data, 5):
plt.axvline(indx, color="green")
plt.show()
def moveout_check(template, nodes, lags, threshold, thresh_type, lowcut, highcut, filt_order):
"""
Function to check different moveouts for detectability
"""
# Generate random noise and seed with the template
from copy import deepcopy
from obspy.signal.filter import bandpass
from joblib import Parallel, delayed
parallel = True
image = deepcopy(template)
for i in xrange(len(image)):
image[i].data = np.random.randn(
len(image[i].data) + (86400 * image[i].stats.sampling_rate) - len(image[i].data)
)
image[i].data[
(len(image[i].data) - len(template[i].data)) / 2 : (len(image[i].data) - len(template[i].data)) / 2
+ len(template[i].data)
] = (
image[i].data[
(len(image[i].data) - len(template[i].data)) / 2 : (len(image[i].data) - len(template[i].data)) / 2
+ len(template[i].data)
]
+ template[i].data / np.mean(template[i].data ** 2) ** 0.5
)
image[i].data = bandpass(image[i].data, lowcut, highcut, image[i].stats.sampling_rate, filt_order)
ccc = ccc_gen(image, template)
# Lags
possible_locations = []
freq = image[0].stats.sampling_rate
if not parallel:
for i in xrange(len(nodes)):
possible_locations += _node_loop(freq, nodes[i], ccc, lags[:, [i]], threshold, thresh_type)
else:
possible_locations = Parallel(n_jobs=10, verbose=5)(
delayed(_node_loop)(freq, nodes[i], ccc, lags[:, [i]], threshold, thresh_type) for i in xrange(len(nodes))
)
return possible_locations
def moveout_check(template, nodes, lags, threshold, thresh_type, lowcut,\
highcut, filt_order):
"""
Function to check different moveouts for detectability
"""
# Generate random noise and seed with the template
from copy import deepcopy
from obspy.signal.filter import bandpass
from joblib import Parallel, delayed
parallel = True
image = deepcopy(template)
for i in xrange(len(image)):
image[i].data=np.random.randn(len(image[i].data)+\
(86400*image[i].stats.sampling_rate)\
-len(image[i].data))
image[i].data[(len(image[i].data)-len(template[i].data))/2:\
(len(image[i].data)-len(template[i].data))/2+len(template[i].data)]=\
image[i].data[(len(image[i].data)-len(template[i].data))/2:\
(len(image[i].data)-len(template[i].data))/2+len(template[i].data)]+\
template[i].data/np.mean(template[i].data**2)**0.5
image[i].data=bandpass(image[i].data, lowcut, highcut,\
image[i].stats.sampling_rate, filt_order)
ccc = ccc_gen(image, template)
# Lags
possible_locations = []
freq = image[0].stats.sampling_rate
if not parallel:
for i in xrange(len(nodes)):
possible_locations+=_node_loop(freq, nodes[i], ccc, lags[:,[i]],\
threshold, thresh_type)
else:
possible_locations = Parallel(n_jobs=10, verbose=5)(delayed(_node_loop)\
(freq, nodes[i], ccc,\
lags[:,[i]], threshold,\
thresh_type)\
for i in xrange(len(nodes)))
return possible_locations
def saxstr(
self, start, end, station, network, channel,
interval=50, alphabet="abcdefghi", lp=1, hp=20):
interval = int(interval)
cp.response.headers['Content-Type'] = "text/plain"
ts = self._get(
metric='Z',
start=start,
end=end,
station=station,
network=network
)
t = np.array(ts['t'])
v = np.array(ts['v'])
start_dt = dt.datetime.fromtimestamp(float(start))
# TODO - move this! - maybe
v = bandpass(data=v, freqmin=lp, freqmax=hp, df=50)
p = Paa(t, v, interval)
s = Sax(p, alphabet)
return s.string
def main(sta1, sta2, filterid, components, mov_stack=1, ampli=5, seismic=False,
show=False, outfile=None, envelope=False, refilter=None):
db = connect()
maxlag = float(get_config(db, 'maxlag'))
samples = get_maxlag_samples(db)
cc_sampling_rate = float(get_config(db, 'cc_sampling_rate'))
start, end, datelist = build_movstack_datelist(db)
base = mdates.date2num(start)
plt.figure(figsize=(12, 9))
sta1 = sta1.replace('.', '_')
sta2 = sta2.replace('.', '_')
t = np.arange(samples)/cc_sampling_rate - maxlag
if refilter:
freqmin, freqmax = refilter.split(':')
freqmin = float(freqmin)
freqmax = float(freqmax)
if sta2 >= sta1:
pair = "%s:%s" % (sta1, sta2)
print("New Data for %s-%s-%i-%i" % (pair, components, filterid,
mov_stack))
nstack, stack_total = get_results(db, sta1, sta2, filterid, components,
datelist, mov_stack, format="matrix")
ax = plt.subplot(111)
for i, line in enumerate(stack_total):
if np.all(np.isnan(line)):
continue
if refilter:
line = bandpass(line, freqmin, freqmax, cc_sampling_rate,
zerophase=True)
if envelope:
line = obspy_envelope(line)
line /= line.max()
plt.plot(t, line * ampli + i + base, c='k')
if seismic:
y1 = np.ones(len(line)) * i
y2 = line*ampli + i + base
plt.fill_between(t, y1, y2, where=y2 >= y1, facecolor='k',
interpolate=True)
for filterdb in get_filters(db, all=True):
if filterid == filterdb.ref:
low = float(filterdb.low)
high = float(filterdb.high)
break
plt.xlabel("Lag Time (s)")
plt.axhline(0, lw=0.5, c='k')
plt.grid()
title = '%s : %s, %s, Filter %d (%.2f - %.2f Hz), Stack %d' %\
(sta1.replace('_', '.'), sta2.replace('_', '.'), components,
filterid, low, high, mov_stack)
if refilter:
title += ", Re-filtered (%.2f - %.2f Hz)" % (freqmin, freqmax)
plt.title(title)
plt.scatter(0, [start, ], alpha=0)
plt.ylim(start-datetime.timedelta(days=ampli),
end+datetime.timedelta(days=ampli))
plt.xlim(-maxlag, maxlag)
ax.fmt_ydata = mdates.DateFormatter('%Y-%m-%d')
cursor = Cursor(ax, useblit=True, color='red', linewidth=1.2)
if outfile:
if outfile.startswith("?"):
pair = pair.replace(':', '-')
outfile = outfile.replace('?', '%s-%s-f%i-m%i' % (pair,
components,
filterid,
mov_stack))
outfile = "ccftime " + outfile
print("output to:", outfile)
plt.savefig(outfile)
if show:
plt.show()
def FilterData(channel):
b_pass = filters.bandpass(channel, freqmin = 2, freqmax = 30, df = 500)
# b_stop =filters.bandstop(b_pass, freqmin = 49 ,freqmax = 51, df = 500)
return b_pass
def main(sta1, sta2, filterid, components, mov_stack=1, ampli=5, show=False,
outfile=False, refilter=None, startdate=None, enddate=None, **kwargs):
db = connect()
cc_sampling_rate = float(get_config(db, 'cc_sampling_rate'))
start, end, datelist = build_movstack_datelist(db)
base = mdates.date2num(start)
sta1 = sta1.replace('.', '_')
sta2 = sta2.replace('.', '_')
# TODO: Height adjustment of the plot for large number of stacks.
# Preferably interactive
fig = plt.figure(figsize=(12, 9))
if refilter:
freqmin, freqmax = refilter.split(':')
freqmin = float(freqmin)
freqmax = float(freqmax)
if sta2 >= sta1:
pair = "%s:%s" % (sta1, sta2)
print("New Data for %s-%s-%i-%i" % (pair, components, filterid,
mov_stack))
nstack, stack_total = get_results(db, sta1, sta2, filterid, components,
datelist, mov_stack, format="matrix")
ax = fig.add_subplot(111)
for i, line in enumerate(stack_total):
if np.all(np.isnan(line)):
continue
if refilter:
line = bandpass(line, freqmin, freqmax, cc_sampling_rate,
zerophase=True)
freq, line = prepare_abs_postitive_fft(line, cc_sampling_rate)
line /= line.max()
ax.plot(freq, line * ampli + i + base, c='k')
for filterdb in get_filters(db, all=True):
if filterid == filterdb.ref:
low = float(filterdb.low)
high = float(filterdb.high)
break
ax.set_ylim(start-datetime.timedelta(days=ampli),
end+datetime.timedelta(days=ampli))
ax.yaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
if "xlim" in kwargs:
plt.xlim(kwargs["xlim"][0],kwargs["xlim"][1])
ax.set_xlabel("Frequency [Hz]")
ax.set_xscale('log')
ax.grid()
title = '%s : %s, %s, Filter %d (%.2f - %.2f Hz), Stack %d' %\
(sta1.replace('_', '.'), sta2.replace('_', '.'), components,
filterid, low, high, mov_stack)
if refilter:
title += ", Re-filtered (%.2f - %.2f Hz)" % (freqmin, freqmax)
ax.set_title(title)
cursor = Cursor(ax, useblit=True, color='red', linewidth=1.2)
print(outfile)
if outfile:
if outfile.startswith("?"):
pair = pair.replace(':', '-')
outfile = outfile.replace('?', '%s-%s-f%i-m%i' % (pair,
components,
filterid,
mov_stack))
outfile = "spectime" + outfile
print("output to:", outfile)
plt.savefig(outfile)
if show:
plt.show()
else:
plt.close(fig)
def main(sta1, sta2, filterid, components, mov_stack=1, show=True,
outfile=None, refilter=None):
db = connect()
maxlag = float(get_config(db, 'maxlag'))
cc_sampling_rate = float(get_config(db, 'cc_sampling_rate'))
start, end, datelist = build_movstack_datelist(db)
if refilter:
freqmin, freqmax = refilter.split(':')
freqmin = float(freqmin)
freqmax = float(freqmax)
plt.figure(figsize=(12, 9))
sta1 = sta1.replace('.', '_')
sta2 = sta2.replace('.', '_')
if sta2 >= sta1:
pair = "%s:%s" % (sta1, sta2)
print("New Data for %s-%s-%i-%i" % (pair, components, filterid,
mov_stack))
nstack, stack_total = get_results(db, sta1, sta2, filterid, components,
datelist, mov_stack, format="matrix")
xextent = (date2num(start), date2num(end), -maxlag, maxlag)
ax = plt.subplot(111)
data = stack_total
if refilter:
for i, d in enumerate(data):
data[i] = bandpass(data[i], freqmin, freqmax, cc_sampling_rate,
zerophase=True)
plt.imshow(data.T, extent=xextent, aspect="auto",
interpolation='none', origin='lower', cmap='seismic',
vmin=-1e-2, vmax=1e-2)
plt.ylabel("Lag Time (s)")
plt.axhline(0, lw=0.5, c='k')
plt.grid()
ax.xaxis.set_major_locator(YearLocator())
ax.xaxis.set_major_formatter(DateFormatter('%Y-%m'))
for filterdb in get_filters(db, all=True):
if filterid == filterdb.ref:
low = float(filterdb.low)
high = float(filterdb.high)
break
plt.ylim(-maxlag, maxlag)
title = '%s : %s, %s, Filter %d (%.2f - %.2f Hz), Stack %d' % \
(sta1.replace('_', '.'), sta2.replace('_', '.'), components,
filterid, low, high, mov_stack)
if refilter:
title += ", Re-filtered (%.2f - %.2f Hz)" % (freqmin, freqmax)
plt.title(title)
cursor = Cursor(ax, useblit=True, color='black', linewidth=1.2)
if outfile:
if outfile.startswith("?"):
pair = pair.replace(':', '-')
outfile = outfile.replace('?', '%s-%s-f%i-m%i' % (pair,
components,
filterid,
mov_stack))
outfile = "interferogram " + outfile
print("output to:", outfile)
plt.savefig(outfile)
if show:
plt.show()
def FilterData(channel, _freqmin, _freqmax):
b_pass = filters.bandpass(channel, freqmin = _freqmin, freqmax = _freqmax, df = 250)
# b_stop =filters.bandstop(b_pass, freqmin = 49 ,freqmax = 51, df = 500)
return b_pass
this_seis[2].data = detrend(this_seis[2].data, type='constant')
# ------- Rotate ENZ to RTZ ---------#
if comp == 1 or comp == 2 or comp == 3:
E = this_seis[2].data
N = this_seis[1].data
Z = this_seis[0].data
else:
E = this_seis[0].data
N = this_seis[1].data
Z = this_seis[2].data
(this_seis[0].data, this_seis[1].data, this_seis[2].data) = seispy.geo.rotateSeisENZtoTRZ(E, N, Z, bazi)
this_seis[0].stats.channel = "R"
this_seis[1].stats.channel = "T"
this_seis[2].stats.channel = "Z"
# ------- Bandpass Filter ------------#
R_filter = bandpass(this_seis[0].data, freqmin, freqmax, 1/dt, corners=3, zerophase=True)
T_filter = bandpass(this_seis[1].data, freqmin, freqmax, 1/dt, corners=3, zerophase=True)
Z_filter = bandpass(this_seis[2].data, freqmin, freqmax, 1/dt, corners=3, zerophase=True)
# ------- Calculate P arrival time ---#
arrivals = model.get_travel_times(source_depth_in_km=dep, distance_in_degree=dis, phase_list=["P"])
ttime_P = arrivals[0].time - O
rayp = seispy.geo.srad2skm(arrivals[0].ray_param)
# this_seis[0].stats.sac.t1 = ttime_P
# this_seis[1].stats.sac.t1 = ttime_P
# this_seis[2].stats.sac.t1 = ttime_P
# ----------- filter by snr ----------#
snrbegin = np.floor((ttime_P-50)/dt)
snrend = np.floor((ttime_P+50)/dt)
snro = np.floor(ttime_P/dt)
snr_R = seispy.geo.snr(R_filter[snro:snrend], R_filter[snrbegin:snro])
snr_T = seispy.geo.snr(T_filter[snro:snrend], T_filter[snrbegin:snro])
def bandpass_filter(self, freqmin, freqmax, *args, **kwargs):
self.filtered_signal = filter.bandpass(self.data, freqmin, freqmax, self.fs, *args, **kwargs)
return self.filtered_signal
def resampleFilterAndCutTraces(stream, resampling_rate, lowpass_value,
highpass_value, zerophase, corners, starttime,
endtime, message_function=None):
"""
Resamples, filters and cuts all Traces in a Stream object.
It will always apply each operation to every trace in the order described
above.
:param stream: obspy.core.stream object
Will be altered and has to contain at least one Trace.
:param resampling_rate: float
Desired new sample rate.
:param lowpass_value: float
High filter frequency.
:param highpass_value: float
Low filter frequency.
:param zerophase: bool
Whether or not to use a zerophase filter.
:param corners: int
Number of corners for the used Butterworth-Filter.
:param starttime: obspy.core.UTCDateTime
New starttime of each Trace.
:param endtime: obspy.core.UTCDateTime
New endtime of each Trace.
:param message_function: Python function
If given, a string will be passed to this function to document the
current progress.
"""
# Convert to floats for more exact handling. Also level the data.
for trace in stream:
trace.data = np.require(trace.data, 'float32')
trace.data -= np.linspace(trace.data[0], trace.data[-1], len(trace.data))
# The first step is to resample the data. This is done before trimming
# so that any boundary effects that might occur can be cut away later
# on.
if resampling_rate != stream[0].stats.sampling_rate:
time_range = stream[0].stats.endtime - \
stream[0].stats.starttime
new_npts = time_range / \
(1 / resampling_rate) + 1
new_freq = 1.0 / (time_range / float(new_npts - 1))
for _i, trace in enumerate(stream):
if message_function:
msg = 'Resampling traces to %.2f Hz [%i/%i]...' % \
(resampling_rate, _i + 1, len(stream))
message_function(msg)
# Use scipy to resample the traces.
trace.data = resample(trace.data, new_npts, window='hamming')
trace.stats.sampling_rate = new_freq
# Filter the trace. Differentiate between low-, high-, and bandpass
if lowpass_value and highpass_value:
if message_function:
msg = 'Bandpass filtering traces from %.2f Hz to %.2f Hz...' % \
(highpass_value, highpass_value)
message_function(msg)
for trace in stream:
trace.data = bandpass(trace.data, highpass_value,
lowpass_value, trace.stats.sampling_rate,
corners=corners, zerophase=zerophase)
elif lowpass_value:
if message_function:
msg = 'Lowpass filtering traces with %.2f Hz...' % lowpass_value
message_function(msg)
for trace in stream:
trace.data = lowpass(trace.data, lowpass_value,
trace.stats.sampling_rate,
corners=corners, zerophase=zerophase)
elif highpass_value:
if message_function:
msg = 'Highpass filtering traces with %.2f Hz...' % highpass_value
message_function(msg)
for trace in stream:
trace.data = highpass(trace.data, highpass_value,
trace.stats.sampling_rate,
corners=corners, zerophase=zerophase)
# Trim the trace if it is necessary.
if message_function:
message_function('Trimming traces...')
stream.trim(starttime, endtime)
def process(tr, lowcut, highcut, filt_order, samp_rate, debug,
starttime=False, full_day=False):
r"""Basic function to bandpass, downsample and check headers and length \
of trace to ensure files start at the start of a day and are daylong.
Works in place on data. This is employed to ensure all parts of the data \
are processed in the same way.
.. note:: Usually this function is called via dayproc or shortproc.
:type tr: obspy.Trace
:param tr: Trace to process
:type highcut: float
:param highcut: High cut in Hz, if set to None and lowcut is set, will \
use a highpass filter.
:type lowcut: float
:type lowcut: Low cut in Hz, if set to None and highcut is set, will use \
a lowpass filter.
:type filt_order: int
:param filt_order: Number of corners for filter.
:type samp_rate: float
:param samp_rate: Desired sampling rate in Hz
:type debug: int
:param debug: Debug output level from 0-5, higher numbers = more output
:type starttime: obspy.UTCDateTime
:param starttime: Desired start of trace
:type full_day: bool
:param full_day: Whether to expect, and enforce a full day of data or not.
:return: obspy.Stream
.. note:: Will convert channel names to two charecters long.
"""
import warnings
from obspy.signal.filter import bandpass, lowpass, highpass
# Add sanity check
if highcut and highcut >= 0.5*samp_rate:
raise IOError('Highcut must be lower than the nyquist')
# Define the start-time
if starttime:
day = starttime.date
else:
day = tr.stats.starttime.date
if debug >= 2:
print('Working on: '+tr.stats.station+'.'+tr.stats.channel)
if debug >= 5:
tr.plot()
# Do a brute force quality check
qual = _check_daylong(tr)
if not qual:
msg = ("Data have more zeros than actual data, please check the raw",
" data set-up and manually sort it")
raise ValueError(msg)
tr = tr.detrend('simple') # Detrend data before filtering
# If there is one sample too many remove the first sample - this occurs
# at station FOZ where the first sample is zero when it shouldn't be,
# Not real sample: generated during data download
# if full_day:
# if len(tr.data) == (86400 * tr.stats.sampling_rate) + 1:
# tr.data = tr.data[1:len(tr.data)]
if debug > 0:
print('I have '+str(len(tr.data))+' data points for ' +
tr.stats.station+'.'+tr.stats.channel+' before processing')
# Sanity check to ensure files are daylong
if float(tr.stats.npts / tr.stats.sampling_rate) != 86400.0\
and full_day:
if debug >= 2:
print('Data for '+tr.stats.station+'.'+tr.stats.channel +
' is not of daylong length, will zero pad')
# Work out when the trace thinks it is starting
# traceday = UTCDateTime(str(tr.stats.starttime.year)+'-' +
# str(tr.stats.starttime.month)+'-' +
# str(tr.stats.starttime.day))
# Use obspy's trim function with zero padding
tr = tr.trim(starttime, starttime+86400, pad=True, fill_value=0,
nearest_sample=True)
# If there is one sample too many after this remove the last one
# by convention
if len(tr.data) == (86400 * tr.stats.sampling_rate) + 1:
tr.data = tr.data[1:len(tr.data)]
if not tr.stats.sampling_rate * 86400 == tr.stats.npts:
raise ValueError('Data are not daylong for '+tr.stats.station +
'.'+tr.stats.channel)
print('I now have '+str(len(tr.data)) +
' data points after enforcing day length')
# Check sampling rate and resample
if tr.stats.sampling_rate != samp_rate:
if debug >= 2:
print('Resampling')
tr.resample(samp_rate)
# Filtering section
tr = tr.detrend('simple') # Detrend data again before filtering
if highcut and lowcut:
if debug >= 2:
print('Bandpassing')
tr.data = bandpass(tr.data, lowcut, highcut,
tr.stats.sampling_rate, filt_order, True)
elif highcut:
if debug >= 2:
print('Lowpassing')
tr.data = lowpass(tr.data, highcut, tr.stats.sampling_rate,
filt_order, True)
elif lowcut:
if debug >= 2:
print('Highpassing')
tr.data = highpass(tr.data, lowcut, tr.stats.sampling_rate,
filt_order, True)
else:
warnings.warn('No filters applied')
# Account for two letter channel names in s-files and therefore templates
tr.stats.channel = tr.stats.channel[0]+tr.stats.channel[-1]
# Sanity check the time header
if tr.stats.starttime.day != day != day and full_day:
warnings.warn("Time headers do not match expected date: " +
str(tr.stats.starttime))
# Sanity check to ensure files are daylong
if float(tr.stats.npts / tr.stats.sampling_rate) != 86400.0 and full_day:
if debug >= 2:
print('Data for '+tr.stats.station+'.'+tr.stats.channel +
' is not of daylong length, will zero pad')
# Use obspy's trim function with zero padding
tr = tr.trim(starttime, starttime+86400, pad=True, fill_value=0,
nearest_sample=True)
# If there is one sample too many after this remove the last one
# by convention
if len(tr.data) == (86400 * tr.stats.sampling_rate) + 1:
tr.data = tr.data[1:len(tr.data)]
if not tr.stats.sampling_rate*86400 == tr.stats.npts:
raise ValueError('Data are not daylong for '+tr.stats.station +
'.'+tr.stats.channel)
# Final visual check for debug
if debug >= 4:
tr.plot()
return tr
def sax(
self, start, end, station, network, channel,
interval=50, alphabet="abcdefghi", lp=1, hp=20):
interval = int(interval)
cp.response.headers['Content-Type'] = "image/png"
ts = self._get(
metric='Z',
start=start,
end=end,
station=station,
network=network
)
t = np.array(ts['t'])
v = np.array(ts['v'])
start_dt = dt.datetime.fromtimestamp(float(start))
# TODO - move this! - maybe
v = bandpass(data=v, freqmin=lp, freqmax=hp, df=50)
p = Paa(t, v, interval)
s = Sax(p, alphabet)
ram = BytesIO()
fig = plt.figure(figsize=(14, 8), dpi=100, facecolor='black')
fig.suptitle("{}.{} {}".format(
network, station, start_dt.strftime("%Y-%m-%d")), color='white')
title1 = "SAX (PAA Interval: {}ms, alphabet: {})".format(
interval, len(alphabet))
title2 = "Raw (Bandpass {}/{}Hz)".format(lp, hp)
ax1 = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
ax1.set_title(label=title1, color='white')
ax2.set_title(label=title2, color='white')
ax1.step(p.as_time(p.paa_time), p.paa_value, color='lime', lw=1)
ax2.plot(p.as_time(p.in_time), p.in_value, color='c', lw=1)
for ax in (ax1, ax2):
plt.setp(
ax.get_xticklabels(), rotation=10, fontsize=8, color='white')
plt.setp(
ax.get_yticklabels(), fontsize=8, color='white')
ax.minorticks_on()
ax.grid(b=True, which='major', color='grey', ls='-', lw=1)
ax.grid(b=True, which='minor', color='grey', ls=':', lw=.5)
ax.set_axis_bgcolor('black')
ax.tick_params(colors='white')
ax.spines['bottom'].set_color('white')
ax.spines['top'].set_color('white')
ax.spines['left'].set_color('white')
ax.spines['right'].set_color('white')
ax1.set_ylim(
s.breakpoints[0] - (s.breakpoints[1] - s.breakpoints[0]) / 2,
s.breakpoints[-1] + (s.breakpoints[-1] - s.breakpoints[-2]) / 2,
)
a = 0
for y in s.breakpoints:
ax1.axhline(y, color='darkred', lw=.75)
ax1.text(
s.paa.times[0], y, va='center', color='darkred',
s=" {}\n {}".format(alphabet[a + 1], alphabet[a]))
a += 1
# for i in range(1, len(s.sax_str)):
# plt.text(
# s.paa.times[i] - dt.timedelta(milliseconds=(interval/2)), 0,
# s.sax_str[i], va='center', ha='center',
# family='monospace', weight='bold', size=8)
plt.savefig(ram, format='png', facecolor='black')
plt.close()
ram.seek(0)
return ram.read()
def crossc(dstart,dend,ch1,ch2,day):
# here you load all the functions you need to use
from obspy.seg2.seg2 import readSEG2
from obspy.core import Stream
import numpy as np
from obspy.signal.cross_correlation import xcorr
from numpy import sign
from obspy.signal.filter import lowpass
from obspy.signal.filter import highpass
from obspy.signal.filter import bandstop
from obspy.signal.filter import bandpass
dataDir = "/import/three-data/hadzii/STEINACH/STEINACH_longtime/"
outdir = "/home/jsalvermoser/Desktop/Processing/bands_SNR/" + "CH" + str(ch1) + "_CH" + str(ch2) + "/" + "JAN" + str(day) + "/"
# loading the info for outfile-name
stream_start = readSEG2(dataDir + str(dstart) + ".dat")
t_start = stream_start[ch1].stats.seg2.ACQUISITION_TIME
stream_end = readSEG2(dataDir + str(dend) + ".dat")
t_end = stream_end[ch1].stats.seg2.ACQUISITION_TIME
# initialization of the arrays and variables
TR = []
rms = []
sq = []
ncalm = 1
nbeat = 1
corr128_calm = 0
corr128_beat = 0
nerror = 0
mu1c=0
mu2c=0
mu3c=0
mu1b=0
mu2b=0
mu3b=0
var1c=0
var2c=0
var3c=0
var1b=0
var2b=0
var3b=0
SNR_calm_b1=[]
SNR_calm_b2=[]
SNR_calm_b3=[]
SNR_beat_b1=[]
SNR_beat_b2=[]
SNR_beat_b3=[]
#TAPER
taper_percentage=0.05
taper= np.blackman(int(len(time_vector) * taper_percentage))
taper_left, taper_right = np.array_split(taper,2)
taper = np.concatenate([taper_left,np.ones(len(time_vector)-len(taper)),taper_right])
for j in range(0, dend-dstart):
sq.append([])
for k in range(dstart, dend, 4):
start = k
end = k + 5 # only used to merge 5-1 = 4 files to one stream
try:
st1 = merge_single(ch1,start,end)
st2 = merge_single(ch2,start,end)
st1.detrend('linear')
st2.detrend('linear')
# calculate squares for rms
r = k-dstart
sq[r] = 0
for h in range(0,64000):
sq[r] += (st1[0].data[h])**2
# lowpass-filter the crossc_beat correlation function
st1.filter('lowpass',freq = 24, zerophase=True, corners=8)
st1.filter('highpass', freq= 0.05, zerophase=True, corners=2) #had to be reduced from 0.1Hz
st1.filter('bandstop', freqmin=8, freqmax=14, corners=4, zerophase=True)
st2.filter('lowpass',freq = 24, zerophase=True, corners=8)
st2.filter('highpass', freq= 0.05, zerophase=True, corners=2) #had to be reduced from 0.1Hz
st2.filter('bandstop', freqmin=8, freqmax=14, corners=4, zerophase=True)
# sometimes channels seem to fail, so I put this to prevent crashing of the program
# 1-bit normalization
tr1 = sign(st1[0].data)
tr2 = sign(st2[0].data)
# cross-correlation
index, value, acorr = xcorr(tr1, tr2, 25000, full_xcorr=True)
print sq[r]
# check sanity
if np.max(acorr)>1:
acorr = zeros(50001)
# sort the 128sec files into calm and beat:
# the value was chosen after observing calm files
if sq[r] < 1000000000000:
corr128_calm += acorr
ncalm += 1.
else:
corr128_beat += acorr
nbeat += 1.
print ncalm, nbeat # just to check if calm or noisy
except:
nerror += 1
print "%d : ERROR" %(r)
if ncalm<8:
corr128_calm = np.zeros(50001)
# normalization
else:
corr128_calm = (corr128_calm/ncalm) * taper
corr128_beat = (corr128_beat/nbeat) * taper
# filter again and divide into 3 bands which can be investigated separately
corr128_calm_band1 = highpass(corr128_calm, freq=0.1, corners=4, zerophase=True, df=500.)
corr128_calm_band1 = lowpass(corr128_calm_band1, freq=2, corners=4, zerophase=True, df=500.)
corr128_calm_band2 = bandpass(corr128_calm, freqmin=2, freqmax=8, df=500., corners=4, zerophase=True)
corr128_calm_band3 = bandpass(corr128_calm, freqmin=8, freqmax=24, df=500., corners=4, zerophase=True)
corr128_beat_band1 = highpass(corr128_beat, freq=0.1, df=500., corners=4, zerophase=True)
corr128_beat_band1 = lowpass(corr128_beat_band1, freq=2, corners=4, zerophase=True, df=500.)
corr128_beat_band2 = bandpass(corr128_beat, freqmin=2, freqmax=8, df=500., corners=4, zerophase=True)
corr128_beat_band3 = bandpass(corr128_beat, freqmin=8, freqmax=24, df=500., corners=4, zerophase=True)
# SNR (Signal-to-Noise Ratio):print 222222
# for the signal-to-noise ratio one divides the maximum of the signal by the
# variance of a late window (noise). As we don't know which window has the
# lowest signal fraction, we loop over some windows. We need windows of
# different lengths for the different bands as different frequencies are
# contained. For every band the minimum-frequency fmin is chosen (e.g. 4Hz), then
# the time for one cyle is 1/fc (e.g. 0.25s) and as we take windows of 3-4
# cycles we choose a window length of 4*0.25s = 1s
## CALM + BEAT
for isnrb1 in range(45000,50000,2500): # steps of half a windowlength
endwb1=isnrb1 + 2500 # 5s window
SNR_calm_b1.append(np.max(np.abs(corr128_calm_band1))/np.std(corr128_calm_band1[isnrb1:endwb1]))
SNR_beat_b1.append(np.max(np.abs(corr128_beat_band1))/np.std(corr128_beat_band1[isnrb1:endwb1]))
SNR_calm_b1 = max(SNR_calm_b1)
SNR_beat_b1 = max(SNR_beat_b1)
for isnrb2 in range(45000,49001,500): # steps of half a windowlength
endwb2=isnrb2 + 1000 # 2s windows
SNR_calm_b2.append(np.max(np.abs(corr128_calm_band2))/np.std(corr128_calm_band2[isnrb2:endwb2]))
SNR_beat_b2.append(np.max(np.abs(corr128_beat_band2))/np.std(corr128_beat_band2[isnrb2:endwb2]))
SNR_beat_b2 = max(SNR_beat_b2)
SNR_calm_b2 = max(SNR_calm_b2)
for isnrb3 in range(45000,49751,125): # steps of half a windowlength
endwb3=isnrb3 + 250 # 0.5s windows
SNR_calm_b3.append(np.max(np.abs(corr128_calm_band3))/np.std(corr128_calm_band3[isnrb3:endwb3]))
SNR_beat_b3.append(np.max(np.abs(corr128_beat_band3))/np.std(corr128_beat_band3[isnrb3:endwb3]))
SNR_beat_b3 = max(SNR_beat_b3)
SNR_calm_b3 = max(SNR_calm_b3)
if ncalm<8:
SNR_calm_b1 = 0
SNR_calm_b2 = 0
SNR_calm_b3 = 0
print SNR_calm_b1, SNR_calm_b2, SNR_calm_b3
print SNR_beat_b1, SNR_beat_b2, SNR_beat_b3
# RMS for histogram and sifting:
#for s in range(0,dend-dstart):
# rms.append((sq[s]/16000)**(0.5))
# save into files:
np.save(outdir + t_start + "-" + t_end + "CH" + str(ch1) + "_" +"xcorr128s_beat_0-2Hz" + "_" + "CH" + str(ch2), corr128_beat_band1)
np.save(outdir + t_start + "-" + t_end + "CH" + str(ch1) + "_" +"xcorr128s_beat_2-8Hz" + "_" + "CH" + str(ch2), corr128_beat_band2)
np.save(outdir + t_start + "-" + t_end + "CH" + str(ch1) + "_" +"xcorr128s_beat_8-24Hz" + "_" + "CH" + str(ch2), corr128_beat_band3)
np.save(outdir + t_start + "-" + t_end + "CH" + str(ch1) + "_" +"xcorr128s_calm_0-2Hz" + "_" + "CH" + str(ch2), corr128_calm_band1)
np.save(outdir + t_start + "-" + t_end + "CH" + str(ch1) + "_" +"xcorr128s_calm_2-8Hz" + "_" + "CH" + str(ch2), corr128_calm_band2)
np.save(outdir + t_start + "-" + t_end + "CH" + str(ch1) + "_" +"xcorr128s_calm_8-24Hz" + "_" + "CH" + str(ch2), corr128_calm_band3)
# np.save(outdir + "JAN_"+"CH" + str(ch1) + "_" +"RMS" + "_" + "CH" + str(ch2) + str(dstart) + "-" + str(dend), rms)
return corr128_beat_band1,corr128_beat_band2,corr128_beat_band3, corr128_calm_band1,corr128_calm_band2,corr128_calm_band3, ncalm, nbeat, SNR_beat_b1, SNR_beat_b2, SNR_beat_b3, SNR_calm_b1, SNR_calm_b2, SNR_calm_b3
def dayproc(tr, lowcut, highcut, filt_order, samp_rate, debug, starttime):
"""
Basic function to bandpass, downsample and check headers and length of trace
to ensure files start at the start of a day and are daylong. Works in place
on data. This is employed to ensure all parts of the data are processed
in the same way.
:type tr: obspy.Trace
:param tr: Trace to process
:type highcut: float
:param highcut: High cut in Hz for bandpass
:type lowcut: float
:type lowcut: Low cut in Hz for bandpass
:type filt_order: int
:param filt_order: Corners for bandpass
:type samp_rate: float
:param samp_rate: Desired sampling rate in Hz
:type debug: int
:param debug: Debug output level from 0-5, higher numbers = more output
:type starttime: obspy.UTCDateTime
:param starttime: Desired start of trace
:return: obspy.Stream
..rubric:: Note
Will convert channel names to two charectars long
"""
# Add sanity check
if highcut >= 0.5*samp_rate:
raise IOError('Highcut must be lower than the nyquist')
day=str(starttime.year)+str(starttime.month).zfill(2)+\
str(starttime.day).zfill(2)
if debug>=2:
print 'Working on: '+tr.stats.station+'.'+tr.stats.channel
if debug >= 5:
tr.plot()
# Do a brute force quality check
qual=_check_daylong(tr)
if not qual:
msg="Data have more zeros than actual data, please check the raw" +\
"data set-up and manually sort it"
raise ValueError(msg)
tr=tr.detrend('simple') # Detrend data before filtering
# If there is one sample too many remove the first sample - this occurs
# at station FOZ where the first sample is zero when it shouldn't be,
# Not real sample generated during data download
if len(tr.data)==(86400*tr.stats.sampling_rate)+1:
tr.data=tr.data[1:len(tr.data)]
print 'I have '+str(len(tr.data))+' data points for '+tr.stats.station+'.'\
+tr.stats.channel+' before processing'
# Sanity check to ensure files are daylong
if float(tr.stats.npts/tr.stats.sampling_rate) != 86400.0:
if debug >= 2:
print 'Data for '+tr.stats.station+'.'+tr.stats.channel+\
' is not of daylong length, will zero pad'
# Work out when the trace thinks it is starting - Aaron's time headers
# are often wrong
traceday=UTCDateTime(str(tr.stats.starttime.year)+'-'+\
str(tr.stats.starttime.month)+'-'+\
str(tr.stats.starttime.day))
# Use obspy's trim function with zero padding
tr=tr.trim(traceday,traceday+86400,pad=True,fill_value=0,\
nearest_sample=True)
# If there is one sample too many after this remove the last one
# by convention
if len(tr.data)==(86400*tr.stats.sampling_rate)+1:
tr.data=tr.data[1:len(tr.data)]
if not tr.stats.sampling_rate*86400 == tr.stats.npts:
raise ValueError ('Data are not daylong for '+tr.stats.station+\
'.'+tr.stats.channel)
print 'I now have '+str(len(tr.data))+' data points after enforcing day length'
# Check sampling rate and resample
if tr.stats.sampling_rate != samp_rate:
if debug>=2:
print 'Resampling'
tr.resample(samp_rate)
# Filtering section
tr=tr.detrend('simple') # Detrend data before filtering
if debug>=2:
print 'Bandpassing'
tr.data=bandpass(tr.data, lowcut, highcut,
tr.stats.sampling_rate, filt_order, True)
# Account for two letter channel names in s-files and therefore templates
tr.stats.channel=tr.stats.channel[0]+tr.stats.channel[-1]
# Sanity check the time header
if str(tr.stats.starttime.year)+str(tr.stats.starttime.month).zfill(2)+\
str(tr.stats.starttime.day).zfill(2) != day:
if debug >= 2:
print "Time headers are wrong: "+str(tr.stats.starttime)
print "Correcting to: "+str(starttime)
tr.stats.starttime=starttime
# Sanity check to ensure files are daylong
if float(tr.stats.npts/tr.stats.sampling_rate) != 86400.0:
if debug >= 2:
print 'Data for '+tr.stats.station+'.'+tr.stats.channel+\
' is not of daylong length, will zero pad'
# Work out when the trace thinks it is starting - Aaron's time headers
# are often wrong
traceday=UTCDateTime(str(tr.stats.starttime.year)+'-'+\
str(tr.stats.starttime.month)+'-'+\
str(tr.stats.starttime.day))
# Use obspy's trim function with zero padding
tr=tr.trim(traceday,traceday+86400,pad=True,fill_value=0,\
nearest_sample=True)
# If there is one sample too many after this remove the last one
# by convention
if len(tr.data)==(86400*tr.stats.sampling_rate)+1:
tr.data=tr.data[1:len(tr.data)]
if not tr.stats.sampling_rate*86400 == tr.stats.npts:
raise ValueError ('Data are not daylong for '+tr.stats.station+\
'.'+tr.stats.channel)
# Final visual check for debug
if debug >= 4:
tr.plot()
return tr
def process(tr,
lowcut,
highcut,
filt_order,
samp_rate,
debug,
starttime=False,
full_day=False):
r"""Basic function to bandpass, downsample and check headers and length \
of trace to ensure files start at the start of a day and are daylong.
Works in place on data. This is employed to ensure all parts of the data \
are processed in the same way.
.. note:: Usually this function is called via dayproc or shortproc.
:type tr: obspy.Trace
:param tr: Trace to process
:type highcut: float
:param highcut: High cut in Hz, if set to None and lowcut is set, will \
use a highpass filter.
:type lowcut: float
:type lowcut: Low cut in Hz, if set to None and highcut is set, will use \
a lowpass filter.
:type filt_order: int
:param filt_order: Number of corners for filter.
:type samp_rate: float
:param samp_rate: Desired sampling rate in Hz
:type debug: int
:param debug: Debug output level from 0-5, higher numbers = more output
:type starttime: obspy.UTCDateTime
:param starttime: Desired start of trace
:type full_day: bool
:param full_day: Whether to expect, and enforce a full day of data or not.
:return: obspy.Stream
.. note:: Will convert channel names to two charecters long.
"""
import warnings
from obspy.signal.filter import bandpass, lowpass, highpass
# Add sanity check
if highcut and highcut >= 0.5 * samp_rate:
raise IOError('Highcut must be lower than the nyquist')
# Define the start-time
if starttime:
day = starttime.date
else:
day = tr.stats.starttime.date
if debug >= 2:
print('Working on: ' + tr.stats.station + '.' + tr.stats.channel)
if debug >= 5:
tr.plot()
# Do a brute force quality check
qual = _check_daylong(tr)
if not qual:
msg = ("Data have more zeros than actual data, please check the raw",
" data set-up and manually sort it")
raise ValueError(msg)
tr = tr.detrend('simple') # Detrend data before filtering
# If there is one sample too many remove the first sample - this occurs
# at station FOZ where the first sample is zero when it shouldn't be,
# Not real sample: generated during data download
# if full_day:
# if len(tr.data) == (86400 * tr.stats.sampling_rate) + 1:
# tr.data = tr.data[1:len(tr.data)]
if debug > 0:
print('I have ' + str(len(tr.data)) + ' data points for ' +
tr.stats.station + '.' + tr.stats.channel + ' before processing')
# Sanity check to ensure files are daylong
if float(tr.stats.npts / tr.stats.sampling_rate) != 86400.0\
and full_day:
if debug >= 2:
print('Data for ' + tr.stats.station + '.' + tr.stats.channel +
' is not of daylong length, will zero pad')
# Work out when the trace thinks it is starting
# traceday = UTCDateTime(str(tr.stats.starttime.year)+'-' +
# str(tr.stats.starttime.month)+'-' +
# str(tr.stats.starttime.day))
# Use obspy's trim function with zero padding
tr = tr.trim(starttime,
starttime + 86400,
pad=True,
fill_value=0,
nearest_sample=True)
# If there is one sample too many after this remove the last one
# by convention
if len(tr.data) == (86400 * tr.stats.sampling_rate) + 1:
tr.data = tr.data[1:len(tr.data)]
if not tr.stats.sampling_rate * 86400 == tr.stats.npts:
raise ValueError('Data are not daylong for ' + tr.stats.station +
'.' + tr.stats.channel)
print('I now have ' + str(len(tr.data)) +
' data points after enforcing day length')
# Check sampling rate and resample
if tr.stats.sampling_rate != samp_rate:
if debug >= 2:
print('Resampling')
tr.resample(samp_rate)
# Filtering section
tr = tr.detrend('simple') # Detrend data again before filtering
if highcut and lowcut:
if debug >= 2:
print('Bandpassing')
tr.data = bandpass(tr.data, lowcut, highcut, tr.stats.sampling_rate,
filt_order, True)
elif highcut:
if debug >= 2:
print('Lowpassing')
tr.data = lowpass(tr.data, highcut, tr.stats.sampling_rate, filt_order,
True)
elif lowcut:
if debug >= 2:
print('Highpassing')
tr.data = highpass(tr.data, lowcut, tr.stats.sampling_rate, filt_order,
True)
else:
warnings.warn('No filters applied')
# Account for two letter channel names in s-files and therefore templates
tr.stats.channel = tr.stats.channel[0] + tr.stats.channel[-1]
# Sanity check the time header
if tr.stats.starttime.day != day != day and full_day:
warnings.warn("Time headers do not match expected date: " +
str(tr.stats.starttime))
# Sanity check to ensure files are daylong
if float(tr.stats.npts / tr.stats.sampling_rate) != 86400.0 and full_day:
if debug >= 2:
print('Data for ' + tr.stats.station + '.' + tr.stats.channel +
' is not of daylong length, will zero pad')
# Use obspy's trim function with zero padding
tr = tr.trim(starttime,
starttime + 86400,
pad=True,
fill_value=0,
nearest_sample=True)
# If there is one sample too many after this remove the last one
# by convention
if len(tr.data) == (86400 * tr.stats.sampling_rate) + 1:
tr.data = tr.data[1:len(tr.data)]
if not tr.stats.sampling_rate * 86400 == tr.stats.npts:
raise ValueError('Data are not daylong for ' + tr.stats.station +
'.' + tr.stats.channel)
# Final visual check for debug
if debug >= 4:
tr.plot()
return tr
def analyze_coda(trace, fm=6, cycle=10, noise_level=16, Lw=50, noise_duration=5, subwdw_length=5,
subwdw_length_rec=2.5):
"""
Return the correlation coefficient of the coda part of the signal : the onset of the coda is selected as the maximum amplitude time and the coda duration is Lw.
NOTE: this function accepts also streams objects (see @stream_compliant decorator in
stream2segments.mseeds)
:param trace: an obspy.core.Trace object
:return: a list of tuples of the form:
(slope_start_time, slope, intercept, R, pvalue, stderr)
where slope_start_time is an obspy UTCDateTime object. For the other values, see:
http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.linregress.html
for details
:rtype: see return field
"""
st = trace
try:
st.data = bandpass(st.data, freqmin=4, freqmax=8, df=st.stats. sampling_rate, corners=2)
except ValueError:
return None
if (st.stats.npts*st.stats.delta) > 100:
st.detrend('demean') # on ramene le signal a 0
energy = st.data * st.data
t = st.times()
st_smooth, t_smooth = mysmooth(energy, t, fm, cycle, st.stats.delta)
imax = st_smooth.index(max(st_smooth))
new_dt = round(t_smooth[1]-t_smooth[0], 2)
sec = int(noise_duration/new_dt) # on prend 10seconde de debut de signal
noise = st_smooth[0:sec] # on prend 5 seconde pour la moyenne de bruit
# df=st.stats.sampling_rate
# df = 1/new_dt
# valeur que j'ai prise= 2 et 5 (en echantillon)
cft = classic_sta_lta(noise, nsta=2, nlta=5)
stalta = np.where(cft > 3)[0] # valeur que j'ai prise =1.5
# si on detecte effectivement du signal dans la fenetre de bruit: ca va pas
if len(stalta) > 0:
return None # on ne peut pas definir une bonne moyenne de bruit
else:
noisedata = noise
# ----fin definition moyenne du bruit ----------------------------------------
# ##### duree de la coda = du maximum de l'enveloppe ------> ratio signal/bruit<4 #######
j = 0
start = imax
end_ = start+int(subwdw_length/new_dt) # on prend 5s de fenetre glissante
# rec_window = new_dt/2. # 50% de recouvrement
n_rec = int(subwdw_length_rec/new_dt) # nombre de pts de recouvrement : on choisit 2.5s
ratio = []
while j < len(st_smooth[imax:imax+int(Lw/new_dt)]):
ratio.append(np.mean(st_smooth[start:end_]) / np.mean(noisedata))
j = j+n_rec
start = start+n_rec
end_ = start+int(subwdw_length/new_dt)
# ou est ce que le signal dans les 80s de fenetre de coda est superieur au niveau de bruit
indok = np.where(np.array(ratio) > noise_level)[0]
ret_vals = None
if len(indok) > 0:
doublons = list(group(indok))
if (len(doublons) == 1) and (doublons[0][-1] == len(ratio)-1) or (doublons[0][0] == 0) \
and (doublons[0][-1] == len(ratio)-1):
# ca veut dire qu'il detecte une coda ou du moins un ratio>4 et
# on choisi une longueur de au moins 20 seconde
coda = st_smooth[imax:imax+int(Lw/new_dt)] # donnee lissee
# tcoda = t_smooth[imax:imax+int(Lw/new_dt)]
# raw=st.data[imax:imax+int(Lw/new_dt)]# donnee brut
# test sur la coda pour voir si on a bien une "pente" :
# on joue avec le coeff de correlation
# tr is the coda trace
coda = np.log10(coda) # on travaille en log avec la coda pour avoir une pente
n_pts = len(coda) # nombre de point dans la coda
# window=5
# rec=2.5
# nombre de pts dans la fenetre de 5 seconde
wdw_npts = int(subwdw_length / new_dt)
# nombre de point pour la fenetre de recouvrement:
wdw_rec = int(subwdw_length_rec / new_dt)
# borne maximale a atteindre pour faire des fenetres de 5 seconde:
n_max = np.floor(n_pts / wdw_npts)
start = 0
end = wdw_npts
means = []
x_means = []
k = 0
while end < n_max * wdw_npts:
means.append(np.mean(coda[start: end]))
x_means.append(k)
k = k + 1
start = start + wdw_rec
end = end + wdw_rec
slope, intercept, R, pvalue, stderr = sc.stats.linregress(x_means, means)
start_time = st.stats.starttime + t_smooth[imax]
ret_vals = (start_time, slope, intercept, R, pvalue, stderr)
return ret_vals