def _compute_tapers(bw, n, fs, ntapers, correct=False):
"""Compute tapers.
Parameters
----------
bw : float
Requested bandwidth
n : int
Number of samples in signal
fs : float
Sampling frequency
ntapers : int
Requested number of tapers
correct : bool
Correct number of tapers and bandwidth if requested values are
out of range.
Returns
-------
tapers : ndarray
"""
ntapers, bw = _check_ntapers(ntapers, n, bw, fs, correct)
tapers = spectrum.dpss(int(n), int(bw * n / fs),
int(ntapers))[0] * np.sqrt(fs)
return tapers
def fft_lfp(data, Fs = 1000): #print "Calculating multi-taper fft of lfp data..." data = data.squeeze() ##get rid of singleton dimensions for the next step if len(data.shape) > 1: ##if there is more than one trace, set N and numTraces appropriately N = data.shape[0] numTraces = data.shape[1] else: ##if there is only 1, set N and numTraces N = len(data) numTraces = 1 ##add a singleton dimension to make data "2d" data = data[:,None] nfft = 2**spec.nextpow2(N) ##next power of 2 from length of data (makes computation faster) tapers, eigs = spec.dpss(N, 3, 5) ##produce multi-taper windows. don't need to normalize like in chronux (integral of the square of each taper = 1) tapers = tapers*np.sqrt(Fs) tapers2 = np.zeros([tapers.shape[0], tapers.shape[1], numTraces]) ##add trial indices to tapers for i in range(tapers2.shape[2]): tapers2[:,:,i] = tapers H = np.fft.fft(tapers,nfft,axis = 0) ##fouier transform of the tapers Nsp = data.sum(axis = 0) ##number of spikes in each trial Msp = Nsp/N ##mean rate for each channel data2 = np.zeros([N, numTraces, tapers.shape[1]]) ##add taper indices to data for i in range(data2.shape[2]): data2[:,:,i] = data data2 = np.transpose(data2,(0,2,1)) ##get data into the same dimensions as tapers2 data_proj = data2*tapers2 ##multiply data by tapers J = np.fft.fft(data_proj,nfft, axis = 0)/Fs ##fft of projected data J #print'...Done!' return J
def _pmtm(self, x, y, NW=None, k=None, NFFT=None, e=None, v=None, method='eigen', show=True):
'''Multitapering spectral estimation
:param array x: the data
:param array y: the data
:param float NW: The time half bandwidth parameter (typical values are 2.5,3,3.5,4).
Must be provided otherwise the tapering windows and eigen values
(outputs of dpss) must be provided.
:param int k: uses the first k Slepian sequences. If *k* is not provided, *k* is set to *NW*2*.
:param NW
:param e: the matrix containing the tapering windows
:param v: the window concentrations (eigenvalues)
:param str method: set how the eigenvalues are used. Must be in ['unity', 'adapt', 'eigen']
:param bool show: plot results
'''
debug = False
assert method in ['adapt', 'eigen', 'unity']
N = len(x)
# if dpss not provided, compute them
if e is None and v is None:
if NW is not None:
[tapers, eigenvalues] = spectrum.dpss(N, NW, k=k)
else:
raise ValueError("NW must be provided (e.g. 2.5, 3, 3.5, 4")
elif e is not None and v is not None:
if debug:
print 'Using given tapers.'
eigenvalues = v[:]
tapers = e[:]
else:
raise ValueError(
"if e provided, v must be provided as well and viceversa.")
# length of the eigen values vector to be used later
nwin = len(eigenvalues)
# set the NFFT
if NFFT is None:
NFFT = max(256, 2 ** spectrum.nextpow2(N))
# si nfft smaller than N, cut otherwise add zero.
# compute
if method == 'unity':
weights = numpy.ones((nwin, 1))
elif method == 'eigen':
# The S_k spectrum can be weighted by the eigenvalues, as in Park
# et al.
weights = numpy.array([_x / float(i + 1)
for i, _x in enumerate(eigenvalues)])
weights = weights.reshape(nwin, 1)
xin = numpy.fft.fft(numpy.multiply(tapers.transpose(), x), NFFT)
yin = numpy.fft.fft(numpy.multiply(tapers.transpose(), y), NFFT)
Sk = numpy.multiply(xin, numpy.conj(yin))
Sk = numpy.mean(Sk * weights, axis=0)
# clf(); p.plot(); plot(arange(0,0.5,1./512),20*log10(res[0:256]))
if show is True:
spectrum.semilogy(Sk)
return Sk
def _cached_get_window(name, nfft):
if name.startswith('dpss'):
assert name in ['dpss1', 'dpss2']
type = int(name[4:]) - 1
tapers, eigen = dpss(nfft, 1.5, 2)
return tapers[:, type]
else:
return signal.get_window(name, nfft)
def _dpsschk(tapers, N, fs):
"""
Helper function to calculate tapers and
if precalculated tapers are provided,
to check that they are the of same
length in time as the time series being studied.
"""
tapers, eigs = dpss(N, tapers[0], tapers[1])
tapers = np.multiply(tapers, np.sqrt(fs))
return tapers
def _dpsschk(tapers, N, fs): """ Helper function to calculate tapers and if precalculated tapers are provided, to check that they are the of same length in time as the time series being studied. """ tapers, eigs = dpss(N, tapers[0], tapers[1]) tapers = np.multiply(tapers, np.sqrt(fs)) return tapers
def psd(data):
N = len(data)
NW = 4 #haf bandwidth parameter 2.5, 3, 3.5, 4
# k=4
dt = 1
[tapers, eigen] = dpss(N, NW)
Sk_complex, weights, eigenvalues = pmtm(data,
e=eigen,
v=tapers,
NFFT=N,
show=False)
Sk = abs(Sk_complex)**2
Sk = np.mean(Sk * np.transpose(weights), axis=0) * dt
Sk = Sk / np.max(Sk)
return Sk
def compute(self, bw=2.5):
''' Low-level method that computes the multitaper estimate
'''
N = np.size(self.dpss, 0)
Nmt = np.size(self.dpss, 1)
dt = (self.t[1] - self.t[0])
# data tapers
print('Number of data tapers: ' + str(Nmt))
[self.dpss, eigens] = dpss(N, bw, Nmt)
# compute Nmt tapered periodograms
for n in range(Nmt):
QFTx = qfft.Qfft(self.signal * self.dpss[:, n]) # tapered QFT
self.densities[:, n] = dt * (np.norm(QFTx) +
utils.StokesNorm(QFTx) * quaternion.y)
def fft_spikes(data): Fs = 1000.0 #print 'Calculating multi-taper fft of spike data...' data = data.squeeze() ##get rid of singleton dimensions for the next step if len(data.shape) > 1: ##if there is more than one trace, set N and numTraces appropriately N = data.shape[0] numTraces = data.shape[1] else: ##if there is only 1, set N and numTraces N = len(data) numTraces = 1 ##add a singleton dimension to make data "2d" data = data[:,None] nfft = 2**spec.nextpow2(N) tapers, eigs = spec.dpss(N, 3, 5) ##produce multi-taper windows. don't need to normalize like in chronux (integral of the square of each taper = 1) tapers = tapers*np.sqrt(Fs) tapers2 = np.zeros([tapers.shape[0], tapers.shape[1], data.shape[1]]) ##add trial indices to tapers for i in range(tapers2.shape[2]): tapers2[:,:,i] = tapers fft_tapers = np.fft.fft(tapers, nfft, axis = 0) ##take the fft of the tapers H = np.zeros([fft_tapers.shape[0],fft_tapers.shape[1],numTraces],dtype = np.complex64) ##add trace/trial indices to fft_tapers for i in range(numTraces): H[:,:,i] = fft_tapers Nsp = np.sum(data, axis = 0) ##number of spikes in each trial Msp = Nsp/N ##mean rate for each channel meansp = np.zeros([Msp.shape[0],tapers.shape[1],H.shape[0]]) ##add taper and freq indices to meansp for i in range(meansp.shape[1]): for j in range(meansp.shape[2]): meansp[:,i,j] = Msp meansp = np.transpose(meansp) data2 = np.zeros([data.shape[0],data.shape[1],tapers.shape[1]]) ##add taper indices to data for i in range(data2.shape[2]): data2[:,:,i] = data data2 = np.transpose(data2,(0,2,1)) ##get data into the same dimensions as H data_proj = data2*tapers2 ##multiply data by tapers J = np.fft.fft(data_proj,nfft,axis = 0) ##fft of projected data #account for spike rate J = J-H*meansp #print '...Done!' return J, Msp, Nsp
def methods_coherence_slepian_sequences(T, TW, filename=None):
# Generate the slepian sequences, here using the time-halfbandwith-product
[w, eigens] = spectrum.dpss(T, TW / 2, 4)
# fig, ax = plt.subplots(figsize=(8, 5))
fig, ax = plt.subplots(figsize=(6, 4.4))
# ax.canvas(200, 350)
sns.lineplot(data=w, ax=ax)
ax.set_xlim([100, 900])
ax.set_xlabel(r"Time t in $ms$")
ax.set_ylim([-0.12, 0.12])
ax.legend(['1st sequence', '2nd sequence', '3rd sequence', '4th sequence'])
# ax.set_title(f"Slepian sequences for T={T}, TW={TW}")
fig.show()
plu.save_figure(fig, filename, T=T, TW=TW)
def get_taper(nfft, taper, *args):
"""
# Return taper with desired parameters
# Input:
# nfft: Taper length
# taper: Taper type
# *args: depending on the taper
# Output:
# taper to be applied to window
"""
if taper == 'slepian':
[tapers, eigen] = spectrum.dpss(nfft, args[0], 1)
tapers = np.asarray(tapers)[:, 0]
if taper == 'hamming':
tapers = spectrum.window.create_window(nfft, 'hamming')
if taper == 'window_tukey':
tapers = spectrum.window.create_window(nfft, 'tuckey')
if taper == 'null':
tapers = np.asarray([1. for i in range(nfft)])
return tapers
def get_power_spectra(sample_freq, data):
# Get robust power spectra based on multitaper method
# Input:
# sample_freq: sample frequency of input data
# data: Input data from which the power spectrum has to be derived
# Output:
# freq: frequency array of power spectrum
# spec: robust power spectra
nfft = 1024
tbw = 3
[tapers, eigen] = spectrum.dpss(nfft, tbw, 1)
#res = spectrum.pmtm(data, e=tapers, v=eigen, show=False)
amp, weights, _ = spectrum.pmtm(data, NW=3, show=False)
freq = np.linspace(0, 1, len(amp[0])) * sample_freq
spec = [0. for i in range(len(amp[0]))]
for i in range(len(amp[0])):
for j in range(len(amp)):
spec[i] += amp[j][i] * weights[i][j]
return freq, np.abs(spec)
def _pmtm(self,
x,
y,
NW=None,
k=None,
NFFT=None,
e=None,
v=None,
method='eigen',
show=True):
'''Multitapering spectral estimation
:param array x: the data
:param array y: the data
:param float NW: The time half bandwidth parameter (typical values are 2.5,3,3.5,4).
Must be provided otherwise the tapering windows and eigen values
(outputs of dpss) must be provided.
:param int k: uses the first k Slepian sequences. If *k* is not provided, *k* is set to *NW*2*.
:param NW
:param e: the matrix containing the tapering windows
:param v: the window concentrations (eigenvalues)
:param str method: set how the eigenvalues are used. Must be in ['unity', 'adapt', 'eigen']
:param bool show: plot results
'''
debug = False
assert method in ['adapt', 'eigen', 'unity']
N = len(x)
# if dpss not provided, compute them
if e is None and v is None:
if NW is not None:
[tapers, eigenvalues] = spectrum.dpss(N, NW, k=k)
else:
raise ValueError("NW must be provided (e.g. 2.5, 3, 3.5, 4")
elif e is not None and v is not None:
if debug:
print 'Using given tapers.'
eigenvalues = v[:]
tapers = e[:]
else:
raise ValueError(
"if e provided, v must be provided as well and viceversa.")
# length of the eigen values vector to be used later
nwin = len(eigenvalues)
# set the NFFT
if NFFT is None:
NFFT = max(256, 2**spectrum.nextpow2(N))
# si nfft smaller than N, cut otherwise add zero.
# compute
if method == 'unity':
weights = numpy.ones((nwin, 1))
elif method == 'eigen':
# The S_k spectrum can be weighted by the eigenvalues, as in Park
# et al.
weights = numpy.array(
[_x / float(i + 1) for i, _x in enumerate(eigenvalues)])
weights = weights.reshape(nwin, 1)
xin = numpy.fft.fft(numpy.multiply(tapers.transpose(), x), NFFT)
yin = numpy.fft.fft(numpy.multiply(tapers.transpose(), y), NFFT)
Sk = numpy.multiply(xin, numpy.conj(yin))
Sk = numpy.mean(Sk * weights, axis=0)
# clf(); p.plot(); plot(arange(0,0.5,1./512),20*log10(res[0:256]))
if show is True:
spectrum.semilogy(Sk)
return Sk
def computeRandomCal(self):
debug = False
if (self.dbconn is not None):
# divide by 10000 when getting the cal_duration from the database
duration = self.cal_duration / 10000.0
else:
duration = self.cal_duration
if debug:
print 'cal duration = ' + str(duration)
# read data for the random calibration
stime = UTCDateTime(self.startdate)
stOUT = read(self.dataOutLoc,
starttime=stime,
endtime=stime + duration)
stOUT.merge()
stIN = read(self.dataInLoc, starttime=stime, endtime=stime + duration)
stIN.merge()
trIN = stIN[0]
trOUT = stOUT[0]
if debug:
trIN.plot()
trOUT.plot()
# remove the linear trend from the data
trIN.split().detrend('constant')
trOUT.split().detrend('constant')
samplerate = trOUT.stats.sampling_rate
segLength = int(
math.pow(2,
math.floor(math.log(math.floor(len(trIN.data) / 1.3),
2))))
offset = int(0.8 * segLength)
cnt = 0
if debug:
print 'Here is the segment length: ' + str(segLength)
print 'Here is the offset: ' + str(offset)
runningTotal = numpy.zeros(segLength)
numSegments = 0
[tapers, eigen] = spectrum.dpss(segLength, 12, 12)
while (cnt + segLength < len(trIN.data)):
x = trIN.data[cnt:cnt + segLength]
y = trOUT.data[cnt:cnt + segLength]
# perform the multi-taper method on both the input and output
# traces
pxx = self._pmtm(x,
x,
e=tapers,
v=eigen,
NFFT=segLength,
show=False)
pxy = self._pmtm(x,
y,
e=tapers,
v=eigen,
NFFT=segLength,
show=False)
# determine the frequency response by dividing the output by the
# input
res = numpy.divide(pxy, pxx)
# create a running total of all responses
runningTotal = numpy.add(runningTotal, res)
if (cnt + segLength > len(trIN.data)):
cnt = len(trIN.data) - segLength
else:
cnt = cnt + (segLength - offset)
if debug:
print 'Here is the cnt: ' + str(cnt)
numSegments = numSegments + 1
# find the average of segments
res = runningTotal / numSegments
# determine sensor type
if self.sentype is None:
self.sentype = self._determineSensorType()
# determine the response based off of the poles and zeros values
resPaz = self._getRespFromModel(self._pzvals(self.sentype), len(res),
trOUT.stats.delta)
# compute best fit
resBfPolesList = fmin(
self._resiFreq,
numpy.array(self._pazDictToList(self._pzvals(self.sentype))),
args=(res, samplerate),
xtol=10**-8,
ftol=10**-3,
disp=False)
resBfPoles = self._pazListToDict(resBfPolesList)
resBf = self._getRespFromModel(resBfPoles, len(res), trOUT.stats.delta)
# generate the frequency array
freq = numpy.multiply(numpy.fft.fftfreq(len(res)), samplerate)
# only grab positive frequencies
freq = freq[freq > 0]
# get the index of the frequency closest to 20 seconds period (0.05 Hz)
freq20Array = freq[(freq >= (1. / 20.))]
min20Freq = numpy.min(freq20Array)
freq20Index = numpy.where(freq == min20Freq)[0]
# get the index of the frequency closest to 1000 seconds period (0.001
# Hz)
freq1000Array = freq[(freq >= (1. / 1000.))]
min1000Freq = numpy.min(freq1000Array)
freq1000Index = numpy.where(freq == min1000Freq)[0]
# limit to data between 20s and 1000s period
freq = freq[freq1000Index:freq20Index]
res = res[freq1000Index:freq20Index]
resPaz = resPaz[freq1000Index:freq20Index]
resBf = resBf[freq1000Index:freq20Index]
#convert to degrees
res = res * (2. * math.pi * freq)
resPaz = resPaz * (2. * math.pi * freq)
resBf = resBf * (2. * math.pi * freq)
# get index where frequency closest to 50 seconds (0.02 Hz)
freq50Array = freq[(freq >= (1. / 50.))]
min50Freq = numpy.min(freq50Array)
res50Index = numpy.where(freq == min50Freq)[0]
res, resPhase = self._respToFAP(res, res50Index)
resPaz, resPazPhase = self._respToFAP(resPaz, res50Index)
resBf, resBfPhase = self._respToFAP(resBf, res50Index)
# calculate the free period
fp = 2 * math.pi / abs(numpy.min(self._pzvals(self.sentype)['poles']))
# determine the damping ratio of the signal
damping = abs(
numpy.real(numpy.min(self._pzvals(self.sentype)['poles'])) /
(2 * math.pi / fp))
fig = plt.figure()
ax = fig.add_subplot(121)
ax.semilogx(numpy.divide(1, freq), res, label='Actual')
ax.semilogx(numpy.divide(1, freq), resPaz, label='Nominal')
ax.semilogx(numpy.divide(1, freq), resBf, label='Best Fit')
ax.set_xlabel('Period (seconds)')
ax.set_ylabel('Amplitude [DB]')
ax.legend(loc=9, ncol=2, mode="expand", borderaxespad=0.)
ax = fig.add_subplot(122)
ax.semilogx(numpy.divide(1, freq), resPhase, label='Actual')
ax.semilogx(numpy.divide(1, freq), -resPazPhase, label='Nominal')
ax.semilogx(numpy.divide(1, freq), -resBfPhase, label='Best Fit')
ax.set_xlabel('Period (seconds)')
ax.set_ylabel('Phase [radian]')
plt.legend(loc=9, ncol=2, mode="expand", borderaxespad=0.)
plt.subplots_adjust(wspace=0.3, top=0.85)
title = 'Frequency Response of a ' + self.sentype + ' Seismometer for \n Station = ' \
+ self.network + '_' + self.station + ', Location = ' + self.location \
+ ', Channel = ' + self.outChannel + ', Start-time = ' + str(self.startdate) \
+ '\nh = ' + str(damping) + ', fp= ' + str(fp)
plt.suptitle(title, fontsize=11)
plt.show()
def spectrogram(self,data,window_width=None,incr=None,window='Hann',equal_loudness=False,mean_normalise=True,onesided=True,multitaper=False,need_even=False):
""" Compute the spectrogram from amplitude data
Returns the power spectrum, not the density -- compute 10.*log10(sg) before plotting.
Uses absolute value of the FT, not FT*conj(FT), 'cos it seems to give better discrimination
Options: multitaper version, but it's slow, mean normalised, even, one-sided.
This version is faster than the default versions in pylab and scipy.signal
Assumes that the values are not normalised.
TODO: Makes a copy of the data and uses that to ensure can be inverted. This is memory wasteful. Is that a problem?
"""
if data is None:
print ("Error")
datacopy = data.astype('float')
if window_width is None:
window_width = self.window_width
if incr is None:
incr = self.incr
# Set of window options
if window=='Hann':
# This is the Hann window
window = 0.5 * (1 - np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1)))
elif window=='Parzen':
# Parzen (window_width even)
n = np.arange(window_width) - 0.5*window_width
window = np.where(np.abs(n)<0.25*window_width,1 - 6*(n/(0.5*window_width))**2*(1-np.abs(n)/(0.5*window_width)), 2*(1-np.abs(n)/(0.5*window_width))**3)
elif window=='Welch':
# Welch
window = 1.0 - ((np.arange(window_width) - 0.5*(window_width-1))/(0.5*(window_width-1)))**2
elif window=='Hamming':
# Hamming
alpha = 0.54
beta = 1.-alpha
window = alpha - beta*np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1))
elif window=='Blackman':
# Blackman
alpha = 0.16
a0 = 0.5*(1-alpha)
a1 = 0.5
a2 = 0.5*alpha
window = a0 - a1*np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1)) + a2*np.cos(4 * np.pi * np.arange(window_width) / (window_width - 1))
elif window=='BlackmanHarris':
# Blackman-Harris
a0 = 0.358375
a1 = 0.48829
a2 = 0.14128
a3 = 0.01168
window = a0 - a1*np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1)) + a2*np.cos(4 * np.pi * np.arange(window_width) / (window_width - 1)) - a3*np.cos(6 * np.pi * np.arange(window_width) / (window_width - 1))
elif window=='Ones':
window = np.ones(window_width)
else:
print("Unknown window, using Hann")
window = 0.5 * (1 - np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1)))
if equal_loudness:
datacopy = self.equalLoudness(datacopy)
if mean_normalise:
datacopy -= datacopy.mean()
starts = range(0, len(datacopy) - window_width, incr)
if multitaper:
from spectrum import dpss, pmtm
[tapers, eigen] = dpss(window_width, 2.5, 4)
counter = 0
sg = np.zeros((len(starts),window_width // 2))
for start in starts:
Sk, weights, eigen = pmtm(datacopy[start:start + window_width], v=tapers, e=eigen, show=False)
Sk = abs(Sk)**2
Sk = np.mean(Sk.T * weights, axis=1)
sg[counter:counter + 1,:] = Sk[window_width // 2:].T
counter += 1
sg = np.fliplr(sg)
else:
if need_even:
starts = np.hstack((starts, np.zeros((window_width - len(datacopy) % window_width),dtype=int)))
ft = np.zeros((len(starts), window_width))
for i in starts:
ft[i // incr, :] = window * datacopy[i:i + window_width]
ft = fft.fft(ft)
#ft = np.fft.fft(ft)
if onesided:
sg = np.absolute(ft[:, :window_width // 2])
else:
sg = np.absolute(ft)
#sg = (ft*np.conj(ft))[:,window_width // 2:].T
return sg
def spectral_derivative(self,data,sampleRate,window_width,incr,K=2,threshold=0.5,returnAll=False):
""" Compute the spectral derivative """
from spectrum import dpss
# Compute the set of multi-tapered spectrograms
starts = range(0, len(data) - window_width, incr)
[tapers, eigen] = dpss(window_width, 2.5, K)
sg = np.zeros((len(starts), window_width,K),dtype=complex)
for k in range(K):
for i in starts:
sg[i // incr, :,k] = tapers[:,k] * data[i:i + window_width]
sg[:,:,k] = fft.fft(sg[:,:,k])
sg = sg[:,window_width//2:,:]
# Spectral derivative is the real part of exp(i \phi) \sum_ k s_k conj(s_{k+1}) where s_k is the k-th tapered spectrogram
# and \phi is the direction of maximum change (tan inverse of the ratio of pure time and pure frequency components)
S = np.sum(sg[:,:,:-1]*np.conj(sg[:,:,1:]),axis=2)
timederiv = np.real(S)
freqderiv = np.imag(S)
# Frequency modulation is the angle $\pi/2 - direction of max change$
fm = np.arctan(np.max(timederiv**2,axis=0) / np.max(freqderiv**2,axis=0))
spectral_deriv = -timederiv*np.sin(fm) + freqderiv*np.cos(fm)
sg = np.sum(np.real(sg*np.conj(sg)),axis=2)
sg /= np.max(sg)
# Suppress the noise (spectral continuity)
# Compute the zero crossings of the spectral derivative in all directions
# Pixel is a contour pixel if it is at a zero crossing and both neighbouring pixels in that direction are > threshold
sdt = spectral_deriv * np.roll(spectral_deriv,1,0)
sdf = spectral_deriv * np.roll(spectral_deriv,1,1)
sdtf = spectral_deriv * np.roll(spectral_deriv,1,(0,1))
sdft = spectral_deriv * np.roll(spectral_deriv,(1,-1),(0,1))
indt,indf = np.where(((sdt < 0) | (sdf < 0) | (sdtf < 0) | (sdft < 0)) & (spectral_deriv < 0))
# Noise reduction using a threshold
we = np.abs(self.wiener_entropy(sg))
freqs,mf = self.mean_frequency(sampleRate,timederiv,freqderiv)
# Given a time and frequency bin
contours = np.zeros(np.shape(spectral_deriv))
for i in range(len(indf)):
f = indf[i]
t = indt[i]
if (t>0) & (t<(np.shape(sg)[0]-1)) & (f>0) & (f<(np.shape(sg)[1]-1)):
thr = threshold*we[t]/np.abs(freqs[f] - mf[t])
if (sdt[t,f]<0) & (sg[t-1,f]>thr) & (sg[t+1,f]>thr):
contours[t,f] = 1
if (sdf[t,f] < 0) & (sg[t,f-1]>thr) & (sg[t,f+1]>thr):
contours[t,f] = 1
if (sdtf[t,f] < 0) & (sg[t-1,f-1]>thr) & (sg[t+1,f+1]>thr):
contours[t,f] = 1
if (sdft[t,f] < 0) & (sg[t-1,f+1]>thr) & (sg[t-1,f+1]>thr):
contours[t,f] = 1
if returnAll:
return spectral_deriv, sg, fm, we, mf, np.fliplr(contours)
else:
return np.fliplr(contours)
def computeRandomCal(self):
debug = False
if(self.dbconn is not None):
# divide by 10000 when getting the cal_duration from the database
duration = self.cal_duration / 10000.0
else:
duration = self.cal_duration
if debug:
print 'cal duration = ' + str(duration)
# read data for the random calibration
stime = UTCDateTime(self.startdate)
stOUT = read(
self.dataOutLoc, starttime=stime, endtime=stime + duration)
stOUT.merge()
stIN = read(self.dataInLoc, starttime=stime, endtime=stime + duration)
stIN.merge()
trIN = stIN[0]
trOUT = stOUT[0]
if debug:
trIN.plot()
trOUT.plot()
# remove the linear trend from the data
trIN.split().detrend('constant')
trOUT.split().detrend('constant')
samplerate = trOUT.stats.sampling_rate
segLength = int(
math.pow(2, math.floor(math.log(math.floor(len(trIN.data) / 1.3), 2))))
offset = int(0.8 * segLength)
cnt = 0
if debug:
print 'Here is the segment length: ' + str(segLength)
print 'Here is the offset: ' + str(offset)
runningTotal = numpy.zeros(segLength)
numSegments = 0
[tapers, eigen] = spectrum.dpss(segLength, 12, 12)
while (cnt + segLength < len(trIN.data)):
x = trIN.data[cnt:cnt + segLength]
y = trOUT.data[cnt:cnt + segLength]
# perform the multi-taper method on both the input and output
# traces
pxx = self._pmtm(
x, x, e=tapers, v=eigen, NFFT=segLength, show=False)
pxy = self._pmtm(
x, y, e=tapers, v=eigen, NFFT=segLength, show=False)
# determine the frequency response by dividing the output by the
# input
res = numpy.divide(pxy, pxx)
# create a running total of all responses
runningTotal = numpy.add(runningTotal, res)
if(cnt + segLength > len(trIN.data)):
cnt = len(trIN.data) - segLength
else:
cnt = cnt + (segLength - offset)
if debug:
print 'Here is the cnt: ' + str(cnt)
numSegments = numSegments + 1
# find the average of segments
res = runningTotal / numSegments
# determine sensor type
if self.sentype is None:
self.sentype = self._determineSensorType()
# determine the response based off of the poles and zeros values
resPaz = self._getRespFromModel(
self._pzvals(self.sentype), len(res), trOUT.stats.delta)
# compute best fit
resBfPolesList = fmin(self._resiFreq, numpy.array(self._pazDictToList(self._pzvals(self.sentype))), args=(res, samplerate),
xtol=10 ** -8, ftol=10 ** -3, disp=False)
resBfPoles = self._pazListToDict(resBfPolesList)
resBf = self._getRespFromModel(
resBfPoles, len(res), trOUT.stats.delta)
# generate the frequency array
freq = numpy.multiply(numpy.fft.fftfreq(len(res)), samplerate)
# only grab positive frequencies
freq = freq[freq > 0]
# get the index of the frequency closest to 20 seconds period (0.05 Hz)
freq20Array = freq[(freq >= (1. / 20.))]
min20Freq = numpy.min(freq20Array)
freq20Index = numpy.where(freq == min20Freq)[0]
# get the index of the frequency closest to 1000 seconds period (0.001
# Hz)
freq1000Array = freq[(freq >= (1. / 1000.))]
min1000Freq = numpy.min(freq1000Array)
freq1000Index = numpy.where(freq == min1000Freq)[0]
# limit to data between 20s and 1000s period
freq = freq[freq1000Index: freq20Index]
res = res[freq1000Index: freq20Index]
resPaz = resPaz[freq1000Index: freq20Index]
resBf = resBf[freq1000Index: freq20Index]
#convert to degrees
res = res * (2. * math.pi * freq)
resPaz = resPaz * (2. * math.pi * freq)
resBf = resBf * (2. * math.pi * freq)
# get index where frequency closest to 50 seconds (0.02 Hz)
freq50Array = freq[(freq >= (1. / 50.))]
min50Freq = numpy.min(freq50Array)
res50Index = numpy.where(freq == min50Freq)[0]
res, resPhase = self._respToFAP(res, res50Index)
resPaz, resPazPhase = self._respToFAP(resPaz, res50Index)
resBf, resBfPhase = self._respToFAP(resBf, res50Index)
# calculate the free period
fp = 2 * math.pi / abs(numpy.min(self._pzvals(self.sentype)['poles']))
# determine the damping ratio of the signal
damping = abs(
numpy.real(numpy.min(self._pzvals(self.sentype)['poles'])) / (2 * math.pi / fp))
fig = plt.figure()
ax = fig.add_subplot(121)
ax.semilogx(numpy.divide(1, freq), res, label='Actual')
ax.semilogx(numpy.divide(1, freq), resPaz, label='Nominal')
ax.semilogx(numpy.divide(1, freq), resBf, label='Best Fit')
ax.set_xlabel('Period (seconds)')
ax.set_ylabel('Amplitude [DB]')
ax.legend(loc=9, ncol=2, mode="expand", borderaxespad=0.)
ax = fig.add_subplot(122)
ax.semilogx(numpy.divide(1, freq), resPhase, label='Actual')
ax.semilogx(numpy.divide(1, freq), -resPazPhase, label='Nominal')
ax.semilogx(numpy.divide(1, freq), -resBfPhase, label='Best Fit')
ax.set_xlabel('Period (seconds)')
ax.set_ylabel('Phase [radian]')
plt.legend(loc=9, ncol=2, mode="expand", borderaxespad=0.)
plt.subplots_adjust(wspace=0.3, top=0.85)
title = 'Frequency Response of a ' + self.sentype + ' Seismometer for \n Station = ' \
+ self.network + '_' + self.station + ', Location = ' + self.location \
+ ', Channel = ' + self.outChannel + ', Start-time = ' + str(self.startdate) \
+ '\nh = ' + str(damping) + ', fp= ' + str(fp)
plt.suptitle(title, fontsize=11)
plt.show()
def pmtmPH(x, dt=1., nw=3, nfft=None):
"""
function [P,s,ci] = pmtmPH(x,dt,nw,qplot,nfft);
Computes the power spectrum using the multi-taper method with adaptive weighting.
Inputs:
x - Input data vector.
dt - Sampling interval, default is 1.
nw - Time bandwidth product, acceptable values are
0:.5:length(x)/2-1, default is 3. 2*nw-1 dpss tapers
are applied except if nw=0 a boxcar window is applied
and if nw=.5 (or 1) a single dpss taper is applied.
qplot - Generate a plot: 1 for yes, else no.
nfft - Number of frequencies to evaluate P at, default is
length(x) for the two-sided transform.
Outputs:
P - Power spectrum computed via the multi-taper method.
s - Frequency vector.
ci - 95% confidence intervals. Note that both the degrees of freedom
calculated by pmtm.m and chi2conf.m, which pmtm.m calls, are
incorrect. Here a quick approximation method is used to
determine the chi-squared 95% confidence limits for v degrees
of freedom. The degrees of freedom are close to but no larger
than (2*nw-1)*2; if the degrees of freedom are greater than
roughly 30, the chi-squared distribution is close to Gaussian.
The vertical ticks at the top of the plot indicate the size of
the full band-width. The distance between ticks would remain
fixed in a linear plot. For an accurate spectral estimate,
the true spectra should not vary abruptly on scales less than
the full-bandwidth.
Other toolbox functions called: dpps.m; and if nfft does not equal length(x) , cz.m
Peter Huybers
MIT, 2003
[email protected]
Adapted from Matlab to Python by Nicolas Barrier"""
if nfft is None:
nfft = len(x)
nx = len(x)
k = np.min([np.round(2 * nw), nx])
k = np.max([k - 1, 1])
s = np.arange(0, 1 / dt, 1 / (nfft * dt))
w = nw / (dt * nx) # half-bandwidth of the dpss
E, V = sp.dpss(nx, NW=nw, k=k)
if nx <= nfft:
tempx = np.transpose(np.tile(x, (k, 1)))
Pk = np.abs(np.fft.fft(E * tempx, n=nfft, axis=0))**2
else:
raise IOError('Not implemented yet')
#Iteration to determine adaptive weights:
if k > 1:
xmat = np.mat(x).T
sig2 = xmat.T * xmat / nx
# power
P = (Pk[:, 0] + Pk[:, 1]) / 2.
# initial spectrum estimate
Ptemp = np.zeros(nfft)
P1 = np.zeros(nfft)
tol = .0005 * sig2 / nfft
a = sig2 * (1 - V)
while np.sum(np.abs(P - P1) / nfft) > tol:
Pmat = np.mat(P).T
Vmat = np.mat(V)
amat = np.mat(a)
temp1 = np.mat(np.ones((1, k)))
temp2 = np.mat(np.ones((nfft, 1)))
b = (Pmat * temp1) / (Pmat * Vmat + temp2 * amat)
# weights
temp3 = np.mat(np.ones((nfft, 1))) * Vmat
temp3 = np.array(temp3)
b = np.array(b)
wk = b**2 * temp3
P1 = np.sum(wk * Pk, axis=1) / np.sum(wk, axis=1)
Ptemp = P1
P1 = P
P = Ptemp
# swap P and P1
#b2=b**2
#temp1=np.mat(np.ones((nfft,1)))*V
temp1 = b**2
temp2 = np.mat(np.ones((nfft, 1))) * Vmat
num = 2 * np.sum(temp1 * np.array(temp2), axis=1)**2
temp1 = b**4
temp2 = np.mat(np.ones((nfft, 1))) * np.mat(V**2)
den = np.sum(temp1 * np.array(temp2), axis=1)
v = num / den
select = np.arange(0, (nfft + 1) / 2 + 1).astype(np.int64)
P = P[select]
s = s[select]
v = v[select]
temp1 = 1 / (1 - 2 / (9 * v) - 1.96 * np.sqrt(2. / (9 * v)))**3
temp2 = 1 / (1 - 2 / (9 * v) + 1.96 * np.sqrt(2 / (9 * v)))**3
ci = np.array([temp1, temp2])
return P, s, ci
def multi_taper_ps( x, nfft, dt = 1, nw = 3 ): """ Based on script available at Peter Huybers' website: http://www.people.fas.harvard.edu/~phuybers/Mfiles/index.html Inputs: x - Input data vector. nfft - Number of frequencies to evaluate P at, set to length(x) for the two-sided transform. dt - Sampling interval, default is 1. nw - Time bandwidth product, acceptable values are 0:.5:length(x)/2-1, default is 3. 2*nw-1 dpss tapers are applied except if nw=0 a boxcar window is applied and if nw=.5 (or 1) a single dpss taper is applied. Outputs: P - Power spectrum computed via the multi-taper method. s - Frequency vector. ci - 95% confidence intervals. Note that both the degrees of freedom calculated by pmtm.m and chi2conf.m, which pmtm.m calls, are incorrect. Here a quick approximation method is used to determine the chi-squared 95% confidence limits for v degrees of freedom. The degrees of freedom are close to but no larger than (2*nw-1)*2; if the degrees of freedom are greater than roughly 30, the chi-squared distribution is close to Gaussian. The vertical ticks at the top of the plot indicate the size of the full band-width. The distance between ticks would remain fixed in a linear plot. For an accurate spectral estimate, the true spectra should not vary abruptly on scales less than the full-bandwidth. """ nx = len(x) k = min( round(2 * nw), nx ) k = int( max( k - 1, 1) ) #number of windows w = float(nw) / float(dt*nx) #half-bandwidth of the dpss s = np.arange(0., 1. / dt, 1. / nfft / dt ) #Compute the discrete prolate spheroidal sequences E, V = spectrum.dpss( nx, nw, k ) #Compute the windowed DFTs. #if nx<=nfft Pk = np.zeros( ( k, nx ) ) for i in range( k ): fx = np.fft.fft( E[:, i] * x[:], nfft) Pk[i, :] = abs( np.fft.fft( E[:, i] * x[:], nfft) ) ** 2 #else #compute DFT on nfft evenly spaced samples around unit circle: # Pk=abs(czt(E(:,1:k).*x(:,ones(1,k)),nfft)).^2; ####python doesn't have czt function. Can find scripts online, but for now just leave it #Iteration to determine adaptive weights: if k > 1: P, b, wk = adaptive_weights( x, Pk, nfft, V ) #Determine equivalent degrees of freedom, see p. of Percival and Walden 1993. v = (2. * np.sum( (b ** 2) * V[np.newaxis, :], axis = 2 ) ** 2 ) / (np.sum( (b ** 4) * ( V[np.newaxis, :] ** 2 ), axis = 2 ) ) else: #simply the periodogram; P = Pk v = 2.* np.ones( nfft ) #cut records P = P[:(nfft + 1) / 2 + 1] s = s[:(nfft + 1) / 2 + 1] v = v[0, :(nfft + 1) / 2 + 1] #Chi-squared 95% confidence interval #approximation from Chambers et al 1983; see Percival and Walden p.256, 1993 ci = np.zeros( ( 2, len( v ) ) ) ci[0, :] = 1. / (1. - 2. / (9. * v ) - 1.96 * np.sqrt( 2. / ( 9 * v ) ) ) ** 3 ci[1, :] = 1. / (1. - 2. / (9. * v ) + 1.96 * np.sqrt( 2. / ( 9 * v ) ) ) ** 3 return P, s, ci
def multi_taper_coh( x, y, dt = 1, nw = 8, qbias = 0, confn = 0 ): """ Based on script available at Peter Huybers' website: http://www.people.fas.harvard.edu/~phuybers/Mfiles/index.html Inputs: x - Input data vector 1. y - Input data vector 2. dt - Sampling interval (default 1) nw - Number of windows to use (default 8) qbias - Correct coherence estimate for bias (yes, 1) (no, 0, default). confn - Number of iterations to use in estimating phase uncertainty using a Monte Carlo method. (default 0) Outputs: s - frequency c - coherence ph - phase a - regression co-efficients ci - 95% coherence confidence level phi - 95% phase confidence interval, bias correct (add and subtract phi from ph). ############################## ############################## Note: the coherence bias and confidence intervals are difficult to calculate - Peter Huybers includes a script which estimates these that takes about an hour to run on a 2Ghz machine. For now, don't calculate the bias and confidence intervals. ############################## ############################## """ if nw < 1.5: print "WARNING: nw must be >= 1.5" print "Number of windows: ", nw if qbias == 1: print "Bias correction: On" else: print "Bias correction: Off" print "Confidence iterations: ", confn x -= np.mean( x ) y -= np.mean( y ) if len( x ) != len( y ): print "WARNING: length x != length y" #define some parameters nx = len(x); k = min( round(2 * nw), nx ) k = int( max( k - 1, 1) ) #number of windows s = np.arange(0., 1. / dt, 1. / nx / dt ) pls = np.arange(2, (nx + 1) / 2 + 1 ) v = (2. * nw - 1) #approximate degrees of freedom if len(y)%2 == 1: pls = pls[:len(pls) - 1] #Compute the discrete prolate spheroidal sequences E, V = spectrum.dpss( nx, nw, k ) #Compute the windowed DFTs. fkx = np.zeros( ( k, nx ) ).astype(complex) fky = np.zeros( ( k, nx ) ).astype(complex) for i in range( k ): fkx[i, :] = np.fft.fft( E[:, i] * x[:], nx) fky[i, :] = np.fft.fft( E[:, i] * y[:], nx) Pkx = abs( fkx ) ** 2 Pky = abs( fky ) ** 2 #Iteration to determine adaptive weights: for i in range( 2 ): if i == 0: P, b, wk = adaptive_weights( x, Pkx, nx, V ) fkx = np.sqrt( k ) * np.sqrt( wk[0] ) * fkx / (np.sum( np.sqrt( wk[0] ), axis = 0 ) )[np.newaxis, :] Fx = P #Power density spectral estimate of x elif i == 1: P, b, wk = adaptive_weights( y, Pky, nx, V ) fky = np.sqrt( k ) * np.sqrt( wk[0] ) * fky / (np.sum( np.sqrt( wk[0] ), axis = 0 ) )[np.newaxis, :] Fy = P #Power density spectral estimate of y #Compute coherence and phase Cxy = np.sum( fkx * np.conj( fky ), axis = 0 ) Cxy = np.conj( Cxy ) ph = np.angle( Cxy ) * 180. / np.pi c = abs( Cxy ) / np.sqrt( np.sum( abs( fkx ) ** 2, axis = 0) * np.sum( abs( fky ) ** 2, axis = 0 ) ) #Compute regression co-efficients a = a[1:(nx + 1) / 2 + 1] #cut records c = c[1:(nx + 1) / 2 + 1] s = s[1:(nx + 1) / 2 + 1] ph = ph[1:(nx + 1) / 2 + 1] phi = np.zeros( len(pls) ) ci = np.zeros( len( c ) ) return s, c, ph, a, ci, phi
def _decon(parent, daughter1, daughter2, noise, nn, method):
# Get length, zero padding parameters and frequencies
dt = parent.stats.delta
# Wiener or Water level deconvolution
if method == 'wiener' or method == 'water':
# npad = _npow2(nn*2)
npad = nn
freqs = np.fft.fftfreq(npad, d=dt)
# Fourier transform
Fp = np.fft.fft(parent.data, n=npad)
Fd1 = np.fft.fft(daughter1.data, n=npad)
Fd2 = np.fft.fft(daughter2.data, n=npad)
Fn = np.fft.fft(noise.data, n=npad)
# Auto and cross spectra
Spp = np.real(Fp * np.conjugate(Fp))
Sd1p = Fd1 * np.conjugate(Fp)
Sd2p = Fd2 * np.conjugate(Fp)
Snn = np.real(Fn * np.conjugate(Fn))
# Final processing depends on method
if method == 'wiener':
Sdenom = Spp + Snn
elif method == 'water':
phi = np.amax(Spp) * wlevel
Sdenom = Spp
Sdenom[Sdenom < phi] = phi
# Multitaper deconvolution
elif method == 'multitaper':
from spectrum import dpss
npad = nn
# Re-check length and pad with zeros if necessary
if not np.allclose([
tr.stats.npts
for tr in [parent, daughter1, daughter2, noise]
], npad):
parent.data = _pad(parent.data, npad)
daughter1.data = _pad(daughter1.data, npad)
daughter2.data = _pad(daughter2.data, npad)
noise.data = _pad(noise.data, npad)
freqs = np.fft.fftfreq(npad, d=dt)
NW = 2.5
Kmax = int(NW * 2 - 2)
[tapers, eigenvalues] = dpss(npad, NW, Kmax)
# Get multitaper spectrum of data
Fp = np.fft.fft(np.multiply(tapers.transpose(), parent.data))
Fd1 = np.fft.fft(
np.multiply(tapers.transpose(), daughter1.data))
Fd2 = np.fft.fft(
np.multiply(tapers.transpose(), daughter2.data))
Fn = np.fft.fft(np.multiply(tapers.transpose(), noise.data))
# Auto and cross spectra
Spp = np.sum(np.real(Fp * np.conjugate(Fp)), axis=0)
Sd1p = np.sum(Fd1 * np.conjugate(Fp), axis=0)
Sd2p = np.sum(Fd2 * np.conjugate(Fp), axis=0)
Snn = np.sum(np.real(Fn * np.conjugate(Fn)), axis=0)
# Denominator
Sdenom = Spp + Snn
else:
print("Method not implemented")
pass
# Apply Gaussian filter?
if gfilt:
gauss = _gauss_filt(dt, npad, gfilt)
gnorm = np.sum(gauss) * (freqs[1] - freqs[0]) * dt
else:
gauss = np.ones(npad)
gnorm = 1.
# Copy traces
rfp = parent.copy()
rfd1 = daughter1.copy()
rfd2 = daughter2.copy()
# Spectral division and inverse transform
rfp.data = np.fft.fftshift(
np.real(np.fft.ifft(gauss * Spp / Sdenom)) / gnorm)
rfd1.data = np.fft.fftshift(
np.real(np.fft.ifft(gauss * Sd1p / Sdenom)) / gnorm)
rfd2.data = np.fft.fftshift(
np.real(np.fft.ifft(gauss * Sd2p / Sdenom)) / gnorm)
# rfd1.data = np.fft.fftshift(np.real(np.fft.ifft(
# gauss*Sd1p/Sdenom))/np.amax(rfp.data)/gnorm)
# rfd2.data = np.fft.fftshift(np.real(np.fft.ifft(
# gauss*Sd2p/Sdenom))/np.amax(rfp.data)/gnorm)
return rfp, rfd1, rfd2
def cmtm(x, y, dt=1.0, NW=8, qbias=0.0, confn=0.0, qplot=True):
"""
s, c, ph, ci, phi = cmtm(x,y,dt,NW,qbias,confn,qplot)
Multi-taper method coherence using adaptive weighting and correcting
for the bias inherent to coherence estimates. The 95% coherence
confidence level is computed by cohconf.py. In addition, a
Monte Carlo estimation procedure is available to estimate phase 95%
confidence limits.
Args:
x - Input data vector 1.
y - Input data vector 2.
dt - Sampling interval (default 8)
NW - Number of windows to use (default 8)
qbias - Correct coherence estimate for bias (default 0).
confn - Number of iterations to use in estimating phase
uncertainty using a Monte Carlo method. (default 0)
qplot - Plot the results. The upper tickmarks indicate the
bandwidth of the coherence and phase estimates.
Returns:
s - frequency
c - coherence
ph - phase
ci - 95% coherence confidence level
phi - 95% phase confidence interval, bias corrected
(add and subtract phi from ph).
required: cohconf.py, cohbias.py, cohbias.nc, scipy signal processing.
"""
# Local Variables: ci, fx, cb, qplot, Pk, E, qbias, vari, ys, phut, Fx, Fy,
# phl, ds, fkx, fky, tol, Ptemp, ph, phlt, pl, NW, phi, P1, pls, xs, i1,fy,
# wk, N, P, V, dt, confn, phu, a, c, b, Cxy, Pkx, Pky, iter, col, s, w, v,
# y, x, h, k
# Function calls: disp, cmtm, dpss, cohconf, conv, fill, fft, set, conj,
# repmat, find, size, plot, angle, figure, cohbias, min, axis, sum, si,
# sqrt, abs, zeros, rem, xlabel, pi, ciph, real, max, ylabel, sort,
# nargin, ones, randn, subplot, ifft, clf, gcf, fliplr, length, num2str,
# title, round, mean
# pre-checks
if NW < 1.5:
raise ValueError("Warning: NW must be greater or equal to 1.5")
print('Number of windows: ', NW)
if qbias == 1.:
print('Bias correction: On')
else:
print('Bias correction: Off')
print('Confidence Itera.: ', confn)
if qplot == 1.:
print('Plotting: On')
else:
print('Plotting: Off')
x = x.flatten(1)-np.mean(x)
y = y.flatten(1)-np.mean(y)
if x.shape[0] != y.shape[0]:
raise ValueError('Warning: the lengths of x and y must be equal.')
# define some parameters
N = x.shape[0]
k = np.max(np.round((2.*NW)), N)
k = np.max((k-1.), 1.)
s = np.arange(0., (1./dt-1./np.dot(N, dt)) +
(1./np.dot(N, dt)), 1./np.dot(N, dt)).conj().T
pls = np.arange(2., ((N+1.)/2.+1.)+1)
v = 2*NW-1 # approximate degrees of freedom
if y.shape % 2 == 1:
pls = pls[0:0-1.]
# Compute the discrete prolate spheroidal sequences,
# requires the spectral analysis toolbox.
[E, V] = dpss(N, NW, k)
# Compute the windowed DFTs.
fkx = np.fft((E[:, 0:k]*x[:, int(np.ones(1., k))-1]), N)
fky = np.fft((E[:, 0:k]*y[:, int(np.ones(1., k))-1]), N)
Pkx = np.abs(fkx)**2.
Pky = np.abs(fky)**2.
# Iteration to determine adaptive weights:
for i1 in np.arange(1, 3):
if i1 == 1:
vari = np.dot(x.conj().T, x)/N
Pk = Pkx
if i1 == 2:
vari = np.dot(y.conj().T, y)/N
Pk = Pky
P = (Pk[:, 0]+Pk[:, 1])/2.
# initial spectrum estimate
Ptemp = np.zeros((N, 1.))
P1 = np.zeros((N, 1.))
tol = np.dot(.0005, vari)/N
# usually within tolerance in about three iterations,
# see equations from [2] (P&W pp 368-370).
a = np.dot(vari, 1.-V)
while np.sum(np.abs((P-P1))/N) > tol:
b = np.dot(P, np.ones(1., k))/(np.dot(P, V.conj().T) +
np.dot(np.ones(N, 1.), a.conj().T))
# weights
wk = b**2.*np.dot(np.ones(N, 1.), V.conj().T)
# new spectral estimate
P1 = (np.sum((wk.conj().T*Pk.conj().T))/
np.sum(wk.conj().T)).conj().T
Ptemp = P1
P1 = P
P = Ptemp
# swap P and P1
if i1 == 1:
dotp1 = np.dot(np.sqrt(k), np.sqrt(wk))
fkxtmp = np.sum(np.sqrt(wk.conj().T)).conj().T
# fkx = dotp1*fkx/matcompat.repmat(fkxtmp, 1., k)
fkx = dotp1*fkx/np.kron(np.ones((1, k)), fkxtmp)
Fx = P
# Power density spectral estimate of x
if i1 == 2:
dotp1 = np.dot(np.sqrt(k), np.sqrt(wk))
fkytmp = np.sum(np.sqrt(wk.conj().T)).conj().T
# fky = dotp1*fky/matcompat.repmat(fkytmp, 1., k)
fky = dotp1*fky/np.kron(np.ones((1, k)), fkytmp)
Fy = P
# Power density spectral estimate of y
# As a check, the quantity sum(abs(fkx(pls,:))'.^2) is the same as Fx and
# the spectral estimate from pmtmPH.
# Compute coherence
Cxy = np.sum(np.array(np.hstack((fkx*np.conj(fky)))).conj().T)
ph = np.divide(np.angle(Cxy)*180., np.pi)
c = np.abs(Cxy)/np.sqrt((np.sum((np.abs(fkx.conj().T)**2.)) *
np.sum((np.abs(fky.conj().T)**2.))))
# correct for the bias of the estimate
if qbias == 1:
c = cohbias(v, c).conj().T
# Phase uncertainty estimates via Monte Carlo analysis.
if confn > 1:
cb = cohbias(v, c).conj().T
nlist = np.arange(1., (confn)+1)
ciph = np.zeros((nlist, x.shape[0])) # not sure about the cmtm return length
phi = np.zeros((nlist, x.shape[0])) # not sure about the cmtm return length
for iter in nlist:
if plt.rem(iter, 10.) == 0.:
print('phase confidence iteration: ', iter)
fx = np.fft(np.randn(x.shape)+1.)
fx = np.divide(fx, np.sum(np.abs(fx)))
fy = np.fft(np.randn(y.shape)+1.)
fy = np.divide(fy, np.sum(np.abs(fy)))
ys = np.real(np.ifft((fy*np.sqrt((1.-cb.conj().T**2.)))))
ys = ys+np.real(np.ifft((fx*cb.conj().T)))
xs = np.real(np.ifft(fx))
si, ciph[iter, :], phi[iter, :] = cmtm(xs, ys, dt, NW)
pl = np.round(np.dot(.975, iter))
# sorting and averaging to determine confidence levels.
phi = np.sort(phi)
phi = np.array(np.vstack((np.hstack((phi[int(pl)-1, :])), np.hstack((-phi[int((iter-pl+1.))-1, :])))))
phi = np.mean(phi)
phi = plt.conv(phi[0:], (np.array(np.hstack((1., 1., 1.)))/3.))
phi = phi[1:0-1.]
else:
phi = np.zeros(pls.shape[0])
# Cut to one-sided funtions
c = c[int(pls)-1]
s = s[int(pls)-1].conj().T
ph = ph[int(pls)-1]
# Coherence confidence level
ci = cohconf(v, .95)
# not corrected for bias, this is conservative.
ci = np.dot(ci, np.ones((c.shape[0])))
# plotting
if qplot:
phl = ph-phi
phu = ph+phi
# coherence
print('coherence plot')
# phase
print('phase plot')
return s, c, ph, ci, phi
def psd_calc_other_methods(df, prm: Mapping[str, Any]):
## scipy
windows = signal.windows.dpss(180000, 2.5, Kmax=9, norm=2)
signal.spectrogram
## Welch
nperseg = 1024 * 8
freqs, psd_Ve = signal.welch(df.Ve, prm['fs'], nperseg=nperseg)
freqs, psd_Vn = signal.welch(df.Vn, prm['fs'], nperseg=nperseg)
## use Spectrum module
from spectrum import dpss
def pmtm(x, eigenvalues, tapers, n_fft=None, method='adapt'):
"""Multitapering spectral estimation
:param array x: the data
:param eigenvalues: the window concentrations (eigenvalues)
:param tapers: the matrix containing the tapering windows
:param str method: set how the eigenvalues are used. Must be
in ['unity', 'adapt', 'eigen']
:return: Sk (each complex), weights, eigenvalues
Usually in spectral estimation the mean to reduce bias is to use tapering
window. In order to reduce variance we need to average different spectrum.
The problem is that we have only one set of data. Thus we need to
decompose a set into several segments. Such method are well-known: simple
daniell's periodogram, Welch's method and so on. The drawback of such
methods is a loss of resolution since the segments used to compute the
spectrum are smaller than the data set.
The interest of multitapering method is to keep a good resolution while
reducing bias and variance.
How does it work? First we compute different simple periodogram with the
whole data set (to keep good resolution) but each periodgram is computed
with a differenttapering windows. Then, we average all these spectrum.
To avoid redundancy and bias due to the tapers mtm use special tapers.
from spectrum import data_cosine, dpss, pmtm
data = data_cosine(N=2048, A=0.1, sampling=1024, freq=200)
[tapers, eigen] = dpss(2048, 2.5, 4)
res = pmtm(data, eigenvalues=eigen, tapers=tapers, show=False)
.. versionchanged:: 0.6.2a
The most of spectrum.pmtm original code is to calc PSD but it is not returns so here we return it
+ Removed redandand functionality (calling semilogy plot and that what included in spectrum.dpss)
"""
assert method in ['adapt', 'eigen', 'unity']
N = len(x)
if eigenvalues is not None and tapers is not None:
eig = eigenvalues[:]
tapers = tapers[:]
else:
raise ValueError(
"if eigenvalues provided, v must be provided as well and viceversa."
)
nwin = len(eig) # length of the eigen values vector to be used later
if n_fft is None:
n_fft = max(256, 2**np.ceil(np.log2(N)).astype('int'))
Sk_complex = np.fft.fft(tapers.transpose() * x, n_fft)
# if nfft < N, cut otherwise add zero.
Sk = (Sk_complex * Sk_complex.conj()).real # abs() ** 2
if method in ['eigen', 'unity']:
if method == 'unity':
weights = np.ones((nwin, 1))
elif method == 'eigen':
# The S_k spectrum can be weighted by the eigenvalues, as in Park et al.
weights = np.array(
[_x / float(i + 1) for i, _x in enumerate(eig)])
weights = weights.reshape(nwin, 1)
Sk = np.mean(Sk * weights, axis=0)
elif method == 'adapt':
# This version uses the equations from [2] (P&W pp 368-370).
Sk = Sk.transpose()
S = Sk[:, :2].mean() # Initial spectrum estimate
# Set tolerance for acceptance of spectral estimate:
sig2 = np.dot(x, x) / float(N)
tol = 0.0005 * sig2 / float(n_fft)
a = sig2 * (1 - eig)
# Wrap the data modulo nfft if N > nfft
S = S.reshape(n_fft, 1)
for i in range(
100): # converges very quickly but for safety; set i<100
# calculate weights
b1 = np.multiply(S, np.ones((1, nwin)))
b2 = np.multiply(S, eig.transpose()) + np.ones(
(n_fft, 1)) * a.transpose()
b = b1 / b2
# calculate new spectral estimate
weights = (b**2) * (np.ones((n_fft, 1)) * eig.transpose())
S1 = ((weights * Sk).sum(axis=1, keepdims=True) /
weights.sum(axis=1, keepdims=True))
S, S1 = S1, S
if np.abs(S - S1).sum() / n_fft < tol:
break
Sk = (weights * Sk).mean(axis=1)
if np.isrealobj(
x
): # Double to account for the energy in the negative frequencies
if prm['n_fft'] % 2 == 0:
Sk = 2 * Sk[:int(prm['n_fft'] / 2 + 1)]
else:
Sk = 2 * Sk[:int((prm['n_fft'] + 1) / 2)]
return Sk_complex, Sk, weights
prm['dpss_sp'], prm['eigvals_sp'] = dpss(prm['n_fft'], 3.5)
sk_complex_Ve, sk_Ve_, weights_Ve = pmtm(
df.Ve.values, prm['eigvals_sp'], prm['dpss_sp']) # n_fft=prm['n_fft']
sk_complex_Vn, sk_Vn_, weights_Vn = pmtm(df.Vn.values, prm['eigvals_sp'],
prm['dpss_sp'])
# Convert Power Spectrum to Power Spectral Density
record_time_length = prm['length'] * prm['dt']
sk_Ve = sk_Ve_ / record_time_length
sk_Vn = sk_Vn_ / record_time_length
def spectrogram(self, window_width=None,incr=None,window='Hann',equal_loudness=False,mean_normalise=True,onesided=True,multitaper=False,need_even=False):
""" Compute the spectrogram from amplitude data
Returns the power spectrum, not the density -- compute 10.*log10(sg) 10.*log10(sg) before plotting.
Uses absolute value of the FT, not FT*conj(FT), 'cos it seems to give better discrimination
Options: multitaper version, but it's slow, mean normalised, even, one-sided.
This version is faster than the default versions in pylab and scipy.signal
Assumes that the values are not normalised.
"""
if self.data is None or len(self.data)==0:
print("ERROR: attempted to calculate spectrogram without audiodata")
return
if window_width is None:
window_width = self.window_width
if incr is None:
incr = self.incr
# clean handling of very short segments:
if len(self.data) <= window_width:
window_width = len(self.data) - 1
self.sg = np.copy(self.data)
if self.sg.dtype != 'float':
self.sg = self.sg.astype('float')
# Set of window options
if window=='Hann':
# This is the Hann window
window = 0.5 * (1 - np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1)))
elif window=='Parzen':
# Parzen (window_width even)
n = np.arange(window_width) - 0.5*window_width
window = np.where(np.abs(n)<0.25*window_width,1 - 6*(n/(0.5*window_width))**2*(1-np.abs(n)/(0.5*window_width)), 2*(1-np.abs(n)/(0.5*window_width))**3)
elif window=='Welch':
# Welch
window = 1.0 - ((np.arange(window_width) - 0.5*(window_width-1))/(0.5*(window_width-1)))**2
elif window=='Hamming':
# Hamming
alpha = 0.54
beta = 1.-alpha
window = alpha - beta*np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1))
elif window=='Blackman':
# Blackman
alpha = 0.16
a0 = 0.5*(1-alpha)
a1 = 0.5
a2 = 0.5*alpha
window = a0 - a1*np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1)) + a2*np.cos(4 * np.pi * np.arange(window_width) / (window_width - 1))
elif window=='BlackmanHarris':
# Blackman-Harris
a0 = 0.358375
a1 = 0.48829
a2 = 0.14128
a3 = 0.01168
window = a0 - a1*np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1)) + a2*np.cos(4 * np.pi * np.arange(window_width) / (window_width - 1)) - a3*np.cos(6 * np.pi * np.arange(window_width) / (window_width - 1))
elif window=='Ones':
window = np.ones(window_width)
else:
print("Unknown window, using Hann")
window = 0.5 * (1 - np.cos(2 * np.pi * np.arange(window_width) / (window_width - 1)))
if equal_loudness:
self.sg = self.equalLoudness(self.sg)
if mean_normalise:
self.sg -= self.sg.mean()
starts = range(0, len(self.sg) - window_width, incr)
if multitaper:
[tapers, eigen] = dpss(window_width, 2.5, 4)
counter = 0
out = np.zeros((len(starts),window_width // 2))
for start in starts:
Sk, weights, eigen = pmtm(self.sg[start:start + window_width], v=tapers, e=eigen, show=False)
Sk = abs(Sk)**2
Sk = np.mean(Sk.T * weights, axis=1)
out[counter:counter + 1,:] = Sk[window_width // 2:].T
counter += 1
self.sg = np.fliplr(out)
else:
if need_even:
starts = np.hstack((starts, np.zeros((window_width - len(self.sg) % window_width),dtype=int)))
# this mode is optimized for speed, but reportedly sometimes
# results in crashes when lots of large files are batch processed.
# The FFTs here could be causing this, but I'm not sure.
# hi_mem = False should switch FFTs to go over smaller vectors
# and possibly use less caching, at the cost of 1.5x longer CPU time.
hi_mem = True
if hi_mem:
ft = np.zeros((len(starts), window_width))
for i in starts:
ft[i // incr, :] = self.sg[i:i + window_width]
ft = np.multiply(window, ft)
if onesided:
self.sg = np.absolute(fft.fft(ft)[:, :window_width //2])
else:
self.sg = np.absolute(fft.fft(ft))
else:
if onesided:
ft = np.zeros((len(starts), window_width//2))
for i in starts:
winddata = window * self.sg[i:i + window_width]
ft[i // incr, :] = fft.fft(winddata)[:window_width//2]
else:
ft = np.zeros((len(starts), window_width))
for i in starts:
winddata = window * self.sg[i:i + window_width]
ft[i // incr, :] = fft.fft(winddata)
self.sg = np.absolute(ft)
del ft
gc.collect()
#sg = (ft*np.conj(ft))[:,window_width // 2:].T
return self.sg
def spectral_derivative(self, window_width, incr, K=2, threshold=0.5, returnAll=False):
""" Compute the spectral derivative """
if self.data is None or len(self.data)==0:
print("ERROR: attempted to calculate spectrogram without audiodata")
return
# Compute the set of multi-tapered spectrograms
starts = range(0, len(self.data) - window_width, incr)
[tapers, eigen] = dpss(window_width, 2.5, K)
sg = np.zeros((len(starts), window_width, K), dtype=complex)
for k in range(K):
for i in starts:
sg[i // incr, :, k] = tapers[:, k] * self.data[i:i + window_width]
sg[:, :, k] = fft.fft(sg[:, :, k])
sg = sg[:, window_width//2:, :]
# Spectral derivative is the real part of exp(i \phi) \sum_ k s_k conj(s_{k+1}) where s_k is the k-th tapered spectrogram
# and \phi is the direction of maximum change (tan inverse of the ratio of pure time and pure frequency components)
S = np.sum(sg[:, :, :-1]*np.conj(sg[:, :, 1:]), axis=2)
timederiv = np.real(S)
freqderiv = np.imag(S)
# Frequency modulation is the angle $\pi/2 - direction of max change$
mfd = np.max(freqderiv**2, axis=0)
mfd = np.where(mfd==0,1,mfd)
fm = np.arctan(np.max(timederiv**2, axis=0) / mfd)
spectral_deriv = -timederiv*np.sin(fm) + freqderiv*np.cos(fm)
sg = np.sum(np.real(sg*np.conj(sg)), axis=2)
sg /= np.max(sg)
# Suppress the noise (spectral continuity)
# Compute the zero crossings of the spectral derivative in all directions
# Pixel is a contour pixel if it is at a zero crossing and both neighbouring pixels in that direction are > threshold
sdt = spectral_deriv * np.roll(spectral_deriv, 1, 0)
sdf = spectral_deriv * np.roll(spectral_deriv, 1, 1)
sdtf = spectral_deriv * np.roll(spectral_deriv, 1, (0, 1))
sdft = spectral_deriv * np.roll(spectral_deriv, (1, -1), (0, 1))
indt, indf = np.where(((sdt < 0) | (sdf < 0) | (sdtf < 0) | (sdft < 0)) & (spectral_deriv < 0))
# Noise reduction using a threshold
we = np.abs(self.wiener_entropy(sg))
freqs, mf = self.mean_frequency(self.sampleRate, timederiv, freqderiv)
# Given a time and frequency bin
contours = np.zeros(np.shape(spectral_deriv))
for i in range(len(indf)):
f = indf[i]
t = indt[i]
if (t > 0) & (t < (np.shape(sg)[0]-1)) & (f > 0) & (f < (np.shape(sg)[1]-1)):
thr = threshold*we[t]/np.abs(freqs[f] - mf[t])
if (sdt[t, f] < 0) & (sg[t-1, f] > thr) & (sg[t+1, f] > thr):
contours[t, f] = 1
if (sdf[t, f] < 0) & (sg[t, f-1] > thr) & (sg[t, f+1] > thr):
contours[t, f] = 1
if (sdtf[t, f] < 0) & (sg[t-1, f-1] > thr) & (sg[t+1, f+1] > thr):
contours[t, f] = 1
if (sdft[t, f] < 0) & (sg[t-1, f+1] > thr) & (sg[t-1, f+1] > thr):
contours[t, f] = 1
if returnAll:
return spectral_deriv, sg, fm, we, mf, np.fliplr(contours)
else:
return np.fliplr(contours)
def test_dpss(self):
cls = self.__class__
tapers, eigen = dpss(cls.nfft, 1.5, 2)
self.assertTrue(np.allclose(tapers, cls.tapers))
def get_spectrogram(self, nframes, lframes):
"""
Go through the data frame by frame and perform transformation. They can be plotted using pcolormesh
x, y and z are ndarrays and have the same shape. In order to access the contents use these kind of
indexing as below:
#Slices parallel to frequency axis
nrows = np.shape(x)[0]
for i in range (nrows):
plt.plot(x[i,:], z[i,:])
#Slices parallel to time axis
ncols = np.shape(y)[1]
for i in range (ncols):
plt.plot(y[:,i], z[:, i])
:return: frequency, time and power for XYZ plot,
"""
from spectrum import dpss, pmtm
assert self.method in ['fft', 'welch', 'mtm']
x = self.data_array
fs = self.fs
# define an empty np-array for appending
pout = np.zeros(nframes * lframes)
if self.method == 'fft':
# go through the data array section wise and create a results array
for i in range(nframes):
f, p, _ = self.get_fft(x[i * lframes:(i + 1) * lframes] *
self.get_window(lframes))
pout[i * lframes:(i + 1) * lframes] = p
elif self.method == 'welch':
# go through the data array section wise and create a results array
for i in range(nframes):
f, p = self.get_pwelch(x[i * lframes:(i + 1) * lframes] *
self.get_window(lframes))
pout[i * lframes:(i + 1) * lframes] = p
elif self.method == 'mtm':
[tapers, eigen] = dpss(lframes, NW=2)
f = self.get_fft_freqs_only(x[0:lframes])
# go through the data array section wise and create a results array
for i in range(nframes):
p = pmtm(x[i * lframes:(i + 1) * lframes] *
self.get_window(lframes),
e=tapers,
v=eigen,
method='adapt',
show=False)
# pay attention that mtm uses padding, so we have to cut the output array
pout[i * lframes:(i + 1) * lframes] = np.fft.fftshift(
p[0:lframes, 0])
# create a mesh grid from 0 to nframes -1 in Y direction
xx, yy = np.meshgrid(f, np.arange(nframes))
# fold the results array to the mesh grid
zz = np.reshape(pout, (nframes, lframes))
return xx, yy * lframes / fs, zz
