def THinv5(self,phi,K,M,eps):
"""
function G=THinv5(phi,K,M,eps)
%
%%%% Implements fast (complexity O(M*K^2))
%%%% computation of the following piece of code:
%
%C=[];
%for im=1:M
% A=toeplitz(phi(1:K,im),phi(1:K,im)')+hankel(phi(1:K,im),phi(K:2*K-1,im)')+eps(im)*eye(K);
% C=[C inv(A)];
%end
%
% DEFAULT PARAMETERS: M=2; phi=randn(2*K-1,M); eps=randn(1,2);
% SIZE of phi SHOULD BE (2*K-1,M).
% SIZE of eps SHOULD BE (1,M).
"""
# %C=[];
C = []
# %for im=1:M
# % A=toeplitz(phi(1:K,im),phi(1:K,im)')+hankel(phi(1:K,im),phi(K:2*K-1,im)')+eps(im)*eye(K);
# % C=[C inv(A)];
# %end
for im in xrange(M):
A = (toeplitz(phi[:K,im],phi[:K,im].T) +
hankel(phi[:K,im],phi[K-1:2*K,im].T) +
eps[im]*np.eye(K))
C.append(np.linalg.inv(A))
return np.concatenate(C,axis=1)
def _firls_even(numtaps, bands, desired):
# This function implements an algorithm similar to the one of the
# SciPy `firls` function, but for even-length filters rather than
# odd-length ones. See paper notes entitled "Least squares FIR
# filter design for even N" for derivation. The derivation is
# similar to that of Ivan Selesnick's "Linear-Phase FIR Filter
# Design By Least Squares" (available online at
# http://cnx.org/contents/eb1ecb35-03a9-4610-ba87-41cd771c95f2@7),
# with due alteration of detail for even filter lengths.
bands.shape = (-1, 2)
desired.shape = (-1, 2)
weights = np.ones(len(desired))
M = int(numtaps / 2)
# Compute M x M matrix Q (actually twice Q).
n = np.arange(numtaps)[:, np.newaxis, np.newaxis]
q = np.dot(np.diff(np.sinc(bands * n) * bands, axis=2)[:, :, 0], weights)
Q1 = linalg.toeplitz(q[:M])
Q2 = linalg.hankel(q[1:M + 1], q[M:])
Q = Q1 + Q2
# Compute M-vector b.
k = np.arange(M) + .5
e = bands[1]
b = np.diff(e * np.sinc(e * k[:, np.newaxis])).reshape(-1)
# Compute a (actually half a).
a = np.dot(linalg.pinv(Q), b)
# Compute h.
h = np.concatenate((np.flipud(a), a))
return h
def coffee_pred(step):
n=1290-step
fea=20
cof=1
from sklearn import svm
from scipy.linalg import hankel
import matplotlib.pyplot as plt
from sklearn import metrics
import math
import numpy as np
#input data from csv file
path='/Users/royyang/Desktop/time_series_forecasting/csv_files/coffee_ls.txt'
path1 = '/Users/royyang/Desktop/time_series_forecasting/csv_files/coffee_ls_nor.txt'
result_tem=[]
with open(path) as f:
next(f)
for line in f:
item=line.replace('\n','').split(' ')
result_tem.append(float(item[1]))
mean = np.mean(result_tem)
sd = np.std(result_tem)
result = (result_tem-mean)/sd
#form hankel matrix
X=hankel(result[0:-fea-step+1], result[-1-fea:-1])
y=result[fea+step-1:]
#split data into training and testing
Xtrain=X[:n]
ytrain=y[:n]
Xtest=X[n:]
ytest=y[n:]
# linear kernal
svr_lin1 = svm.SVR(kernel='linear', C=cof)
y_lin1 = svr_lin1.fit(Xtrain, ytrain)
return int(y_lin1.predict(result[-1-fea:-1])*sd+mean)
def firls(m, bands, desired, weight=None):
if weight==None : weight = ones(len(bands)/2)
bands, desired, weight = array(bands), array(desired), array(weight)
#if not desired[-1] == 0 and bands[-1] == 1 and m % 2 == 1:
if m % 2 == 1:
m = m + 1
M = m/2
w = kron(weight, [-1,1])
omega = bands * pi
i1 = arange(1,M+1)
# generate the matrix q
# as illustrated in the above-cited reference, the matrix can be
# expressed as the sum of a hankel and toeplitz matrix. a factor of
# 1/2 has been dropped and the final filter hficients multiplied
# by 2 to compensate.
cos_ints = append(omega, sin(mat(arange(1,m+1)).T*mat(omega))).reshape((-1,omega.shape[0]))
q = append(1, 1.0/arange(1.0,m+1)) * array(mat(cos_ints) * mat(w).T).T[0]
q = toeplitz(q[:M+1]) + hankel(q[:M+1], q[M : ])
# the vector b is derived from solving the integral:
#
# _ w
# / 2
# b = / w(w) d(w) cos(kw) dw
# k / w
# - 1
#
# since we assume that w(w) is constant over each band (if not, the
# computation of q above would be considerably more complex), but
# d(w) is allowed to be a linear function, in general the function
# w(w) d(w) is linear. the computations below are derived from the
# fact that:
# _
# / a ax + b
# / (ax + b) cos(nx) dx = --- cos (nx) + ------ sin(nx)
# / 2 n
# - n
#
enum = append(omega[::2]**2 - omega[1::2]**2, cos(mat(i1).T * mat(omega[1::2])) - cos(mat(i1).T * mat(omega[::2]))).flatten()
deno = mat(append(2, i1)).T * mat(omega[1::2] - omega[::2])
cos_ints2 = enum.reshape(deno.shape)/array(deno)
d = zeros_like(desired)
d[::2] = -weight * desired[::2]
d[1::2] = weight * desired[1::2]
b = append(1, 1.0/i1) * array(mat(kron (cos_ints2, [1, 1]) + cos_ints[:M+1,:]) * mat(d).T)[:,0]
# having computed the components q and b of the matrix equation,
# solve for the filter hficients.
a = (array(inv(q)*mat(b).T).T)[0]
h = append( a[:0:-1], append(2*a[0], a[1:]))
return h
def update(self, x_n, d_n):
# Update the internal buffers
self.n += 1
slot = self.L - ((self.n-1) % self.L) - 1
self.x[slot] = x_n
self.d[slot] = d_n
# Block update
if self.n % self.L == 0:
# block-update parameters
X = la.hankel(self.x[:self.L],r=self.x[self.L-1:])
e = self.d - np.dot(X, self.w)
if self.nlms:
norm = np.linalg.norm(X, axis=1)**2
if self.L == 1:
X = X/norm[0]
else:
X = (X.T/norm).T
self.w += self.mu*np.dot(X.T, e)
# Remember a few values
self.x[-self.length+1:] = self.x[0:self.length-1]
def update(self, x_n, d_n):
# Update the internal buffers
self.n += 1
slot = self.L - ((self.n-1) % self.L) - 1
self.x[slot] = x_n
self.d[slot] = d_n
# Block update
if self.n % self.L == 0:
# block-update parameters
X = la.hankel(self.x[:self.L],r=self.x[self.L-1:])
Lambda_diag = (self.lmbd**np.arange(self.L))
lmbd_L = self.lmbd**self.L
alpha = self.d - np.dot(X, self.w)
pi = np.dot(self.P, X.T)
g = np.linalg.solve((lmbd_L*np.diag(1/Lambda_diag) + np.dot(X,pi)).T, pi.T).T
self.w += np.dot(g, alpha)
self.P = self.P/lmbd_L - np.dot(g, pi.T)/lmbd_L
# Remember a few values
self.x[-self.length+1:] = self.x[0:self.length-1]
def embed(self, embedding_dimension=None, suspected_frequency=None, verbose=False, return_df=True):
'''Embed the time series with embedding_dimension window size.
Optional: suspected_frequency changes embedding_dimension such that it is divisible by suspected frequency'''
if not embedding_dimension:
self.embedding_dimension = self.ts_N//2
else:
self.embedding_dimension = embedding_dimension
if suspected_frequency:
self.suspected_frequency = suspected_frequency
self.embedding_dimension = (self.embedding_dimension//self.suspected_frequency)*self.suspected_frequency
self.K = self.ts_N-self.embedding_dimension+1
self.X = np.matrix(linalg.hankel(self.ts, np.zeros(self.embedding_dimension))).T[:,:self.K]
self.X_df = pd.DataFrame(self.X)
self.X_complete = self.X_df.dropna(axis=1)
self.X_com = np.matrix(self.X_complete.values)
self.X_missing = self.X_df.drop(self.X_complete.columns, axis=1)
self.X_miss = np.matrix(self.X_missing.values)
self.trajectory_dimentions = self.X_df.shape
self.complete_dimensions = self.X_complete.shape
self.missing_dimensions = self.X_missing.shape
self.no_missing = self.missing_dimensions[1]==0
if verbose:
msg1 = 'Embedding dimension\t: {}\nTrajectory dimensions\t: {}'
msg2 = 'Complete dimension\t: {}\nMissing dimension \t: {}'
msg1 = msg1.format(self.embedding_dimension, self.trajectory_dimentions)
msg2 = msg2.format(self.complete_dimensions, self.missing_dimensions)
self._printer('EMBEDDING SUMMARY', msg1, msg2)
if return_df:
return self.X_df
def eigen_pairs(self, T, k):
v = onp.fromfunction(lambda i: 1.0 / ((i + 2)**3 - (i + 2)),
(2 * T - 1, ))
Z = 2 * la.hankel(v[:T], v[T - 1:])
eigen_values, eigen_vectors = np.linalg.eigh(Z)
return np.flip(eigen_values[-k:], axis=0), np.flip(eigen_vectors[:,
-k:],
axis=1)
def ssa(x,
r,
fullMatr=True,
comp_uv=False): # x is nt x 1 vector, r is embedding dimension
#nt = len(x); mean = sum(x)/nt
#x = x - ones(nt)*mean
H = scilinalg.hankel(x, zeros(r)) #Construct Hankel matrix
return scilinalg.svd(H, full_matrices=fullMatr, compute_uv=comp_uv)
def test_norm(n=5, m=10):
# Build Hankel matrices
c = (10 * np.random.randn(n)) + 1j * np.random.randn(n)
r = (10 * np.random.randn(m)) + 1j * np.random.randn(m)
H_dense = hankel(c, r)
H = Hankel(c, r)
assert np.allclose(H.norm(), np.linalg.norm(H_dense, 2))
def detect_clearsky_helper_data():
samples_per_window = 3
sample_interval = 1
x = pd.Series(np.arange(0, 7)**2.)
# line length between adjacent points
sqt = pd.Series(np.sqrt(np.array([np.nan, 2., 10., 26., 50., 82, 122.])))
H = hankel(np.arange(samples_per_window),
np.arange(samples_per_window - 1, len(sqt)))
return x, samples_per_window, sample_interval, H
def make_duplication_matrix(T, tau):
H = hankel(range(tau), range(tau - 1, T))
T2 = np.prod(H.shape)
h = np.reshape(H, [1, T2])
h2 = np.array([range(T2)])
index = np.concatenate([h, h2], axis=0)
S = np.zeros([T, T2], dtype='uint64')
S[tuple(index)] = 1
return S.T
def MomsFromHankelMoms(hm):
"""
Returns the raw moments given the Hankel moments.
The raw moments are: `m_i=E(\mathcal{X}^i)`
The ith Hankel moment is the determinant of matrix
`\Delta_{i/2}`, if i is even,
and it is the determinant of `\Delta^{(1)}_{(i+1)/2}`,
if i is odd. For the definition of matrices `\Delta`
and `\Delta^{(1)}` see [1]_.
Parameters
----------
hm : vector of doubles
The list of Hankel moments (starting with the first
moment)
Returns
-------
m : vector of doubles
The list of raw moments
References
----------
.. [1] http://en.wikipedia.org/wiki/Stieltjes_moment_problem
"""
m = [hm[0]]
for i in range(1, len(hm)):
if i % 2 == 0:
N = i // 2 + 1
H = la.hankel(m[0:N], m[N - 1:2 * N - 2] + [0])
else:
N = (i + 1) // 2 + 1
H = la.hankel([1] + m[0:N - 1], m[N - 2:2 * N - 3] + [0])
h = hm[i]
rH = np.delete(H, N - 1, 0)
for j in range(N):
cofactor = (-1)**(N + j - 1) * la.det(np.delete(rH, j, 1))
if j < N - 1:
h -= cofactor * H[N - 1, j]
else:
m.append(h / cofactor)
return m
def MomsFromHankelMoms (hm):
"""
Returns the raw moments given the Hankel moments.
The raw moments are: `m_i=E(\mathcal{X}^i)`
The ith Hankel moment is the determinant of matrix
`\Delta_{i/2}`, if i is even,
and it is the determinant of `\Delta^{(1)}_{(i+1)/2}`,
if i is odd. For the definition of matrices `\Delta`
and `\Delta^{(1)}` see [1]_.
Parameters
----------
hm : vector of doubles
The list of Hankel moments (starting with the first
moment)
Returns
-------
m : vector of doubles
The list of raw moments
References
----------
.. [1] http://en.wikipedia.org/wiki/Stieltjes_moment_problem
"""
m = [hm[0]]
for i in range(1,len(hm)):
if i%2 == 0:
N = i//2 + 1
H = la.hankel(m[0:N], m[N-1:2*N-2] + [0])
else:
N = (i+1) // 2 + 1
H = la.hankel([1] + m[0:N-1], m[N-2:2*N-3] + [0])
h = hm[i]
rH = np.delete(H, N-1, 0)
for j in range(N):
cofactor = (-1)**(N+j-1) * la.det(np.delete(rH, j, 1))
if j<N-1:
h -= cofactor * H[N-1,j]
else:
m.append(h/cofactor)
return m
def hankel_matrix(n):
hm = hankel(n)
if len(hm) % 2 == 1:
dim = len(hm) / 2 + 1
else:
dim = len(hm) / 2 # last element will be lost (0)
h = hm[0:dim, 0:dim] # real hankel matrix
eigenvalues, eigenvectors = LA.eig(h)
return eigenvalues, eigenvectors
def all_pairs_unique_sum(vec):
"""gives all pairs (x, y) of items in vec."""
# generate all pairs
A = hankel(vec, np.roll(vec, 1))
A = A[1:, :].flatten()
n = int(len(A) / 2)
A = np.vstack([A[:n], np.tile(vec, int((len(A) / len(vec))))[:n]])
# remove duplicates
A.sort(axis=0)
return np.unique(A.T, axis=0)
def quadmom(moments, dettol=0, inf=1e10):
""" compute quadrature from moments
ref: Gene Golub, John Welsch, Calculation of Gaussian Quadrature Rules
Args:
m: moments sequence
dettol: tolerant of singularity of moments sequence (quantified by determinant of moment matrix)
INF: infinity
Returns:
U: quadrature
"""
moments = np.asarray(moments)
# INF = float('inf')
inf = 1e10
if len(moments) % 2 == 1:
moments = np.append(moments, inf)
num = int(len(moments) / 2)
moments = np.insert(moments, 0, 1)
h_mat = hankel(moments[:num + 1:], moments[num::]) # Hankel matrix
for i in range(len(h_mat)):
# check positive definite and decide to use how many moments
if np.linalg.det(
h_mat[0:i + 1, 0:i +
1]) <= dettol: # alternative: less than some threshold
h_mat = h_mat[0:i + 1, 0:i + 1]
h_mat[i, i] = inf
num = i
break
r_mat = np.transpose(
np.linalg.cholesky(h_mat)) # upper triangular Cholesky factor
# Compute alpha and beta from r, using Golub and Welsch's formula.
alpha = np.zeros(num)
alpha[0] = r_mat[0][1] / r_mat[0][0]
for i in range(1, num):
alpha[i] = r_mat[i][i + 1] / r_mat[i][i] - r_mat[i - 1][i] / r_mat[
i - 1][i - 1]
beta = np.zeros(num - 1)
for i in range(num - 1):
beta[i] = r_mat[i + 1][i + 1] / r_mat[i][i]
jacobi = np.diag(alpha, 0) + np.diag(beta, 1) + np.diag(beta, -1)
eigval, eigvec = np.linalg.eig(jacobi)
atoms = eigval
weights = moments[0] * np.power(eigvec[0], 2)
return DiscreteRV(w=weights, x=atoms)
def buildHank(c, k):
""" Build a Hankel matrix for separable PSF and reflexive boundary
conditions."""
n = len(c)
col = zeros(n, 'd')
col[0:n - k - 1] = c[k + 1:n]
row = zeros(n, 'd')
row[n - k:n] = c[0:k]
H = hankel(col, row)
return H
def trajectory_matrix(self, X):
# a window of 1 just recreates X,
# i.e. no temporal analysis takes place
if self.window == 1:
return X
n, d = X.shape
new_n = n - self.window + 1
lag_copies = [linalg.hankel(X[:new_n, j],
X[new_n-1:, j]) for j in range(d)]
return np.concatenate(lag_copies, axis=1)
def smooth(X, L):
"""Spatial Smoothing of Correlated Sources
Args:
L: The number of elements in each subarray
"""
T, M = X.shape
J = M - L + 1
X = [hankel(X[i, :L], X[i, -J:]).T for i in range(T)]
X = np.asarray(X).reshape(-1, L)
return X
def getfile(self):
fname = QFileDialog.getOpenFileName(self, 'Open file')
self.data = detrend(np.loadtxt(str(fname)))
self.maxvec = int(self.textbox1.text())
winsize = int(self.textbox4.text())
tsl = len(self.data)
self.H = hankel(self.data[range(tsl / winsize)], self.data[range(
(tsl / winsize) - 1, tsl)])
self.U, self.s, self.V = randomized_svd(self.H,
n_components=self.maxvec,
n_iter=1,
random_state=None)
def patch_matrix_build_circ(f, l):
'''
Build the patch matrix of f described in A Tale of 2 Bases.
Input
f: Nx1 signal.
l: patch size.
Output
hankel: patch matrix of f.
'''
last_row = np.zeros(l, dtype=f.dtype)
last_row[1:] = f[0:l - 1]
return hankel(f, last_row)
def patch_matrix_build_trcon(f, f2):
'''
Build the Hankel structured patch matrix of f under the truncated linear convolution setup.
Input
f: Nx1 signal.
f2: (l-1)x1 start of next block
Output
hankel: patch matrix of f.
'''
last_row = np.zeros(np.size(f2) + 1, dtype=f.dtype)
last_row[1:] = f2
return hankel(f, last_row)
def CheckMoments(m, prec=1e-14):
"""
Checks if the given moment sequence is valid in the sense
that it belongs to a distribution with support (0,inf).
This procedure checks the determinant of `\Delta_n`
and `\Delta_n^{(1)}` according to [1]_.
Parameters
----------
m : list of doubles, length 2N+1
The (raw) moments to check
(starts with the first moment).
Its length must be odd.
prec : double, optional
Entries with absolute value less than prec are
considered to be zeros. The default value is 1e-14.
Returns
-------
r : bool
The result of the check.
References
----------
.. [1] http://en.wikipedia.org/wiki/Stieltjes_moment_problem
"""
if butools.checkInput and len(m) % 2 == 0:
raise Exception("CheckMoments: the number of moments must be odd!")
m = [1.0] + m
N = math.floor(len(m) / 2) - 1
for n in range(N + 1):
H = la.hankel(m[0:n + 1], m[n:2 * n + 1])
H0 = la.hankel(m[1:n + 2], m[n + 1:2 * n + 2])
if la.det(H) < -prec or la.det(H0) < -butools.checkPrecision:
return False
return True
def CheckMoments (m, prec=1e-14):
"""
Checks if the given moment sequence is valid in the sense
that it belongs to a distribution with support (0,inf).
This procedure checks the determinant of `\Delta_n`
and `\Delta_n^{(1)}` according to [1]_.
Parameters
----------
m : list of doubles, length 2N+1
The (raw) moments to check
(starts with the first moment).
Its length must be odd.
prec : double, optional
Entries with absolute value less than prec are
considered to be zeros. The default value is 1e-14.
Returns
-------
r : bool
The result of the check.
References
----------
.. [1] http://en.wikipedia.org/wiki/Stieltjes_moment_problem
"""
if butools.checkInput and len(m)%2==0:
raise Exception("CheckMoments: the number of moments must be odd!")
m = [1.0] + m
N = math.floor(len(m)/2)-1
for n in range(N+1):
H = la.hankel(m[0:n+1], m[n:2*n+1])
H0 = la.hankel(m[1:n+2], m[n+1:2*n+2])
if la.det(H)<-prec or la.det(H0)<-butools.checkPrecision:
return False
return True
def rolling_window(self, vector=None, window=None):
# create rolling prefs window across any range of space (circular tuning)
if vector is None:
vector = self.all_prefs
if window is None:
# >= -90 to < 90 consists of all possible orientations in standard size feature space
window = self.all_prefs[(self.all_prefs >= -90)
& (self.all_prefs < 90)]
m = len(vector) - len(window)
self.all_prefs_windows_idx = hankel(np.arange(0, m + 1),
np.arange(m, len(vector)))
self.all_prefs_windows = vector[self.all_prefs_windows_idx]
return self.all_prefs_windows, self.all_prefs_windows_idx
def THinv5(self,phi,K,M,eps):
# %C=[];
C = []
# %for im=1:M
# % A=toeplitz(phi(1:K,im),phi(1:K,im)')+hankel(phi(1:K,im),phi(K:2*K-1,im)')+eps(im)*eye(K);
# % C=[C inv(A)];
# %end
for im in range(M):
A = (toeplitz(phi[:K,im],phi[:K,im].T) +
hankel(phi[:K,im],phi[K-1:2*K,im].T) +
eps[im]*np.eye(K))
C.append(np.linalg.inv(A))
return np.concatenate(C,axis=1)
def ESPRIT(s, ordre):
r"""Straight forward implementation of ESPRIT algorithm
ESPRIT stands for "Estimation of Signal Parameters via Rotationnal Invariance Techniques".
Parameters
----------
s: numpy array
Signal to analyse
ordre: int
Order of ESPRIT pole estimation
Returns
-------
f: numpy array
Reduced frequency detected
zeta: numpy array
Corresponding damping
Examples
--------
In this example, we show how to properly use the ESPRIT function by creating
a 10Hz damped cosine with a damping factor of 0.5 and trying to get those
parameters back with ESPRIT.
>>> t = np.linspace(0,1,2000) # One second sampled at 2000Hz
>>> s = np.cos(2*np.pi*10*t) # Compute a cosine at f = 10Hz
>>> d = np.exp(-.5*t) # Compute a damping of 0.5
>>> f,zeta = ESPRIT(s*d,2) # Estimation of f and damping
>>> print(f*2000, zeta*2000) # Multiply by sampling rate
[ 10.013694 -10.013694] [-0.500378 -0.500378]
"""
with torch.no_grad():
s = torch.from_numpy(s)
N = len(s)
H = torch.from_numpy(hankel(np.arange(N // 3), np.arange(N // 3, N)))
H = s[H.long()]
H = torch.mm(H, H.transpose(0, 1))
U, S, V = torch.svd(H, some=False)
phi = torch.pinverse(U[1:, :ordre]).mm(U[:-1, :ordre])
z = torch.eig(phi)[0].cpu().numpy()
z = z[:, 0] + 1j * z[:, 1]
f = np.angle(z) / (2 * np.pi)
zeta = -np.log(abs(z))
return f, zeta
def aa2vect(aa, tamanho_janela_movel=17):
aa_dict = {
'A': 0,
'R': 1,
'N': 2,
'D': 3,
'B': 4,
'C': 5,
'E': 6,
'Q': 7,
'Z': 8,
'G': 9,
'H': 10,
'I': 11,
'L': 12,
'K': 13,
'M': 14,
'F': 15,
'P': 16,
'S': 17,
'T': 18,
'W': 19,
'Y': 20,
'V': 21,
'?': 22
}
aa_num = []
for i, w in enumerate(aa):
aa_num.append(np.array([aa_dict.get(x) for x in aa[i]]))
base_indexador = np.arange(tamanho_janela_movel) * 23
sequencias = []
total_de_sequencias = len(aa)
for i in range(0, total_de_sequencias): #para % cada sequencia
seq = aa_num[i] #obter sequencia atual
janelas = hankel(
seq[:tamanho_janela_movel], seq[tamanho_janela_movel - 1:]
) #obter todas as janelas moveis para a sequencia, cada coluna forma uma
matriz_binaria = np.full(
(np.size(janelas, 1), 23 * tamanho_janela_movel),
False,
dtype=float)
for k in range(0, np.size(
janelas, 1)): #para cada janela movel, ou seja, cada coluna
indexador = np.array(base_indexador + janelas[:, k])
matriz_binaria[
k, indexador] = True #marca como True o valor de cada posição
for j in matriz_binaria:
sequencias.append(j)
return sequencias
def THinv5(self, phi, K, M, eps):
# %C=[];
C = []
# %for im=1:M
# % A=toeplitz(phi(1:K,im),phi(1:K,im)')+hankel(phi(1:K,im),phi(K:2*K-1,im)')+eps(im)*eye(K);
# % C=[C inv(A)];
# %end
for im in range(M):
A = (toeplitz(phi[:K, im], phi[:K, im].T) +
hankel(phi[:K, im], phi[K - 1:2 * K, im].T) +
eps[im] * np.eye(K))
C.append(np.linalg.inv(A))
return np.concatenate(C, axis=1)
def HankelMomsFromMoms (m):
"""
Returns the Hankel moments given the raw moments.
The raw moments are: `m_i=E(\mathcal{X}^i)`
The ith Hankel moment is the determinant of matrix
`\Delta_{i/2}`, if i is even,
and it is the determinant of `\Delta^{(1)}_{(i+1)/2}`,
if i is odd. For the definition of matrices `\Delta`
and `\Delta^{(1)}` see [1]_.
Parameters
----------
m : vector of doubles
The list of raw moments (starting with the first
moment)
Returns
-------
hm : vector of doubles
The list of Hankel moments
References
----------
.. [1] http://en.wikipedia.org/wiki/Stieltjes_moment_problem
"""
hm = []
for i in range(len(m)):
if i%2 == 0:
N = i//2 + 1
H = la.hankel(m[0:N], m[N-1:2*N-1])
else:
N = (i+1) // 2 + 1
H = la.hankel([1] + m[0:N-1], m[N-2:2*N-2])
hm.append (la.det(H))
return hm
def HankelMomsFromMoms(m):
"""
Returns the Hankel moments given the raw moments.
The raw moments are: `m_i=E(\mathcal{X}^i)`
The ith Hankel moment is the determinant of matrix
`\Delta_{i/2}`, if i is even,
and it is the determinant of `\Delta^{(1)}_{(i+1)/2}`,
if i is odd. For the definition of matrices `\Delta`
and `\Delta^{(1)}` see [1]_.
Parameters
----------
m : vector of doubles
The list of raw moments (starting with the first
moment)
Returns
-------
hm : vector of doubles
The list of Hankel moments
References
----------
.. [1] http://en.wikipedia.org/wiki/Stieltjes_moment_problem
"""
hm = []
for i in range(len(m)):
if i % 2 == 0:
N = i // 2 + 1
H = la.hankel(m[0:N], m[N - 1:2 * N - 1])
else:
N = (i + 1) // 2 + 1
H = la.hankel([1] + m[0:N - 1], m[N - 2:2 * N - 2])
hm.append(la.det(H))
return hm
def pronys_ad(inp, threshold=2, period=7):
nfreq = period//2
train = inp[:period]
data = inp[period:]
test_val = inp.iloc[-1]
_data = []
SIGIDX = 3
NORMIDX = 2
MEANIDX = 1
lo = len(data)-period
while lo >= 0:
_data.append([lo, np.mean(data[lo:lo+period]), nl.norm(data[lo:lo+period] - np.mean(data[lo:lo+period])), data[lo:lo+period]])
lo -= period
_data = _data[::-1]
normalized_data = np.hstack([ (_[SIGIDX] - _[MEANIDX])/_[NORMIDX] for _ in _data] )
A = sl.hankel(normalized_data, normalized_data[-period:])[:period, :]
try:
u,s,v = nl.svd(A)
except nl.LinAlgError as e:
raise(e)
_s = s.copy()
_s[2*nfreq+1:]=0
_S = np.zeros((u.shape[1], v.shape[0]))
_S[:len(_s), :u.shape[1]] = np.diag(_s)
_A = u.dot(_S.dot(v))
_b = train - np.mean(train)
_b /= nl.norm(_b)
try:
_x = sl.lstsq(_A, _b)[0]
except ValueError as e:
raise(e)
profile = _A.dot(_x)
sig = _data[-1][SIGIDX]
fit = profile * _data[-1][NORMIDX] + _data[-1][MEANIDX]
prediction = fit[-1]
error = prediction - test_val
inp_mean = np.mean(inp[:-1])
inp_std = np.std(inp[:-1])
inp_median = np.median(inp[:-1])
thres = inp_std*inp_median/inp_mean
E2SD_ratio = np.absolute(error)/thres
metric_values = [prediction, test_val, error, period, inp_mean, inp_std, E2SD_ratio,E2SD_ratio>threshold, 0, -1]
metric_names = [AlphaResultObject.PREDICTION, 'original_value', 'residual', 'model_period', 'mean', 'standard_deviation', AlphaResultObject.ERROR_COEFF, AlphaResultObject.IS_ANOMALY, AlphaResultObject.STATUS_CODE, AlphaResultObject.SEVERITY]
return dict(zip(metric_names, metric_values))
def hankel_matrix(data, p=-1, q=None):
"""
Create the Hankel matrix for a univariate time series. p specifies the width
of the matrix
"""
if p == -1:
p = len(data)
if not q:
q = p
last = data[-p:]
first = data[-(p + q):-p]
h_mat = hankel(first, last)
return h_mat
def generate_window_slices(self, arr):
"""Generate arrays for slicing data into windows.
Arguments
---------
arr: np.array
Returns
-------
slices: np.ndarray
Hankel matrix for slicing data into windows.
"""
slices = linalg.hankel(np.arange(0,
len(arr) - self.window + 1),
np.arange(len(arr) - self.window, len(arr)))
return slices
def hankel_matrix(n):
n_appended = n
# if odd number of elements in "n", we append average of "n" to array "n"
if len(n) % 2 == 1:
n_appended.append(sum(n) / float(len(n)))
dim = (len(n_appended) / 2)
else:
dim = (len(n_appended) / 2)
hm = hankel(n_appended)
h = hm[0:int(dim), 0:int(dim)] # real hankel matrix
# eigenvalues, eigenvectors = LA.eig(h)
u, svd_h, Vh = svd(h)
# return eigenvalues, eigenvectors, svd_h, h
return svd_h, h
def findPoints2(img, toothNb, points, profiles, covariances, lenSearch,
lenProfile):
sobel = applySobel(img)
gray_img = to_grayscale(img)
# Normal points on gray image
itbuffer = getAllNormalPoints(points, lenSearch, gray_img)
xs = itbuffer[:, 0::3]
ys = itbuffer[:, 1::3]
intensities = itbuffer[:, 2::3]
#Normal points on sobel image
itbuffer2 = getAllNormalPoints(points, lenSearch, sobel)
intensities2 = itbuffer2[:, 2::3]
newXs = np.zeros(points.get_list().shape[0] / 2)
newYs = np.zeros(points.get_list().shape[0] / 2)
bestFits = np.zeros(2 * (lenSearch - lenProfile) + 1)
bestFits2 = []
for idx in range(0, 40):
normalizedSearchProfiles = greymodels.calculateProfile(
intensities[idx], intensities2[idx])
subsequences = np.asarray(
hankel(normalizedSearchProfiles[:lenProfile * 2 + 1],
normalizedSearchProfiles[lenProfile * 2:])).T
bestFit, error = findBestFit(profiles[idx], subsequences,
covariances[idx])
bestFits[bestFit] += 1
bestFits2.append(bestFit)
# smooth shape
bestFits2 = signal.medfilt(bestFits2, 5)
for idx in range(0, 40):
newidx = np.rint(bestFits2[idx] + lenProfile + 1).astype(np.uint32)
newXs[idx] = xs[idx][newidx]
newYs[idx] = ys[idx][newidx]
return Landmarks(
np.asarray([val for pair in zip(newXs, newYs)
for val in pair])), np.sum(
bestFits[len(bestFits) / 2 - 1:len(bestFits) / 2 + 2])
def projhankel(Z, H=0, gamma=0):
"""
Orthogonal projection onto the subspace of Hankel matrices
X = PROJHANKEL(Z) determines the orthogonal projection of
Z onto the subspaces of Hankel matrices, i.e., X is the
proximal mapping of (i_{Hankel}(X)), where i_{Hankel}
is the convex indicator function of the set of Hankel matrices.
X = PROJHANKEL(Z,H,gamma) allows to shift Z by -gamma*H, i.e.,
X is projection of Z-gamma*H onto the subspaces of Hankel matrices and is
therefore the proximal mapping of gamma*(i_{Hankel}(X)+trace(X'H)).
"""
import numpy as np
from scipy.linalg import hankel
Z = Z - gamma * H
dim = Z.shape
if dim[0] < dim[1]:
Z = Z.T
transp_Z = 1 # Flag transpositon of Z
dim = np.sort(dim, axis=-1)[::-1]
else:
transp_Z = 0 # Flag Z is not transposed
n = dim[0]
m = dim[1]
# Determine the means k[0],...,k[n+m-2] along the anti-diagonals
B = np.zeros((m + n, m))
B[0:n, :] = Z
N = np.sum(np.reshape(B.flatten('F')[0:(n + m - 1) * m], (n + m - 1, m),
order='F'),
axis=1)
D = np.append(np.append(np.arange(1, m), m * np.ones((1, n - m + 1))),
np.arange(1, m)[::-1])
k = np.divide(N, D)
# Arrange means as Hankel matrix
X = hankel(k[0:n], k[n - 1:])
if transp_Z == 1:
X = X.T
return X
def fft2x(x, y, z, nfft=None):
"""Return Bi-cross-spectrum."""
# 1d-fourier-transforms
# ----------------------------------------------------------
x = np.ravel(x)
y = np.ravel(y)
z = np.ravel(z)
assert len(x) == len(y) and len(x) == len(z), (
"Arrays must have the same length")
if not nfft:
# let's hope it's not too much!
nfft = len(x)
# transform
xfft = fft(x, n=nfft)
if y is x:
yfft = xfft
else:
yfft = fft(y, n=nfft)
if z is x:
zfft = xfft
elif z is y:
zfft = yfft
else:
zfft = fft(z, n=nfft)
# Fourier space product
# ----------------------------------------------------------
## create indices aligned with fftpack's fft2 quadrants
lm = nfft / 2.
i0 = np.roll(np.arange(int(np.ceil(lm - 1)), int(-lm - 1), -1, dtype=int),
int(np.floor(lm)) + 1)
i = i0 * np.ones((nfft, nfft), dtype=int)
j0 = np.arange(0, -nfft, -1, dtype=int)
j = hankel(j0, np.roll(j0, 1))
# B
xfft = np.conjugate(xfft)
return xfft[j] * yfft[i] * zfft[i.T]
def TrainingSet_ML_Prices(prices,
minsize,
lag,
scale=False,
ZigZagFunc=ZigZag):
S = MinMaxScaler()
P = prices
M = minsize
Z = ZigZagFunc(P, M, True)
N = len(P)
T = {
'input': hankel(P[0:lag], P[lag - 1:]).T,
'label': np.full(N - lag + 1, 0)
}
#fliter Z according lag
for i in range(len(Z)):
tmin = Z[i]['tmin']
tmax = Z[i]['tmax']
if (tmin <= lag) and (lag < tmax):
Z[i]['tmin'] = lag
Z = Z[i:]
break
for i in range(len(Z)):
tmin = Z[i]['tmin']
tmax = Z[i]['tmax']
T['label'][tmin - lag:tmax - lag] = np.full(tmax - tmin, Z[i]['label'])
S = MinMaxScaler()
if scale:
P = S.fit_transform(P)
for i in range(len(T)):
T['input'][i] = S.transform(T['input'][i])
return T, S
def PronyExpFit(self, deg, x, y):
'''
Construct a sum of exponentials fit to a given time-sequence y by
using prony's method
input:
[deg]: int, number of exponentials
[x]: numpy array, sequence of regularly spaced points at which y is evaluated
[y]: numpy array, sequence
output:
[a]: numpy array, exponential coefficient
[c]: numpy array, exponentail magnitudes
[rms]: float, root mean square error of the data
'''
# stepsize
h = x[1] - x[0]
#Build matrix
A = la.hankel(y[:-deg],y[-deg-1:])
a = -A[:,:deg]
b = A[:,deg]
#Solve it
s = la.lstsq(a,b)[0]
#Solve polynomial
p = np.flipud(np.hstack((s,1)))
u = np.roots(p)
#Only keep roots in unit circle
inds = np.where(np.logical_and((np.abs(u) < 1.), \
np.logical_not(np.logical_and(np.imag(u) == 0., np.real(u) <= 0.))))[0]
u = u[inds]
#Calc exponential factors
a = np.log(u)/h
#Build power matrix
A = np.power((np.ones((len(y),1))*u[:,None].T),np.arange(len(y))[:,None]*np.ones((1,len(inds))))
#solve it
f = la.lstsq(A,y)[0]
#calc amplitudes
c = f/np.exp(a*x[0])
#build x, approx and calc rms
approx = self.sumExp(x, a, c).real
rms = np.sqrt(((approx-y)**2).sum() / len(y))
return a, c, rms
def bispectrumdx(x, y, z, nfft=None, wind=None, nsamp=None, overlap=None):
"""
Parameters:
x - data vector or time-series
y - data vector or time-series (same dimensions as x)
z - data vector or time-series (same dimensions as x)
nfft - fft length [default = power of two > segsamp]
wind - window specification for frequency-domain smoothing
if 'wind' is a scalar, it specifies the length of the side
of the square for the Rao-Gabr optimal window [default=5]
if 'wind' is a vector, a 2D window will be calculated via
w2(i,j) = wind(i) * wind(j) * wind(i+j)
if 'wind' is a matrix, it specifies the 2-D filter directly
segsamp - samples per segment [default: such that we have 8 segments]
- if x is a matrix, segsamp is set to the number of rows
overlap - percentage overlap, allowed range [0,99]. [default = 50];
- if x is a matrix, overlap is set to 0.
Output:
Bspec - estimated bispectrum: an nfft x nfft array, with origin
at the center, and axes pointing down and to the right.
waxis - vector of frequencies associated with the rows and columns
of Bspec; sampling frequency is assumed to be 1.
"""
(lx, lrecs) = x.shape
(ly, nrecs) = y.shape
(lz, krecs) = z.shape
if lx != ly or lrecs != nrecs or ly != lz or nrecs != krecs:
raise Exception('x, y and z should have identical dimensions')
if ly == 1:
x = x.reshape(1,-1)
y = y.reshape(1,-1)
z = z.reshape(1,-1)
ly = nrecs
nrecs = 1
if not overlap: overlap = 50
overlap = max(0,min(overlap,99))
if nrecs > 1: overlap = 0
if not nsamp: nsamp = 0
if nrecs > 1: nsamp = ly
if nrecs == 1 and nsamp <= 0:
nsamp = np.fix(ly/ (8 - 7 * overlap/100))
if nfft < nsamp:
nfft = 2**nextpow2(nsamp)
overlap = np.fix(overlap/100 * nsamp)
nadvance = nsamp - overlap
nrecs = np.fix((ly*nrecs - overlap) / nadvance)
# create the 2-D window
if not wind: wind = 5
m = n = 0
try:
(m, n) = wind.shape
except ValueError:
(m,) = wind.shape
n = 1
except AttributeError:
m = n = 1
window = wind
# scalar: wind is size of Rao-Gabr window
if max(m, n) == 1:
winsize = wind
if winsize < 0: winsize = 5 # the window size L
winsize = winsize - (winsize%2) + 1 # make it odd
if winsize > 1:
mwind = np.fix(nfft/winsize) # the scale parameter M
lby2 = (winsize - 1)/2
theta = np.array([np.arange(-1*lby2, lby2+1)]) # force a 2D array
opwind = np.ones([winsize, 1]) * (theta**2) # w(m,n) = m**2
opwind = opwind + opwind.transpose() + (np.transpose(theta) * theta) # m**2 + n**2 + mn
opwind = 1 - ((2*mwind/nfft)**2) * opwind
Hex = np.ones([winsize,1]) * theta
Hex = abs(Hex) + abs(np.transpose(Hex)) + abs(Hex + np.transpose(Hex))
Hex = (Hex < winsize)
opwind = opwind * Hex
opwind = opwind * (4 * mwind**2) / (7 * np.pi**2)
else:
opwind = 1
# 1-D window passed: convert to 2-D
elif min(m, n) == 1:
window = window.reshape(1,-1)
if np.any(np.imag(window)) != 0:
print "1-D window has imaginary components: window ignored"
window = 1
if np.any(window) < 0:
print "1-D window has negative components: window ignored"
window = 1
lwind = np.size(window)
w = window.ravel(order='F')
# the full symmetric 1-D
windf = np.array(w[range(lwind-1, 0, -1) + [window]])
window = np.array([window], np.zeros([lwind-1,1]))
# w(m)w(n)w(m+n)
opwind = (windf * np.transpose(windf)) * hankel(np.flipud(window), window)
winsize = np.size(window)
# 2-D window passed: use directly
else:
winsize = m
if m != n:
print "2-D window is not square: window ignored"
window = 1
winsize = m
if m%2 == 0:
print "2-D window does not have odd length: window ignored"
window = 1
winsize = m
opwind = window
# accumulate triple products
Bspec = np.zeros([nfft, nfft]) # the hankel mask (faster)
mask = hankel(np.arange(nfft),np.array([nfft-1]+range(nfft-1)))
locseg = np.arange(nsamp).transpose()
x = x.ravel(order='F')
y = y.ravel(order='F')
z = z.ravel(order='F')
for krec in xrange(nrecs):
xseg = x[locseg].reshape(1,-1)
yseg = y[locseg].reshape(1,-1)
zseg = z[locseg].reshape(1,-1)
Xf = np.fft.fft(xseg - np.mean(xseg), nfft) / nsamp
Yf = np.fft.fft(yseg - np.mean(yseg), nfft) / nsamp
CZf = np.fft.fft(zseg - np.mean(zseg), nfft) / nsamp
CZf = np.conjugate(CZf).ravel(order='F')
Bspec = Bspec + \
flat_eq(Bspec, (Xf * np.transpose(Yf)) * CZf[mask].reshape(nfft, nfft))
locseg = locseg + int(nadvance)
Bspec = np.fft.fftshift(Bspec) / nrecs
# frequency-domain smoothing
if winsize > 1:
lby2 = int((winsize-1)/2)
Bspec = convolve2d(Bspec,opwind)
Bspec = Bspec[range(lby2+1,lby2+nfft+1), :][:, np.arange(lby2+1,lby2+nfft+1)]
if nfft%2 == 0:
waxis = np.transpose(np.arange(-1*nfft/2, nfft/2)) / nfft
else:
waxis = np.transpose(np.arange(-1*(nfft-1)/2, (nfft-1)/2+1)) / nfft
# cont1 = plt.contour(abs(Bspec), 4, waxis, waxis)
cont = plt.contourf(waxis, waxis, abs(Bspec), 100, cmap=plt.cm.Spectral_r)
plt.colorbar(cont)
plt.title('Bispectrum estimated via the direct (FFT) method')
plt.xlabel('f1')
plt.ylabel('f2')
plt.show()
return (Bspec, waxis)
def calc_ss_radiation(self, max_order=10, r2_thresh=0.95):
'''Function to calculate the state space reailization of the wave
radiation IRF.
Args:
max_order : int
Maximum order of the state space reailization fit
r2_thresh : float
The R^2 threshold used for the fit. THe value must be between 0
and 1
Returns:
No variables are directily returned by thi function. The following
internal variables are calculated:
Ass :
time-invariant state matrix
Bss :
time-invariant input matrix
Css :
time-invariant output matrix
Dss :
time-invariant feedthrough matrix
k_ss_est :
Impusle response function as cacluated from state space
approximation
status :
status of the realization, 0 - zero hydrodynamic oefficients, 1
- state space realization meets R2 thresholdm 2 - state space
realization does not meet R2 threshold and at ss_max limit
Examples:
'''
dt = self.rd.irf.t[2] - self.rd.irf.t[1]
r2bt = np.zeros([self.am.inf.shape[0], self.am.inf.shape[0], self.rd.irf.t.size])
k_ss_est = np.zeros(self.rd.irf.t.size)
self.rd.ss.irk_bss = np.zeros([self.am.inf.shape[0], self.am.inf.shape[0], self.rd.irf.t.size])
self.rd.ss.A = np.zeros([6, self.am.inf.shape[1], max_order, max_order])
self.rd.ss.B = np.zeros([6, self.am.inf.shape[1], max_order, 1])
self.rd.ss.C = np.zeros([6, self.am.inf.shape[1], 1, max_order])
self.rd.ss.D = np.zeros([6, self.am.inf.shape[1], 1])
self.rd.ss.irk_bss = np.zeros([6, self.am.inf.shape[1], self.rd.irf.t.size])
self.rd.ss.rad_conv = np.zeros([6, self.am.inf.shape[1]])
self.rd.ss.it = np.zeros([6, self.am.inf.shape[1]])
self.rd.ss.r2t = np.zeros([6, self.am.inf.shape[1]])
pbar = ProgressBar(widgets=['Radiation damping state space realization for ' + self.name + ':', Percentage(), Bar()], maxval=self.am.inf.shape[0] * self.am.inf.shape[1]).start()
count = 0
for i in xrange(self.am.inf.shape[0]):
for j in xrange(self.am.inf.shape[1]):
r2bt = np.linalg.norm(
self.rd.irf.K[i, j, :] - self.rd.irf.K.mean(axis=2)[i, j])
ss = 2 # Initial state space order
if r2bt != 0.0:
while True:
# Perform Hankel Singular Value Decomposition
y = dt * self.rd.irf.K[i, j, :]
h = hankel(y[1::])
u, svh, v = np.linalg.svd(h)
u1 = u[0:self.rd.irf.t.size - 2, 0:ss]
v1 = v.T[0:self.rd.irf.t.size - 2, 0:ss]
u2 = u[1:self.rd.irf.t.size - 1, 0:ss]
sqs = np.sqrt(svh[0:ss].reshape(ss, 1))
invss = 1 / sqs
ubar = np.dot(u1.T, u2)
a = ubar * np.dot(invss, sqs.T)
b = v1[0, :].reshape(ss, 1) * sqs
c = u1[0, :].reshape(1, ss) * sqs.T
d = y[0]
CoeA = dt / 2
CoeB = 1
CoeC = -CoeA
CoeD = 1
# (T/2*I + T/2*A)^{-1} = 2/T(I + A)^{-1}
iidd = np.linalg.inv(CoeA * np.eye(ss) - CoeC * a)
# (A-I)2/T(I + A)^{-1} = 2/T(A-I)(I + A)^{-1}
ac = np.dot(CoeB * a - CoeD * np.eye(ss), iidd)
# (T/2+T/2)*2/T(I + A)^{-1}B = 2(I + A)^{-1}B
bc = (CoeA * CoeB - CoeC * CoeD) * np.dot(iidd, b)
# C * 2/T(I + A)^{-1} = 2/T(I + A)^{-1}
cc = np.dot(c, iidd)
# D - T/2C (2/T(I + A)^{-1})B = D - C(I + A)^{-1})B
dc = d + CoeC * np.dot(np.dot(c, iidd), b)
for jj in xrange(self.rd.irf.t.size):
# Calculate impulse response function from state space
# approximation
k_ss_est[jj] = np.dot(np.dot(cc, expm(ac * dt * jj)), bc)
# Calculate 2 norm of the difference between know and estimated
# values impulse response function
R2TT = np.linalg.norm(self.rd.irf.K[i, j, :] - k_ss_est)
# Calculate the R2 value for impulse response function
R2T = 1 - np.square(R2TT / r2bt)
# Check to see if threshold for the impulse response is meet
if R2T >= r2_thresh:
status = 1 # %Set status
break
# Check to see if limit on the state space order has been reached
if ss == max_order:
status = 2 # %Set status
break
ss = ss + 1 # Increase state space order
self.rd.ss.A[i, j, 0:ac.shape[0], 0:ac.shape[0]] = ac
self.rd.ss.B[i, j, 0:bc.shape[0], 0] = bc[:, 0]
self.rd.ss.C[i, j, 0, 0:cc.shape[1]] = cc[0, :]
self.rd.ss.D[i, j] = dc
self.rd.ss.irk_bss[i, j, :] = k_ss_est
self.rd.ss.rad_conv[i, j] = status
self.rd.ss.r2t[i, j] = R2T
self.rd.ss.it[i, j] = ss
count += 1
pbar.update(count)
pbar.finish()
def update(self, x_n, d_n):
self.n += 1
self.update_buf(x_n, d_n)
# get this randomness
pr = np.random.uniform(size=(self.N))
I = np.arange(self.N)[pr < self.q]
if I.shape[0] > 0:
# extract data from buffer
L = self.buf_ind
x = self.x[(self.length-1)+self.buf_ind-1::-1]
d = self.d[self.buf_ind-1::-1]
# compute more power of lambda if needed
if L >= self.lmbd_pwr.shape[0]:
self.lmbd_pwr = np.concatenate((self.lmbd_pwr, self.lmbd**np.arange(self.lmbd_pwr.shape[0], L+1)))
# block-update parameters
X = la.hankel(x[:L],r=x[L-1:])
Lambda_diag = self.lmbd_pwr[:L]
lmbd_L = self.lmbd_pwr[L]
""" As of now, using BLAS dot is more efficient than the FFT based method toeplitz_multiplication.
rhs = (self.lmbd**np.arange(L) * X.T).T
self.R = (self.lmbd**L)*self.R + hankel_multiplication(x[:self.length], x[self.length-1:], rhs)
""" # Use thus dot instead
# block-update the auto-correlation matrix
rhs = (Lambda_diag * X.T).T
self.R = lmbd_L*self.R + np.dot(X.T, rhs)
# block-update of the cross-correlation vector here
self.r_dx = lmbd_L*self.r_dx + np.dot(X.T, Lambda_diag*d)
# update the sketches as needed
for i in np.arange(self.N):
if pr[i] < self.q:
self.last_update[i] = self.n
buf = np.zeros(self.length)
if self.sketch == 'RowSample':
buf = x[:self.length]
elif self.sketch == 'Binary':
data = (np.sqrt(Lambda_diag) * X.T)
buf = np.dot(data, np.random.choice([-1,1], size=self.L))*np.sqrt(self.L)
x_up = buf.copy()
if self.mem_buf_size > 0:
for row in self.mem_buf:
x_up += np.random.choice([-1,1])*row*np.sqrt(lmbd_L)
# update the sketch buffer
self.mem_buf *= np.sqrt(lmbd_L)
self.mem_buf[1:,:] = self.mem_buf[0:-1,:]
self.mem_buf[0,:] = buf
# update the inverse correlation matrix
mu = 1/lmbd_L
pi = np.dot(self.P[i,:,:], x_up)
denom = (self.q*lmbd_L + np.inner(x_up, pi))
np.outer(pi, pi/denom, out=self.out_buf)
self.P[i,:,:] = self.P[i,:,:] - self.out_buf
self.P[i,:,:] *= mu
# do IHS
self.w = self.w + \
np.dot(self.P[i,:,:], self.r_dx - np.dot(self.R, self.w))/self.n
# Flush the buffers
self.x[:self.length-1] = self.x[self.buf_ind:self.buf_ind+self.length-1]
self.x[self.length-1:] = 0
self.d[:] = 0
self.buf_ind = 0
def fit_direct(x, y, F=0, weighted=True, _weights=None):
"""Fit a Gaussian to the given data.
Returns a fit so that y ~ gauss(x, A, mu, sigma)
Parameters
----------
x : ndarray
Sampling positions.
y : ndarray
Sampled values.
F : float
Ignore values of y <= F.
weighted : bool
Whether to use weighted least squares. If True, weigh
the error function by y, ensuring that small values
has less influence on the outcome.
Additional Parameters
---------------------
_weights : ndarray
Weights used in weighted least squares. For internal use
by fit_iterative.
Returns
-------
A : float
Amplitude.
mu : float
Mean.
std : float
Standard deviation.
"""
mask = (y > F)
x = x[mask]
y = y[mask]
if _weights is None:
_weights = y
else:
_weights = _weights[mask]
# We do not want to risk working with negative values
np.clip(y, 1e-10, np.inf, y)
e = np.ones(len(x))
if weighted:
e = e * (_weights**2)
v = (np.sum(np.vander(x, 5) * e[:, None], axis=0))[::-1]
A = v[sl.hankel([0, 1, 2], [2, 3, 4])]
ly = e * np.log(y)
ls = np.sum(ly)
x_ls = np.sum(ly * x)
xx_ls = np.sum(ly * x**2)
B = np.array([ls, x_ls, xx_ls])
(a, b, c), res, rank, s = np.linalg.lstsq(A, B)
A = np.exp(a - (b**2 / (4 * c)))
mu = -b / (2 * c)
sigma = sp.sqrt(-1 / (2 * c))
return A, mu, sigma
def ts_rf(n,fea,step,ntrees,njobs):
#Random Forest Model for time series prediction
#from sklearn import svm
import math
from sklearn import metrics
import matplotlib.pyplot as plt
from scipy.linalg import hankel
import numpy as np
from sklearn.ensemble.forest import RandomForestRegressor
#input data from csv file
#use n datapoints
#n=1100
# # of features of training set
## fre=50
# # how many steps to predict
#step=29
#fea=50
path='/Users/royyang/Desktop/time_series_forecasting/csv_files/coffee_ls.txt'
path1 = '/Users/royyang/Desktop/time_series_forecasting/csv_files/coffee_ls_nor.txt'
result_tem=[]
date = []
with open(path) as f:
next(f)
for line in f:
item=line.replace('\n','').split(' ')
result_tem.append(float(item[1]))
date.append(item[2])
mean = np.mean(result_tem)
sd = np.std(result_tem)
result=(result_tem-mean)/sd
#form hankel matrix
X=hankel(result[0:-fea-step+1], result[-1-fea:-1])
y=result[fea+step-1:]
#split data into training and testing
Xtrain=X[:n]
ytrain=y[:n]
Xtest=X[n:]
ytest=y[n:]
# random forest
rf = RandomForestRegressor(n_estimators = ntrees, n_jobs=njobs)
rf_pred = rf.fit(Xtrain, ytrain).predict(Xtest)
#a = rf.transform(Xtrain,'median')
#plot results
LABELS = [x[-6:] for x in date[n+fea+step-1:n+fea+step-1+len(ytest)]]
t=range(n,n+len(ytest))
# plt.show()
# plt.plot(t,y_lin1,'r--',t,ytest,'b^-')
# plt.plot(t,y_lin2,'g--',t,ytest,'b^-')
ypred = rf_pred*sd+mean
ytest = ytest*sd+mean
line1, = plt.plot(t,ypred,'r*-')
plt.xticks(t, LABELS)
line2, = plt.plot(t,ytest,'b*-')
# plt.xlim([500,510])
plt.legend([line1, line2], ["Predicted", "Actual"], loc=2)
#plt.show()
#plt.plot(xrange(n),result[0:n],'r--',t,y_lin3,'b--',t,ytest,'r--')
y_true = ytest
y_pred = ypred
metrics_result = {'rf_MAE':metrics.mean_absolute_error(y_true, y_pred),'rf_MSE':metrics.mean_squared_error(y_true, y_pred),
'rf_MAPE':np.mean(np.abs((y_true - y_pred) / y_true)) * 100}
print metrics_result
def ssa(x, r, fullMatr=True, comp_uv=False): # x is nt x 1 vector, r is embedding dimension
#nt = len(x); mean = sum(x)/nt
#x = x - ones(nt)*mean
H = scilinalg.hankel(x, zeros(r)) #Construct Hankel matrix
return scilinalg.svd(H, full_matrices=fullMatr, compute_uv=comp_uv)
def detect_clearsky(measured, clearsky, times, window_length,
mean_diff=75, max_diff=75,
lower_line_length=-5, upper_line_length=10,
var_diff=0.005, slope_dev=8, max_iterations=20,
return_components=False):
"""
Detects clear sky times according to the algorithm developed by Reno
and Hansen for GHI measurements [1]. The algorithm was designed and
validated for analyzing GHI time series only. Users may attempt to
apply it to other types of time series data using different filter
settings, but should be skeptical of the results.
The algorithm detects clear sky times by comparing statistics for a
measured time series and an expected clearsky time series.
Statistics are calculated using a sliding time window (e.g., 10
minutes). An iterative algorithm identifies clear periods, uses the
identified periods to estimate bias in the clearsky data, scales the
clearsky data and repeats.
Clear times are identified by meeting 5 criteria. Default values for
these thresholds are appropriate for 10 minute windows of 1 minute
GHI data.
Parameters
----------
measured : array or Series
Time series of measured values.
clearsky : array or Series
Time series of the expected clearsky values.
times : DatetimeIndex
Times of measured and clearsky values.
window_length : int
Length of sliding time window in minutes. Must be greater than 2
periods.
mean_diff : float, default 75
Threshold value for agreement between mean values of measured
and clearsky in each interval, see Eq. 6 in [1].
max_diff : float, default 75
Threshold value for agreement between maxima of measured and
clearsky values in each interval, see Eq. 7 in [1].
lower_line_length : float, default -5
Lower limit of line length criterion from Eq. 8 in [1].
Criterion satisfied when
lower_line_length < line length difference < upper_line_length
upper_line_length : float, default 10
Upper limit of line length criterion from Eq. 8 in [1].
var_diff : float, default 0.005
Threshold value in Hz for the agreement between normalized
standard deviations of rate of change in irradiance, see Eqs. 9
through 11 in [1].
slope_dev : float, default 8
Threshold value for agreement between the largest magnitude of
change in successive values, see Eqs. 12 through 14 in [1].
max_iterations : int, default 20
Maximum number of times to apply a different scaling factor to
the clearsky and redetermine clear_samples. Must be 1 or larger.
return_components : bool, default False
Controls if additional output should be returned. See below.
Returns
-------
clear_samples : array or Series
Boolean array or Series of whether or not the given time is
clear. Return type is the same as the input type.
components : OrderedDict, optional
Dict of arrays of whether or not the given time window is clear
for each condition. Only provided if return_components is True.
alpha : scalar, optional
Scaling factor applied to the clearsky_ghi to obtain the
detected clear_samples. Only provided if return_components is
True.
References
----------
[1] Reno, M.J. and C.W. Hansen, "Identification of periods of clear
sky irradiance in time series of GHI measurements" Renewable Energy,
v90, p. 520-531, 2016.
Notes
-----
Initial implementation in MATLAB by Matthew Reno. Modifications for
computational efficiency by Joshua Patrick and Curtis Martin. Ported
to Python by Will Holmgren, Tony Lorenzo, and Cliff Hansen.
Differences from MATLAB version:
* no support for unequal times
* automatically determines sample_interval
* requires a reference clear sky series instead calculating one
from a user supplied location and UTCoffset
* parameters are controllable via keyword arguments
* option to return individual test components and clearsky scaling
parameter
"""
# calculate deltas in units of minutes (matches input window_length units)
deltas = np.diff(times.values) / np.timedelta64(1, '60s')
# determine the unique deltas and if we can proceed
unique_deltas = np.unique(deltas)
if len(unique_deltas) == 1:
sample_interval = unique_deltas[0]
else:
raise NotImplementedError('algorithm does not yet support unequal '
'times. consider resampling your data.')
intervals_per_window = int(window_length / sample_interval)
# generate matrix of integers for creating windows with indexing
from scipy.linalg import hankel
H = hankel(np.arange(intervals_per_window), # noqa: N806
np.arange(intervals_per_window - 1, len(times)))
# calculate measurement statistics
meas_mean = np.mean(measured[H], axis=0)
meas_max = np.max(measured[H], axis=0)
meas_diff = np.diff(measured[H], n=1, axis=0)
meas_slope = np.diff(measured[H], n=1, axis=0) / sample_interval
# matlab std function normalizes by N-1, so set ddof=1 here
meas_slope_nstd = np.std(meas_slope, axis=0, ddof=1) / meas_mean
meas_line_length = np.sum(np.sqrt(
meas_diff * meas_diff +
sample_interval * sample_interval), axis=0)
# calculate clear sky statistics
clear_mean = np.mean(clearsky[H], axis=0)
clear_max = np.max(clearsky[H], axis=0)
clear_diff = np.diff(clearsky[H], n=1, axis=0)
clear_slope = np.diff(clearsky[H], n=1, axis=0) / sample_interval
from scipy.optimize import minimize_scalar
alpha = 1
for iteration in range(max_iterations):
clear_line_length = np.sum(np.sqrt(
alpha * alpha * clear_diff * clear_diff +
sample_interval * sample_interval), axis=0)
line_diff = meas_line_length - clear_line_length
# evaluate comparison criteria
c1 = np.abs(meas_mean - alpha*clear_mean) < mean_diff
c2 = np.abs(meas_max - alpha*clear_max) < max_diff
c3 = (line_diff > lower_line_length) & (line_diff < upper_line_length)
c4 = meas_slope_nstd < var_diff
c5 = np.max(np.abs(meas_slope -
alpha * clear_slope), axis=0) < slope_dev
c6 = (clear_mean != 0) & ~np.isnan(clear_mean)
clear_windows = c1 & c2 & c3 & c4 & c5 & c6
# create array to return
clear_samples = np.full_like(measured, False, dtype='bool')
# find the samples contained in any window classified as clear
clear_samples[np.unique(H[:, clear_windows])] = True
# find a new alpha
previous_alpha = alpha
clear_meas = measured[clear_samples]
clear_clear = clearsky[clear_samples]
def rmse(alpha):
return np.sqrt(np.mean((clear_meas - alpha*clear_clear)**2))
alpha = minimize_scalar(rmse).x
if round(alpha*10000) == round(previous_alpha*10000):
break
else:
import warnings
warnings.warn('failed to converge after %s iterations'
% max_iterations, RuntimeWarning)
# be polite about returning the same type as was input
if isinstance(measured, pd.Series):
clear_samples = pd.Series(clear_samples, index=times)
if return_components:
components = OrderedDict()
components['mean_diff_flag'] = c1
components['max_diff_flag'] = c2
components['line_length_flag'] = c3
components['slope_nstd_flag'] = c4
components['slope_max_flag'] = c5
components['mean_nan_flag'] = c6
components['windows'] = clear_windows
components['mean_diff'] = np.abs(meas_mean - alpha * clear_mean)
components['max_diff'] = np.abs(meas_max - alpha * clear_max)
components['line_length'] = meas_line_length - clear_line_length
components['slope_nstd'] = meas_slope_nstd
components['slope_max'] = (np.max(
meas_slope - alpha * clear_slope, axis=0))
return clear_samples, components, alpha
else:
return clear_samples
def firls(numtaps, bands, desired, weight=None, nyq=None, fs=None):
"""
FIR filter design using least-squares error minimization.
Calculate the filter coefficients for the linear-phase finite
impulse response (FIR) filter which has the best approximation
to the desired frequency response described by `bands` and
`desired` in the least squares sense (i.e., the integral of the
weighted mean-squared error within the specified bands is
minimized).
Parameters
----------
numtaps : int
The number of taps in the FIR filter. `numtaps` must be odd.
bands : array_like
A monotonic nondecreasing sequence containing the band edges in
Hz. All elements must be non-negative and less than or equal to
the Nyquist frequency given by `nyq`.
desired : array_like
A sequence the same size as `bands` containing the desired gain
at the start and end point of each band.
weight : array_like, optional
A relative weighting to give to each band region when solving
the least squares problem. `weight` has to be half the size of
`bands`.
nyq : float, optional
*Deprecated. Use `fs` instead.*
Nyquist frequency. Each frequency in `bands` must be between 0
and `nyq` (inclusive). Default is 1.
fs : float, optional
The sampling frequency of the signal. Each frequency in `bands`
must be between 0 and ``fs/2`` (inclusive). Default is 2.
Returns
-------
coeffs : ndarray
Coefficients of the optimal (in a least squares sense) FIR filter.
See also
--------
firwin
firwin2
minimum_phase
remez
Notes
-----
This implementation follows the algorithm given in [1]_.
As noted there, least squares design has multiple advantages:
1. Optimal in a least-squares sense.
2. Simple, non-iterative method.
3. The general solution can obtained by solving a linear
system of equations.
4. Allows the use of a frequency dependent weighting function.
This function constructs a Type I linear phase FIR filter, which
contains an odd number of `coeffs` satisfying for :math:`n < numtaps`:
.. math:: coeffs(n) = coeffs(numtaps - 1 - n)
The odd number of coefficients and filter symmetry avoid boundary
conditions that could otherwise occur at the Nyquist and 0 frequencies
(e.g., for Type II, III, or IV variants).
.. versionadded:: 0.18
References
----------
.. [1] Ivan Selesnick, Linear-Phase Fir Filter Design By Least Squares.
OpenStax CNX. Aug 9, 2005.
http://cnx.org/contents/eb1ecb35-03a9-4610-ba87-41cd771c95f2@7
Examples
--------
We want to construct a band-pass filter. Note that the behavior in the
frequency ranges between our stop bands and pass bands is unspecified,
and thus may overshoot depending on the parameters of our filter:
>>> from scipy import signal
>>> import matplotlib.pyplot as plt
>>> fig, axs = plt.subplots(2)
>>> fs = 10.0 # Hz
>>> desired = (0, 0, 1, 1, 0, 0)
>>> for bi, bands in enumerate(((0, 1, 2, 3, 4, 5), (0, 1, 2, 4, 4.5, 5))):
... fir_firls = signal.firls(73, bands, desired, fs=fs)
... fir_remez = signal.remez(73, bands, desired[::2], fs=fs)
... fir_firwin2 = signal.firwin2(73, bands, desired, fs=fs)
... hs = list()
... ax = axs[bi]
... for fir in (fir_firls, fir_remez, fir_firwin2):
... freq, response = signal.freqz(fir)
... hs.append(ax.semilogy(0.5*fs*freq/np.pi, np.abs(response))[0])
... for band, gains in zip(zip(bands[::2], bands[1::2]),
... zip(desired[::2], desired[1::2])):
... ax.semilogy(band, np.maximum(gains, 1e-7), 'k--', linewidth=2)
... if bi == 0:
... ax.legend(hs, ('firls', 'remez', 'firwin2'),
... loc='lower center', frameon=False)
... else:
... ax.set_xlabel('Frequency (Hz)')
... ax.grid(True)
... ax.set(title='Band-pass %d-%d Hz' % bands[2:4], ylabel='Magnitude')
...
>>> fig.tight_layout()
>>> plt.show()
""" # noqa
nyq = 0.5 * _get_fs(fs, nyq)
numtaps = int(numtaps)
if numtaps % 2 == 0 or numtaps < 1:
raise ValueError("numtaps must be odd and >= 1")
M = (numtaps-1) // 2
# normalize bands 0->1 and make it 2 columns
nyq = float(nyq)
if nyq <= 0:
raise ValueError('nyq must be positive, got %s <= 0.' % nyq)
bands = np.asarray(bands).flatten() / nyq
if len(bands) % 2 != 0:
raise ValueError("bands must contain frequency pairs.")
bands.shape = (-1, 2)
# check remaining params
desired = np.asarray(desired).flatten()
if bands.size != desired.size:
raise ValueError("desired must have one entry per frequency, got %s "
"gains for %s frequencies."
% (desired.size, bands.size))
desired.shape = (-1, 2)
if (np.diff(bands) <= 0).any() or (np.diff(bands[:, 0]) < 0).any():
raise ValueError("bands must be monotonically nondecreasing and have "
"width > 0.")
if (bands[:-1, 1] > bands[1:, 0]).any():
raise ValueError("bands must not overlap.")
if (desired < 0).any():
raise ValueError("desired must be non-negative.")
if weight is None:
weight = np.ones(len(desired))
weight = np.asarray(weight).flatten()
if len(weight) != len(desired):
raise ValueError("weight must be the same size as the number of "
"band pairs (%s)." % (len(bands),))
if (weight < 0).any():
raise ValueError("weight must be non-negative.")
# Set up the linear matrix equation to be solved, Qa = b
# We can express Q(k,n) = 0.5 Q1(k,n) + 0.5 Q2(k,n)
# where Q1(k,n)=q(k−n) and Q2(k,n)=q(k+n), i.e. a Toeplitz plus Hankel.
# We omit the factor of 0.5 above, instead adding it during coefficient
# calculation.
# We also omit the 1/π from both Q and b equations, as they cancel
# during solving.
# We have that:
# q(n) = 1/π ∫W(ω)cos(nω)dω (over 0->π)
# Using our nomalization ω=πf and with a constant weight W over each
# interval f1->f2 we get:
# q(n) = W∫cos(πnf)df (0->1) = Wf sin(πnf)/πnf
# integrated over each f1->f2 pair (i.e., value at f2 - value at f1).
n = np.arange(numtaps)[:, np.newaxis, np.newaxis]
q = np.dot(np.diff(np.sinc(bands * n) * bands, axis=2)[:, :, 0], weight)
# Now we assemble our sum of Toeplitz and Hankel
Q1 = toeplitz(q[:M+1])
Q2 = hankel(q[:M+1], q[M:])
Q = Q1 + Q2
# Now for b(n) we have that:
# b(n) = 1/π ∫ W(ω)D(ω)cos(nω)dω (over 0->π)
# Using our normalization ω=πf and with a constant weight W over each
# interval and a linear term for D(ω) we get (over each f1->f2 interval):
# b(n) = W ∫ (mf+c)cos(πnf)df
# = f(mf+c)sin(πnf)/πnf + mf**2 cos(nπf)/(πnf)**2
# integrated over each f1->f2 pair (i.e., value at f2 - value at f1).
n = n[:M + 1] # only need this many coefficients here
# Choose m and c such that we are at the start and end weights
m = (np.diff(desired, axis=1) / np.diff(bands, axis=1))
c = desired[:, [0]] - bands[:, [0]] * m
b = bands * (m*bands + c) * np.sinc(bands * n)
# Use L'Hospital's rule here for cos(nπf)/(πnf)**2 @ n=0
b[0] -= m * bands * bands / 2.
b[1:] += m * np.cos(n[1:] * np.pi * bands) / (np.pi * n[1:]) ** 2
b = np.dot(np.diff(b, axis=2)[:, :, 0], weight)
# Now we can solve the equation (use pinv because Q can be rank deficient)
a = np.dot(pinv(Q), b)
# make coefficients symmetric (linear phase)
coeffs = np.hstack((a[:0:-1], 2 * a[0], a[1:]))
return coeffs
def construct_Hankel_matrices(self, y):
ind0 = int(len(y)/2)
# original and shifted hankel matrix
H0 = la.hankel(y[0:ind0], y[ind0-1:2*ind0-1])
H1 = la.hankel(y[1:ind0+1], y[ind0:2*ind0])
return H0, H1
def ssa(x, r): # x is nt x 1 vector, r is embedding dimension
#nt = len(x); mean = sum(x)/nt
#x = x - ones(nt)*mean
H = linalg.hankel(x, zeros(r)) #Construct Hankel matrix
return linalg.svd(H, compute_uv=False)
def bicoherence(y, nfft=None, wind=None, nsamp=None, overlap=None):
"""
Direct (FD) method for estimating bicoherence
Parameters:
y - data vector or time-series
nfft - fft length [default = power of two > segsamp]
actual size used is power of two greater than 'nsamp'
wind - specifies the time-domain window to be applied to each
data segment; should be of length 'segsamp' (see below);
otherwise, the default Hanning window is used.
segsamp - samples per segment [default: such that we have 8 segments]
- if x is a matrix, segsamp is set to the number of rows
overlap - percentage overlap, allowed range [0,99]. [default = 50];
- if x is a matrix, overlap is set to 0.
Output:
bic - estimated bicoherence: an nfft x nfft array, with origin
at the center, and axes pointing down and to the right.
waxis - vector of frequencies associated with the rows and columns
of bic; sampling frequency is assumed to be 1.
"""
# Parameter checks
(ly, nrecs) = y.shape
if ly == 1:
y = y.reshape(1, -1)
ly = nrecs
nrecs = 1
if not nfft: nfft = 128
if not overlap: overlap = 50
if nrecs > 1: overlap = 0
if not nsamp: nsamp = 0
if nrecs > 1: nsamp = ly
if nrecs > 1 and nsamp <= 0:
nsamp = np.fix(ly / (8 - 7 * overlap/100))
if nfft < nsamp:
nfft = 2**nextpow2(nsamp)
overlap = np.fix(nsamp * overlap/100)
nadvance = nsamp - overlap
nrecs = np.fix ((ly*nrecs - overlap) / nadvance)
if not wind:
wind = np.hanning(nsamp)
try:
(rw, cw) = wind.shape
except ValueError:
(rw,) = wind.shape
cw = 1
if min(rw, cw) == 1 or max(rw, cw) == nsamp:
print "Segment size is " + str(nsamp)
print "Wind array is " + str(rw) + " by " + str(cw)
print "Using default Hanning window"
wind = np.hanning(nsamp)
wind = wind.reshape(1,-1)
# Accumulate triple products
bic = np.zeros([nfft, nfft])
Pyy = np.zeros([nfft,1])
mask = hankel(np.arange(nfft),np.array([nfft-1]+range(nfft-1)))
Yf12 = np.zeros([nfft,nfft])
ind = np.arange(nsamp)
y = y.ravel(order='F')
for k in xrange(nrecs):
ys = y[ind]
ys = (ys.reshape(1,-1) - np.mean(ys)) * wind
Yf = np.fft.fft(ys, nfft)/nsamp
CYf = np.conjugate(Yf)
Pyy = Pyy + flat_eq(Pyy, (Yf*CYf))
Yf12 = flat_eq(Yf12, CYf.ravel(order='F')[mask])
bic = bic + ((Yf * np.transpose(Yf)) * Yf12)
ind = ind + int(nadvance)
bic = bic / nrecs
Pyy = Pyy / nrecs
mask = flat_eq(mask, Pyy.ravel(order='F')[mask])
bic = abs(bic)**2 / ((Pyy * np.transpose(Pyy)) * mask)
bic = np.fft.fftshift(bic)
if nfft%2 == 0:
waxis = np.transpose(np.arange(-1*nfft/2, nfft/2)) / nfft
else:
waxis = np.transpose(np.arange(-1*(nfft-1)/2, (nfft-1)/2+1)) / nfft
cont = plt.contourf(waxis,waxis,bic,100, cmap=plt.cm.Spectral_r)
plt.colorbar(cont)
plt.title('Bicoherence estimated via the direct (FFT) method')
plt.xlabel('f1')
plt.ylabel('f2')
colmax, row = bic.max(0), bic.argmax(0)
maxval, col = colmax.max(0), colmax.argmax(0)
print 'Max: bic('+str(waxis[col])+','+str(waxis[col])+') = '+str(maxval)
plt.show()
return (bic, waxis)
def bicoherencex(w, x, y, nfft=None, wind=None, nsamp=None, overlap=None):
"""
Direct (FD) method for estimating cross-bicoherence
Parameters:
w,x,y - data vector or time-series
- should have identical dimensions
nfft - fft length [default = power of two > nsamp]
actual size used is power of two greater than 'nsamp'
wind - specifies the time-domain window to be applied to each
data segment; should be of length 'segsamp' (see below);
otherwise, the default Hanning window is used.
segsamp - samples per segment [default: such that we have 8 segments]
- if x is a matrix, segsamp is set to the number of rows
overlap - percentage overlap, 0 to 99 [default = 50]
- if y is a matrix, overlap is set to 0.
Output:
bic - estimated cross-bicoherence: an nfft x nfft array, with
origin at center, and axes pointing down and to the right.
waxis - vector of frequencies associated with the rows and columns
of bic; sampling frequency is assumed to be 1.
"""
if w.shape != x.shape or x.shape != y.shape:
raise ValueError('w, x and y should have identical dimentions')
(ly, nrecs) = y.shape
if ly == 1:
ly = nrecs
nrecs = 1
w = w.reshape(1,-1)
x = x.reshape(1,-1)
y = y.reshape(1,-1)
if not nfft:
nfft = 128
if not overlap: overlap = 50
overlap = max(0,min(overlap,99))
if nrecs > 1: overlap = 0
if not nsamp: nsamp = 0
if nrecs > 1: nsamp = ly
if nrecs == 1 and nsamp <= 0:
nsamp = np.fix(ly/ (8 - 7 * overlap/100))
if nfft < nsamp:
nfft = 2**nextpow2(nsamp)
overlap = np.fix(overlap/100 * nsamp)
nadvance = nsamp - overlap
nrecs = np.fix((ly*nrecs - overlap) / nadvance)
if not wind:
wind = np.hanning(nsamp)
try:
(rw, cw) = wind.shape
except ValueError:
(rw,) = wind.shape
cw = 1
if min(rw, cw) != 1 or max(rw, cw) != nsamp:
print "Segment size is " + str(nsamp)
print "Wind array is " + str(rw) + " by " + str(cw)
print "Using default Hanning window"
wind = np.hanning(nsamp)
wind = wind.reshape(1,-1)
# Accumulate triple products
bic = np.zeros([nfft, nfft])
Pyy = np.zeros([nfft,1])
Pww = np.zeros([nfft,1])
Pxx = np.zeros([nfft,1])
mask = hankel(np.arange(nfft),np.array([nfft-1]+range(nfft-1)))
Yf12 = np.zeros([nfft,nfft])
ind = np.transpose(np.arange(nsamp))
w = w.ravel(order='F')
x = x.ravel(order='F')
y = y.ravel(order='F')
for k in xrange(nrecs):
ws = w[ind]
ws = (ws - np.mean(ws)) * wind
Wf = np.fft.fft(ws, nfft) / nsamp
CWf = np.conjugate(Wf)
Pww = Pww + flat_eq(Pww, (Wf*CWf))
xs = x[ind]
xs = (xs - np.mean(xs)) * wind
Xf = np.fft.fft(xs, nfft) / nsamp
CXf = np.conjugate(Xf)
Pxx = Pxx + flat_eq(Pxx, (Xf*CXf))
ys = y[ind]
ys = (ys - np.mean(ys)) * wind
Yf = np.fft.fft(ys, nfft) / nsamp
CYf = np.conjugate(Yf)
Pyy = Pyy + flat_eq(Pyy, (Yf*CYf))
Yf12 = flat_eq(Yf12, CYf.ravel(order='F')[mask])
bic = bic + (Wf * np.transpose(Xf)) * Yf12
ind = ind + int(nadvance)
bic = bic / nrecs
Pww = Pww / nrecs
Pxx = Pxx / nrecs
Pyy = Pyy / nrecs
mask = flat_eq(mask, Pyy.ravel(order='F')[mask])
bic = abs(bic)**2 / ((Pww * np.transpose(Pxx)) * mask)
bic = np.fft.fftshift(bic)
# Contour plot of magnitude bispectrum
if nfft%2 == 0:
waxis = np.transpose(np.arange(-1*nfft/2, nfft/2)) / nfft
else:
waxis = np.transpose(np.arange(-1*(nfft-1)/2, (nfft-1)/2+1)) / nfft
cont = plt.contourf(waxis,waxis,bic,100, cmap=plt.cm.Spectral_r)
plt.colorbar(cont)
plt.title('Bicoherence estimated via the direct (FFT) method')
plt.xlabel('f1')
plt.ylabel('f2')
colmax, row = bic.max(0), bic.argmax(0)
maxval, col = colmax.max(0), colmax.argmax(0)
print 'Max: bic('+str(waxis[col])+','+str(waxis[col])+') = '+str(maxval)
plt.show()
return (bic, waxis)
#how many data points do we have?
len(result)
for i,item in enumerate(result):
if i % 52 == 0:
print i
# use 10% to test
# feature # of training set
fre=52
#form hankel matrix
from scipy.linalg import hankel
X=hankel(result[0:-fre], result[-1-fre:-1])
y=result[fre:]
#use n datapoints
n=430
#split data into training and testing
Xtrain=X[:n]
ytrain=result[:n]
Xtest=X[n:]
ytest=y[n:]
#check if the split is correct
len(X)
len(y)
len(X[0])
def firls(m, bands, desired, weight=None):
"""
FIR filter design using least squares method.
Inputs :
m : oder of FIR filter
bands : A montonic sequence containing the band edges. All elements
must be non-negative and less than 1 the sampling frequency
as given in pi units.
desired : A sequency with the same size of bands containing the desired gain
in each of the specified bands
weight : A relative weighting to give to each band region.
Output :
h : coefficients of length m+1 fir filter.
Example :
h = firls(50, [0,0.2,0.3,1.0], [1,1,0,0],[1,5.0])
Calculate impulse response for 51 tabs lowpass filter using least squares method
with passband == [0,0.2*pi], stopband == [0.3*pi, pi],
weight to passband == 1, weight to stopband == 5
Note : This function is modified from signal package for octave
(http://octave.sourceforge.net/signal/index.html)
"""
if weight==None : weight = ones(len(bands)/2)
bands, desired, weight = array(bands), array(desired), array(weight)
M = m/2;
w = kron(weight, [-1,1])
omega = bands * pi
i1 = arange(1,M+1)
# generate the matrix q
# as illustrated in the above-cited reference, the matrix can be
# expressed as the sum of a hankel and toeplitz matrix. a factor of
# 1/2 has been dropped and the final filter hficients multiplied
# by 2 to compensate.
cos_ints = append(omega, sin(mat(arange(1,m+1)).T*mat(omega))).reshape((-1,omega.shape[0]))
q = append(1, 1.0/arange(1.0,m+1)) * array(mat(cos_ints) * mat(w).T).T[0]
q = toeplitz(q[:M+1]) + hankel(q[:M+1], q[M : ])
# the vector b is derived from solving the integral:
#
# _ w
# / 2
# b = / w(w) d(w) cos(kw) dw
# k / w
# - 1
#
# since we assume that w(w) is constant over each band (if not, the
# computation of q above would be considerably more complex), but
# d(w) is allowed to be a linear function, in general the function
# w(w) d(w) is linear. the computations below are derived from the
# fact that:
# _
# / a ax + b
# / (ax + b) cos(nx) dx = --- cos (nx) + ------ sin(nx)
# / 2 n
# - n
#
enum = append(omega[::2]**2 - omega[1::2]**2, cos(mat(i1).T * mat(omega[1::2])) - cos(mat(i1).T * mat(omega[::2]))).flatten()
deno = mat(append(2, i1)).T * mat(omega[1::2] - omega[::2])
cos_ints2 = enum.reshape(deno.shape)/array(deno)
d = zeros_like(desired)
d[::2] = -weight * desired[::2]
d[1::2] = weight * desired[1::2]
b = append(1, 1.0/i1) * array(mat(kron (cos_ints2, [1, 1]) + cos_ints[:M+1,:]) * mat(d).T)[:,0]
# having computed the components q and b of the matrix equation,
# solve for the filter hficients.
a = (array(inv(q)*mat(b).T).T)[0]
h = append( a[:0:-1], append(2*a[0], a[1:]))
return h
def time_hankel(self, size):
sl.hankel(self.x)
def test_basic(self):
y = hankel([1,2,3])
assert_array_equal(y, [[1,2,3], [2,3,0], [3,0,0]])
y = hankel([1,2,3], [3,4,5])
assert_array_equal(y, [[1,2,3], [2,3,4], [3,4,5]])
def pbdesign(n, keep=None):
"""
Generate a Plackett-Burman design
Parameter
---------
n : int
The multiple of 4 higher than the number of factors.
Optional
--------
keep : int
The actual number of factors that the matrix will be used for
(default: n).
Returns
-------
H : 2d-array
An orthogonal design matrix with n rows, and (n-1) columns.
Example
-------
Create a 5-factor design::
>>> pbdesign(8) # since 8 is the next heigher multiple of 4
array([[ 1., 1., 1., 1., 1., 1., 1.],
[-1., 1., -1., 1., -1., 1., -1.],
[ 1., -1., -1., 1., 1., -1., -1.],
[-1., -1., 1., 1., -1., -1., 1.],
[ 1., 1., 1., -1., -1., -1., -1.],
[-1., 1., -1., -1., 1., -1., 1.],
[ 1., -1., -1., -1., -1., 1., 1.],
[-1., -1., 1., -1., 1., 1., -1.]])
And if we only want to keep the needed five columns::
>>> pbdesign(8, keep=5)
array([[ 1., 1., 1., 1., 1.],
[-1., 1., -1., 1., -1.],
[ 1., -1., -1., 1., 1.],
[-1., -1., 1., 1., -1.],
[ 1., 1., 1., -1., -1.],
[-1., 1., -1., -1., 1.],
[ 1., -1., -1., -1., -1.],
[-1., -1., 1., -1., 1.]])
"""
f, e = np.frexp([n, n/12., n/20.])
k = [idx for idx, val in enumerate(np.logical_and(f==0.5, e>0)) if val]
assert isinstance(n, int) and k!=[], 'Invalid inputs. n must be a multiple of 4.'
k = k[0]
e = e[k] - 1
if k==0: # N = 1*2**e
H = np.ones((1, 1))
elif k==1: # N = 12*2**e
H = np.vstack((np.ones((1, 12)), np.hstack((np.ones((11, 1)),
toeplitz([-1, -1, 1, -1, -1, -1, 1, 1, 1, -1, 1],
[-1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1])))))
elif k==2: # N = 20*2**e
H = np.vstack((np.ones((1, 20)), np.hstack((np.ones((19, 1)),
hankel(
[-1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1],
[1, -1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1])
))))
# Kronecker product construction
for i in xrange(e):
H = np.vstack((np.hstack((H, H)), np.hstack((H, -H))))
if keep is not None:
assert keep<=(H.shape[1]-1), 'Too many variables specified in "keep" for matrix'
return H[:, 1:(keep + 1)]
else:
return H[:, 1:]
def pbdesign(n):
"""
Generate a Plackett-Burman design
Parameter
---------
n : int
The number of factors to create a matrix for.
Returns
-------
H : 2d-array
An orthogonal design matrix with n columns, one for each factor, and
the number of rows being the next multiple of 4 higher than n (e.g.,
for 1-3 factors there are 4 rows, for 4-7 factors there are 8 rows,
etc.)
Example
-------
A 3-factor design::
>>> pbdesign(3)
array([[-1., -1., 1.],
[ 1., -1., -1.],
[-1., 1., -1.],
[ 1., 1., 1.]])
A 5-factor design::
>>> pbdesign(5)
array([[-1., -1., 1., -1., 1.],
[ 1., -1., -1., -1., -1.],
[-1., 1., -1., -1., 1.],
[ 1., 1., 1., -1., -1.],
[-1., -1., 1., 1., -1.],
[ 1., -1., -1., 1., 1.],
[-1., 1., -1., 1., -1.],
[ 1., 1., 1., 1., 1.]])
"""
assert n>0, 'Number of factors must be a positive integer'
keep = int(n)
n = 4*(int(n/4) + 1) # calculate the correct number of rows (multiple of 4)
f, e = np.frexp([n, n/12., n/20.])
k = [idx for idx, val in enumerate(np.logical_and(f==0.5, e>0)) if val]
assert isinstance(n, int) and k!=[], 'Invalid inputs. n must be a multiple of 4.'
k = k[0]
e = e[k] - 1
if k==0: # N = 1*2**e
H = np.ones((1, 1))
elif k==1: # N = 12*2**e
H = np.vstack((np.ones((1, 12)), np.hstack((np.ones((11, 1)),
toeplitz([-1, -1, 1, -1, -1, -1, 1, 1, 1, -1, 1],
[-1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1])))))
elif k==2: # N = 20*2**e
H = np.vstack((np.ones((1, 20)), np.hstack((np.ones((19, 1)),
hankel(
[-1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1],
[1, -1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1])
))))
# Kronecker product construction
for i in range(e):
H = np.vstack((np.hstack((H, H)), np.hstack((H, -H))))
# Reduce the size of the matrix as needed
H = H[:, 1:(keep + 1)]
return np.flipud(H)
def get_hankel_matrix(y, L=None):
N = len(y)
if L is None:
L = N // 2 + 1
M = N - L + 1
return la.hankel(y)[:L,:M]
