def backPropagate_(self):
'''Back-Propagate errors (and node responsibilities).'''
errors3 = multiply(-(self.xs__ - self.a3__), 1 - power(self.a3__, 2))
d3 = multiply(errors3, self.xs__)
errors2 = multiply(d3 * self.weights2_, 1 - power(self.a2__, 2))
d2 = multiply(errors2, self.a2__)
# average the errors
d3 = np.sum(d3,0) / d3.shape[0]
d2 = np.sum(d2,0) / d2.shape[0]
self.d3__ = d3.T
self.d2__ = d2.T
def SPIRIT(A, lamb, energy, k0=1, holdOffTime=0, reorthog=False, evalMetrics="F"):
A = np.mat(A)
n = A.shape[1]
totalTime = A.shape[0]
Proj = npm.ones((totalTime, n)) * np.nan
recon = npm.zeros((totalTime, n))
# initialize w_i to unit vectors
W = npm.eye(n)
d = 0.01 * npm.ones((n, 1))
m = k0 # number of eigencomponents
relErrors = npm.zeros((totalTime, 1))
sumYSq = 0.0
E_t = []
sumXSq = 0.0
E_dash_t = []
res = {}
k_hist = []
W_hist = []
anomalies = []
# incremental update W
lastChangeAt = 0
for t in range(totalTime):
k_hist.append(m)
# update W for each y_t
x = A[t, :].T # new data as column vector
for j in range(m):
W[:, j], d[j], x = updateW(x, W[:, j], d[j], lamb)
Wj = W[:, j]
# Grams smit reorthog
if reorthog == True:
W[:, :m], R = npm.linalg.qr(W[:, :m])
# compute low-D projection, reconstruction and relative error
Y = W[:, :m].T * A[t, :].T # project to m-dimensional space
xActual = A[t, :].T # actual vector of the current time
xProj = W[:, :m] * Y # reconstruction of the current time
Proj[t, :m] = Y.T
recon[t, :] = xProj.T
xOrth = xActual - xProj
relErrors[t] = npm.sum(npm.power(xOrth, 2)) / npm.sum(npm.power(xActual, 2))
# update energy
sumYSq = lamb * sumYSq + npm.sum(npm.power(Y, 2))
E_dash_t.append(sumYSq)
sumXSq = lamb * sumXSq + npm.sum(npm.power(A[t, :], 2))
E_t.append(sumXSq)
# Record RSRE
if t == 0:
top = 0.0
bot = 0.0
top = top + npm.power(npm.linalg.norm(xActual - xProj), 2)
bot = bot + npm.power(npm.linalg.norm(xActual), 2)
new_RSRE = top / bot
if t == 0:
RSRE = new_RSRE
else:
RSRE = npm.vstack((RSRE, new_RSRE))
### Metric EVALUATION ###
# deviation from truth
if evalMetrics == "T":
Qt = W[:, :m]
if t == 0:
res["subspace_error"] = npm.zeros((totalTime, 1))
res["orthog_error"] = npm.zeros((totalTime, 1))
res["angle_error"] = npm.zeros((totalTime, 1))
Cov_mat = npm.zeros([n, n])
# Calculate Covarentce Matrix of data up to time t
Cov_mat = lamb * Cov_mat + npm.dot(xActual, xActual.T)
# Get eigenvalues and eigenvectors
WW, V = npm.linalg.eig(Cov_mat)
# Use this to sort eigenVectors in according to deccending eigenvalue
eig_idx = WW.argsort() # Get sort index
eig_idx = eig_idx[::-1] # Reverse order (default is accending)
# v_r = highest r eigen vectors (accoring to thier eigenvalue if sorted).
V_k = V[:, eig_idx[:m]]
# Calculate subspace error
C = npm.dot(V_k, V_k.T) - npm.dot(Qt, Qt.T)
res["subspace_error"][t, 0] = 10 * np.log10(npm.trace(npm.dot(C.T, C))) # frobenius norm in dB
# Calculate angle between projection matrixes
D = npm.dot(npm.dot(npm.dot(V_k.T, Qt), Qt.T), V_k)
eigVal, eigVec = npm.linalg.eig(D)
angle = npm.arccos(np.sqrt(max(eigVal)))
res["angle_error"][t, 0] = angle
# Calculate deviation from orthonormality
F = npm.dot(Qt.T, Qt) - npm.eye(m)
res["orthog_error"][t, 0] = 10 * np.log10(npm.trace(npm.dot(F.T, F))) # frobenius norm in dB
# Energy thresholding
######################
# check the lower bound of energy level
if sumYSq < energy[0] * sumXSq and lastChangeAt < t - holdOffTime and m < n:
lastChangeAt = t
m = m + 1
anomalies.append(t)
# print 'Increasing m to %d at time %d (ratio %6.2f)\n' % (m, t, 100 * sumYSq/sumXSq)
# check the upper bound of energy level
elif sumYSq > energy[1] * sumXSq and lastChangeAt < t - holdOffTime and m < n and m > 1:
lastChangeAt = t
m = m - 1
# print 'Decreasing m to %d at time %d (ratio %6.2f)\n' % (m, t, 100 * sumYSq/sumXSq)
W_hist.append(W[:, :m])
# set outputs
# Grams smit reorthog
if reorthog == True:
W[:, :m], R = npm.linalg.qr(W[:, :m])
# Data Stores
res2 = {
"hidden": Proj, # Array for hidden Variables
"E_t": np.array(E_t), # total energy of data
"E_dash_t": np.array(E_dash_t), # hidden var energy
"e_ratio": np.array(E_dash_t) / np.array(E_t), # Energy ratio
"rel_orth_err": relErrors, # orthoX error
"RSRE": RSRE, # Relative squared Reconstruction error
"recon": recon, # reconstructed data
"r_hist": k_hist, # history of r values
"W_hist": W_hist, # history of Weights
"anomalies": anomalies,
}
res.update(res2)
return res
def SPIRIT(A,
lamb,
energy,
k0=1,
holdOffTime=0,
reorthog=False,
evalMetrics='F'):
A = np.mat(A)
n = A.shape[1]
totalTime = A.shape[0]
Proj = npm.ones((totalTime, n)) * np.nan
recon = npm.zeros((totalTime, n))
# initialize w_i to unit vectors
W = npm.eye(n)
d = 0.01 * npm.ones((n, 1))
m = k0 # number of eigencomponents
relErrors = npm.zeros((totalTime, 1))
sumYSq = 0.
E_t = []
sumXSq = 0.
E_dash_t = []
res = {}
k_hist = []
W_hist = []
anomalies = []
# incremental update W
lastChangeAt = 0
for t in range(totalTime):
k_hist.append(m)
# update W for each y_t
x = A[t, :].T # new data as column vector
for j in range(m):
W[:, j], d[j], x = updateW(x, W[:, j], d[j], lamb)
Wj = W[:, j]
# Grams smit reorthog
if reorthog == True:
W[:, :m], R = npm.linalg.qr(W[:, :m])
# compute low-D projection, reconstruction and relative error
Y = W[:, :m].T * A[t, :].T # project to m-dimensional space
xActual = A[t, :].T # actual vector of the current time
xProj = W[:, :m] * Y # reconstruction of the current time
Proj[t, :m] = Y.T
recon[t, :] = xProj.T
xOrth = xActual - xProj
relErrors[t] = npm.sum(npm.power(xOrth, 2)) / npm.sum(
npm.power(xActual, 2))
# update energy
sumYSq = lamb * sumYSq + npm.sum(npm.power(Y, 2))
E_dash_t.append(sumYSq)
sumXSq = lamb * sumXSq + npm.sum(npm.power(A[t, :], 2))
E_t.append(sumXSq)
# Record RSRE
if t == 0:
top = 0.0
bot = 0.0
top = top + npm.power(npm.linalg.norm(xActual - xProj), 2)
bot = bot + npm.power(npm.linalg.norm(xActual), 2)
new_RSRE = top / bot
if t == 0:
RSRE = new_RSRE
else:
RSRE = npm.vstack((RSRE, new_RSRE))
### Metric EVALUATION ###
#deviation from truth
if evalMetrics == 'T':
Qt = W[:, :m]
if t == 0:
res['subspace_error'] = npm.zeros((totalTime, 1))
res['orthog_error'] = npm.zeros((totalTime, 1))
res['angle_error'] = npm.zeros((totalTime, 1))
Cov_mat = npm.zeros([n, n])
# Calculate Covarentce Matrix of data up to time t
Cov_mat = lamb * Cov_mat + npm.dot(xActual, xActual.T)
# Get eigenvalues and eigenvectors
WW, V = npm.linalg.eig(Cov_mat)
# Use this to sort eigenVectors in according to deccending eigenvalue
eig_idx = WW.argsort() # Get sort index
eig_idx = eig_idx[::-1] # Reverse order (default is accending)
# v_r = highest r eigen vectors (accoring to thier eigenvalue if sorted).
V_k = V[:, eig_idx[:m]]
# Calculate subspace error
C = npm.dot(V_k, V_k.T) - npm.dot(Qt, Qt.T)
res['subspace_error'][t, 0] = 10 * np.log10(
npm.trace(npm.dot(C.T, C))) #frobenius norm in dB
# Calculate angle between projection matrixes
D = npm.dot(npm.dot(npm.dot(V_k.T, Qt), Qt.T), V_k)
eigVal, eigVec = npm.linalg.eig(D)
angle = npm.arccos(np.sqrt(max(eigVal)))
res['angle_error'][t, 0] = angle
# Calculate deviation from orthonormality
F = npm.dot(Qt.T, Qt) - npm.eye(m)
res['orthog_error'][t, 0] = 10 * np.log10(
npm.trace(npm.dot(F.T, F))) #frobenius norm in dB
# Energy thresholding
######################
# check the lower bound of energy level
if sumYSq < energy[
0] * sumXSq and lastChangeAt < t - holdOffTime and m < n:
lastChangeAt = t
m = m + 1
anomalies.append(t)
# print 'Increasing m to %d at time %d (ratio %6.2f)\n' % (m, t, 100 * sumYSq/sumXSq)
# check the upper bound of energy level
elif sumYSq > energy[
1] * sumXSq and lastChangeAt < t - holdOffTime and m < n and m > 1:
lastChangeAt = t
m = m - 1
# print 'Decreasing m to %d at time %d (ratio %6.2f)\n' % (m, t, 100 * sumYSq/sumXSq)
W_hist.append(W[:, :m])
# set outputs
# Grams smit reorthog
if reorthog == True:
W[:, :m], R = npm.linalg.qr(W[:, :m])
# Data Stores
res2 = {
'hidden': Proj, # Array for hidden Variables
'E_t': np.array(E_t), # total energy of data
'E_dash_t': np.array(E_dash_t), # hidden var energy
'e_ratio': np.array(E_dash_t) / np.array(E_t), # Energy ratio
'rel_orth_err': relErrors, # orthoX error
'RSRE': RSRE, # Relative squared Reconstruction error
'recon': recon, # reconstructed data
'r_hist': k_hist, # history of r values
'W_hist': W_hist, # history of Weights
'anomalies': anomalies
}
res.update(res2)
return res
def backPropagate_(self):
'''Back-Propagate errors (and node responsibilities).'''
self.d3__ = multiply(-(self.xs__ - self.a3__), 1 - power(self.a3__, 2))
self.d2__ = multiply(self.weights2_.T * self.d3__, 1 - power(self.a2__, 2))
def test_suite_1():
n1 = 100
n2 = 100
n = n1+n2
d = 5
eta = .1
degree = 3
iterations = 1
results = mat.zeros((8,5))
times = mat.zeros((1,5))
sigma = 2
# 1st col is non-kernelized
# 2nd col is poly-kernel
for itr in xrange(iterations):
X = mat.randn(n1,d)
Phi_X = poly.phi(X, degree)
D0 = X + mat.rand(n2,d) / 1000
# Verify identity K(X,X) = 1
D1 = mat.randn(n2,d)
# How does kernel perform iid data
D2 = mat.rand(n2,d)
# Uniform rather than normal distribution
D3 = mat.randn(n2,d) * 2 + 2
# Linear transformation
D4 = mat.power(mat.randn(n2,d) + 1 ,3)
#Non-linear transformation
D5 = mat.power(X+1,3)
#non-linear transformation of the D0 dataset;
D6 = mat.rand(n2,d)/100 + mat.eye(n2,d)
#Totally different data - should have low similarity
D7 = mat.rand(n2,d)/100 + mat.eye(n2,d)*5
# Scaled version of D7
Data = [D0, D1, D2, D3, D4, D5, D6, D7]
for idx in xrange(8):
D = Data[idx]
start = time.time()
results[idx, 0] += nk_bhatta(X, D, 0)
nk = time.time()
emp = time.time()
results[idx, 1] += Bhattacharrya(X,D,gaussk(sigma),eta,5)
e5 = time.time()
results[idx, 2] += Bhattacharrya(X,D,gaussk(sigma),eta,15)
e15 = time.time()
results[idx, 3] += Bhattacharrya(X,D,gaussk(sigma),eta,25)
e25 = time.time()
nktime = nk-start
emptime = emp-nk
e5time = e5-emp
e15time = e15-e5
e25time = e25-e15
print "nk: {:.1f}, emp: {:.1f}, e5: {:.1f}, e15: {:.1f}, e25: {:.1f}".format(nktime, emptime, e5time, e15time, e25time)
times[0,0]+= nktime
times[0,4]+= emptime
times[0,1]+= e5time
times[0,2]+= e15time
times[0,3]+= e25time
def backPropagate_(self):
'''Back-Propagate errors (and node responsibilities).'''
self.d3__ = multiply(-(self.xs__ - self.a3__), 1 - power(self.a3__, 2))
self.d2__ = multiply(self.weights2_.T * self.d3__,
1 - power(self.a2__, 2))