def experiment_anomaly_detection(train, test, comb, num_train, anom_prob, labels): phi = calc_feature_vecs(comb.X) print phi.size # bayes classifier (DIMS, N) = phi.size w_bayes = co.matrix(1.0, (DIMS, 1)) pred = w_bayes.trans()*phi[:,num_train:] (fpr, tpr, thres) = metric.roc_curve(labels[num_train:], pred.trans()) bayes_auc = metric.auc(fpr, tpr) # train one-class svm kern = Kernel.get_kernel(phi[:,0:num_train], phi[:,0:num_train]) ocsvm = OCSVM(kern, C=1.0/(num_train*anom_prob)) ocsvm.train_dual() kern = Kernel.get_kernel(phi, phi) (oc_as, foo) = ocsvm.apply_dual(kern[num_train:,ocsvm.get_support_dual()]) (fpr, tpr, thres) = metric.roc_curve(labels[num_train:], oc_as) base_auc = metric.auc(fpr, tpr) if (base_auc<0.5): base_auc = 1.0-base_auc # train structured anomaly detection #sad = StructuredOCSVM(train, C=1.0/(num_train*anom_prob)) sad = StructuredOCSVM(train, C=1.0/(num_train*0.5)) (lsol, lats, thres) = sad.train_dc(max_iter=50) (pred_vals, pred_lats) = sad.apply(test) (fpr, tpr, thres) = metric.roc_curve(labels[num_train:], pred_vals) auc = metric.auc(fpr, tpr) if (auc<0.5): auc = 1.0-auc return (auc, base_auc, bayes_auc)
def test_ocsvm(phi, kern, train, test, num_train, anom_prob, labels):
auc = 0.5
ocsvm = OCSVM(kern[:num_train, :num_train], C=1.0 / (num_train * anom_prob))
msg = ocsvm.train_dual()
if not msg == OCSVM.MSG_ERROR:
(oc_as, foo) = ocsvm.apply_dual(kern[num_train:, ocsvm.get_support_dual()])
(fpr, tpr, thres) = metric.roc_curve(labels[num_train:], oc_as)
auc = metric.auc(fpr, tpr)
return auc
def experiment_anomaly_detection(train, test, comb, num_train, anom_prob, labels): # train one-class svm phi = calc_feature_vecs(comb.X) kern = Kernel.get_kernel(phi[:,0:num_train], phi[:,0:num_train]) ocsvm = OCSVM(kern, C=1.0/(num_train*anom_prob)) ocsvm.train_dual() kern = Kernel.get_kernel(phi, phi) (oc_as, foo) = ocsvm.apply_dual(kern[num_train:,ocsvm.get_support_dual()]) (fpr, tpr, thres) = metric.roc_curve(labels[num_train:], oc_as) base_auc = metric.auc(fpr, tpr) # train structured anomaly detection sad = StructuredOCSVM(train, C=1.0/(num_train*anom_prob)) (lsol, lats, thres) = sad.train_dc(max_iter=40) (pred_vals, pred_lats) = sad.apply(test) (fpr, tpr, thres) = metric.roc_curve(labels[num_train:], pred_vals) auc = metric.auc(fpr, tpr) return (auc, base_auc)
def perf_ocsvm(phi, marker, train, test, anom_prob, ord=1):
#phi = phi[phi_inds.tolist(),:]
print('(a) normalize features...')
phi = normalize_features(phi, ord=ord)
print('(b) Build kernel...')
kern = Kernel.get_kernel(phi, phi)
print('(c) Train OCSVM...')
ocsvm = OCSVM(kern[train, train],
C=1.0 / (float(len(train)) * (1.0 - anom_prob)))
ocsvm.train_dual()
print('(d) Apply OCSVM...')
(oc_as, foo) = ocsvm.apply_dual(kern[test,
train[ocsvm.get_support_dual()]])
(fpr, tpr, thres) = metric.roc_curve(co.matrix(marker)[test], oc_as)
auc = metric.auc(fpr, tpr)
print('(e) Return AUC={0}...'.format(auc))
return auc
def experiment_anomaly_detection(train, test, comb, num_train, anom_prob,
labels):
# train one-class svm
phi = calc_feature_vecs(comb.X)
kern = Kernel.get_kernel(phi[:, 0:num_train], phi[:, 0:num_train])
ocsvm = OCSVM(kern, C=1.0 / (num_train * anom_prob))
ocsvm.train_dual()
kern = Kernel.get_kernel(phi, phi)
(oc_as, foo) = ocsvm.apply_dual(kern[num_train:, ocsvm.get_support_dual()])
(fpr, tpr, thres) = metric.roc_curve(labels[num_train:], oc_as)
base_auc = metric.auc(fpr, tpr)
# train structured anomaly detection
sad = StructuredOCSVM(train, C=1.0 / (num_train * anom_prob))
(lsol, lats, thres) = sad.train_dc(max_iter=40)
(pred_vals, pred_lats) = sad.apply(test)
(fpr, tpr, thres) = metric.roc_curve(labels[num_train:], pred_vals)
auc = metric.auc(fpr, tpr)
return (auc, base_auc)
def train_dc(self, zero_shot=False, max_iter=50, hotstart=matrix([])):
""" Solve the optimization problem with a
sequential convex programming/DC-programming
approach:
Iteratively, find the most likely configuration of
the latent variables and then, optimize for the
model parameter using fixed latent states.
"""
N = self.sobj.get_num_samples()
DIMS = self.sobj.get_num_dims()
# intermediate solutions
# latent variables
latent = [0.0]*N
#setseed(0)
sol = self.sobj.get_hotstart_sol()
#sol[0:4] *= 0.01
if hotstart.size==(DIMS,1):
print('New hotstart position defined.')
sol = hotstart
psi = matrix(0.0, (DIMS,N)) # (dim x exm)
old_psi = matrix(0.0, (DIMS,N)) # (dim x exm)
threshold = 0
obj = -1
iter = 0
allobjs = []
restarts = 0
# terminate if objective function value doesn't change much
while iter < max_iter and (iter < 2 or sum(sum(abs(np.array(psi-old_psi)))) >= 0.001):
print('Starting iteration {0}.'.format(iter))
print(sum(sum(abs(np.array(psi-old_psi)))))
iter += 1
old_psi = matrix(psi)
old_sol = sol
# 1. linearize
# for the current solution compute the
# most likely latent variable configuration
for i in range(N):
(foo, latent[i], psi[:,i]) = self.sobj.argmax(sol, i, add_prior=True)
#print psi[:,i]
#psi[:4,i] /= 600.0
#psi[:,i] /= 600.0
#psi[:4,i] = psi[:4,i]/np.linalg.norm(psi[:4,i],ord=2)
#psi[4:,i] = psi[4:,i]/np.linalg.norm(psi[4:,i],ord=2)
psi[:,i] /= np.linalg.norm(psi[:, i], ord=self.norm_ord)
#psi[:,i] /= np.max(np.abs(psi[:,i]))
#psi[:,i] /= 600.0
#if i>10:
# (foo, latent[i], psi[:,i]) = self.sobj.argmax(sol,i)
#else:
# psi[:,i] = self.sobj.get_joint_feature_map(i)
# latent[i] = self.sobj.y[i]
print psi
# 2. solve the intermediate convex optimization problem
kernel = Kernel.get_kernel(psi, psi)
svm = OCSVM(kernel, self.C)
svm.train_dual()
threshold = svm.get_threshold()
#inds = svm.get_support_dual()
#alphas = svm.get_support_dual_values()
#sol = phi[:,inds]*alphas
self.svs_inds = svm.get_support_dual()
#alphas = svm.get_support_dual_values()
sol = psi*svm.get_alphas()
print matrix([sol.trans(), old_sol.trans()]).trans()
if len(self.svs_inds) == N and self.C > (1.0 / float(N)):
print('###################################')
print('Degenerate solution.')
print('###################################')
restarts += 1
if (restarts>10):
print('###################################')
print 'Too many restarts...'
print('###################################')
# calculate objective
self.threshold = threshold
slacks = [max([0.0, np.single(threshold - sol.trans()*psi[:,i]) ]) for i in xrange(N)]
obj = 0.5*np.single(sol.trans()*sol) - np.single(threshold) + self.C*sum(slacks)
print("Iter {0}: Values (Threshold-Slacks-Objective) = {1}-{2}-{3}".format(int(iter),np.single(threshold),np.single(sum(slacks)),np.single(obj)))
allobjs.append(float(np.single(obj)))
break
# intermediate solutions
# latent variables
latent = [0.0]*N
#setseed(0)
sol = self.sobj.get_hotstart_sol()
#sol[0:4] *= 0.01
if hotstart.size==(DIMS,1):
print('New hotstart position defined.')
sol = hotstart
psi = matrix(0.0, (DIMS,N)) # (dim x exm)
old_psi = matrix(0.0, (DIMS,N)) # (dim x exm)
threshold = 0
obj = -1
iter = 0
allobjs = []
# calculate objective
self.threshold = threshold
slacks = [max([0.0, np.single(threshold - sol.trans()*psi[:,i]) ]) for i in xrange(N)]
obj = 0.5*np.single(sol.trans()*sol) - np.single(threshold) + self.C*sum(slacks)
print("Iter {0}: Values (Threshold-Slacks-Objective) = {1}-{2}-{3}".format(int(iter),np.single(threshold),np.single(sum(slacks)),np.single(obj)))
allobjs.append(float(np.single(obj)))
# zero shot learning: single iteration, hence random
# structure coefficient
if zero_shot:
print('LatentOcSvm: Zero shot learning.')
break
print '+++++++++'
print threshold
print slacks
print obj
print '+++++++++'
self.slacks = slacks
print allobjs
print(sum(sum(abs(np.array(psi-old_psi)))))
print '+++++++++ SAD END'
self.sol = sol
self.latent = latent
return sol, latent, threshold
def test_ocsvm(phi, kern, train, test, num_train, anom_prob, labels):
startTime = timer.time()
ocsvm = OCSVM(kern[:num_train, :num_train],
C=1.0 / (num_train * anom_prob))
msg = ocsvm.train_dual()
return timer.time() - startTime
from ocsvm import OCSVM
from kernel import Kernel
if __name__ == '__main__':
# kernel parameter and type
kparam = 0.1
ktype = 'rbf'
# generate raw training data
Dtrain = co.normal(2,100)
# build kernel
kernel = Kernel.get_kernel(Dtrain,Dtrain,ktype,kparam)
# train svdd
svm = OCSVM(kernel,0.1)
svm.train_dual()
delta = 0.07
x = np.arange(-4.0, 4.0, delta)
y = np.arange(-4.0, 4.0, delta)
X, Y = np.meshgrid(x, y)
(sx,sy) = X.shape
Xf = np.reshape(X,(1,sx*sy))
Yf = np.reshape(Y,(1,sx*sy))
Dtest = np.append(Xf,Yf,axis=0)
print(Dtest.shape)
print('halloooo')
foo = 3 * delta
# build test kernel
plt.figure()
scores = []
for i in range(train.samples):
LENS = len(train.y[i])
(anom_score, scores_exm) = train.get_scores(lsol, i, lats[i])
scores.append(anom_score)
plt.plot(range(LENS),scores_exm.trans() + i*8,'-g')
plt.plot(range(LENS),train.y[i].trans() + i*8,'-b')
plt.plot(range(LENS),lats[i].trans() + i*8,'-r')
if i==0:
plt.plot(range(LENS),train.X[i][0,:].trans() + i*8,'-k')
(fpr, tpr, thres) = metric.roc_curve(label, co.matrix(scores))
auc = metric.auc(fpr, tpr)
print auc
# train one-class svm
kern = Kernel.get_kernel(phi, phi)
ocsvm = OCSVM(kern, C=1.0/(EXMS_TRAIN*0.1))
ocsvm.train_dual()
(oc_as, foo) = ocsvm.apply_dual(kern[:,ocsvm.get_support_dual()])
(fpr, tpr, thres) = metric.roc_curve(label[0:EXMS_TRAIN], oc_as)
base_auc = metric.auc(fpr, tpr)
print base_auc
plt.show()
print('finished')
new_data = data[data[:,2] == 2]
new_data = new_data[:, [0,1]]
# only reserve unique elements here
new_data = np.unique(new_data, axis=0)
# generate artificial outlier and target datasets
data_shifting = Data_Shifting(new_data)
pseudo_outliers = data_shifting.outlier_generation()
pseudo_targets = data_shifting.target_generation()
# initialization for error calculations
nu_list = [0.01, 0.02, 0.05, 0.08, 0.1]
gamma_list = [2,5,8,10,15]
ocsvm = OCSVM()
error_array = np.zeros((len(nu_list),len(gamma_list)))
full_err_array = np.zeros((len(nu_list),len(gamma_list), 2))
grid_size = len(nu_list)*len(gamma_list)
err_min = 1.0
best_err = [0.0, 0.0]
best_param = [0.0,0.0]
training_time_sum = 0.0
predicting_time_outliers = 0.0
predicting_time_targets = 0.0
# grid search
for index, i in enumerate(nu_list):
for jndex, j in enumerate(gamma_list):
print("nu=%r, gamma=%r"%(i,j))
def train_dc(self, max_iter=50, hotstart=matrix([])):
""" Solve the optimization problem with a
sequential convex programming/DC-programming
approach:
Iteratively, find the most likely configuration of
the latent variables and then, optimize for the
model parameter using fixed latent states.
"""
N = self.sobj.get_num_samples()
DIMS = self.sobj.get_num_dims()
# intermediate solutions
# latent variables
latent = [0.0]*N
#setseed(0)
sol = self.sobj.get_hotstart_sol()
#sol[0:4] *= 0.01
if hotstart.size==(DIMS,1):
print('New hotstart position defined.')
sol = hotstart
psi = matrix(0.0, (DIMS,N)) # (dim x exm)
old_psi = matrix(0.0, (DIMS,N)) # (dim x exm)
threshold = 0
obj = -1
iter = 0
allobjs = []
# terminate if objective function value doesn't change much
while iter<max_iter and (iter<3 or sum(sum(abs(np.array(psi-old_psi))))>=0.001):
print('Starting iteration {0}.'.format(iter))
print(sum(sum(abs(np.array(psi-old_psi)))))
iter += 1
old_psi = matrix(psi)
old_sol = sol
# 1. linearize
# for the current solution compute the
# most likely latent variable configuration
for i in range(N):
(foo, latent[i], psi[:,i]) = self.sobj.argmax(sol, i, add_prior=True)
norm = np.linalg.norm(psi[:,i],2)
psi[:,i] /= norm
#if i>10:
# (foo, latent[i], psi[:,i]) = self.sobj.argmax(sol,i)
#else:
# psi[:,i] = self.sobj.get_joint_feature_map(i)
# latent[i] = self.sobj.y[i]
# 2. solve the intermediate convex optimization problem
psi_star = matrix(psi)
#psi_star[0:7,:] *= 4.0
#psi_star[0:3,:] *= 0.01
#psi_star[0,:] *= 1.2
#psi_star[2,:] *= 2.4
kernel = Kernel.get_kernel(psi_star, psi_star)
svm = OCSVM(kernel, self.C)
svm.train_dual()
threshold = svm.get_threshold()
#inds = svm.get_support_dual()
#alphas = svm.get_support_dual_values()
#sol = phi[:,inds]*alphas
#inds = svm.get_support_dual()
#alphas = svm.get_support_dual_values()
sol = psi_star*svm.get_alphas()
print matrix([sol.trans(), old_sol.trans()]).trans()
# calculate objective
slacks = [max([0.0, np.single(threshold - sol.trans()*psi[:,i]) ]) for i in xrange(N)]
obj = 0.5*np.single(sol.trans()*sol) - np.single(threshold) + self.C*sum(slacks)
print("Iter {0}: Values (Threshold-Slacks-Objective) = {1}-{2}-{3}".format(int(iter),np.single(threshold),np.single(sum(slacks)),np.single(obj)))
allobjs.append(float(np.single(obj)))
print '+++++++++'
print threshold
print slacks
print obj
print '+++++++++'
print allobjs
print(sum(sum(abs(np.array(psi-old_psi)))))
print '+++++++++ SAD END'
self.sol = sol
self.latent = latent
return (sol, latent, threshold)
def train_dc(self, max_iter=50, hotstart=matrix([])):
""" Solve the optimization problem with a
sequential convex programming/DC-programming
approach:
Iteratively, find the most likely configuration of
the latent variables and then, optimize for the
model parameter using fixed latent states.
"""
N = self.sobj.get_num_samples()
DIMS = self.sobj.get_num_dims()
# intermediate solutions
# latent variables
latent = [0.0] * N
#setseed(0)
sol = self.sobj.get_hotstart_sol()
#sol[0:4] *= 0.01
if hotstart.size == (DIMS, 1):
print('New hotstart position defined.')
sol = hotstart
psi = matrix(0.0, (DIMS, N)) # (dim x exm)
old_psi = matrix(0.0, (DIMS, N)) # (dim x exm)
threshold = 0
obj = -1
iter = 0
allobjs = []
# terminate if objective function value doesn't change much
while iter < max_iter and (
iter < 3 or sum(sum(abs(np.array(psi - old_psi)))) >= 0.001):
print('Starting iteration {0}.'.format(iter))
print(sum(sum(abs(np.array(psi - old_psi)))))
iter += 1
old_psi = matrix(psi)
old_sol = sol
# 1. linearize
# for the current solution compute the
# most likely latent variable configuration
for i in range(N):
(foo, latent[i], psi[:, i]) = self.sobj.argmax(sol,
i,
add_prior=True)
norm = np.linalg.norm(psi[:, i], 2)
psi[:, i] /= norm
#if i>10:
# (foo, latent[i], psi[:,i]) = self.sobj.argmax(sol,i)
#else:
# psi[:,i] = self.sobj.get_joint_feature_map(i)
# latent[i] = self.sobj.y[i]
# 2. solve the intermediate convex optimization problem
psi_star = matrix(psi)
#psi_star[0:7,:] *= 4.0
#psi_star[0:3,:] *= 0.01
#psi_star[0,:] *= 1.2
#psi_star[2,:] *= 2.4
kernel = Kernel.get_kernel(psi_star, psi_star)
svm = OCSVM(kernel, self.C)
svm.train_dual()
threshold = svm.get_threshold()
#inds = svm.get_support_dual()
#alphas = svm.get_support_dual_values()
#sol = phi[:,inds]*alphas
#inds = svm.get_support_dual()
#alphas = svm.get_support_dual_values()
sol = psi_star * svm.get_alphas()
print matrix([sol.trans(), old_sol.trans()]).trans()
# calculate objective
slacks = [
max([0.0, np.single(threshold - sol.trans() * psi[:, i])])
for i in xrange(N)
]
obj = 0.5 * np.single(sol.trans() * sol) - np.single(
threshold) + self.C * sum(slacks)
print(
"Iter {0}: Values (Threshold-Slacks-Objective) = {1}-{2}-{3}".
format(int(iter), np.single(threshold), np.single(sum(slacks)),
np.single(obj)))
allobjs.append(float(np.single(obj)))
print '+++++++++'
print threshold
print slacks
print obj
print '+++++++++'
print allobjs
print(sum(sum(abs(np.array(psi - old_psi)))))
print '+++++++++ SAD END'
self.sol = sol
self.latent = latent
return (sol, latent, threshold)
from ocsvm import OCSVM
from kernel import Kernel
if __name__ == '__main__':
# kernel parameter and type
kparam = 0.1
ktype = 'rbf'
# generate raw training data
Dtrain = co.normal(2, 100)
# build kernel
kernel = Kernel.get_kernel(Dtrain, Dtrain, ktype, kparam)
# train svdd
svm = OCSVM(kernel, 0.1)
svm.train_dual()
delta = 0.07
x = np.arange(-4.0, 4.0, delta)
y = np.arange(-4.0, 4.0, delta)
X, Y = np.meshgrid(x, y)
(sx, sy) = X.shape
Xf = np.reshape(X, (1, sx * sy))
Yf = np.reshape(Y, (1, sx * sy))
Dtest = np.append(Xf, Yf, axis=0)
print(Dtest.shape)
print('halloooo')
foo = 3 * delta
# build test kernel