def k_obj(X, y):
N, D = X.shape
k_scale = np.linspace(0.1, 1, 10)
t_subg = []
alg2 = my_svm.SVM(method='qp_primal', max_iter=3000)
alg2.fit(X, y)
true_obj = np.ones(3000) * alg2.res['objective']
fig = pyplot.figure()
a = fig.add_subplot(1, 1, 0.5)
a.set_yscale('log')
for k in k_scale:
subsample = int(k * N)
alg1 = my_svm.SVM(method='subgradient_1',
alpha=1e-2,
max_iter=3000,
tol=1e-1,
stochastic=True,
k=subsample)
alg1.fit(X, y)
x_max = len(alg1.res['objective_curve'])
pyplot.plot(np.arange(x_max), alg1.res['objective_curve'])
pyplot.plot(np.arange(3000), true_obj)
pyplot.ylabel('Objective value', fontsize=20)
pyplot.xlabel('Iteration', fontsize=20)
k_scale = list(k_scale)
pyplot.legend(k_scale + ['true value'])
pyplot.show()
def alpha_beta_obj(X, y, alpha, beta1, beta2):
alg1 = my_svm.SVM(method='subgradient_1',
alpha=alpha,
beta=beta1,
max_iter=2000,
tol=1e-6)
alg1.fit(X, y)
alg2 = my_svm.SVM(method='qp_primal', max_iter=2000)
alg2.fit(X, y)
alg3 = my_svm.SVM(method='subgradient_1',
alpha=alpha,
beta=beta2,
max_iter=2000,
tol=1e-6)
alg3.fit(X, y)
x_max = min(len(alg1.res['objective_curve']),
len(alg3.res['objective_curve']))
true_obj = np.ones(x_max) * alg2.res['objective']
fig = pyplot.figure()
fig.suptitle('Subgradient with alpha and beta', fontsize=30)
a = fig.add_subplot(1, 1, 0.5)
a.set_yscale('log')
pyplot.plot(np.arange(x_max), alg1.res['objective_curve'][0:x_max],
np.arange(x_max), true_obj, np.arange(x_max),
alg3.res['objective_curve'][0:x_max])
pyplot.ylabel('Objective value', fontsize=20)
pyplot.xlabel('Iteration', fontsize=20)
pyplot.legend([
'Subgradient with the best beta', 'True value',
'Subgradient with the worst beta'
])
pyplot.show()
def section_2():
for n in n_sample:
for d in dim:
X, y = generate_data(n, n, d)
qp_dual = my_svm.SVM(method='qp_dual',
kernel='rbf_kernel',
gamma=0.01)
qp_dual.fit(X, y)
rbf_t_qp_dual[n / 100 - 1, d / 2 - 1] = qp_dual.res['time']
rbf_obj_qp_dual[n / 100 - 1, d / 2 - 1] = qp_dual.res['objective']
libsvm = my_svm.SVM(method='libsvm',
kernel='rbf_kernel',
gamma=0.01)
libsvm.fit(X, y)
rbf_t_libsvm[n / 100 - 1, d / 2 - 1] = libsvm.res['time']
rbf_obj_libsvm[n / 100 - 1, d / 2 - 1] = libsvm.res['objective']
def k_graph(X, y):
N, D = X.shape
k_scale = np.linspace(0.1, 1, 10)
t_subg = []
for k in k_scale:
subsample = int(k * N)
alg = my_svm.SVM(method='subgradient_1',
alpha=1e-2,
max_iter=3000,
tol=1e-1,
stochastic=True,
k=subsample)
alg.fit(X, y)
t_subg += [alg.res['time']]
fig = pyplot.figure()
fig.suptitle('Stochastic subgradient with different subsample',
fontsize=30)
pyplot.plot(k_scale, t_subg)
pyplot.ylabel('Time, s', fontsize=20)
pyplot.xlabel('Size of subsample, portion', fontsize=20)
pyplot.show()
t_subg = np.asarray(t_subg)
return (k_scale[np.argmin(t_subg)], k_scale[np.argmax(t_subg)])
def visualize_subg(X, y):
fig = pyplot.figure()
fig.suptitle('Subgradient descent', fontsize=30)
alg = my_svm.SVM(kernel='libear_kernel',
method='subgradient',
alpha=1e-3,
tol=1e-3)
alg.fit(X, y)
alg.visualize(X, y)
def visualize_subg_stoch(X, y):
fig = pyplot.figure()
fig.suptitle('Stochastic subgradient descent', fontsize=30)
alg = my_svm.SVM(kernel='libear_kernel',
method='subgradient',
stochastic=True,
alpha=1e-3,
tol=1e-3,
k=100)
alg.fit(X, y)
alg.visualize(X, y)
def section_1():
for n in n_sample:
for d in dim:
X, y = generate_data(n, n, d)
qp_primal = my_svm.SVM(method='qp_primal')
qp_primal.fit(X, y)
t_qp_primal[n / 100 - 1, d / 2 - 1] = qp_primal.res['time']
obj_qp_primal[n / 100 - 1, d / 2 - 1] = qp_primal.res['objective']
qp_dual = my_svm.SVM(method='qp_dual')
qp_dual.fit(X, y)
t_qp_dual[n / 100 - 1, d / 2 - 1] = qp_dual.res['time']
obj_qp_dual[n / 100 - 1, d / 2 - 1] = qp_dual.res['objective']
subg = my_svm.SVM(method='subgradient', alpha=1e-3, tol=1e-2)
subg.fit(X, y)
t_subg[n / 100 - 1, d / 2 - 1] = subg.res['time']
obj_subg[n / 100 - 1, d / 2 - 1] = subg.res['objective']
subg_stoch = my_svm.SVM(method='subgradient',
alpha=1e-3,
tol=1e-2,
stochastic=True,
k=100)
subg_stoch.fit(X, y)
t_subg_stoch[n / 100 - 1, d / 2 - 1] = subg_stoch.res['time']
obj_subg_stoch[n / 100 - 1,
d / 2 - 1] = subg_stoch.res['objective']
liblinear = my_svm.SVM(method='liblinear')
liblinear.fit(X, y)
t_liblinear[n / 100 - 1, d / 2 - 1] = liblinear.res['time']
obj_liblinear[n / 100 - 1, d / 2 - 1] = liblinear.res['objective']
libsvm = my_svm.SVM(method='libsvm')
libsvm.fit(X, y)
t_libsvm[n / 100 - 1, d / 2 - 1] = libsvm.res['time']
obj_libsvm[n / 100 - 1, d / 2 - 1] = libsvm.res['objective']
def alpha_graph(X, y):
alpha_scale = np.linspace(0.0001, 0.01, 100)
t_subg = []
for alpha in alpha_scale:
alg = my_svm.SVM(method='subgradient_1', alpha=alpha, max_iter=2000)
alg.fit(X, y)
t_subg += [alg.res['time']]
fig = pyplot.figure()
fig.suptitle('Subgradient with alpha', fontsize=30)
pyplot.plot(alpha_scale, t_subg)
pyplot.ylabel('Time, s', fontsize=20)
pyplot.xlabel('Alpha', fontsize=20)
pyplot.show()
t_subg = np.asarray(t_subg)
return (alpha_scale[np.argmin(t_subg)], alpha_scale[np.argmax(t_subg)])
def alpha_beta_graph(X, y, alpha):
beta_scale = np.linspace(1e-6, 1e-5, 50)
t_subg = []
for beta in beta_scale:
alg = my_svm.SVM(method='subgradient_1',
alpha=alpha,
beta=beta,
max_iter=2000,
tol=1e-6)
alg.fit(X, y)
t_subg += [alg.res['time']]
fig = pyplot.figure()
fig.suptitle('Subgradient with alpha and beta', fontsize=30)
pyplot.plot(beta_scale, t_subg)
pyplot.ylabel('Time, s', fontsize=20)
pyplot.xlabel('Beta', fontsize=20)
pyplot.show()
t_subg = np.asarray(t_subg)
return (beta_scale[np.argmin(t_subg)], beta_scale[np.argmax(t_subg)])
def cross_val_C_gamma_bad():
X_bad, y_bad = generate_bad_data(2000, 2000, 2)
C_vals = np.linspace(1, 100, 100)
quality_vals_C_bad = np.array([])
for c in C_vals:
alg = my_svm.SVM(kernel='rbf_kernel', gamma=0.136, C=c)
qual = []
cross_v = KFold(n=len(X_bad), n_folds=5)
for train_idx, test_idx in cross_v:
alg.fit(X_bad[train_idx], y_bad[train_idx])
qual += [alg.precision(X_bad[test_idx], y_bad[test_idx])]
qual = np.asarray(qual)
quality_vals_C_bad = np.append(quality_vals_C_bad, qual.mean())
fig = pyplot.figure()
fig.suptitle('Bad separable, gamma=0.136', fontsize=30)
pyplot.plot(C_vals, quality_vals_C_bad)
pyplot.xlabel("C vals", fontsize=20)
pyplot.ylabel("Precision", fontsize=20)
pyplot.show()
return C_vals[np.argmax(quality_vals_C_bad)]
def cross_val_C_great():
X_great, y_great = generate_great_data(2000, 2000, 2)
C_vals = np.linspace(1, 100, 100)
quality_vals_C_great = np.array([])
for c in C_vals:
alg = my_svm.SVM(kernel='linear_kernel', C=c)
qual = []
cross_v = KFold(n=len(X_great), n_folds=5)
for train_idx, test_idx in cross_v:
alg.fit(X_great[train_idx], y_great[train_idx])
qual += [alg.precision(X_great[test_idx], y_great[test_idx])]
qual = np.asarray(qual)
quality_vals_C_great = np.append(quality_vals_C_great, qual.mean())
fig = pyplot.figure()
fig.suptitle('Linear separable', fontsize=30)
pyplot.plot(C_vals, quality_vals_C_great)
pyplot.xlabel("C vals", fontsize=20)
pyplot.ylabel("Precision", fontsize=20)
pyplot.show()
return C_vals[np.argmax(quality_vals_C_great)]
line = line.strip().split(" ")
labels.append(float(line[0]))
i = 1
for word in line[1:]:
feature, value = word.split(":")
while int(feature) != i:
tmp.append(float(0))
i += 1
tmp.append(float(value))
i += 1
data_set.append(tmp)
return (np.mat(data_set), np.mat(labels).T)
def run():
pass
if __name__ == "__main__":
train_x, train_y = load_data("heart_scale")
# print(train_y,train_x)
svm = my_svm.SVM(C=0.6, kernel_option=("rbf", 0.431029))
svm = svm.SVM_training(
train_x,
train_y,
)
# print(svm.alphas,svm.b)
accuracy = svm.get_train_accracy()
print("The training accuracy is: %.3f%%" % (accuracy * 100))
i = 1
for word in line[1:]:
feature,value = word.split(":")
while int(feature) != i:
tmp.append(float(0))
i += 1
tmp.append(float(value))
i += 1
data_set.append(tmp)
return (np.mat(data_set),np.mat(labels).T)
def run():
pass
if __name__ == "__main__":
x,y = load_data("heart_scale")
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.4,random_state=0)
kernel_ = kernel.Kernel.gaussian(0.5)
# print(train_y,train_x)
svm = my_svm.SVM(kernel=kernel_,c=0.431029,)
svm = svm.training(X_train,y_train)
accuracy = svm.calc_accuracy(X_train,y_train)
print("The training accuracy is: %.3f%%" % (accuracy * 100))
accuracy = svm.calc_accuracy(X_test,y_test)
print("The testing accuracy is: %.3f%%" % (accuracy * 100))
def spam_classification_svm():
import time
start_time = time.time()
print(start_time)
errors = []
# 对于线程核函数，epsilon和tolerance是要分开的；
# 线性核的η会比较大，所有α会很小，更新值也很小，因此epsilon要小；
# 但对于tolerance，线性核与非线性核代表的度量是一样的。
# svm_my = my_svm.SVM(0.1, 0.000005, 100, my_svm.Linear)
svm_my = my_svm.SVM(0.1, 0.01, 100, my_svm.RBF)
# my_svm.DEBUG = False
test_set_x, test_feature_names, test_set_y = read_data(
'data_set\\MATRIX.TEST')
data_set_x, feature_names, data_set_y = read_data(
'data_set\\MATRIX.TRAIN.50')
svm_my.train(data_set_x, data_set_y)
false_count = svm_my.test(test_set_x, test_set_y)
errors.append(false_count * 100.0 / len(test_set_y))
data_set_x, feature_names, data_set_y = read_data(
'data_set\\MATRIX.TRAIN.100')
svm_my.train(data_set_x, data_set_y)
false_count = svm_my.test(test_set_x, test_set_y)
errors.append(false_count * 100.0 / len(test_set_y))
data_set_x, feature_names, data_set_y = read_data(
'data_set\\MATRIX.TRAIN.200')
svm_my.train(data_set_x, data_set_y)
false_count = svm_my.test(test_set_x, test_set_y)
errors.append(false_count * 100.0 / len(test_set_y))
data_set_x, feature_names, data_set_y = read_data(
'data_set\\MATRIX.TRAIN.400')
svm_my.train(data_set_x, data_set_y)
false_count = svm_my.test(test_set_x, test_set_y)
errors.append(false_count * 100.0 / len(test_set_y))
data_set_x, feature_names, data_set_y = read_data(
'data_set\\MATRIX.TRAIN.800')
svm_my.train(data_set_x, data_set_y)
false_count = svm_my.test(test_set_x, test_set_y)
errors.append(false_count * 100.0 / len(test_set_y))
data_set_x, feature_names, data_set_y = read_data(
'data_set\\MATRIX.TRAIN.1400')
svm_my.train(data_set_x, data_set_y)
false_count = svm_my.test(test_set_x, test_set_y)
errors.append(false_count * 100.0 / len(test_set_y))
duration = time.time() - start_time
print(duration)
x = [50, 100, 200, 400, 800, 1400]
y = errors
plt.title('SVM')
plt.xlabel('Data Size')
plt.ylabel('Error(%)')
plt.plot(x, y)
plt.show()
return
def two_dimension_svm():
errors = []
# svm_my = my_svm.SVM(0.1, 0.01, 100, my_svm.Linear)
svm_my = my_svm.SVM(0.00001, 0.00001, 100000, my_svm.RBF)
import time
start_time = time.time()
print(start_time)
data_set_x, feature_names, data_set_y = read_tow_dimension_matrix(
'data_set\\TWODIMENSION.TRAIN')
test_set_x, test_feature_names, test_set_y = read_tow_dimension_matrix(
'data_set\\TWODIMENSION.TRAIN')
# data_set_x, feature_names, data_set_y = read_tow_dimension_matrix('data_set\\TWODIMENSION.TRAIN.3')
# test_set_x, test_feature_names, test_set_y = read_tow_dimension_matrix('data_set\\TWODIMENSION.TRAIN.3')
# data_set_x, feature_names, data_set_y = read_tow_dimension_matrix('data_set\\TWODIMENSION.TRAIN.5')
# test_set_x, test_feature_names, test_set_y = read_tow_dimension_matrix('data_set\\TWODIMENSION.TRAIN.5')
svm_my.train(data_set_x, data_set_y)
false_count = svm_my.test(test_set_x, test_set_y)
errors.append(false_count * 100.0 / len(test_set_y))
duration = time.time() - start_time
print(duration)
omega = svm_my.compute_omega()
# normal_omega = np.sqrt(omega.dot(omega.T))
print(omega)
# output = svm_my.predict(test_set_x)
# from matplotlib.patches import Circle
# ax = plt.figure().add_subplot(111)
# for index, data_x in enumerate(test_set_x):
# print(abs(output[0][index]) / normal_omega)
# circle = Circle(xy=(data_x[0], data_x[1]), radius=abs(output[0][index]) / normal_omega, alpha=0.4)
# ax.add_patch(circle)
# plt.axis('scaled')
# plt.axis('equal')
x0 = 0
y0 = (-svm_my.b - omega[0] * x0) / omega[1]
x3 = 7
y3 = (-svm_my.b - omega[0] * x3) / omega[1]
plt.title("SVM")
plt.xlim(xmax=7, xmin=0)
plt.ylim(ymax=7, ymin=0)
plt.xlabel("x")
plt.ylabel("y")
x = data_set_x.T[0]
y = data_set_x.T[1]
for i, data_y in enumerate(data_set_y):
if data_y == 1:
plt.plot(x[i], y[i], 'r.')
else:
plt.plot(x[i], y[i], 'b.')
plt.plot([x0, x3], [y0, y3])
plt.show()
return
def visualize_libsvm(X, y):
fig = pyplot.figure()
fig.suptitle('LibSVM', fontsize=30)
alg = my_svm.SVM(kernel='linear_kernel')
alg.fit(X, y)
alg.visualize(X, y)
def visualize_qp_dual(X, y):
fig = pyplot.figure()
fig.suptitle('QP dual with support vectors', fontsize=30)
alg = my_svm.SVM(kernel='linear_kernel', method='qp_dual')
alg.fit(X, y)
alg.visualize(X, y, support=True)
def visualize_libsvm_rbf(X, y):
fig = pyplot.figure()
fig.suptitle('LibSVM with RBF-kernel', fontsize=30)
alg = my_svm.SVM(kernel='rbf_kernel', gamma=0.02)
alg.fit(X, y)
alg.visualize(X, y, rbf_kernel=True)
