def svm():
# ********************* load the dataset and divide to X&y ***********************
from sklearn.datasets import make_blobs
X, Y = make_blobs(cluster_std=0.9,
random_state=20,
n_samples=1000,
centers=10,
n_features=10)
from Algorithms.ML_.helper.data_helper import split_train_val_test
X, Xv, y, Yv, Xt, Yt = split_train_val_test(X, Y)
print(X.shape, y.shape, Xv.shape, Yv.shape, Xt.shape, Yt.shape)
# ********************* build model ***********************
from model import SVM
from activation import Activation, Softmax, Hinge
from regularization import Regularization, L1, L2, L12
from optimizer import Vanilla
model = SVM()
learning_rate, reg_rate = 1e-3, 5e-1
model.compile(alpha=learning_rate,
lambda_=reg_rate,
activation=Softmax(),
reg=L2(),
opt=Vanilla())
model.describe()
# ********************* train ***********************
loss_train, loss_val = model.train(X,
y,
val=(Xv, Yv),
iter_=1000,
return_loss=True,
verbose=True,
eps=1e-3)
import matplotlib.pyplot as plt
plt.plot(range(len(loss_train)), loss_train)
plt.plot(range(len(loss_val)), loss_val)
plt.legend(['train', 'val'])
plt.xlabel('Iteration')
plt.ylabel('Training loss')
plt.title('Training Loss history')
plt.show()
# ********************* predict ***********************
pred_train = model.predict(X)
pred_val = model.predict(Xv)
pred_test = model.predict(Xt)
import metrics
print('train accuracy=', metrics.accuracy(y, pred_train))
print('val accuracy=', metrics.accuracy(Yv, pred_val))
print('test accuracy=', metrics.accuracy(Yt, pred_test))
print('null accuracy=', metrics.null_accuracy(y))
import metrics
metrics.print_metrics(Yt, pred_test)
X, Xv, Xte, Xd = X - mu, Xv - mu, Xte - mu, Xd - mu
# ********************* train ***********************
# model = SVM()
# model.compile(lambda_=2.5e4, alpha=1e-7) # 1e-7, reg=2.5e4,
# loss_history = model.train(X, y, eps=0.001, batch=200, iter_=1500)
#
# plt.plot(range(len(loss_history)), loss_history)
# plt.xlabel('Iteration number')
# plt.ylabel('Loss value')
# plt.show()
# print(loss_history[::100])
# lr, rg = SVM.ff(X, y, Xv, Yv, [1e-7, 1e-6],[2e4, 2.5e4, 3e4, 3.5e4, 4e4, 4.5e4, 5e4, 6e4])
# print(lr, rg)
model = SVM()
model.compile(alpha=1e-7, lambda_=2, activation=Softmax, reg=L2)
# model.compile(alpha=0, lambda_=0, activation=Hinge, Reg=L2, dReg=dL2)
history = model.train(Xd, Yd, iter_=0, eps=0.0001)
print(model.loss(model.X, model.y, add_ones=False),
np.sum(model.grad(model.X, model.y, False)))
L, dW = model.grad(model.X, model.y, True)
print(L, np.sum(dW))
# print(np.sum(model.W))
# print(np.sum(model.grad(model.X, model.y, loss_=False)))
# print(np.sum(model.grad1(model.X, model.y)))
# L, dW = model.activation.loss_grad_loop(model.X, model.W, model.y)
# print(L, np.sum(dW))
loss_history = model.train(X, y, eps=0.0001, batch=200, iter_=1500)
plt.plot(range(len(loss_history)), loss_history)
