def test_rbf_kernel(self):
# Tests RBF kernel of svc.
X1 = Distribution.radial_binary(pts=100,
mean=[0, 0],
st=1,
ed=2,
seed=100)
X2 = Distribution.radial_binary(pts=100,
mean=[0, 0],
st=4,
ed=5,
seed=100)
Y1 = np.ones(X1.shape[0])
Y2 = -np.ones(X1.shape[0])
X_train = np.vstack((X1, X2))
y_train = np.hstack((Y1, Y2))
clf = svm.SVC(kernel='rbf', gamma=10)
clf.fit(X_train, y_train)
X1 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=1,
ed=2,
seed=100)
X2 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=4,
ed=5,
seed=100)
Y1 = np.ones(X1.shape[0])
Y2 = -np.ones(X2.shape[0])
X_test = np.vstack((X1, X2))
y_test = np.hstack((Y1, Y2))
predictions, projections = clf.predict(X_test, return_projection=True)
expected_projections = np.array([
1.2630574, 1.3302442, 1.502788, 1.2003369, 1.4567516, 1.0555044,
1.434326, 1.4227715, 1.1069533, 1.104987, -1.6992458, -1.5001097,
-1.0005158, -1.8284273, -1.0863144, -2.238042, -1.2274336,
-1.2235101, -2.1250129, -2.0870237
], )
self.assertTrue(np.allclose(projections, expected_projections))
self.assertTrue(np.allclose(predictions, y_test))
def fit(self, X):
"""Fits the kmeans unsupervised clustering algorithm.
Parameters
----------
X : numpy.ndarray
The training features.
"""
self.X = X
center_ids = np.random.randint(self.X.shape[0], size=self.n_clusters)
centers = self.X[center_ids]
self.labels = -np.ones(self.X.shape[0], dtype=np.float64)
converged = False
for i in range(self.max_iter):
if converged:
break
converged = True
for j, x in enumerate(self.X):
min_dist_center = np.linalg.norm(x - centers[0])
label = 0
for i, center in enumerate(centers):
if min_dist_center > np.linalg.norm(x - center):
min_dist_center = np.linalg.norm(x - center)
label = i
self.labels[j] = label
centers, converged = self.__get_new_centers(centers)
return self
def test_linear_kernel(self):
# Tests linear kernel of svc.
X1 = Distribution.linear(pts=100,
mean=[8, 10],
covr=[[1.5, 1], [1, 1.5]],
seed=100)
X2 = Distribution.linear(pts=100,
mean=[9, 5],
covr=[[1.5, 1], [1, 1.5]],
seed=100)
Y1 = np.ones(X1.shape[0])
Y2 = -np.ones(X2.shape[0])
X_train = np.vstack((X1, X2))
y_train = np.hstack((Y1, Y2))
clf_lin = svm.SVC(kernel='linear')
clf_lin.fit(X_train, y_train)
X1 = Distribution.linear(pts=10,
mean=[8, 10],
covr=[[1.5, 1], [1, 1.5]],
seed=100)
X2 = Distribution.linear(pts=10,
mean=[9, 5],
covr=[[1.5, 1], [1, 1.5]],
seed=100)
Y1 = np.ones(X1.shape[0])
Y2 = -np.ones(X2.shape[0])
X_test = np.vstack((X1, X2))
y_test = np.hstack((Y1, Y2))
predictions, projections = clf_lin.predict(X_test,
return_projection=True)
expected_projections = np.array([
5.2844825, 2.8846788, 3.898558, 2.4527097, 4.271367, 4.6425023,
5.170607, 3.3408344, 5.3939104, 2.779106, -2.909471, -5.3092747,
-4.2953954, -5.7412434, -3.9225864, -3.551451, -3.0233462,
-4.853119, -2.8000426, -5.4148474
])
self.assertTrue(np.allclose(projections, expected_projections))
self.assertTrue(np.allclose(predictions, y_test))
def setUp(self):
X11 = Distribution.radial_binary(pts=300,
mean=[0, 0],
st=1,
ed=2,
seed=20)
X22 = Distribution.radial_binary(pts=300,
mean=[0, 0],
st=4,
ed=5,
seed=10)
Y11 = np.ones(X11.shape[0])
Y22 = np.zeros(X11.shape[0])
X = np.vstack((X11, X22))
y = np.hstack((Y11, Y22))
y = to_onehot(y)
self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
X, y, test_size=50, random_state=42)
def fit(self, X, y, multiplier_threshold=1e-5):
"""Fits the svc model on training data.
Parameters
----------
X : numpy.array
The training features.
y : numpy.array
The training labels.
multiplier_threshold : float
The threshold for selecting lagrange multipliers.
Returns
-------
kernel_matrix : list of svm.SVC
A list of all the classifiers used for multi class classification
"""
X = np.array(X)
self.y = y
self.n = self.y.shape[0]
self.uniques, self.ind = np.unique(self.y, return_index=True)
self.n_classes = len(self.uniques)
# Do multi class classification
if sorted(self.uniques) != [-1, 1]:
y_list = [np.where(self.y == u, 1, -1) for u in self.uniques]
for y_i in y_list:
# Copy the current initializer
clf = SVC()
clf.kernel = self.kernel
clf.C = self.C
self.classifiers.append(clf.fit(X, y_i))
return
# create a gram matrix by taking the outer product of y
gram_matrix_y = np.outer(self.y, self.y)
K = self.__create_kernel_matrix(X)
gram_matrix_xy = gram_matrix_y * K
P = cvxopt.matrix(gram_matrix_xy)
q = cvxopt.matrix(-np.ones(self.n))
G1 = cvxopt.spmatrix(-1.0, range(self.n), range(self.n))
G2 = cvxopt.spmatrix(1, range(self.n), range(self.n))
G = cvxopt.matrix([[G1, G2]])
h1 = cvxopt.matrix(np.zeros(self.n))
h2 = cvxopt.matrix(np.ones(self.n) * self.C)
h = cvxopt.matrix([[h1, h2]])
A = cvxopt.matrix(self.y.astype(np.double)).trans()
b = cvxopt.matrix(0.0)
lagrange_multipliers = np.array(
list(cvxopt.solvers.qp(P, q, G, h, A, b)['x']))
lagrange_multiplier_indices = np.greater_equal(lagrange_multipliers,
multiplier_threshold)
lagrange_multiplier_indices = list(
map(list, lagrange_multiplier_indices.nonzero()))[0]
# self.support_vectors = np.take(X, lagrange_multiplier_indices, axis=1)
self.support_vectors = X[lagrange_multiplier_indices]
# print(X)
# print(lagrange_multiplier_indices)
# print(self.support_vectors)
# self.support_vectors_y = np.take(self.y, lagrange_multiplier_indices)
self.support_vectors_y = self.y[lagrange_multiplier_indices]
# self.support_lagrange_multipliers = np.take(lagrange_multipliers, lagrange_multiplier_indices)
self.support_lagrange_multipliers = lagrange_multipliers[
lagrange_multiplier_indices]
self.b = 0
self.n_support_vectors = self.support_vectors.shape[0]
for i in range(self.n_support_vectors):
kernel_trick = K[[lagrange_multiplier_indices[i]],
lagrange_multiplier_indices]
self.b += self.support_vectors_y[i] - np.sum(
self.support_lagrange_multipliers * self.support_vectors_y *
kernel_trick)
self.b /= self.n_support_vectors
self.classifiers = [self]
return self
def test_poly_kernel(self):
# Tests polynomial kernel of svc.
X1 = Distribution.linear(pts=50,
mean=[8, 20],
covr=[[1.5, 1], [1, 2]],
seed=100)
X2 = Distribution.linear(pts=50,
mean=[8, 15],
covr=[[1.5, -1], [-1, 2]],
seed=100)
X3 = Distribution.linear(pts=50,
mean=[15, 20],
covr=[[1.5, 1], [1, 2]],
seed=100)
X4 = Distribution.linear(pts=50,
mean=[15, 15],
covr=[[1.5, -1], [-1, 2]],
seed=100)
X1 = np.vstack((X1, X2))
X2 = np.vstack((X3, X4))
Y1 = np.ones(X1.shape[0])
Y2 = -np.ones(X2.shape[0])
X_train = np.vstack((X1, X2))
y_train = np.hstack((Y1, Y2))
clf = svm.SVC(kernel='polynomial', const=1, degree=2)
clf.fit(X_train, y_train)
X1 = Distribution.linear(pts=5,
mean=[8, 20],
covr=[[1.5, 1], [1, 2]],
seed=100)
X2 = Distribution.linear(pts=5,
mean=[8, 15],
covr=[[1.5, -1], [-1, 2]],
seed=100)
X3 = Distribution.linear(pts=5,
mean=[15, 20],
covr=[[1.5, 1], [1, 2]],
seed=100)
X4 = Distribution.linear(pts=5,
mean=[15, 15],
covr=[[1.5, -1], [-1, 2]],
seed=100)
X1 = np.vstack((X1, X2))
X2 = np.vstack((X3, X4))
Y1 = np.ones(X1.shape[0])
Y2 = -np.ones(X2.shape[0])
X_test = np.vstack((X1, X2))
y_test = np.hstack((Y1, Y2))
predictions, projections = clf.predict(X_test, return_projection=True)
expected_projections = np.array([
1.2630574, 1.3302442, 1.502788, 1.2003369, 1.4567516, 1.0555044,
1.434326, 1.4227715, 1.1069533, 1.104987, -1.6992458, -1.5001097,
-1.0005158, -1.8284273, -1.0863144, -2.238042, -1.2274336,
-1.2235101, -2.1250129, -2.0870237
])
expected_projections = np.array([
1.9282368, 4.1053743, 4.449601, 2.8149981, 3.337817, 1.5934888,
4.237419, 3.699658, 3.8548565, 2.8402433, -6.7378554, -2.9163127,
-2.5978136, -4.833237, -4.421687, -5.2333884, -2.2744238,
-3.0598483, -2.4422958, -3.890006
], )
self.assertTrue(np.allclose(projections, expected_projections))
self.assertTrue(np.allclose(predictions, y_test))
def test_multiclass(self):
X1 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=1,
ed=2,
seed=100)
X2 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=4,
ed=5,
seed=100)
X3 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=6,
ed=7,
seed=100)
X4 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=8,
ed=9,
seed=100)
Y1 = -np.ones(X1.shape[0])
Y2 = np.ones(X2.shape[0])
Y3 = 2 * np.ones(X3.shape[0])
Y4 = 3000 * np.ones(X4.shape[0])
X_train = np.vstack((X1, X2, X3, X4))
y_train = np.hstack((Y1, Y2, Y3, Y4))
clf = svm.SVC(kernel='rbf', gamma=10)
clf.fit(X_train, y_train)
X1 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=1,
ed=2,
seed=100)
X2 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=4,
ed=5,
seed=100)
X3 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=6,
ed=7,
seed=100)
X4 = Distribution.radial_binary(pts=10,
mean=[0, 0],
st=8,
ed=9,
seed=100)
X_test = np.vstack((X1, X2, X3, X4))
_, projections = clf.predict(X_test, return_projection=True)
expected_projections = np.array([
1.23564788, 1.15519477, 1.32441802, 1.04496554, 1.29740627, 0.,
1.25561797, 1.22925452, 0., 1.11920321, 0.2991908, 0.23818634,
0.55359011, 0.29655677, 0., 0.59992803, 0.52733203, 0.30456398,
0.6027897, 0.33755249, 0., 0.04997651, 0.12099712, 0.12276944, 0.,
0.19631702, 0.11836214, 0.06221966, 0.24539362, 0., 1.00000106,
1.0000021, 1.00000092, 1.19952335, 1.00000283, 1.17741522,
1.40596479, 1.60945299, 1.41534644, 1.27928235
])
self.assertTrue(np.allclose(projections, expected_projections))
