def test_rbf_kernel(self): # Tests RBF kernel of svc. X1 = Distribution.radial_binary(pts=100, mean=[0, 0], st=1, ed=2, seed=100) X2 = Distribution.radial_binary(pts=100, mean=[0, 0], st=4, ed=5, seed=100) Y1 = np.ones(X1.shape[0]) Y2 = -np.ones(X1.shape[0]) X_train = np.vstack((X1, X2)) y_train = np.hstack((Y1, Y2)) clf = svm.SVC(kernel='rbf', gamma=10) clf.fit(X_train, y_train) X1 = Distribution.radial_binary(pts=10, mean=[0, 0], st=1, ed=2, seed=100) X2 = Distribution.radial_binary(pts=10, mean=[0, 0], st=4, ed=5, seed=100) Y1 = np.ones(X1.shape[0]) Y2 = -np.ones(X2.shape[0]) X_test = np.vstack((X1, X2)) y_test = np.hstack((Y1, Y2)) predictions, projections = clf.predict(X_test, return_projection=True) expected_projections = np.array([ 1.2630574, 1.3302442, 1.502788, 1.2003369, 1.4567516, 1.0555044, 1.434326, 1.4227715, 1.1069533, 1.104987, -1.6992458, -1.5001097, -1.0005158, -1.8284273, -1.0863144, -2.238042, -1.2274336, -1.2235101, -2.1250129, -2.0870237 ], ) self.assertTrue(np.allclose(projections, expected_projections)) self.assertTrue(np.allclose(predictions, y_test))
def fit(self, X): """Fits the kmeans unsupervised clustering algorithm. Parameters ---------- X : numpy.ndarray The training features. """ self.X = X center_ids = np.random.randint(self.X.shape[0], size=self.n_clusters) centers = self.X[center_ids] self.labels = -np.ones(self.X.shape[0], dtype=np.float64) converged = False for i in range(self.max_iter): if converged: break converged = True for j, x in enumerate(self.X): min_dist_center = np.linalg.norm(x - centers[0]) label = 0 for i, center in enumerate(centers): if min_dist_center > np.linalg.norm(x - center): min_dist_center = np.linalg.norm(x - center) label = i self.labels[j] = label centers, converged = self.__get_new_centers(centers) return self
def test_linear_kernel(self): # Tests linear kernel of svc. X1 = Distribution.linear(pts=100, mean=[8, 10], covr=[[1.5, 1], [1, 1.5]], seed=100) X2 = Distribution.linear(pts=100, mean=[9, 5], covr=[[1.5, 1], [1, 1.5]], seed=100) Y1 = np.ones(X1.shape[0]) Y2 = -np.ones(X2.shape[0]) X_train = np.vstack((X1, X2)) y_train = np.hstack((Y1, Y2)) clf_lin = svm.SVC(kernel='linear') clf_lin.fit(X_train, y_train) X1 = Distribution.linear(pts=10, mean=[8, 10], covr=[[1.5, 1], [1, 1.5]], seed=100) X2 = Distribution.linear(pts=10, mean=[9, 5], covr=[[1.5, 1], [1, 1.5]], seed=100) Y1 = np.ones(X1.shape[0]) Y2 = -np.ones(X2.shape[0]) X_test = np.vstack((X1, X2)) y_test = np.hstack((Y1, Y2)) predictions, projections = clf_lin.predict(X_test, return_projection=True) expected_projections = np.array([ 5.2844825, 2.8846788, 3.898558, 2.4527097, 4.271367, 4.6425023, 5.170607, 3.3408344, 5.3939104, 2.779106, -2.909471, -5.3092747, -4.2953954, -5.7412434, -3.9225864, -3.551451, -3.0233462, -4.853119, -2.8000426, -5.4148474 ]) self.assertTrue(np.allclose(projections, expected_projections)) self.assertTrue(np.allclose(predictions, y_test))
def setUp(self): X11 = Distribution.radial_binary(pts=300, mean=[0, 0], st=1, ed=2, seed=20) X22 = Distribution.radial_binary(pts=300, mean=[0, 0], st=4, ed=5, seed=10) Y11 = np.ones(X11.shape[0]) Y22 = np.zeros(X11.shape[0]) X = np.vstack((X11, X22)) y = np.hstack((Y11, Y22)) y = to_onehot(y) self.X_train, self.X_test, self.y_train, self.y_test = train_test_split( X, y, test_size=50, random_state=42)
def fit(self, X, y, multiplier_threshold=1e-5): """Fits the svc model on training data. Parameters ---------- X : numpy.array The training features. y : numpy.array The training labels. multiplier_threshold : float The threshold for selecting lagrange multipliers. Returns ------- kernel_matrix : list of svm.SVC A list of all the classifiers used for multi class classification """ X = np.array(X) self.y = y self.n = self.y.shape[0] self.uniques, self.ind = np.unique(self.y, return_index=True) self.n_classes = len(self.uniques) # Do multi class classification if sorted(self.uniques) != [-1, 1]: y_list = [np.where(self.y == u, 1, -1) for u in self.uniques] for y_i in y_list: # Copy the current initializer clf = SVC() clf.kernel = self.kernel clf.C = self.C self.classifiers.append(clf.fit(X, y_i)) return # create a gram matrix by taking the outer product of y gram_matrix_y = np.outer(self.y, self.y) K = self.__create_kernel_matrix(X) gram_matrix_xy = gram_matrix_y * K P = cvxopt.matrix(gram_matrix_xy) q = cvxopt.matrix(-np.ones(self.n)) G1 = cvxopt.spmatrix(-1.0, range(self.n), range(self.n)) G2 = cvxopt.spmatrix(1, range(self.n), range(self.n)) G = cvxopt.matrix([[G1, G2]]) h1 = cvxopt.matrix(np.zeros(self.n)) h2 = cvxopt.matrix(np.ones(self.n) * self.C) h = cvxopt.matrix([[h1, h2]]) A = cvxopt.matrix(self.y.astype(np.double)).trans() b = cvxopt.matrix(0.0) lagrange_multipliers = np.array( list(cvxopt.solvers.qp(P, q, G, h, A, b)['x'])) lagrange_multiplier_indices = np.greater_equal(lagrange_multipliers, multiplier_threshold) lagrange_multiplier_indices = list( map(list, lagrange_multiplier_indices.nonzero()))[0] # self.support_vectors = np.take(X, lagrange_multiplier_indices, axis=1) self.support_vectors = X[lagrange_multiplier_indices] # print(X) # print(lagrange_multiplier_indices) # print(self.support_vectors) # self.support_vectors_y = np.take(self.y, lagrange_multiplier_indices) self.support_vectors_y = self.y[lagrange_multiplier_indices] # self.support_lagrange_multipliers = np.take(lagrange_multipliers, lagrange_multiplier_indices) self.support_lagrange_multipliers = lagrange_multipliers[ lagrange_multiplier_indices] self.b = 0 self.n_support_vectors = self.support_vectors.shape[0] for i in range(self.n_support_vectors): kernel_trick = K[[lagrange_multiplier_indices[i]], lagrange_multiplier_indices] self.b += self.support_vectors_y[i] - np.sum( self.support_lagrange_multipliers * self.support_vectors_y * kernel_trick) self.b /= self.n_support_vectors self.classifiers = [self] return self
def test_poly_kernel(self): # Tests polynomial kernel of svc. X1 = Distribution.linear(pts=50, mean=[8, 20], covr=[[1.5, 1], [1, 2]], seed=100) X2 = Distribution.linear(pts=50, mean=[8, 15], covr=[[1.5, -1], [-1, 2]], seed=100) X3 = Distribution.linear(pts=50, mean=[15, 20], covr=[[1.5, 1], [1, 2]], seed=100) X4 = Distribution.linear(pts=50, mean=[15, 15], covr=[[1.5, -1], [-1, 2]], seed=100) X1 = np.vstack((X1, X2)) X2 = np.vstack((X3, X4)) Y1 = np.ones(X1.shape[0]) Y2 = -np.ones(X2.shape[0]) X_train = np.vstack((X1, X2)) y_train = np.hstack((Y1, Y2)) clf = svm.SVC(kernel='polynomial', const=1, degree=2) clf.fit(X_train, y_train) X1 = Distribution.linear(pts=5, mean=[8, 20], covr=[[1.5, 1], [1, 2]], seed=100) X2 = Distribution.linear(pts=5, mean=[8, 15], covr=[[1.5, -1], [-1, 2]], seed=100) X3 = Distribution.linear(pts=5, mean=[15, 20], covr=[[1.5, 1], [1, 2]], seed=100) X4 = Distribution.linear(pts=5, mean=[15, 15], covr=[[1.5, -1], [-1, 2]], seed=100) X1 = np.vstack((X1, X2)) X2 = np.vstack((X3, X4)) Y1 = np.ones(X1.shape[0]) Y2 = -np.ones(X2.shape[0]) X_test = np.vstack((X1, X2)) y_test = np.hstack((Y1, Y2)) predictions, projections = clf.predict(X_test, return_projection=True) expected_projections = np.array([ 1.2630574, 1.3302442, 1.502788, 1.2003369, 1.4567516, 1.0555044, 1.434326, 1.4227715, 1.1069533, 1.104987, -1.6992458, -1.5001097, -1.0005158, -1.8284273, -1.0863144, -2.238042, -1.2274336, -1.2235101, -2.1250129, -2.0870237 ]) expected_projections = np.array([ 1.9282368, 4.1053743, 4.449601, 2.8149981, 3.337817, 1.5934888, 4.237419, 3.699658, 3.8548565, 2.8402433, -6.7378554, -2.9163127, -2.5978136, -4.833237, -4.421687, -5.2333884, -2.2744238, -3.0598483, -2.4422958, -3.890006 ], ) self.assertTrue(np.allclose(projections, expected_projections)) self.assertTrue(np.allclose(predictions, y_test))
def test_multiclass(self): X1 = Distribution.radial_binary(pts=10, mean=[0, 0], st=1, ed=2, seed=100) X2 = Distribution.radial_binary(pts=10, mean=[0, 0], st=4, ed=5, seed=100) X3 = Distribution.radial_binary(pts=10, mean=[0, 0], st=6, ed=7, seed=100) X4 = Distribution.radial_binary(pts=10, mean=[0, 0], st=8, ed=9, seed=100) Y1 = -np.ones(X1.shape[0]) Y2 = np.ones(X2.shape[0]) Y3 = 2 * np.ones(X3.shape[0]) Y4 = 3000 * np.ones(X4.shape[0]) X_train = np.vstack((X1, X2, X3, X4)) y_train = np.hstack((Y1, Y2, Y3, Y4)) clf = svm.SVC(kernel='rbf', gamma=10) clf.fit(X_train, y_train) X1 = Distribution.radial_binary(pts=10, mean=[0, 0], st=1, ed=2, seed=100) X2 = Distribution.radial_binary(pts=10, mean=[0, 0], st=4, ed=5, seed=100) X3 = Distribution.radial_binary(pts=10, mean=[0, 0], st=6, ed=7, seed=100) X4 = Distribution.radial_binary(pts=10, mean=[0, 0], st=8, ed=9, seed=100) X_test = np.vstack((X1, X2, X3, X4)) _, projections = clf.predict(X_test, return_projection=True) expected_projections = np.array([ 1.23564788, 1.15519477, 1.32441802, 1.04496554, 1.29740627, 0., 1.25561797, 1.22925452, 0., 1.11920321, 0.2991908, 0.23818634, 0.55359011, 0.29655677, 0., 0.59992803, 0.52733203, 0.30456398, 0.6027897, 0.33755249, 0., 0.04997651, 0.12099712, 0.12276944, 0., 0.19631702, 0.11836214, 0.06221966, 0.24539362, 0., 1.00000106, 1.0000021, 1.00000092, 1.19952335, 1.00000283, 1.17741522, 1.40596479, 1.60945299, 1.41534644, 1.27928235 ]) self.assertTrue(np.allclose(projections, expected_projections))