def solve(self, A, y, x0, as_signs): print 'cvxopt QP: # of params=%d' % A.shape[1] #flip sign of columns of A that have negative usefulness, a trick #Patrick Gill uses to halve the # of parameters for the QP solver A *= as_signs #the sign of x0 is unflipped for the nonzero elements, flip it x0 *= as_signs p = A.shape[1] H = cvxopt_matrix(np.dot(A.transpose(), A)) f = cvxopt_matrix(self.lambda_val - np.dot(A.transpose(), y)) b = cvxopt_matrix(np.zeros(p)) Q = cvxopt_matrix(-np.eye(p)) stime = time.time() sol = cvxopt_solvers.qp(H, f, Q, b, None, None, None, x0) etime = time.time() - stime print 'QP solver took %d seconds' % int(etime) xnew = np.array(sol['x']).squeeze() #unflip elements of xnew xnew *= as_signs return xnew
def get_lazy_policy_action( self, r_plus_gamma_v_nexts, actions, ): """ Get optimal action using the lazy action model """ actions_augmented = np.concatenate((np.ones( (len(actions), 1)), actions), axis=-1) sol = qp( cvxopt_matrix( np.sum(actions_augmented[:, :, None] * actions_augmented[:, None, :], axis=0) + self.local_model_regularization * np.eye(3)), cvxopt_matrix( np.sum( -(r_plus_gamma_v_nexts[:, None] * actions_augmented[:, :]), axis=0))) if 'optimal' not in sol['status']: raise Exception('Optimal solution not found') alphabeta = np.array(sol['x']).reshape(-1) return self.environment.action_length / np.linalg.norm( alphabeta[1:]) * alphabeta[1:]
def get_lazy_model_value( self, r_plus_gamma_v_nexts, actions, ): """ Predict value of V-function, performing the maximum over the available actions using the lazy action model """ actions_augmented = np.concatenate((np.ones( (len(actions), 1)), actions), axis=-1) sol = qp( cvxopt_matrix( np.sum(actions_augmented[:, :, None] * actions_augmented[:, None, :], axis=0) + self.local_model_regularization * np.eye(3)), cvxopt_matrix( np.sum( -(r_plus_gamma_v_nexts[:, None] * actions_augmented[:, :]), axis=0))) if 'optimal' not in sol['status']: raise Exception('Optimal solution not found') alphabeta = np.array(sol['x']).reshape(-1) # Instability countermeasure: The target cant be higher than the maximum # r_plus_gamma_V_next in the vicinity return min([ alphabeta[0] + self.environment.action_length * np.linalg.norm(alphabeta[1:]), np.max(r_plus_gamma_v_nexts) ])
def primal_svm(X, Y, C=1.0): n = len(X[0]) m = len(Y) d = n + m + 1 P = cvxopt_matrix(0.0, (d, d)) for i in range(n): P[i, i] = 1.0 q = cvxopt_matrix(0.0, (d, 1)) for i in range(n, n + m): q[i] = C q[-1] = 0.0 h = cvxopt_matrix(-1.0, (m + m, 1)) h[m:] = 0.0 # print m G = cvxopt_matrix(0.0, (2 * m, d)) for i in range(m): G[i, :n] = -Y[i] * X[i] G[i, n + i] = -1 G[i, -1] = -Y[i] for i in range(m, m + m): G[i, n + i - m] = -1.0 cvxopt_solvers.options['show_progress'] = False vars = cvxopt_solvers.qp(P, q, G, h)['x'] w = vars[0:n] b = vars[-1] return w, b
def solve(self, A, y, x0, as_signs): print('cvxopt QP: # of params=%d' % A.shape[1]) #flip sign of columns of A that have negative usefulness, a trick #Patrick Gill uses to halve the # of parameters for the QP solver A *= as_signs #the sign of x0 is unflipped for the nonzero elements, flip it x0 *= as_signs p = A.shape[1] H = cvxopt_matrix(np.dot(A.transpose(), A)) f = cvxopt_matrix(self.lambda_val - np.dot(A.transpose(), y)) b = cvxopt_matrix(np.zeros(p)) Q = cvxopt_matrix(-np.eye(p)) stime = time.time() sol = cvxopt_solvers.qp(H, f, Q, b, None, None, None, x0) etime = time.time() - stime print('QP solver took %d seconds' % int(etime)) xnew = np.array(sol['x']).squeeze() #unflip elements of xnew xnew *= as_signs return xnew
def cvxopt_qp_from_numpy(P, q, G, h): """ Converts arguments to cvxopt matrices and runs cvxopt's quadratic programming solver """ P = cvxopt_matrix(P) q = cvxopt_matrix(q) G = cvxopt_matrix(G) h = cvxopt_matrix(np.squeeze(h)) sol = cvxopt_qp(P, q, G, h) return np.array(sol['x'])
def perform_cvx_opt(X, y, C=None, soft_threshold=1e-4, kernel='linear', p=3, gamma=1e-1): n_samples, n_features = X.shape K = np.zeros((n_samples, n_samples)) for i in range(n_samples): for j in range(n_samples): if kernel == 'linear': K[i, j] = linear_kernel(X[i], X[j]) elif kernel == 'poly': K[i, j] = polynomial_kernel(X[i], X[j], p) elif kernel == 'rbf': K[i, j] = gaussian_kernel(X[i], X[j], gamma) P = cvxopt_matrix(np.outer(y, y) * K) q = cvxopt_matrix(np.ones(n_samples) * -1) A = cvxopt_matrix(y.reshape(1, -1)) A = cvxopt_matrix(A, (1, n_samples), 'd') b = cvxopt_matrix(0.0) if C is None: G = cvxopt_matrix(np.diag(np.ones(n_samples) * -1)) h = cvxopt_matrix(np.zeros(n_samples)) else: tmp1 = np.diag(np.ones(n_samples) * -1) tmp2 = np.identity(n_samples) G = cvxopt_matrix(np.vstack((tmp1, tmp2))) tmp1 = np.zeros(n_samples) tmp2 = np.ones(n_samples) * C h = cvxopt_matrix(np.hstack((tmp1, tmp2))) # solve QP problem solution = cvxopt_solvers.qp(P, q, G, h, A, b) # Lagrange multipliers a = np.ravel(solution['x']) # Calculate weights w = np.matrix(np.zeros((1, n_features))) for i in range(n_samples): w += a[i] * y[i] * X[i] # Calculate Intercepts intercept = 0 for i in range(n_samples): if a[i] > soft_threshold: intercept = y[i] - w.dot(np.transpose(X[i])) break intercept = float(intercept) return w.T, intercept, a
def svm(X, Y, C=None): if C is not None: C = float(C) #Soft marging condition m, n = X.shape H = np.outer(Y, Y) * kernel(X, X) Y = Y.reshape(-1, 1) * 1. #Converting into cvxopt format P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) A = cvxopt_matrix(Y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) if C is None: G = cvxopt_matrix(-np.eye(m)) h = cvxopt_matrix(np.zeros(m)) else: G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m)))) h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m) * C))) sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) #Getting weights w = sum(y*alphas*x) w = ((Y * alphas).T @ X).reshape(-1, 1) #Selecting the set of indices S corresponding to non zero parameters S = (alphas > 1e-4).flatten() #Computing b b = np.mean(Y[S] - np.dot(X[S], w)) return w, b
def train(X, y, C=10): n_samples, n_features = X.shape H = np.zeros((n_samples, n_samples)) for i in range(n_samples): for j in range(n_samples): H[i, j] = kernel(X[i], X[j]) P = cvxopt_matrix(np.outer(y, y) * H, tc='d') q = cvxopt_matrix(np.ones(n_samples) * -1) A = cvxopt_matrix(y, (1, n_samples), tc='d') b = cvxopt_matrix(0.0, tc='d') G_max = np.identity(n_samples) * -1 G_min = np.identity(n_samples) G = cvxopt_matrix(np.vstack((G_max, G_min))) h_max = cvxopt_matrix(np.zeros(n_samples)) h_min = cvxopt_matrix(np.ones(n_samples) * C) h = cvxopt_matrix(np.vstack((h_max, h_min))) solution = cvxopt_solvers.qp(P, q, G, h, A, b) lm = np.ravel(solution['x']) idx = lm > 1e-5 ind = np.arange(len(lm))[idx] lagr_multipliers = lm[idx] sv = X[idx] sv_labels = y[idx] bias = 0 for n in range(len(lagr_multipliers)): bias += sv_labels[n] bias -= np.sum(lagr_multipliers * sv_labels * H[ind[n], idx]) bias /= len(lagr_multipliers) return lagr_multipliers, sv, sv_labels, bias
def find_alpha(X, Y, C, K, n_l_p=3) -> np.ndarray: # Setting solver parameters (change default to decrease tolerance) cvxopt_solvers.options['show_progress'] = False cvxopt_solvers.options['abstol'] = 1e-10 cvxopt_solvers.options['reltol'] = 1e-10 cvxopt_solvers.options['feastol'] = 1e-10 m, n = X.shape H = _H(X, Y, K, n_l_p) P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) A = cvxopt_matrix(Y, (1, m), 'd') b = cvxopt_matrix(np.zeros(1)) if C == None: # Converting into cvxopt format G = cvxopt_matrix(-np.eye(m)) h = cvxopt_matrix(np.zeros(m)) else: # Converting into cvxopt format - as previously G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m)))) h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m) * C))) # Run solver sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) return alphas
def OptimizeDirection(vdir, lp): """Optimize in one direction.""" lp_G, lp_h = lp lp_q = cvxopt_matrix(-vdir) sol = solve_lp(lp_q, lp_G, lp_h, solver='glpk') z = sol['x'] if z is not None: z = array(z).reshape((lp_q.size[0],)) return True, z else: print "Failed" return False, 0
def svm_train_dual(yTrain, XTrain, c): import numpy as np #Initializing values and computing H. Note the 1. to force to float type n_samples, n_features = XTrain.shape ##normalization of hyperparameter C = c / n_samples ## X_negate just multiplies -1 to all the features i.e negative of the features yTrain = yTrain.reshape(-1, 1) * 1. X_negate = yTrain * XTrain M = np.dot(X_negate, X_negate.T) * 1. #using cvxopt to compute matrices P = cvxopt_matrix(M) q = cvxopt_matrix(-np.ones((n_samples, 1))) A = cvxopt_matrix(yTrain.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) G = cvxopt_matrix(np.vstack((np.eye(n_samples) * -1, np.eye(n_samples)))) h = cvxopt_matrix(np.hstack((np.zeros(n_samples), np.ones(n_samples) * C))) #Run solver solving the quadratic equation solver = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(solver['x']) ## finding w and b w = ((yTrain * alphas).T @ XTrain).reshape(-1, 1) w = w.flatten() S = (alphas > 1e-5).flatten() b = np.mean(yTrain[S] - np.dot(XTrain[S], w)) return w, b
def svm_train_primal(yTrain, XTrain, C): n_samples, n_features = XTrain.shape ##normalization of hyperparameter C = C / n_samples ## finding a matrix m which is the dot product of ith and jth features m = np.zeros((n_samples, n_samples)) for i in range(n_samples): for j in range(n_samples): m[i, j] = np.dot(XTrain[i], XTrain[j]) ## using cvxopt to make variable matrices P = cvxopt_matrix(np.outer(yTrain, yTrain) * m) q = cvxopt_matrix(np.ones(n_samples) * -1) A = cvxopt_matrix(yTrain, (1, n_samples)) b = cvxopt_matrix(0.0) G = cvxopt_matrix( np.vstack((np.diag(np.ones(n_samples) * -1), np.identity(n_samples)))) h = cvxopt_matrix(np.hstack((np.zeros(n_samples), np.ones(n_samples) * C))) solver = cvxopt_solvers.qp(P, q, G, h, A, b) ## lagrange multiplier by solving the quadratic equation alphas = np.ravel(solver['x']) sv = alphas > 1e-5 S = (alphas[sv]).flatten() w = ((yTrain * alphas).T @ XTrain).reshape(-1, 1) b = np.mean(yTrain - np.dot(XTrain, w)) return w, b
def fit(self, X, y): X = np.array(X) y = np.array(y) m, n = X.shape y = y.reshape(-1, 1) * 1. X_dash = y * X H = np.dot(X_dash, X_dash.T) * 1. # Quadratic Programming P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m)))) h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m) * self.C))) A = cvxopt_matrix(y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) # Run solver sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) # Compute the weights and bias self.w = ((y * alphas).T @ X).reshape(-1, 1) S = (alphas > 1e-4).flatten() self.b = np.mean(y[S] - np.dot(X[S], self.w))
def cvxopt_solve(X, y, mems): C = 10 m, n = X.shape y = y.reshape(-1, 1) * 1. X_dash = y * X H = np.dot(X_dash, X_dash.T) * 1. # Converting into cvxopt format - as previously P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m)))) h = cvxopt_matrix(np.hstack((np.zeros(m), mems * C))) A = cvxopt_matrix(y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) # Run solver sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) # ==================Computing and printing parameters===============================# w = ((y * alphas).T @ X).reshape(-1, 1) S = (((C * mems) > alphas.T) > 1e-4).flatten() b = y[S] - np.dot(X[S], w) b = np.mean(b) return (w.flatten(), b)
def SVM(): H = np.zeros((len(Xs), len(Xs))) for i in range(len(Xs)): for j in range(len(Xs)): H[i, j] = Ys[i] * Ys[j] * np.dot(Xs[i], Xs[j]) H = cvxopt_matrix(H) q = cvxopt_matrix(-1 * np.ones(len(Xs))) G = cvxopt_matrix(np.diag(np.ones(len(Xs)) * -1)) h = cvxopt_matrix(np.zeros(len(Xs))) A = cvxopt_matrix(Ys, (1, len(Xs))) b = cvxopt_matrix(0.0) result = cvxopt_solvers.qp(H, q, G, h, A, b) alphas = np.array(result['x']) support_vector_indices = np.where(alphas > 0.0001)[0] print(support_vector_indices) alphas = alphas[support_vector_indices] support_vectors = Xs[support_vector_indices] support_vectors_y = Ys[support_vector_indices] print("%d support vectors out of %d points" % (len(alphas), len(Xs))) print(alphas) weights = np.zeros(2) for i in range(len(alphas)): weights += alphas[i] * support_vectors_y[i] * support_vectors[i] bias = Ys[support_vector_indices] - np.dot(Xs[support_vector_indices], weights) return weights, bias, support_vectors, support_vectors_y
def cvxopt(X, y): m, n = X.shape y = y.reshape(-1, 1) * 1. X_dash = y * X H = np.dot(X_dash, X_dash.T) * 1. P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) G = cvxopt_matrix(-np.eye(m)) h = cvxopt_matrix(np.zeros(m)) A = cvxopt_matrix(y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) # w parameter in vectorized form w = ((y * alphas).T @ X).reshape(-1, 1) # Selecting the set of indices S corresponding to non zero parameters S = (alphas > 1e-4).flatten() # Computing b b = y[S] - np.dot(X[S], w) clf = MyClassifier() clf.coef_.append(w.flatten()) clf.coef_ = np.asarray(clf.coef_) clf.intercept_ = b[0].tolist() clf.support_vectors_ = X[S] return clf, []
def SVM_soft(): H = np.zeros((len(Xs), len(Xs))) for i in range(len(Xs)): for j in range(len(Xs)): H[i, j] = Ys[i] * Ys[j] * kernel_func(Xs[i], Xs[j]) H = cvxopt_matrix(H) q = cvxopt_matrix(-1 * np.ones(len(Xs))) G = cvxopt_matrix(np.diag(np.ones(len(Xs)) * -1)) h = cvxopt_matrix(np.zeros(len(Xs))) A = cvxopt_matrix(Ys, (1, len(Xs))) b = cvxopt_matrix(0.0) result = cvxopt_solvers.qp(H, q, G, h, A, b) alphas = np.array(result['x']) support_vector_indices = np.where(alphas > 0.0001)[0] print(support_vector_indices) alphas = alphas[support_vector_indices] support_vectors = Xs[support_vector_indices] support_vectors_y = Ys[support_vector_indices] print("%d support vectors out of %d points" % (len(alphas), len(Xs))) print(alphas) sum = 0 for i in range(len(support_vector_indices)): sum += alphas[i] * support_vectors_y[i] * kernel_func(support_vectors[i], support_vectors[support_vector_indices[0]]) bias = Ys[support_vector_indices[0]] - sum return alphas, bias, support_vectors, support_vectors_y, support_vector_indices
def l2_norm_LinearSVM_Dual(X, y, C): zero_tol = 1e-7 start_time = time.time() n, p = X.shape y = y.reshape(-1, 1) * 1. X_dash = y * X cvxopt_solvers.options['show_progress'] = False cvxopt_solvers.options['abstol'] = 1e-10 cvxopt_solvers.options['reltol'] = 1e-10 cvxopt_solvers.options['feastol'] = 1e-10 # objective: (1/2) x^T P x + q^T x # constraints: Gx < h # Ax = b # -------- INSERT YOUR CODE HERE -------- # P = cvxopt_matrix(X_dash.dot(X_dash.T) + 1 / C * np.identity(n)) q = cvxopt_matrix(-np.ones((n, 1))) G = cvxopt_matrix(-np.identity(n)) h = cvxopt_matrix(np.zeros(n)) A = cvxopt_matrix(y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) cvxopt_solvers.options['show_progress'] = False cvxopt_solvers.options['abstol'] = 1e-10 cvxopt_solvers.options['reltol'] = 1e-10 cvxopt_solvers.options['feastol'] = 1e-10 sol = cvxopt_solvers.qp(P, q, G, h, A, b) # -- INSERT YOUR CODE HERE -- # alphas = np.array(sol['x']) sol_time = time.time() - start_time return alphas, sol_time
def LinearSVM_Dual(X, y, C): start_time = time.time() n, p = X.shape y = y.reshape(-1, 1) * 1. X_dash = y * X # Complete the following code: # cvxopt_solvers.qp(P, q, G, h, A, b) # objective: (1/2) x^T P x + q^T x # constraints: Gx < h # Ax = b # example could be found here: # https://cvxopt.org/userguide/coneprog.html#quadratic-programming P = cvxopt_matrix(X_dash.dot(X_dash.T)) q = cvxopt_matrix(-np.ones((n, 1))) G = cvxopt_matrix(np.vstack((-np.diag(np.ones(n)), np.identity(n)))) h = cvxopt_matrix(np.hstack((np.zeros(n), np.ones(n) * C))) A = cvxopt_matrix(y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) cvxopt_solvers.options['show_progress'] = False cvxopt_solvers.options['abstol'] = 1e-10 cvxopt_solvers.options['reltol'] = 1e-10 cvxopt_solvers.options['feastol'] = 1e-10 # -- INSERT YOUR CODE HERE -- # sol = cvxopt_solvers.qp(P, q, G, h, A, b) # -- INSERT YOUR CODE HERE -- # alphas = np.array(sol['x']) sol_time = time.time() - start_time return alphas, sol_time
def svr_cvxopt(X_train, y_train, X_test, C, eps, kernel_type, gamma, degree): X = X_train y = y_train m, n = X.shape y = y.reshape(-1, 1) * 1 print(y.shape) q1 = eps - y q2 = eps + y K = kernel(kernel_type, X, X, gamma, degree) p1 = np.hstack((K, K * -1)) P = cvxopt_matrix(np.vstack((p1, p1 * -1))) q = cvxopt_matrix(np.vstack((q1, q2))) G = cvxopt_matrix(np.vstack((np.eye(2 * m) * -1, np.eye(2 * m)))) h = cvxopt_matrix(np.hstack((np.zeros(2 * m), np.ones(2 * m) * C))) A = cvxopt_matrix((np.hstack( (np.ones(m), np.ones(m) * -1))).reshape(1, -1)) # print(A.shape) b = cvxopt_matrix(np.zeros(1)) #Run solver sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) # print(alphas) alphas1 = alphas[:m] alphas2 = alphas[m:2 * m] b = np.array(sol['y']) # print(b) # W = ((alphas1 - alphas2).T @ X).reshape(-1, 1) ## W is a column vector y_pred = kernel(kernel_type, X, X_test, gamma, degree).T @ (alphas1 - alphas2) + b # print(y_pred) return y_pred
def svr_cvxopt_rh(X_train, y_train, X_test, C, eps, kernel_type, gamma, degree): X = X_train y = y_train m, n = X.shape y = y.reshape(-1, 1) * 1 # print(y.shape) q1 = 2 * eps * y K = kernel(kernel_type, X, X, gamma, degree) K1 = K + y @ y.T p1 = np.hstack((K1, K1 * -1)) a1 = np.hstack((np.ones(m), np.zeros(m))) a2 = np.hstack((np.zeros(m), np.ones(m))) P = cvxopt_matrix(np.vstack((p1, p1 * -1))) q = cvxopt_matrix(np.vstack((q1, q1 * -1))) G = cvxopt_matrix(np.vstack((np.eye(2 * m) * -1, np.eye(2 * m)))) h = cvxopt_matrix(np.hstack((np.zeros(2 * m), np.ones(2 * m) * C))) A = cvxopt_matrix(np.vstack((a1, a2))) # print(A.shape) b = cvxopt_matrix(np.ones(2)) #Run solver sol = cvxopt_solvers.qp(P, q, G, h, A, b) variables = np.array(sol['x']) u = variables[:m] v = variables[m:2 * m] delta = (u - v).T @ y + 2 * eps bias = (u - v).T @ (K @ (u + v)) / (2 * delta) + (u + v).T @ y / 2 u1 = u / delta v1 = v / delta y_pred = kernel(kernel_type, X, X_test, gamma, degree).T @ (v1 - u1) + bias # print(y_pred) return y_pred
def __compute_topp_polygon_bretl(cwc, p, ps, pss, sdd_max=None, sd_max=None): """Compute the polygon in the (sddot,sdot^2) plane using Hauser method.""" g = get_gravity() b1 = dot(cwc, hstack([ps, cross(p, ps)])) b2 = dot(cwc, hstack([pss, cross(p, pss)])) b3 = dot(cwc, hstack([g, cross(p, g)])) n = len(b1) G = zeros((n+1, 2)) G[0, :] = [0, -1] G[1:, 0] = b1 G[1:, 1] = b2 h = zeros(n+1) h[1:] = b3 if sdd_max is not None: G = vstack([array([[1, 0], [-1, 0]]), G]) h = hstack([array([sdd_max, sdd_max]), h]) if sd_max is not None: sd_max_2 = sd_max ** 2 G = vstack([array([[0, 1], [0, -1]]), G]) h = hstack([array([sd_max_2, sd_max_2]), h]) lp = cvxopt_matrix(G), cvxopt_matrix(h) res, z1 = OptimizeDirection(array([1., 0.]), lp) if not res: return None res, z2 = OptimizeDirection(array([cos(2*pi/3), sin(2*pi/3)]), lp) if not res: return None res, z3 = OptimizeDirection(array([cos(4*pi/3), sin(4*pi/3)]), lp) if not res: return None v1 = Vertex(z1) v2 = Vertex(z2) v3 = Vertex(z3) P = Polygon() P.fromVertices(v1, v2, v3) P.iter_expand(lp, 100) return P
def soft_margin_rbf(C, gamma, x_vals, y_vals): x_vals = np.concatenate([x_vals[y_vals == 0], x_vals[y_vals == 1]]) y_vals = np.concatenate([y_vals[y_vals == 0], y_vals[y_vals == 1]]) x_vals,y_vals = shuffle(x_vals,y_vals,random_state = 0) y_vals = 2*y_vals - 1 x_vals = x_vals.reshape(x_vals.shape[0],25) y_vals = y_vals.reshape(-1,1)*1 tr_len = int(0.7*len(y_vals)) train_length=tr_len x_train = x_vals[:tr_len, :] y_train = y_vals[:tr_len, :] x_test = x_vals[tr_len:, :] y_test = y_vals[tr_len:, :] m,n = x_train.shape norm_column_squared = np.linalg.norm(x_train, axis = 1).reshape(m,1)**2 norm_row_squared = np.linalg.norm(x_train.T, axis = 0).reshape(1,m)**2 norm_column_extended = norm_column_squared*np.ones((m,m)) norm_row_extended = norm_column_extended.T cross_terms = x_train@x_train.T #rbf matrix has terms of the form aij = exp(-gamma*||xi - xj||^2) X_dash = np.exp(-gamma*(norm_column_extended + norm_row_extended - 2*cross_terms)) #((y_train.reshape(tr_len,1)*X_dash).T) H = (y_train.reshape(tr_len,1))*((y_train.reshape(tr_len,1)*X_dash).T) #Converting into cvxopt format P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) G = cvxopt_matrix(np.vstack((np.eye(m)*-1,np.eye(m)))) h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m) * C))) A = cvxopt_matrix(y_train.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) #Setting solver parameters (change default to decrease tolerance) cvxopt_solvers.options['show_progress'] = False cvxopt_solvers.options['abstol'] = 1e-10 cvxopt_solvers.options['reltol'] = 1e-10 cvxopt_solvers.options['feastol'] = 1e-10 #Run solver sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) w = ((y_train*alphas).T @ x_train).reshape(-1,1) #Selecting the set of indices S corresponding to non zero parameters S = (alphas > 1e-4).flatten() #Computing b b = y_train[S] - np.dot(x_train[S], w) #Display results # print('Alphas = ',alphas[alphas > 1e-4]) # print('w = ', w.flatten()) # print('b = ', b[0]) scores = x_test.dot(w) + b[0] predictions = np.sign(scores) correct_predictions = np.sum(predictions == y_test) val_accuracy = (correct_predictions/len(predictions))*100 return val_accuracy
def fit(self, X, Y): # Keep samples self.X = X self.Y = Y N, _ = X.shape # Calculate Kernel Matrix self.K = self.svc_kernel_mat(self) # Adjust Outputs for cvxopt function A = Y.reshape(-1, 1) * 1 A = A.reshape(1, -1) A = A.astype('float') # Calculate Optmization Matrices P = cvxopt_matrix(self.K) q = cvxopt_matrix(-np.ones((N, 1))) G = cvxopt_matrix(np.vstack((np.eye(N) * -1, np.eye(N)))) h = cvxopt_matrix(np.hstack((np.zeros(N), np.ones(N) * self.C))) A = cvxopt_matrix(A) b = cvxopt_matrix(np.zeros(1)) #Run solver cvxopt_solvers.options['show_progress'] = False sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) # Get support vectors (alphas > epsilon) epsilon = 1e-4 sv = (alphas > epsilon).flatten() Xsv = X[sv, :] Ysv = Y[sv] alphas = alphas[sv, 0] Nsv = len(Ysv) # Calculate Bias b = 0 for i in range(Nsv): b = b + 1 / Ysv[i] for j in range(Nsv): kij = self.kernel_func(Xsv[j, :], Xsv[i, :], kernel=self.kernel_type, gamma=self.gamma, d=self.d) b = b - Ysv[j] * alphas[j] * kij # Hold Results self.Xsv = Xsv self.Ysv = Ysv self.alphas = alphas self.b = b
def hard_margin_linear(x_vals, y_vals): x_vals = np.concatenate([x_vals[y_vals == 0], x_vals[y_vals == 1]]) y_vals = np.concatenate([y_vals[y_vals == 0], y_vals[y_vals == 1]]) x_vals,y_vals = shuffle(x_vals,y_vals,random_state = 0) y_vals = 2*y_vals - 1 x_vals = x_vals.reshape(x_vals.shape[0],25) y_vals = y_vals.reshape(-1,1)*1 tr_len = int(0.7*len(y_vals)) x_train = x_vals[:tr_len, :] y_train = y_vals[0:tr_len, :] x_test = x_vals[tr_len:, :] y_test = y_vals[tr_len:, :] m,n = x_train.shape # x_dash = x_train*y_train # H = x_dash@x_dash.T X_dash = y_train* x_train H = np.dot(X_dash , X_dash.T) * 1. #X_dash = y_train*x_train #H = np.dot(X_dash , X_dash.T) * 1. #Converting into cvxopt format P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) G = cvxopt_matrix(-np.eye(m)) h = cvxopt_matrix(np.zeros(m)) A = cvxopt_matrix(y_train.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) #Setting solver parameters (change default to decrease tolerance) cvxopt_solvers.options['show_progress'] = False cvxopt_solvers.options['abstol'] = 1e-10 cvxopt_solvers.options['reltol'] = 1e-10 cvxopt_solvers.options['feastol'] = 1e-10 #Run solver sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) #w parameter in vectorized form w = ((y_train*alphas).T @ x_train).reshape(-1,1) #Selecting the set of indices S corresponding to non zero parameters S = (alphas > 1e-4).flatten() #Computing b b = y_train[S] - np.dot(x_train[S], w) #Display results # print('Alphas = ',alphas[alphas > 1e-4]) # print('w = ', w.flatten()) # print('b = ', b[0]) scores = x_test.dot(w) + b[0] predictions = np.sign(scores) correct_predictions = np.sum(predictions == np.rint(y_test)) val_accuracy = (correct_predictions/len(predictions))*100 #print('validation_accuracy = ', val_accuracy) return val_accuracy
def svmlin(X, t, C): # Linear SVM Classifier # # INPUT: # X : the dataset (num_samples x dim) # t : labeling (num_samples x 1) # C : penalty factor for slack variables (scalar) # # OUTPUT: # alpha : output of quadprog function (num_samples x 1) # sv : support vectors (boolean) (1 x num_samples) # w : parameters of the classifier (1 x dim) # b : bias of the classifier (scalar) # result : result of classification (1 x num_samples) # slack : points inside the margin (boolean) (1 x num_samples) #####Insert your code here for subtask 2a##### m, n = X.shape y = t.reshape(-1, 1) * 1. X_dash = y * X H = np.dot(X_dash, X_dash.T) * 1. P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) G = cvxopt_matrix(-np.eye(m)) h = cvxopt_matrix(np.zeros(m)) A = cvxopt_matrix(y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) #n = X.shape[0] #q = (-1)* np.ones(n) #G = np.vstack([-np.eye(n), np.eye(n)]) #A = t #b = 0 #h = np.hstack([0,C]) #P = np.full((n,n),0) #for i in range(n): # for j in range(n): # P[i,j] = t[i]*t[j]*np.dot(X[i],X[j]) a = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) # w parameter in vectorized form w = ((y * alphas).T @ X).reshape(-1, 1) # Selecting the set of indices S corresponding to non zero parameters S = (alphas > 1e-4).flatten() # Computing b b = y[S] - np.dot(X[S], w) alpha, sv, w, b, result, slack = svmlin(train['data'], train['label'], C) return alpha, sv, w, b, result, slack
def calculateW(Xtrain, Ytrain, lam): m = len(Ytrain) y = Ytrain.reshape(-1, 1) * 1. x = y * Xtrain P = cvxopt_matrix(x @ x.T * lam * 1.) q = cvxopt_matrix(-np.ones((m, 1))) G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m)))) h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m)))) A = cvxopt_matrix(y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) sol = cvxopt_solvers.qp(P, q, G, h, A, b, options={'show_progress': False}) alphas = np.array(sol['x']) return ((y * alphas).T @ Xtrain).reshape(-1, 1)
def HardMarg(X, y): #Note the problem is to: #max \sigma_i alpha - 0.5 \sigma_{i,j} alpha_i alpha_j y_i y_j x^T_i x_j #subject to alpha_i \geq 0, \sigma \alpha_i y_i = 0 #Form of qt solver (http://cvxopt.org/userguide/coneprog.html): # min 0.5 (alpha^T P alpha) + q alpha # subject to G alpha < h # A alpha = b # In this case (mutiply by -1): # P_ij = y_i y_j x^T_i x_j, P = y^T X^T X y # q = -1 # G = -1 # h = 0 # a = y # b = 0 d = np.shape(X)[1] y = y.astype(np.double) P = cvxopt_matrix(np.matmul(y, y.T) * np.matmul(X.T, X)) q = cvxopt_matrix(-np.ones((d, 1))) G = cvxopt_matrix(-np.eye(d)) h = cvxopt_matrix(np.zeros(d)) a = cvxopt_matrix(y.T) b = cvxopt_matrix(np.zeros(1)) alpha = np.array( cvxopt_solvers.qp(P, q, G, h, a, b, kktsolver='ldl', options={ 'kktreg': 1e-9, 'show_progress': False })['x']) beta = np.sum((alpha * y).T * X, axis=1) count = 0 while (alpha[count] <= 1e-5): count += 1 beta_0 = y[count] - np.matmul(beta, X[:, count:count + 1]) return beta, beta_0
def _convert_to_cvxopt_matrices(self): from cvxopt import matrix as cvxopt_matrix self._quadratic_func = cvxopt_matrix(self._quadratic_func) if self._linear_func is not None: self._linear_func = cvxopt_matrix(self._linear_func) else: # CVXOPT needs this vector to be set even if it is not used, so we put zeros in it! self._linear_func = cvxopt_matrix(np.zeros((self._n_variables, 1))) if self._inequality_constraints_matrix is not None: self._inequality_constraints_matrix = cvxopt_matrix(self._inequality_constraints_matrix) if self._inequality_constraints_values is not None: self._inequality_constraints_values = cvxopt_matrix(self._inequality_constraints_values) if self._equality_constraints_matrix is not None: self._equality_constraints_matrix = cvxopt_matrix(self._equality_constraints_matrix) if self._equality_constraints_values is not None: self._equality_constraints_values = cvxopt_matrix(self._equality_constraints_values)
def l2_norm_LinearSVM_Dual(X, y, C): zero_tol = 1e-4 cluster_1, cluster_2 = split_data(X, y) plt.scatter(cluster_1[:, 0], cluster_1[:, 1], color='blue') plt.scatter(cluster_2[:, 0], cluster_2[:, 1], color='green') for i in range(len(y)): if y[i] == 0: y[i] = -1 start_time = time.time() n, p = X.shape y = y.reshape(-1, 1) * 1. X_dash = y * X extra_term = 1 / C * np.identity(n) P = cvxopt_matrix(X_dash.dot(X_dash.T) + extra_term) q = cvxopt_matrix(-np.ones((n, 1))) G = cvxopt_matrix(np.vstack((-np.diag(np.ones(n))))) h = cvxopt_matrix(np.hstack((np.zeros(n)))) A = cvxopt_matrix(y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) cvxopt_solvers.options['show_progress'] = False cvxopt_solvers.options['abstol'] = 1e-10 cvxopt_solvers.options['reltol'] = 1e-10 cvxopt_solvers.options['feastol'] = 1e-10 sol = cvxopt_solvers.qp(P, q, G, h, A, b) alphas = np.array(sol['x']) sol_time = time.time() - start_time w = ((y * alphas).T @ X).reshape(-1, 1) # Selecting the set of indices S corresponding to non zero parameters S = (alphas > zero_tol).flatten() # Computing b b = y[S] - np.dot(X[S], w) b = b[0] # Display results x = np.linspace(-1, 5, 20) plt.plot(x, (-b - (w[0] * x)) / w[1], 'm') plt.show() return alphas, sol_time
def fit(self,X,Y): m,n = X.shape X_dash = Y*X H = np.dot(X_dash , X_dash.T) * 1. P = cvxopt_matrix(H) q = cvxopt_matrix(-np.ones((m, 1))) G = cvxopt_matrix(-np.eye(m)) h = cvxopt_matrix(np.zeros(m)) A = cvxopt_matrix(Y.reshape(1, -1)) b = cvxopt_matrix(np.zeros(1)) sol = cvxopt_solvers.qp(P, q, G, h, A, b) self.alphas = np.array(sol['x']) #Selecting the set of indices S corresponding to non zero parameters self.S = np.where(self.alphas>1e-5)[0] self.w = ((Y * self.alphas).T @ X).reshape(-1,1) self.b = Y[self.S] - np.dot(X[self.S], self.w)