def solve(self, A, y, x0, as_signs):
print 'cvxopt QP: # of params=%d' % A.shape[1]
#flip sign of columns of A that have negative usefulness, a trick
#Patrick Gill uses to halve the # of parameters for the QP solver
A *= as_signs
#the sign of x0 is unflipped for the nonzero elements, flip it
x0 *= as_signs
p = A.shape[1]
H = cvxopt_matrix(np.dot(A.transpose(), A))
f = cvxopt_matrix(self.lambda_val - np.dot(A.transpose(), y))
b = cvxopt_matrix(np.zeros(p))
Q = cvxopt_matrix(-np.eye(p))
stime = time.time()
sol = cvxopt_solvers.qp(H, f, Q, b, None, None, None, x0)
etime = time.time() - stime
print 'QP solver took %d seconds' % int(etime)
xnew = np.array(sol['x']).squeeze()
#unflip elements of xnew
xnew *= as_signs
return xnew
def get_lazy_policy_action(
self,
r_plus_gamma_v_nexts,
actions,
):
"""
Get optimal action using the lazy action model
"""
actions_augmented = np.concatenate((np.ones(
(len(actions), 1)), actions),
axis=-1)
sol = qp(
cvxopt_matrix(
np.sum(actions_augmented[:, :, None] *
actions_augmented[:, None, :],
axis=0) + self.local_model_regularization * np.eye(3)),
cvxopt_matrix(
np.sum(
-(r_plus_gamma_v_nexts[:, None] * actions_augmented[:, :]),
axis=0)))
if 'optimal' not in sol['status']:
raise Exception('Optimal solution not found')
alphabeta = np.array(sol['x']).reshape(-1)
return self.environment.action_length / np.linalg.norm(
alphabeta[1:]) * alphabeta[1:]
def get_lazy_model_value(
self,
r_plus_gamma_v_nexts,
actions,
):
"""
Predict value of V-function, performing the maximum over the available actions using
the lazy action model
"""
actions_augmented = np.concatenate((np.ones(
(len(actions), 1)), actions),
axis=-1)
sol = qp(
cvxopt_matrix(
np.sum(actions_augmented[:, :, None] *
actions_augmented[:, None, :],
axis=0) + self.local_model_regularization * np.eye(3)),
cvxopt_matrix(
np.sum(
-(r_plus_gamma_v_nexts[:, None] * actions_augmented[:, :]),
axis=0)))
if 'optimal' not in sol['status']:
raise Exception('Optimal solution not found')
alphabeta = np.array(sol['x']).reshape(-1)
# Instability countermeasure: The target cant be higher than the maximum
# r_plus_gamma_V_next in the vicinity
return min([
alphabeta[0] +
self.environment.action_length * np.linalg.norm(alphabeta[1:]),
np.max(r_plus_gamma_v_nexts)
])
def primal_svm(X, Y, C=1.0):
n = len(X[0])
m = len(Y)
d = n + m + 1
P = cvxopt_matrix(0.0, (d, d))
for i in range(n):
P[i, i] = 1.0
q = cvxopt_matrix(0.0, (d, 1))
for i in range(n, n + m):
q[i] = C
q[-1] = 0.0
h = cvxopt_matrix(-1.0, (m + m, 1))
h[m:] = 0.0
# print m
G = cvxopt_matrix(0.0, (2 * m, d))
for i in range(m):
G[i, :n] = -Y[i] * X[i]
G[i, n + i] = -1
G[i, -1] = -Y[i]
for i in range(m, m + m):
G[i, n + i - m] = -1.0
cvxopt_solvers.options['show_progress'] = False
vars = cvxopt_solvers.qp(P, q, G, h)['x']
w = vars[0:n]
b = vars[-1]
return w, b
def solve(self, A, y, x0, as_signs):
print('cvxopt QP: # of params=%d' % A.shape[1])
#flip sign of columns of A that have negative usefulness, a trick
#Patrick Gill uses to halve the # of parameters for the QP solver
A *= as_signs
#the sign of x0 is unflipped for the nonzero elements, flip it
x0 *= as_signs
p = A.shape[1]
H = cvxopt_matrix(np.dot(A.transpose(), A))
f = cvxopt_matrix(self.lambda_val - np.dot(A.transpose(), y))
b = cvxopt_matrix(np.zeros(p))
Q = cvxopt_matrix(-np.eye(p))
stime = time.time()
sol = cvxopt_solvers.qp(H, f, Q, b, None, None, None, x0)
etime = time.time() - stime
print('QP solver took %d seconds' % int(etime))
xnew = np.array(sol['x']).squeeze()
#unflip elements of xnew
xnew *= as_signs
return xnew
def cvxopt_qp_from_numpy(P, q, G, h):
"""
Converts arguments to cvxopt matrices and runs
cvxopt's quadratic programming solver
"""
P = cvxopt_matrix(P)
q = cvxopt_matrix(q)
G = cvxopt_matrix(G)
h = cvxopt_matrix(np.squeeze(h))
sol = cvxopt_qp(P, q, G, h)
return np.array(sol['x'])
def perform_cvx_opt(X,
y,
C=None,
soft_threshold=1e-4,
kernel='linear',
p=3,
gamma=1e-1):
n_samples, n_features = X.shape
K = np.zeros((n_samples, n_samples))
for i in range(n_samples):
for j in range(n_samples):
if kernel == 'linear':
K[i, j] = linear_kernel(X[i], X[j])
elif kernel == 'poly':
K[i, j] = polynomial_kernel(X[i], X[j], p)
elif kernel == 'rbf':
K[i, j] = gaussian_kernel(X[i], X[j], gamma)
P = cvxopt_matrix(np.outer(y, y) * K)
q = cvxopt_matrix(np.ones(n_samples) * -1)
A = cvxopt_matrix(y.reshape(1, -1))
A = cvxopt_matrix(A, (1, n_samples), 'd')
b = cvxopt_matrix(0.0)
if C is None:
G = cvxopt_matrix(np.diag(np.ones(n_samples) * -1))
h = cvxopt_matrix(np.zeros(n_samples))
else:
tmp1 = np.diag(np.ones(n_samples) * -1)
tmp2 = np.identity(n_samples)
G = cvxopt_matrix(np.vstack((tmp1, tmp2)))
tmp1 = np.zeros(n_samples)
tmp2 = np.ones(n_samples) * C
h = cvxopt_matrix(np.hstack((tmp1, tmp2)))
# solve QP problem
solution = cvxopt_solvers.qp(P, q, G, h, A, b)
# Lagrange multipliers
a = np.ravel(solution['x'])
# Calculate weights
w = np.matrix(np.zeros((1, n_features)))
for i in range(n_samples):
w += a[i] * y[i] * X[i]
# Calculate Intercepts
intercept = 0
for i in range(n_samples):
if a[i] > soft_threshold:
intercept = y[i] - w.dot(np.transpose(X[i]))
break
intercept = float(intercept)
return w.T, intercept, a
def svm(X, Y, C=None):
if C is not None: C = float(C) #Soft marging condition
m, n = X.shape
H = np.outer(Y, Y) * kernel(X, X)
Y = Y.reshape(-1, 1) * 1.
#Converting into cvxopt format
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
A = cvxopt_matrix(Y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
if C is None:
G = cvxopt_matrix(-np.eye(m))
h = cvxopt_matrix(np.zeros(m))
else:
G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m))))
h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m) * C)))
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
#Getting weights w = sum(y*alphas*x)
w = ((Y * alphas).T @ X).reshape(-1, 1)
#Selecting the set of indices S corresponding to non zero parameters
S = (alphas > 1e-4).flatten()
#Computing b
b = np.mean(Y[S] - np.dot(X[S], w))
return w, b
def train(X, y, C=10):
n_samples, n_features = X.shape
H = np.zeros((n_samples, n_samples))
for i in range(n_samples):
for j in range(n_samples):
H[i, j] = kernel(X[i], X[j])
P = cvxopt_matrix(np.outer(y, y) * H, tc='d')
q = cvxopt_matrix(np.ones(n_samples) * -1)
A = cvxopt_matrix(y, (1, n_samples), tc='d')
b = cvxopt_matrix(0.0, tc='d')
G_max = np.identity(n_samples) * -1
G_min = np.identity(n_samples)
G = cvxopt_matrix(np.vstack((G_max, G_min)))
h_max = cvxopt_matrix(np.zeros(n_samples))
h_min = cvxopt_matrix(np.ones(n_samples) * C)
h = cvxopt_matrix(np.vstack((h_max, h_min)))
solution = cvxopt_solvers.qp(P, q, G, h, A, b)
lm = np.ravel(solution['x'])
idx = lm > 1e-5
ind = np.arange(len(lm))[idx]
lagr_multipliers = lm[idx]
sv = X[idx]
sv_labels = y[idx]
bias = 0
for n in range(len(lagr_multipliers)):
bias += sv_labels[n]
bias -= np.sum(lagr_multipliers * sv_labels * H[ind[n], idx])
bias /= len(lagr_multipliers)
return lagr_multipliers, sv, sv_labels, bias
def find_alpha(X, Y, C, K, n_l_p=3) -> np.ndarray:
# Setting solver parameters (change default to decrease tolerance)
cvxopt_solvers.options['show_progress'] = False
cvxopt_solvers.options['abstol'] = 1e-10
cvxopt_solvers.options['reltol'] = 1e-10
cvxopt_solvers.options['feastol'] = 1e-10
m, n = X.shape
H = _H(X, Y, K, n_l_p)
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
A = cvxopt_matrix(Y, (1, m), 'd')
b = cvxopt_matrix(np.zeros(1))
if C == None:
# Converting into cvxopt format
G = cvxopt_matrix(-np.eye(m))
h = cvxopt_matrix(np.zeros(m))
else:
# Converting into cvxopt format - as previously
G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m))))
h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m) * C)))
# Run solver
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
return alphas
def OptimizeDirection(vdir, lp):
"""Optimize in one direction."""
lp_G, lp_h = lp
lp_q = cvxopt_matrix(-vdir)
sol = solve_lp(lp_q, lp_G, lp_h, solver='glpk')
z = sol['x']
if z is not None:
z = array(z).reshape((lp_q.size[0],))
return True, z
else:
print "Failed"
return False, 0
def svm_train_dual(yTrain, XTrain, c):
import numpy as np
#Initializing values and computing H. Note the 1. to force to float type
n_samples, n_features = XTrain.shape
##normalization of hyperparameter
C = c / n_samples
## X_negate just multiplies -1 to all the features i.e negative of the features
yTrain = yTrain.reshape(-1, 1) * 1.
X_negate = yTrain * XTrain
M = np.dot(X_negate, X_negate.T) * 1.
#using cvxopt to compute matrices
P = cvxopt_matrix(M)
q = cvxopt_matrix(-np.ones((n_samples, 1)))
A = cvxopt_matrix(yTrain.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
G = cvxopt_matrix(np.vstack((np.eye(n_samples) * -1, np.eye(n_samples))))
h = cvxopt_matrix(np.hstack((np.zeros(n_samples), np.ones(n_samples) * C)))
#Run solver solving the quadratic equation
solver = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(solver['x'])
## finding w and b
w = ((yTrain * alphas).T @ XTrain).reshape(-1, 1)
w = w.flatten()
S = (alphas > 1e-5).flatten()
b = np.mean(yTrain[S] - np.dot(XTrain[S], w))
return w, b
def svm_train_primal(yTrain, XTrain, C):
n_samples, n_features = XTrain.shape
##normalization of hyperparameter
C = C / n_samples
## finding a matrix m which is the dot product of ith and jth features
m = np.zeros((n_samples, n_samples))
for i in range(n_samples):
for j in range(n_samples):
m[i, j] = np.dot(XTrain[i], XTrain[j])
## using cvxopt to make variable matrices
P = cvxopt_matrix(np.outer(yTrain, yTrain) * m)
q = cvxopt_matrix(np.ones(n_samples) * -1)
A = cvxopt_matrix(yTrain, (1, n_samples))
b = cvxopt_matrix(0.0)
G = cvxopt_matrix(
np.vstack((np.diag(np.ones(n_samples) * -1), np.identity(n_samples))))
h = cvxopt_matrix(np.hstack((np.zeros(n_samples), np.ones(n_samples) * C)))
solver = cvxopt_solvers.qp(P, q, G, h, A, b)
## lagrange multiplier by solving the quadratic equation
alphas = np.ravel(solver['x'])
sv = alphas > 1e-5
S = (alphas[sv]).flatten()
w = ((yTrain * alphas).T @ XTrain).reshape(-1, 1)
b = np.mean(yTrain - np.dot(XTrain, w))
return w, b
def fit(self, X, y):
X = np.array(X)
y = np.array(y)
m, n = X.shape
y = y.reshape(-1, 1) * 1.
X_dash = y * X
H = np.dot(X_dash, X_dash.T) * 1.
# Quadratic Programming
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m))))
h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m) * self.C)))
A = cvxopt_matrix(y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
# Run solver
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
# Compute the weights and bias
self.w = ((y * alphas).T @ X).reshape(-1, 1)
S = (alphas > 1e-4).flatten()
self.b = np.mean(y[S] - np.dot(X[S], self.w))
def cvxopt_solve(X, y, mems):
C = 10
m, n = X.shape
y = y.reshape(-1, 1) * 1.
X_dash = y * X
H = np.dot(X_dash, X_dash.T) * 1.
# Converting into cvxopt format - as previously
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m))))
h = cvxopt_matrix(np.hstack((np.zeros(m), mems * C)))
A = cvxopt_matrix(y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
# Run solver
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
# ==================Computing and printing parameters===============================#
w = ((y * alphas).T @ X).reshape(-1, 1)
S = (((C * mems) > alphas.T) > 1e-4).flatten()
b = y[S] - np.dot(X[S], w)
b = np.mean(b)
return (w.flatten(), b)
def SVM():
H = np.zeros((len(Xs), len(Xs)))
for i in range(len(Xs)):
for j in range(len(Xs)):
H[i, j] = Ys[i] * Ys[j] * np.dot(Xs[i], Xs[j])
H = cvxopt_matrix(H)
q = cvxopt_matrix(-1 * np.ones(len(Xs)))
G = cvxopt_matrix(np.diag(np.ones(len(Xs)) * -1))
h = cvxopt_matrix(np.zeros(len(Xs)))
A = cvxopt_matrix(Ys, (1, len(Xs)))
b = cvxopt_matrix(0.0)
result = cvxopt_solvers.qp(H, q, G, h, A, b)
alphas = np.array(result['x'])
support_vector_indices = np.where(alphas > 0.0001)[0]
print(support_vector_indices)
alphas = alphas[support_vector_indices]
support_vectors = Xs[support_vector_indices]
support_vectors_y = Ys[support_vector_indices]
print("%d support vectors out of %d points" % (len(alphas), len(Xs)))
print(alphas)
weights = np.zeros(2)
for i in range(len(alphas)):
weights += alphas[i] * support_vectors_y[i] * support_vectors[i]
bias = Ys[support_vector_indices] - np.dot(Xs[support_vector_indices], weights)
return weights, bias, support_vectors, support_vectors_y
def cvxopt(X, y):
m, n = X.shape
y = y.reshape(-1, 1) * 1.
X_dash = y * X
H = np.dot(X_dash, X_dash.T) * 1.
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
G = cvxopt_matrix(-np.eye(m))
h = cvxopt_matrix(np.zeros(m))
A = cvxopt_matrix(y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
# w parameter in vectorized form
w = ((y * alphas).T @ X).reshape(-1, 1)
# Selecting the set of indices S corresponding to non zero parameters
S = (alphas > 1e-4).flatten()
# Computing b
b = y[S] - np.dot(X[S], w)
clf = MyClassifier()
clf.coef_.append(w.flatten())
clf.coef_ = np.asarray(clf.coef_)
clf.intercept_ = b[0].tolist()
clf.support_vectors_ = X[S]
return clf, []
def SVM_soft():
H = np.zeros((len(Xs), len(Xs)))
for i in range(len(Xs)):
for j in range(len(Xs)):
H[i, j] = Ys[i] * Ys[j] * kernel_func(Xs[i], Xs[j])
H = cvxopt_matrix(H)
q = cvxopt_matrix(-1 * np.ones(len(Xs)))
G = cvxopt_matrix(np.diag(np.ones(len(Xs)) * -1))
h = cvxopt_matrix(np.zeros(len(Xs)))
A = cvxopt_matrix(Ys, (1, len(Xs)))
b = cvxopt_matrix(0.0)
result = cvxopt_solvers.qp(H, q, G, h, A, b)
alphas = np.array(result['x'])
support_vector_indices = np.where(alphas > 0.0001)[0]
print(support_vector_indices)
alphas = alphas[support_vector_indices]
support_vectors = Xs[support_vector_indices]
support_vectors_y = Ys[support_vector_indices]
print("%d support vectors out of %d points" % (len(alphas), len(Xs)))
print(alphas)
sum = 0
for i in range(len(support_vector_indices)):
sum += alphas[i] * support_vectors_y[i] * kernel_func(support_vectors[i], support_vectors[support_vector_indices[0]])
bias = Ys[support_vector_indices[0]] - sum
return alphas, bias, support_vectors, support_vectors_y, support_vector_indices
def l2_norm_LinearSVM_Dual(X, y, C):
zero_tol = 1e-7
start_time = time.time()
n, p = X.shape
y = y.reshape(-1, 1) * 1.
X_dash = y * X
cvxopt_solvers.options['show_progress'] = False
cvxopt_solvers.options['abstol'] = 1e-10
cvxopt_solvers.options['reltol'] = 1e-10
cvxopt_solvers.options['feastol'] = 1e-10
# objective: (1/2) x^T P x + q^T x
# constraints: Gx < h
# Ax = b
# -------- INSERT YOUR CODE HERE -------- #
P = cvxopt_matrix(X_dash.dot(X_dash.T) + 1 / C * np.identity(n))
q = cvxopt_matrix(-np.ones((n, 1)))
G = cvxopt_matrix(-np.identity(n))
h = cvxopt_matrix(np.zeros(n))
A = cvxopt_matrix(y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
cvxopt_solvers.options['show_progress'] = False
cvxopt_solvers.options['abstol'] = 1e-10
cvxopt_solvers.options['reltol'] = 1e-10
cvxopt_solvers.options['feastol'] = 1e-10
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
# -- INSERT YOUR CODE HERE -- #
alphas = np.array(sol['x'])
sol_time = time.time() - start_time
return alphas, sol_time
def LinearSVM_Dual(X, y, C):
start_time = time.time()
n, p = X.shape
y = y.reshape(-1, 1) * 1.
X_dash = y * X
# Complete the following code:
# cvxopt_solvers.qp(P, q, G, h, A, b)
# objective: (1/2) x^T P x + q^T x
# constraints: Gx < h
# Ax = b
# example could be found here:
# https://cvxopt.org/userguide/coneprog.html#quadratic-programming
P = cvxopt_matrix(X_dash.dot(X_dash.T))
q = cvxopt_matrix(-np.ones((n, 1)))
G = cvxopt_matrix(np.vstack((-np.diag(np.ones(n)), np.identity(n))))
h = cvxopt_matrix(np.hstack((np.zeros(n), np.ones(n) * C)))
A = cvxopt_matrix(y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
cvxopt_solvers.options['show_progress'] = False
cvxopt_solvers.options['abstol'] = 1e-10
cvxopt_solvers.options['reltol'] = 1e-10
cvxopt_solvers.options['feastol'] = 1e-10
# -- INSERT YOUR CODE HERE -- #
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
# -- INSERT YOUR CODE HERE -- #
alphas = np.array(sol['x'])
sol_time = time.time() - start_time
return alphas, sol_time
def svr_cvxopt(X_train, y_train, X_test, C, eps, kernel_type, gamma, degree):
X = X_train
y = y_train
m, n = X.shape
y = y.reshape(-1, 1) * 1
print(y.shape)
q1 = eps - y
q2 = eps + y
K = kernel(kernel_type, X, X, gamma, degree)
p1 = np.hstack((K, K * -1))
P = cvxopt_matrix(np.vstack((p1, p1 * -1)))
q = cvxopt_matrix(np.vstack((q1, q2)))
G = cvxopt_matrix(np.vstack((np.eye(2 * m) * -1, np.eye(2 * m))))
h = cvxopt_matrix(np.hstack((np.zeros(2 * m), np.ones(2 * m) * C)))
A = cvxopt_matrix((np.hstack(
(np.ones(m), np.ones(m) * -1))).reshape(1, -1))
# print(A.shape)
b = cvxopt_matrix(np.zeros(1))
#Run solver
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
# print(alphas)
alphas1 = alphas[:m]
alphas2 = alphas[m:2 * m]
b = np.array(sol['y'])
# print(b)
# W = ((alphas1 - alphas2).T @ X).reshape(-1, 1) ## W is a column vector
y_pred = kernel(kernel_type, X, X_test, gamma,
degree).T @ (alphas1 - alphas2) + b
# print(y_pred)
return y_pred
def svr_cvxopt_rh(X_train, y_train, X_test, C, eps, kernel_type, gamma,
degree):
X = X_train
y = y_train
m, n = X.shape
y = y.reshape(-1, 1) * 1
# print(y.shape)
q1 = 2 * eps * y
K = kernel(kernel_type, X, X, gamma, degree)
K1 = K + y @ y.T
p1 = np.hstack((K1, K1 * -1))
a1 = np.hstack((np.ones(m), np.zeros(m)))
a2 = np.hstack((np.zeros(m), np.ones(m)))
P = cvxopt_matrix(np.vstack((p1, p1 * -1)))
q = cvxopt_matrix(np.vstack((q1, q1 * -1)))
G = cvxopt_matrix(np.vstack((np.eye(2 * m) * -1, np.eye(2 * m))))
h = cvxopt_matrix(np.hstack((np.zeros(2 * m), np.ones(2 * m) * C)))
A = cvxopt_matrix(np.vstack((a1, a2)))
# print(A.shape)
b = cvxopt_matrix(np.ones(2))
#Run solver
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
variables = np.array(sol['x'])
u = variables[:m]
v = variables[m:2 * m]
delta = (u - v).T @ y + 2 * eps
bias = (u - v).T @ (K @ (u + v)) / (2 * delta) + (u + v).T @ y / 2
u1 = u / delta
v1 = v / delta
y_pred = kernel(kernel_type, X, X_test, gamma, degree).T @ (v1 - u1) + bias
# print(y_pred)
return y_pred
def __compute_topp_polygon_bretl(cwc, p, ps, pss, sdd_max=None, sd_max=None):
"""Compute the polygon in the (sddot,sdot^2) plane using Hauser method."""
g = get_gravity()
b1 = dot(cwc, hstack([ps, cross(p, ps)]))
b2 = dot(cwc, hstack([pss, cross(p, pss)]))
b3 = dot(cwc, hstack([g, cross(p, g)]))
n = len(b1)
G = zeros((n+1, 2))
G[0, :] = [0, -1]
G[1:, 0] = b1
G[1:, 1] = b2
h = zeros(n+1)
h[1:] = b3
if sdd_max is not None:
G = vstack([array([[1, 0], [-1, 0]]), G])
h = hstack([array([sdd_max, sdd_max]), h])
if sd_max is not None:
sd_max_2 = sd_max ** 2
G = vstack([array([[0, 1], [0, -1]]), G])
h = hstack([array([sd_max_2, sd_max_2]), h])
lp = cvxopt_matrix(G), cvxopt_matrix(h)
res, z1 = OptimizeDirection(array([1., 0.]), lp)
if not res:
return None
res, z2 = OptimizeDirection(array([cos(2*pi/3), sin(2*pi/3)]), lp)
if not res:
return None
res, z3 = OptimizeDirection(array([cos(4*pi/3), sin(4*pi/3)]), lp)
if not res:
return None
v1 = Vertex(z1)
v2 = Vertex(z2)
v3 = Vertex(z3)
P = Polygon()
P.fromVertices(v1, v2, v3)
P.iter_expand(lp, 100)
return P
def soft_margin_rbf(C, gamma, x_vals, y_vals):
x_vals = np.concatenate([x_vals[y_vals == 0], x_vals[y_vals == 1]])
y_vals = np.concatenate([y_vals[y_vals == 0], y_vals[y_vals == 1]])
x_vals,y_vals = shuffle(x_vals,y_vals,random_state = 0)
y_vals = 2*y_vals - 1
x_vals = x_vals.reshape(x_vals.shape[0],25)
y_vals = y_vals.reshape(-1,1)*1
tr_len = int(0.7*len(y_vals))
train_length=tr_len
x_train = x_vals[:tr_len, :]
y_train = y_vals[:tr_len, :]
x_test = x_vals[tr_len:, :]
y_test = y_vals[tr_len:, :]
m,n = x_train.shape
norm_column_squared = np.linalg.norm(x_train, axis = 1).reshape(m,1)**2
norm_row_squared = np.linalg.norm(x_train.T, axis = 0).reshape(1,m)**2
norm_column_extended = norm_column_squared*np.ones((m,m))
norm_row_extended = norm_column_extended.T
cross_terms = x_train@x_train.T
#rbf matrix has terms of the form aij = exp(-gamma*||xi - xj||^2)
X_dash = np.exp(-gamma*(norm_column_extended + norm_row_extended - 2*cross_terms))
#((y_train.reshape(tr_len,1)*X_dash).T)
H = (y_train.reshape(tr_len,1))*((y_train.reshape(tr_len,1)*X_dash).T)
#Converting into cvxopt format
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
G = cvxopt_matrix(np.vstack((np.eye(m)*-1,np.eye(m))))
h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m) * C)))
A = cvxopt_matrix(y_train.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
#Setting solver parameters (change default to decrease tolerance)
cvxopt_solvers.options['show_progress'] = False
cvxopt_solvers.options['abstol'] = 1e-10
cvxopt_solvers.options['reltol'] = 1e-10
cvxopt_solvers.options['feastol'] = 1e-10
#Run solver
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
w = ((y_train*alphas).T @ x_train).reshape(-1,1)
#Selecting the set of indices S corresponding to non zero parameters
S = (alphas > 1e-4).flatten()
#Computing b
b = y_train[S] - np.dot(x_train[S], w)
#Display results
# print('Alphas = ',alphas[alphas > 1e-4])
# print('w = ', w.flatten())
# print('b = ', b[0])
scores = x_test.dot(w) + b[0]
predictions = np.sign(scores)
correct_predictions = np.sum(predictions == y_test)
val_accuracy = (correct_predictions/len(predictions))*100
return val_accuracy
def fit(self, X, Y):
# Keep samples
self.X = X
self.Y = Y
N, _ = X.shape
# Calculate Kernel Matrix
self.K = self.svc_kernel_mat(self)
# Adjust Outputs for cvxopt function
A = Y.reshape(-1, 1) * 1
A = A.reshape(1, -1)
A = A.astype('float')
# Calculate Optmization Matrices
P = cvxopt_matrix(self.K)
q = cvxopt_matrix(-np.ones((N, 1)))
G = cvxopt_matrix(np.vstack((np.eye(N) * -1, np.eye(N))))
h = cvxopt_matrix(np.hstack((np.zeros(N), np.ones(N) * self.C)))
A = cvxopt_matrix(A)
b = cvxopt_matrix(np.zeros(1))
#Run solver
cvxopt_solvers.options['show_progress'] = False
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
# Get support vectors (alphas > epsilon)
epsilon = 1e-4
sv = (alphas > epsilon).flatten()
Xsv = X[sv, :]
Ysv = Y[sv]
alphas = alphas[sv, 0]
Nsv = len(Ysv)
# Calculate Bias
b = 0
for i in range(Nsv):
b = b + 1 / Ysv[i]
for j in range(Nsv):
kij = self.kernel_func(Xsv[j, :],
Xsv[i, :],
kernel=self.kernel_type,
gamma=self.gamma,
d=self.d)
b = b - Ysv[j] * alphas[j] * kij
# Hold Results
self.Xsv = Xsv
self.Ysv = Ysv
self.alphas = alphas
self.b = b
def hard_margin_linear(x_vals, y_vals):
x_vals = np.concatenate([x_vals[y_vals == 0], x_vals[y_vals == 1]])
y_vals = np.concatenate([y_vals[y_vals == 0], y_vals[y_vals == 1]])
x_vals,y_vals = shuffle(x_vals,y_vals,random_state = 0)
y_vals = 2*y_vals - 1
x_vals = x_vals.reshape(x_vals.shape[0],25)
y_vals = y_vals.reshape(-1,1)*1
tr_len = int(0.7*len(y_vals))
x_train = x_vals[:tr_len, :]
y_train = y_vals[0:tr_len, :]
x_test = x_vals[tr_len:, :]
y_test = y_vals[tr_len:, :]
m,n = x_train.shape
# x_dash = x_train*y_train
# H = x_dash@x_dash.T
X_dash = y_train* x_train
H = np.dot(X_dash , X_dash.T) * 1.
#X_dash = y_train*x_train
#H = np.dot(X_dash , X_dash.T) * 1.
#Converting into cvxopt format
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
G = cvxopt_matrix(-np.eye(m))
h = cvxopt_matrix(np.zeros(m))
A = cvxopt_matrix(y_train.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
#Setting solver parameters (change default to decrease tolerance)
cvxopt_solvers.options['show_progress'] = False
cvxopt_solvers.options['abstol'] = 1e-10
cvxopt_solvers.options['reltol'] = 1e-10
cvxopt_solvers.options['feastol'] = 1e-10
#Run solver
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
#w parameter in vectorized form
w = ((y_train*alphas).T @ x_train).reshape(-1,1)
#Selecting the set of indices S corresponding to non zero parameters
S = (alphas > 1e-4).flatten()
#Computing b
b = y_train[S] - np.dot(x_train[S], w)
#Display results
# print('Alphas = ',alphas[alphas > 1e-4])
# print('w = ', w.flatten())
# print('b = ', b[0])
scores = x_test.dot(w) + b[0]
predictions = np.sign(scores)
correct_predictions = np.sum(predictions == np.rint(y_test))
val_accuracy = (correct_predictions/len(predictions))*100
#print('validation_accuracy = ', val_accuracy)
return val_accuracy
def svmlin(X, t, C):
# Linear SVM Classifier
#
# INPUT:
# X : the dataset (num_samples x dim)
# t : labeling (num_samples x 1)
# C : penalty factor for slack variables (scalar)
#
# OUTPUT:
# alpha : output of quadprog function (num_samples x 1)
# sv : support vectors (boolean) (1 x num_samples)
# w : parameters of the classifier (1 x dim)
# b : bias of the classifier (scalar)
# result : result of classification (1 x num_samples)
# slack : points inside the margin (boolean) (1 x num_samples)
#####Insert your code here for subtask 2a#####
m, n = X.shape
y = t.reshape(-1, 1) * 1.
X_dash = y * X
H = np.dot(X_dash, X_dash.T) * 1.
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
G = cvxopt_matrix(-np.eye(m))
h = cvxopt_matrix(np.zeros(m))
A = cvxopt_matrix(y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
#n = X.shape[0]
#q = (-1)* np.ones(n)
#G = np.vstack([-np.eye(n), np.eye(n)])
#A = t
#b = 0
#h = np.hstack([0,C])
#P = np.full((n,n),0)
#for i in range(n):
# for j in range(n):
# P[i,j] = t[i]*t[j]*np.dot(X[i],X[j])
a = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
# w parameter in vectorized form
w = ((y * alphas).T @ X).reshape(-1, 1)
# Selecting the set of indices S corresponding to non zero parameters
S = (alphas > 1e-4).flatten()
# Computing b
b = y[S] - np.dot(X[S], w)
alpha, sv, w, b, result, slack = svmlin(train['data'], train['label'], C)
return alpha, sv, w, b, result, slack
def calculateW(Xtrain, Ytrain, lam):
m = len(Ytrain)
y = Ytrain.reshape(-1, 1) * 1.
x = y * Xtrain
P = cvxopt_matrix(x @ x.T * lam * 1.)
q = cvxopt_matrix(-np.ones((m, 1)))
G = cvxopt_matrix(np.vstack((np.eye(m) * -1, np.eye(m))))
h = cvxopt_matrix(np.hstack((np.zeros(m), np.ones(m))))
A = cvxopt_matrix(y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
sol = cvxopt_solvers.qp(P, q, G, h, A, b, options={'show_progress': False})
alphas = np.array(sol['x'])
return ((y * alphas).T @ Xtrain).reshape(-1, 1)
def HardMarg(X, y):
#Note the problem is to:
#max \sigma_i alpha - 0.5 \sigma_{i,j} alpha_i alpha_j y_i y_j x^T_i x_j
#subject to alpha_i \geq 0, \sigma \alpha_i y_i = 0
#Form of qt solver (http://cvxopt.org/userguide/coneprog.html):
# min 0.5 (alpha^T P alpha) + q alpha
# subject to G alpha < h
# A alpha = b
# In this case (mutiply by -1):
# P_ij = y_i y_j x^T_i x_j, P = y^T X^T X y
# q = -1
# G = -1
# h = 0
# a = y
# b = 0
d = np.shape(X)[1]
y = y.astype(np.double)
P = cvxopt_matrix(np.matmul(y, y.T) * np.matmul(X.T, X))
q = cvxopt_matrix(-np.ones((d, 1)))
G = cvxopt_matrix(-np.eye(d))
h = cvxopt_matrix(np.zeros(d))
a = cvxopt_matrix(y.T)
b = cvxopt_matrix(np.zeros(1))
alpha = np.array(
cvxopt_solvers.qp(P,
q,
G,
h,
a,
b,
kktsolver='ldl',
options={
'kktreg': 1e-9,
'show_progress': False
})['x'])
beta = np.sum((alpha * y).T * X, axis=1)
count = 0
while (alpha[count] <= 1e-5):
count += 1
beta_0 = y[count] - np.matmul(beta, X[:, count:count + 1])
return beta, beta_0
def _convert_to_cvxopt_matrices(self):
from cvxopt import matrix as cvxopt_matrix
self._quadratic_func = cvxopt_matrix(self._quadratic_func)
if self._linear_func is not None:
self._linear_func = cvxopt_matrix(self._linear_func)
else:
# CVXOPT needs this vector to be set even if it is not used, so we put zeros in it!
self._linear_func = cvxopt_matrix(np.zeros((self._n_variables, 1)))
if self._inequality_constraints_matrix is not None:
self._inequality_constraints_matrix = cvxopt_matrix(self._inequality_constraints_matrix)
if self._inequality_constraints_values is not None:
self._inequality_constraints_values = cvxopt_matrix(self._inequality_constraints_values)
if self._equality_constraints_matrix is not None:
self._equality_constraints_matrix = cvxopt_matrix(self._equality_constraints_matrix)
if self._equality_constraints_values is not None:
self._equality_constraints_values = cvxopt_matrix(self._equality_constraints_values)
def l2_norm_LinearSVM_Dual(X, y, C):
zero_tol = 1e-4
cluster_1, cluster_2 = split_data(X, y)
plt.scatter(cluster_1[:, 0], cluster_1[:, 1], color='blue')
plt.scatter(cluster_2[:, 0], cluster_2[:, 1], color='green')
for i in range(len(y)):
if y[i] == 0:
y[i] = -1
start_time = time.time()
n, p = X.shape
y = y.reshape(-1, 1) * 1.
X_dash = y * X
extra_term = 1 / C * np.identity(n)
P = cvxopt_matrix(X_dash.dot(X_dash.T) + extra_term)
q = cvxopt_matrix(-np.ones((n, 1)))
G = cvxopt_matrix(np.vstack((-np.diag(np.ones(n)))))
h = cvxopt_matrix(np.hstack((np.zeros(n))))
A = cvxopt_matrix(y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
cvxopt_solvers.options['show_progress'] = False
cvxopt_solvers.options['abstol'] = 1e-10
cvxopt_solvers.options['reltol'] = 1e-10
cvxopt_solvers.options['feastol'] = 1e-10
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
alphas = np.array(sol['x'])
sol_time = time.time() - start_time
w = ((y * alphas).T @ X).reshape(-1, 1)
# Selecting the set of indices S corresponding to non zero parameters
S = (alphas > zero_tol).flatten()
# Computing b
b = y[S] - np.dot(X[S], w)
b = b[0]
# Display results
x = np.linspace(-1, 5, 20)
plt.plot(x, (-b - (w[0] * x)) / w[1], 'm')
plt.show()
return alphas, sol_time
def fit(self,X,Y):
m,n = X.shape
X_dash = Y*X
H = np.dot(X_dash , X_dash.T) * 1.
P = cvxopt_matrix(H)
q = cvxopt_matrix(-np.ones((m, 1)))
G = cvxopt_matrix(-np.eye(m))
h = cvxopt_matrix(np.zeros(m))
A = cvxopt_matrix(Y.reshape(1, -1))
b = cvxopt_matrix(np.zeros(1))
sol = cvxopt_solvers.qp(P, q, G, h, A, b)
self.alphas = np.array(sol['x'])
#Selecting the set of indices S corresponding to non zero parameters
self.S = np.where(self.alphas>1e-5)[0]
self.w = ((Y * self.alphas).T @ X).reshape(-1,1)
self.b = Y[self.S] - np.dot(X[self.S], self.w)
