def reconstruct(data_matrix, missing_matrix):
"""
Reconstructs the data_matrix after the pixels indicated in the
missing_matrix have been removed.
"""
flipped = (data_matrix.shape[0] < data_matrix.shape[1])
if flipped:
data_matrix = data_matrix.T
missing_matrix = missing_matrix.T
unknown_indices = nonzero(ravel(missing_matrix, order='F'))[0]
known = matrix(data_matrix, dtype=float)
known[nonzero(missing_matrix)] = 0.0
u = len(unknown_indices)
A = spmatrix(1.0, unknown_indices, range(u), (data_matrix.size, u))
B = cvxmat(known)
x = nrmapp(A, B)['x']
if x is not None:
Ax = reshape(array(A*cvxmat(x)), data_matrix.shape, order='F')
retval = array(Ax + B, dtype=data_matrix.dtype)
if flipped:
retval = retval.T
return retval
else:
raise Exception("Error during optimization.")
def reconstruct(data_matrix, missing_matrix):
"""
Reconstructs the data_matrix after the pixels indicated in the
missing_matrix have been removed.
"""
flipped = (data_matrix.shape[0] < data_matrix.shape[1])
if flipped:
data_matrix = data_matrix.T
missing_matrix = missing_matrix.T
unknown_indices = nonzero(ravel(missing_matrix, order='F'))[0]
known = matrix(data_matrix, dtype=float)
known[nonzero(missing_matrix)] = 0.0
u = len(unknown_indices)
A = spmatrix(1.0, unknown_indices, range(u), (data_matrix.size, u))
B = cvxmat(known)
x = nrmapp(A, B)['x']
if x is not None:
Ax = reshape(array(A * cvxmat(x)), data_matrix.shape, order='F')
retval = array(Ax + B, dtype=data_matrix.dtype)
if flipped:
retval = retval.T
return retval
else:
raise Exception("Error during optimization.")
def fit(self, X, y):
y = (2.0*(np.asarray(y) == 1) - 1.0)
X = np.asarray(X)
self.data = X
self.gram_matrix = self.kernel(X, X)
n = len(X)
if self.scale_C:
C = self.C / float(n)
else:
C = self.C
Y = np.diag(y)
YK = sparse(cvxmat(np.dot(Y, self.gram_matrix)))
Y1 = sparse(cvxmat(np.dot(Y, np.ones((n, 1)))))
YI = sparse(cvxmat(np.dot(Y, np.eye(n))))
c = cvxmat(sparse([spo(n), spz(n + 1), spo(n, v=C)]))
G = sparse(t([[spz(n, n), YK, Y1, YI ],
[spz(n, n), spz(n, n), spz(n), spI(n) ],
[spI(n), -spI(n), spz(n), spz(n, n)],
[spI(n), spI(n), spz(n), spz(n, n)],
[spI(n), spz(n, n), spz(n), spz(n, n)]]))
h = cvxmat(sparse([spo(n), spz(4*n)]))
# We want Gx >= h, but solver uses Gx <= h, so negate:
xstar, _ = linprog(c, -G, -h, verbose=self.verbose)
try:
self.w = xstar[n:2*n + 1].reshape((-1,))
except ValueError as e:
print e
self.w = np.zeros(n + 1)
def iterate(cself, alphas, upsilon, svm):
V = to_V(upsilon)
cself.mention('Update QP...')
qp.update_H(D * V * K * V.T * D)
cself.mention('Solve QP...')
alphas, obj = qp.solve(self.verbose)
svm = MICA(kernel=self.kernel, gamma=self.gamma, p=self.p,
verbose=self.verbose, sv_cutoff=self.sv_cutoff)
svm._X = self._X
svm._y = self._y
svm._V = V
svm._alphas = alphas
svm._objective = obj
svm._compute_separator(K)
svm._K = K
cself.mention('Update LP...')
for row, (i, j) in enumerate(slices(bs.pos_groups)):
G[row, i:j] = cvxmat(-svm._dotprods[Ln + i: Ln + j].T)
h[Xp: Xp + Ln] = cvxmat(-(1 + svm._dotprods[:Ln]))
cself.mention('Solve LP...')
sol, _ = linprog(c, G, h, A, b, verbose=self.verbose)
new_upsilon = sol[:Lp]
if cself.check_tolerance(np.linalg.norm(upsilon - new_upsilon)):
return None, svm
return {'alphas': alphas, 'upsilon': new_upsilon, 'svm': svm}, None
def solve_lp(c, G, h, A=None, b=None, solver=__default_solver):
"""
Solve a Linear Program defined by:
minimize
c.T * x
subject to
G * x <= h
A * x == b (optional)
using CVXOPT <http://cvxopt.org/userguide/coneprog.html#linear-programming>.
INPUT:
- ``c`` -- cost vector
- ``G`` -- inequality matrix
- ``h`` -- inequality vector
- ``A`` -- (optional) equality matrix
- ``b`` -- (optional) equality vector
- ``solver`` -- (optional) solver to use, defaults to GLPK if available
"""
args = [cvxmat(c), cvxmat(G), cvxmat(h)]
if A is not None:
args.extend([cvxmat(A), cvxmat(b)])
sol = cvxopt_lp(*args, solver=solver)
if 'optimal' not in sol['status']:
return None
return array(sol['x']).reshape((c.shape[0],))
def lp_emd(X,
Y,
X_weights=None,
Y_weights=None,
distance='euclidean',
D=None,
verbose=False):
if distance != 'precomputed':
n = len(X)
m = len(Y)
D = cdist(X, Y, distance)
if X_weights is None:
X_weights = np.ones(n) / n
elif n != len(X_weights):
raise ValueError('Size mismatch of X and X_weights')
if Y_weights is None:
Y_weights = np.ones(m) / m
elif m != len(Y_weights):
raise ValueError('Size mismatch of Y and Y_weights')
else:
if D is None:
raise ValueError("D must be supplied when distance='precomputed'")
n, m = D.shape
if X_weights is None:
X_weights = np.ones(n) / n
elif n != len(X_weights):
raise ValueError('Size mismatch of D and X_weights')
if Y_weights is None:
Y_weights = np.ones(m) / m
elif m != len(Y_weights):
raise ValueError('Size mismatch of D and Y_weights')
vecD = D.reshape((-1, 1))
# Set up objective function
C = cvxmat(vecD)
# Set up inequality constraints
G = -spI(n * m)
h = cvxmat(np.zeros((n * m, 1)))
# Set up equality constraints
Aeq = np.zeros((n + m - 1, n * m))
# Sum of rows
for r in range(n):
Aeq[r, r * m:(r + 1) * m] = 1
# Sum of columns
# (Exclude final column, since contraint is determined by others)
for c in range(m - 1):
for r in range(n):
Aeq[c + n, r * m + c] = 1
sparseAeq = sparse(cvxmat(Aeq))
b = np.vstack(
[X_weights.reshape((-1, 1)), Y_weights[:-1].reshape((-1, 1))])
sparseb = cvxmat(b)
_, dist = linprog(C, G, h, sparseAeq, sparseb, verbose=verbose)
return dist
def iterate(cself, alphas, upsilon, svm):
V = to_V(upsilon)
cself.mention('Update QP...')
qp.update_H(D * V * K * V.T * D)
cself.mention('Solve QP...')
alphas, obj = qp.solve(self.verbose)
svm = MICA(kernel=self.kernel, gamma=self.gamma, p=self.p,
verbose=self.verbose, sv_cutoff=self.sv_cutoff)
svm._X = self._X
svm._y = self._y
svm._V = V
svm._alphas = alphas
svm._objective = obj
svm._compute_separator(K)
svm._K = K
cself.mention('Update LP...')
for row, (i, j) in enumerate(slices(bs.pos_groups)):
G[row, i:j] = cvxmat(-svm._dotprods[Ln + i: Ln + j].T)
h[Xp: Xp + Ln] = cvxmat(-(1 + svm._dotprods[:Ln]))
cself.mention('Solve LP...')
sol, _ = linprog(c, G, h, A, b, verbose=self.verbose)
new_upsilon = sol[:Lp]
if cself.check_tolerance(np.linalg.norm(upsilon - new_upsilon)):
return None, svm
return {'alphas': alphas, 'upsilon': new_upsilon, 'svm': svm}, None
def _convert(H, f, A, b, Aeq, beq, lb, ub):
"""
Convert everything to
cvxopt-style matrices
"""
P = cvxmat(H)
q = cvxmat(f)
if Aeq is None:
A_ = None
else:
A_ = cvxmat(Aeq)
if beq is None:
b_ = None
else:
b_ = cvxmat(beq)
if lb is None and ub is None:
if A is None:
G = None
h = None
else:
G = cvxmat(A)
h = cvxmat(b)
else:
n = len(lb)
if A is None:
G = sparse([-speye(n), speye(n)])
h = cvxmat(np.vstack([-lb, ub]))
else:
G = sparse([cvxmat(A), -speye(n), speye(n)])
h = cvxmat(np.vstack([b, -lb, ub]))
return P, q, G, h, A_, b_
def lp_emd(X, Y, X_weights=None, Y_weights=None, distance='euclidean', D=None, verbose=False):
if distance != 'precomputed':
n = len(X)
m = len(Y)
D = cdist(X, Y, distance)
if X_weights is None:
X_weights = np.ones(n)/n
elif n != len(X_weights):
raise ValueError('Size mismatch of X and X_weights')
if Y_weights is None:
Y_weights = np.ones(m)/m
elif m != len(Y_weights):
raise ValueError('Size mismatch of Y and Y_weights')
else:
if D is None:
raise ValueError("D must be supplied when distance='precomputed'")
n, m = D.shape
if X_weights is None:
X_weights = np.ones(n)/n
elif n != len(X_weights):
raise ValueError('Size mismatch of D and X_weights')
if Y_weights is None:
Y_weights = np.ones(m)/m
elif m != len(Y_weights):
raise ValueError('Size mismatch of D and Y_weights')
vecD = D.reshape((-1, 1))
# Set up objective function
C = cvxmat(vecD)
# Set up inequality constraints
G = -spI(n*m)
h = cvxmat(np.zeros((n*m, 1)))
# Set up equality constraints
Aeq = np.zeros((n+m-1, n*m))
# Sum of rows
for r in range(n):
Aeq[r, r*m:(r+1)*m] = 1
# Sum of columns
# (Exclude final column, since contraint is determined by others)
for c in range(m-1):
for r in range(n):
Aeq[c+n, r*m + c] = 1
sparseAeq = sparse(cvxmat(Aeq))
b = np.vstack([X_weights.reshape((-1, 1)), Y_weights[:-1].reshape((-1, 1))])
sparseb = cvxmat(b)
_, dist = linprog(C, G, h, sparseAeq, sparseb, verbose=verbose)
return dist
def _convert(H, f, Aeq, beq, lb, ub):
P = cvxmat(H)
q = cvxmat(f)
if Aeq is None:
A = None
else:
A = cvxmat(Aeq)
if beq is None:
b = None
else:
b = cvxmat(beq)
n = lb.size
G = sparse([-speye(n), speye(n)])
h = cvxmat(np.vstack([-lb, ub]))
return P, q, G, h, A, b
def problem_data_prep(problem_data):
'''
'Touch up' the problem data in the following ways:
- Make sure the matrix elements aren't integers
- Make sure they're dense matrices rather than sparse ones, otherwise
there seem to be difficulties constructing block matrices
- Transpose c to be a row vector, which matches the organization of A, b, G, h
(rows are for constraints, columns are for variables)
'''
problem_data['A'] = cvxmat(1. * problem_data['A'])
problem_data['b'] = cvxmat(1. * problem_data['b'])
problem_data['G'] = cvxmat(1. * problem_data['G'])
problem_data['h'] = cvxmat(1. * problem_data['h'])
problem_data['c'] = cvxmat(1. * problem_data['c']).T
return problem_data
def problem_data_prep(problem_data):
'''
'Touch up' the problem data in the following ways:
- Make sure the matrix elements aren't integers
- Make sure they're dense matrices rather than sparse ones, otherwise
there seem to be difficulties constructing block matrices
- Transpose c to be a row vector, which matches the organization of A, b, G, h
(rows are for constraints, columns are for variables)
'''
problem_data['A'] = cvxmat(1. * problem_data['A'])
problem_data['b'] = cvxmat(1. * problem_data['b'])
problem_data['G'] = cvxmat(1. * problem_data['G'])
problem_data['h'] = cvxmat(1. * problem_data['h'])
problem_data['c'] = cvxmat(1. * problem_data['c']).T
return problem_data
def solve_lp(c, G, h, A=None, b=None, solver=GLPK_IF_AVAILABLE):
"""
Solve a linear program defined by:
.. math::
\\begin{eqnarray}
\\mathrm{minimize} & & c^T x \\\\
\\mathrm{subject\\ to} & & G x \\leq h \\\\
& & A x = b
\\end{eqnarray}
using the `CVXOPT
<http://cvxopt.org/userguide/coneprog.html#linear-programming>`_ interface
to LP solvers.
Parameters
----------
c : array, shape=(n,)
Linear-cost vector.
G : array, shape=(m, n)
Linear inequality constraint matrix.
h : array, shape=(m,)
Linear inequality constraint vector.
A : array, shape=(meq, n), optional
Linear equality constraint matrix.
b : array, shape=(meq,), optional
Linear equality constraint vector.
solver : string, optional
Solver to use, default is GLPK if available
Returns
-------
x : array, shape=(n,)
Optimal solution to the LP, if found, otherwise ``None``.
Raises
------
ValueError
If the LP is not feasible.
"""
args = [cvxmat(c), cvxmat(G), cvxmat(h)]
if A is not None:
args.extend([cvxmat(A), cvxmat(b)])
sol = lp(*args, solver=solver)
if 'optimal' not in sol['status']:
raise ValueError("LP optimum not found: %s" % sol['status'])
return array(sol['x']).reshape((c.shape[0], ))
def update_Aeq(self, Aeq):
if Aeq is None:
self.A = None
else:
self.A = cvxmat(Aeq)
# Old results no longer valid
self.last_results = None
def _ensure_pd(self, epsilon):
"""
Add epsilon times identity matrix
to P to ensure numerically it is P.D.
"""
n = self.P.size[0]
self.P = self.P + cvxmat(epsilon * eye(n))
def update_Aeq(self, Aeq):
if Aeq is None:
self.A = None
else:
self.A = cvxmat(Aeq)
# Old results no longer valid
self.last_results = None
def _ensure_pd(self, epsilon):
"""
Add epsilon times identity matrix
to P to ensure numerically it is P.D.
"""
n = self.P.size[0]
self.P = self.P + cvxmat(epsilon * eye(n))
def _convert(H, f, Aeq, beq, lb, ub):
"""
Convert everything to
cvxopt-style matrices
"""
P = cvxmat(H)
q = cvxmat(f)
if Aeq is None:
A = None
else:
A = cvxmat(Aeq)
if beq is None:
b = None
else:
b = cvxmat(beq)
n = lb.size
G = sparse([-speye(n), speye(n)])
h = cvxmat(vstack([-lb, ub]))
return P, q, G, h, A, b
def _convert(H, f, Aeq, beq, lb, ub):
"""
Convert everything to
cvxopt-style matrices
"""
P = cvxmat(H)
q = cvxmat(f)
if Aeq is None:
A = None
else:
A = cvxmat(Aeq)
if beq is None:
b = None
else:
b = cvxmat(beq)
n = lb.size
G = sparse([-speye(n), speye(n)])
h = cvxmat(np.vstack([-lb, ub]))
return P, q, G, h, A, b
def cvxopt_solve_qp(P, q, G, h, A=None, b=None, initvals=None):
"""
Solve a Quadratic Program defined as:
minimize
(1/2) * x.T * P * x + q.T * x
subject to
G * x <= h
A * x == b (optional)
using CVXOPT
<http://cvxopt.org/userguide/coneprog.html#quadratic-programming>.
"""
# CVXOPT only considers the lower entries of P so we project on its
# symmetric part beforehand
P = .5 * (P + P.T)
args = [cvxmat(P), cvxmat(q), cvxmat(G), cvxmat(h)]
if A is not None:
args.extend([cvxmat(A), cvxmat(b)])
sol = cvxopt_qp(*args, initvals=initvals)
if not ('optimal' in sol['status']):
warn("QP optimum not found: %s" % sol['status'])
return None
return array(sol['x']).reshape((P.shape[1],))
def create_param_for_soft_margin(K, H, L, C):
"""
P, q, G, h, A, b = _convert(H, f, Aeq, beq, lb, ub)
return P, q, G, h, A, b
results = qp(P, q, G, h, A, b)
P = cvxmat(H)
q = cvxmat(f)
if Aeq is None:
A = None
else:
A = cvxmat(Aeq)
if beq is None:
b = None
else:
b = cvxmat(beq)
n = lb.size
G = sparse([-matrix.eye(n), matrix.eye(n)])
h = cvxmat(vstack([-lb, ub]))
"""
# To calculate the threshold using a convex quadratic programming
N = len(H)
Q = cvxmat(H)
p = cvxmat(-ones(N))
G = cvxmat(vstack((diag([-1.0] * N), identity(N))))
h = cvxmat(hstack((zeros(N), (ones(N) * C))))
A = cvxmat(array(L), (1, N))
b = cvxmat(0.0)
sol = cvxopt.qp(Q, p, G, h, A, b)
alpha = array(sol['x']).reshape(N)
return alpha
def fit_one_class_svm(delta_X, weights, v, mt_feat_types_):
mt_feat_types = np.asarray(mt_feat_types_, dtype=np.int32)
N = delta_X.shape[0]
p = delta_X.shape[1]
mt_feats = np.arange(p)[
mt_feat_types != 0] #np.asarray(list(incr_feats) + list(decr_feats))
nmt_feats = np.arange(p)[mt_feat_types == 0] #np.asarray(
solvers.options['show_progress'] = False
if N == 0:
return np.zeros(delta_X.shape[1]) #[-99]
else:
# Build QP matrices
# Minimize 1/2 x^T P x + q^T x
# Subject to G x <= h
# A x = b
if weights is None:
weights = np.ones(N)
P = np.zeros([p + 2 * N, p + 2 * N])
lambda_ = N
for ip in nmt_feats:
P[ip, ip] = lambda_ * 0.01
for ip in mt_feats:
P[ip, ip] = lambda_ * 0.0001 # 0.01
q = np.zeros(p + 2 * N)
q[p:p + N] = v
q[p + N:] = (1. - v)
G1a = np.zeros([p, p])
for ip in np.arange(p):
G1a[ip, ip] = -1 if ip in mt_feats else 1
G1 = np.hstack([G1a, np.zeros([p, 2 * N])])
G2 = np.hstack([np.zeros([2 * N, p]), -np.eye(2 * N)])
G3 = np.hstack([-delta_X, -np.eye(N), np.zeros([N, N])])
G4 = np.hstack([delta_X, np.zeros([N, N]), -np.eye(N)])
G = np.vstack([G1, G2, G3, G4])
h = np.zeros([p + 4 * N])
A = np.zeros([1, p + 2 * N])
for ip in np.arange(p):
A[0, ip] = 1 if ip in mt_feats else -1
b = np.asarray([1.])
P = cvxmat(P)
q = cvxmat(q)
A = cvxmat(A)
b = cvxmat(b)
# options['abstol']=1e-20 #(default: 1e-7).
# options['reltol']=1e-11 #(default: 1e-6)
sol = solvers.qp(P, q, cvxmat(G), cvxmat(h), A, b)
if sol['status'] != 'optimal':
print('****** NOT OPTIMAL ' + sol['status'] + ' ******* [N=' +
str(N) + ', p=' + str(p) + ']')
return np.zeros(delta_X.shape[1]) - 99 #[-99]
else:
soln = sol['x']
w = np.ravel(soln[0:p, :])
if np.sum(np.abs(w)) == 0.:
print('what the?')
# err = np.asarray(soln[-N:, :])
return w
def solve_qp(P, q, G, h, A=None, b=None, solver=None, sym_proj=False):
"""
Solve a quadratic program defined as:
.. math::
\\begin{eqnarray}
\\mathrm{minimize} & & (1/2) x^T P x + q^T x \\\\
\\mathrm{subject\\ to} & & G x \\leq h \\\\
& & A x = b
\\end{eqnarray}
using CVXOPT
<http://cvxopt.org/userguide/coneprog.html#quadratic-programming>.
Parameters
----------
P : array, shape=(n, n)
Symmetric quadratic-cost matrix.
q : array, shape=(n,)
Quadratic-cost vector.
G : array, shape=(m, n)
Linear inequality matrix.
h : array, shape=(m,)
Linear inequality vector.
A : array, shape=(meq, n), optional
Linear equality matrix.
b : array, shape=(meq,), optional
Linear equality vector.
solver : string, optional
Set to 'mosek' to run MOSEK rather than CVXOPT.
sym_proj : bool, optional
Set to `True` when the `P` matrix provided is not symmetric.
Returns
-------
x : array, shape=(n,)
Solution to the QP, if found, otherwise ``None``.
Note
----
CVXOPT only considers the lower entries of `P`, assuming it is symmetric. If
that is not the case, set `sym_proj=True` to project it on its symmetric
part beforehand.
"""
if sym_proj:
P = .5 * (P + P.T)
args = [cvxmat(P), cvxmat(q), cvxmat(G), cvxmat(h)]
if A is not None:
args.extend([cvxmat(A), cvxmat(b)])
sol = qp(*args, solver=solver)
if not ('optimal' in sol['status']):
raise ValueError("QP optimum not found: %s" % sol['status'])
return array(sol['x']).reshape((P.shape[1], ))
def fit_one_class_svm_pub(delta_X, delta_y, weights, v, mt_feat_types):
N = delta_X.shape[0]
p = delta_X.shape[1]
mt_feats = np.arange(p)[
mt_feat_types != 0] #np.asarray(list(incr_feats) + list(decr_feats))
nmt_feats = np.arange(p)[mt_feat_types == 0] #np.asarray(
solvers.options['show_progress'] = False
if N == 0:
return np.zeros(delta_X.shape[1]) #[-99]
else:
# Build QP matrices
# Minimize 1/2 x^T P x + q^T x
# Subject to G x <= h
# A x = b
if weights is None:
weights = np.ones(N)
P = np.zeros([p + N, p + N])
for ip in nmt_feats:
P[ip, ip] = 1
#for ip in mt_feats :
# P[ip, ip] = 1
q = 1 / (N * v) * np.ones((N + p, 1))
q[0:p, 0] = 0
q[p:, 0] = q[p:, 0] * weights
G1a = np.zeros([p, p])
for ip in np.arange(p):
G1a[ip, ip] = -1 if ip in mt_feats else 1
G1 = np.hstack([G1a, np.zeros([p, N])])
G2 = np.hstack([np.zeros([N, p]), -np.eye(N)])
G3 = np.hstack([delta_X, -np.eye(N)])
G = np.vstack([G1, G2, G3])
h = np.zeros([p + 2 * N])
A = np.zeros([1, p + N])
for ip in np.arange(p):
A[0, ip] = 1 if ip in mt_feats else -1
b = np.asarray([1.])
#b = np.asarray([0.])
P = cvxmat(P)
q = cvxmat(q)
A = cvxmat(A)
b = cvxmat(b)
# options['abstol']=1e-20 #(default: 1e-7).
# options['reltol']=1e-11 #(default: 1e-6)
sol = solvers.qp(P, q, cvxmat(G), cvxmat(h), A, b)
if sol['status'] != 'optimal':
print('****** NOT OPTIMAL ' + sol['status'] + ' ******* [N=' +
str(N) + ', p=' + str(p) + ']')
return np.zeros(delta_X.shape[1]) #[-99]
else:
soln = sol['x']
w = np.ravel(soln[0:p, :])
# err = np.asarray(soln[-N:, :])
return w
def _convert(H, f, Aeq, beq, lb, ub, x0):
""" Convert everything to cvxopt-style matrices """
P = cvxmat(H)
print H[0]
print P[0]
raw_input()
q = cvxmat(f)
A = cvxmat(Aeq)
b = cvxmat(beq)
n = lb.size
G = sparse([-speye(n),speye(n)])
h = cvxmat(np.vstack[-lb,ub])
x0 = cvxmat(x0)
return P, q, G, h, A, b, x0
def cvxopt_solve_qp(P, q, G, h, A=None, b=None, solver=None, initvals=None):
"""
Solve a Quadratic Program defined as:
.. math::
\\begin{eqnarray}
\\mathrm{minimize} & & (1/2) x^T P x + q^T x \\\\
\\mathrm{subject\\ to} & & G x \leq h \\\\
& & A x = b
\\end{eqnarray}
using CVXOPT
<http://cvxopt.org/userguide/coneprog.html#quadratic-programming>.
Parameters
----------
P : array, shape=(n, n)
Primal quadratic cost matrix.
q : array, shape=(n,)
Primal quadratic cost vector.
G : array, shape=(m, n)
Linear inequality constraint matrix.
h : array, shape=(m,)
Linear inequality constraint vector.
A : array, shape=(meq, n), optional
Linear equality constraint matrix.
b : array, shape=(meq,), optional
Linear equality constraint vector.
solver : string, optional
Set to 'mosek' to run MOSEK rather than CVXOPT.
initvals : array, shape=(n,), optional
Warm-start guess vector.
Returns
-------
x : array, shape=(n,)
Solution to the QP, if found, otherwise ``None``.
"""
# CVXOPT only considers the lower entries of P so we project on its
# symmetric part beforehand
P = .5 * (P + P.T)
args = [cvxmat(P), cvxmat(q), cvxmat(G), cvxmat(h)]
if A is not None:
args.extend([cvxmat(A), cvxmat(b)])
sol = cvxopt_qp(*args, solver=solver, initvals=initvals)
if not ('optimal' in sol['status']):
raise ValueError("QP optimum not found: %s" % sol['status'])
return array(sol['x']).reshape((P.shape[1], ))
def update_ub(self, ub):
self.h = cvxmat(vstack([-self.lb, ub]))
# Old results no longer valid
self.last_results = None
def fit(self, bags, y):
"""
@param bags : a sequence of n bags; each bag is an m-by-k array-like
object containing m instances with k features
@param y : an array-like object of length n containing -1/+1 labels
"""
self._bags = map(np.asmatrix, bags)
bs = BagSplitter(self._bags,
np.asmatrix(y).reshape((-1, 1)))
self._X = bs.instances
Ln = bs.L_n
Lp = bs.L_p
Xp = bs.X_p
m = Ln + Xp
if self.scale_C:
C = self.C / float(len(self._bags))
else:
C = self.C
K = kernel_by_name(self.kernel, gamma=self.gamma, p=self.p)(self._X, self._X)
new_classes = np.matrix(np.vstack([-np.ones((Ln, 1)),
np.ones((Xp, 1))]))
self._y = new_classes
D = spdiag(new_classes)
setup = list(self._setup_svm(new_classes, new_classes, C))[1:]
setup[0] = np.matrix([0])
qp = IterativeQP(*setup)
c = cvxmat(np.hstack([np.zeros(Lp + 1),
np.ones(Xp + Ln)]))
b = cvxmat(np.ones((Xp, 1)))
A = spz(Xp, Lp + 1 + Xp + Ln)
for row, (i, j) in enumerate(slices(bs.pos_groups)):
A[row, i:j] = 1.0
bottom_left = sparse(t([[-spI(Lp), spz(Lp)],
[spz(m, Lp), spz(m)]]))
bottom_right = sparse([spz(Lp, m), -spI(m)])
inst_cons = sparse(t([[spz(Xp, Lp), -spo(Xp)],
[spz(Ln, Lp), spo(Ln)]]))
G = sparse(t([[inst_cons, -spI(m)],
[bottom_left, bottom_right]]))
h = cvxmat(np.vstack([-np.ones((Xp, 1)),
np.zeros((Ln + Lp + m, 1))]))
def to_V(upsilon):
bot = np.zeros((Xp, Lp))
for row, (i, j) in enumerate(slices(bs.pos_groups)):
bot[row, i:j] = upsilon.flat[i:j]
return sp.bmat([[sp.eye(Ln, Ln), None],
[None, sp.coo_matrix(bot)]])
class MICACCCP(CCCP):
def bailout(cself, alphas, upsilon, svm):
return svm
def iterate(cself, alphas, upsilon, svm):
V = to_V(upsilon)
cself.mention('Update QP...')
qp.update_H(D * V * K * V.T * D)
cself.mention('Solve QP...')
alphas, obj = qp.solve(self.verbose)
svm = MICA(kernel=self.kernel, gamma=self.gamma, p=self.p,
verbose=self.verbose, sv_cutoff=self.sv_cutoff)
svm._X = self._X
svm._y = self._y
svm._V = V
svm._alphas = alphas
svm._objective = obj
svm._compute_separator(K)
svm._K = K
cself.mention('Update LP...')
for row, (i, j) in enumerate(slices(bs.pos_groups)):
G[row, i:j] = cvxmat(-svm._dotprods[Ln + i: Ln + j].T)
h[Xp: Xp + Ln] = cvxmat(-(1 + svm._dotprods[:Ln]))
cself.mention('Solve LP...')
sol, _ = linprog(c, G, h, A, b, verbose=self.verbose)
new_upsilon = sol[:Lp]
if cself.check_tolerance(np.linalg.norm(upsilon - new_upsilon)):
return None, svm
return {'alphas': alphas, 'upsilon': new_upsilon, 'svm': svm}, None
best_obj = float('inf')
best_svm = None
for rr in range(self.restarts + 1):
if rr == 0:
if self.verbose:
print 'Non-random start...'
upsilon0 = np.matrix(np.vstack([np.ones((size, 1)) / float(size)
for size in bs.pos_groups]))
else:
if self.verbose:
print 'Random restart %d of %d...' % (rr, self.restarts)
upsilon0 = np.matrix(np.vstack([rand_convex(size).T
for size in bs.pos_groups]))
cccp = MICACCCP(verbose=self.verbose, alphas=None, upsilon=upsilon0,
svm=None, max_iters=self.max_iters)
svm = cccp.solve()
if svm is not None:
obj = float(svm._objective)
if obj < best_obj:
best_svm = svm
best_obj = obj
if best_svm is not None:
self._V = best_svm._V
self._alphas = best_svm._alphas
self._objective = best_svm._objective
self._compute_separator(best_svm._K)
self._bag_predictions = self.predict(self._bags)
def update_H(self, H):
self.P = cvxmat(H)
def fit(self, bags, y):
"""
@param bags : a sequence of n bags; each bag is an m-by-k array-like
object containing m instances with k features
@param y : an array-like object of length n containing -1/+1 labels
"""
def transform(mx):
"""
Transform into np.matrix if array/list
ignore scipy.sparse matrix
"""
if issparse(mx):
return mx.todense()
return np.asmatrix(mx)
self._bags = [transform(bag) for bag in bags]
y = np.asmatrix(y).reshape((-1, 1))
bs = BagSplitter(self._bags, y)
best_obj = float('inf')
best_svm = None
for rr in range(self.restarts + 1):
if rr == 0:
if self.verbose:
print('Non-random start...')
pos_bag_avgs = np.vstack(
[np.average(bag, axis=0) for bag in bs.pos_bags])
else:
if self.verbose:
print('Random restart %d of %d...' % (rr, self.restarts))
pos_bag_avgs = np.vstack(
[rand_convex(len(bag)) * bag for bag in bs.pos_bags])
intial_instances = np.vstack([bs.neg_instances, pos_bag_avgs])
classes = np.vstack([-np.ones((bs.L_n, 1)), np.ones((bs.X_p, 1))])
# Setup SVM and QP
if self.scale_C:
C = self.C / float(len(intial_instances))
else:
C = self.C
setup = self._setup_svm(intial_instances, classes, C)
K = setup[0]
qp = IterativeQP(*setup[1:])
# Fix Gx <= h
neg_cons = spzeros(bs.X_n, bs.L_n)
for b, (l, u) in enumerate(slices(bs.neg_groups)):
neg_cons[b, l:u] = 1.0
pos_cons = speye(bs.X_p)
bot_left = spzeros(bs.X_p, bs.L_n)
top_right = spzeros(bs.X_n, bs.X_p)
half_cons = sparse([[neg_cons, bot_left], [top_right, pos_cons]])
qp.G = sparse([-speye(bs.X_p + bs.L_n), half_cons])
qp.h = cvxmat(
np.vstack([
np.zeros((bs.X_p + bs.L_n, 1)), C * np.ones(
(bs.X_p + bs.X_n, 1))
]))
# Precompute kernel for all positive instances
kernel = kernel_by_name(self.kernel, gamma=self.gamma, p=self.p)
K_all = kernel(bs.instances, bs.instances)
neg_selectors = np.array(range(bs.L_n))
class MISVMCCCP(CCCP):
def bailout(cself, svm, selectors, instances, K):
return svm
def iterate(cself, svm, selectors, instances, K):
cself.mention('Training SVM...')
alphas, obj = qp.solve(cself.verbose)
# Construct SVM from solution
svm = SVM(kernel=self.kernel,
gamma=self.gamma,
p=self.p,
verbose=self.verbose,
sv_cutoff=self.sv_cutoff)
svm._X = instances
svm._y = classes
svm._alphas = alphas
svm._objective = obj
svm._compute_separator(K)
svm._K = K
cself.mention('Recomputing classes...')
p_confs = svm.predict(bs.pos_instances)
pos_selectors = bs.L_n + np.array([
l + np.argmax(p_confs[l:u])
for l, u in slices(bs.pos_groups)
])
new_selectors = np.hstack([neg_selectors, pos_selectors])
if selectors is None:
sel_diff = len(new_selectors)
else:
sel_diff = np.nonzero(new_selectors -
selectors)[0].size
cself.mention('Selector differences: %d' % sel_diff)
if sel_diff == 0:
return None, svm
elif sel_diff > 5:
# Clear results to avoid a
# bad starting point in
# the next iteration
qp.clear_results()
cself.mention('Updating QP...')
indices = (new_selectors, )
K = K_all[indices].T[indices].T
D = spdiag(classes)
qp.update_H(D * K * D)
return {
'svm': svm,
'selectors': new_selectors,
'instances': bs.instances[indices],
'K': K
}, None
cccp = MISVMCCCP(verbose=self.verbose,
svm=None,
selectors=None,
instances=intial_instances,
K=K,
max_iters=self.max_iters)
svm = cccp.solve()
if svm is not None:
obj = float(svm._objective)
if obj < best_obj:
best_svm = svm
best_obj = obj
if best_svm is not None:
self._X = best_svm._X
self._y = best_svm._y
self._alphas = best_svm._alphas
self._objective = best_svm._objective
self._compute_separator(best_svm._K)
def fit(self, bags, y):
"""
@param bags : a sequence of n bags; each bag is an m-by-k array-like
object containing m instances with k features
@param y : an array-like object of length n containing -1/+1 labels
"""
self._bags = map(np.asmatrix, bags)
bs = BagSplitter(self._bags,
np.asmatrix(y).reshape((-1, 1)))
self._X = bs.instances
Ln = bs.L_n
Lp = bs.L_p
Xp = bs.X_p
m = Ln + Xp
if self.scale_C:
C = self.C / float(len(self._bags))
else:
C = self.C
K = kernel_by_name(self.kernel, gamma=self.gamma, p=self.p)(self._X, self._X)
new_classes = np.matrix(np.vstack([-np.ones((Ln, 1)),
np.ones((Xp, 1))]))
self._y = new_classes
D = spdiag(new_classes)
setup = list(self._setup_svm(new_classes, new_classes, C))[1:]
setup[0] = np.matrix([0])
qp = IterativeQP(*setup)
c = cvxmat(np.hstack([np.zeros(Lp + 1),
np.ones(Xp + Ln)]))
b = cvxmat(np.ones((Xp, 1)))
A = spz(Xp, Lp + 1 + Xp + Ln)
for row, (i, j) in enumerate(slices(bs.pos_groups)):
A[row, i:j] = 1.0
bottom_left = sparse(t([[-spI(Lp), spz(Lp)],
[spz(m, Lp), spz(m)]]))
bottom_right = sparse([spz(Lp, m), -spI(m)])
inst_cons = sparse(t([[spz(Xp, Lp), -spo(Xp)],
[spz(Ln, Lp), spo(Ln)]]))
G = sparse(t([[inst_cons, -spI(m)],
[bottom_left, bottom_right]]))
h = cvxmat(np.vstack([-np.ones((Xp, 1)),
np.zeros((Ln + Lp + m, 1))]))
def to_V(upsilon):
bot = np.zeros((Xp, Lp))
for row, (i, j) in enumerate(slices(bs.pos_groups)):
bot[row, i:j] = upsilon.flat[i:j]
return sp.bmat([[sp.eye(Ln, Ln), None],
[None, sp.coo_matrix(bot)]])
class MICACCCP(CCCP):
def bailout(cself, alphas, upsilon, svm):
return svm
def iterate(cself, alphas, upsilon, svm):
V = to_V(upsilon)
cself.mention('Update QP...')
qp.update_H(D * V * K * V.T * D)
cself.mention('Solve QP...')
alphas, obj = qp.solve(self.verbose)
svm = MICA(kernel=self.kernel, gamma=self.gamma, p=self.p,
verbose=self.verbose, sv_cutoff=self.sv_cutoff)
svm._X = self._X
svm._y = self._y
svm._V = V
svm._alphas = alphas
svm._objective = obj
svm._compute_separator(K)
svm._K = K
cself.mention('Update LP...')
for row, (i, j) in enumerate(slices(bs.pos_groups)):
G[row, i:j] = cvxmat(-svm._dotprods[Ln + i: Ln + j].T)
h[Xp: Xp + Ln] = cvxmat(-(1 + svm._dotprods[:Ln]))
cself.mention('Solve LP...')
sol, _ = linprog(c, G, h, A, b, verbose=self.verbose)
new_upsilon = sol[:Lp]
if cself.check_tolerance(np.linalg.norm(upsilon - new_upsilon)):
return None, svm
return {'alphas': alphas, 'upsilon': new_upsilon, 'svm': svm}, None
best_obj = float('inf')
best_svm = None
for rr in range(self.restarts + 1):
if rr == 0:
if self.verbose:
print('Non-random start...')
upsilon0 = np.matrix(np.vstack([np.ones((size, 1)) / float(size)
for size in bs.pos_groups]))
else:
if self.verbose:
print('Random restart %d of %d...' % (rr, self.restarts))
upsilon0 = np.matrix(np.vstack([rand_convex(size).T
for size in bs.pos_groups]))
cccp = MICACCCP(verbose=self.verbose, alphas=None, upsilon=upsilon0,
svm=None, max_iters=self.max_iters)
svm = cccp.solve()
if svm is not None:
obj = float(svm._objective)
if obj < best_obj:
best_svm = svm
best_obj = obj
if best_svm is not None:
self._V = best_svm._V
self._alphas = best_svm._alphas
self._objective = best_svm._objective
self._compute_separator(best_svm._K)
self._bag_predictions = self.predict(self._bags)
def solve_qp(P, q, G, h,
A=None, b=None, sym_proj=False,
solver='cvxopt'):
if sym_proj:
P = .5 * (P + P.T)
cvxmat(P)
cvxmat(q)
cvxmat(G)
cvxmat(h)
args = [cvxmat(P), cvxmat(q), cvxmat(G), cvxmat(h)]
if A is not None:
args.extend([cvxmat(A), cvxmat(b)])
sol = qp(*args, solver=solver)
if not ('optimal' in sol['status']):
raise ValueError('QP optimum not found: %s' % sol['status'])
return np.array(sol['x']).reshape((P.shape[1],))
def fit(self, bags, y):
"""
@param bags : a sequence of n bags; each bag is an m-by-k array-like
object containing m instances with k features
@param y : an array-like object of length n containing -1/+1 labels
"""
def transform(mx):
"""
Transform into np.matrix if array/list
ignore scipy.sparse matrix
"""
if issparse(mx):
return mx.todense()
return np.asmatrix(mx)
self._bags = [transform(bag) for bag in bags]
y = np.asmatrix(y).reshape((-1, 1))
bs = BagSplitter(self._bags, y)
best_obj = float('inf')
best_svm = None
for rr in range(self.restarts + 1):
if rr == 0:
if self.verbose:
print 'Non-random start...'
pos_bag_avgs = np.vstack([np.average(bag, axis=0) for bag in bs.pos_bags])
else:
if self.verbose:
print 'Random restart %d of %d...' % (rr, self.restarts)
pos_bag_avgs = np.vstack([rand_convex(len(bag)) * bag for bag in bs.pos_bags])
intial_instances = np.vstack([bs.neg_instances, pos_bag_avgs])
classes = np.vstack([-np.ones((bs.L_n, 1)),
np.ones((bs.X_p, 1))])
# Setup SVM and QP
if self.scale_C:
C = self.C / float(len(intial_instances))
else:
C = self.C
setup = self._setup_svm(intial_instances, classes, C)
K = setup[0]
qp = IterativeQP(*setup[1:])
# Fix Gx <= h
neg_cons = spzeros(bs.X_n, bs.L_n)
for b, (l, u) in enumerate(slices(bs.neg_groups)):
neg_cons[b, l:u] = 1.0
pos_cons = speye(bs.X_p)
bot_left = spzeros(bs.X_p, bs.L_n)
top_right = spzeros(bs.X_n, bs.X_p)
half_cons = sparse([[neg_cons, bot_left],
[top_right, pos_cons]])
qp.G = sparse([-speye(bs.X_p + bs.L_n), half_cons])
qp.h = cvxmat(np.vstack([np.zeros((bs.X_p + bs.L_n, 1)),
C * np.ones((bs.X_p + bs.X_n, 1))]))
# Precompute kernel for all positive instances
kernel = kernel_by_name(self.kernel, gamma=self.gamma, p=self.p)
K_all = kernel(bs.instances, bs.instances)
neg_selectors = np.array(range(bs.L_n))
class MISVMCCCP(CCCP):
def bailout(cself, svm, selectors, instances, K):
return svm
def iterate(cself, svm, selectors, instances, K):
cself.mention('Training SVM...')
alphas, obj = qp.solve(cself.verbose)
# Construct SVM from solution
svm = SVM(kernel=self.kernel, gamma=self.gamma, p=self.p,
verbose=self.verbose, sv_cutoff=self.sv_cutoff)
svm._X = instances
svm._y = classes
svm._alphas = alphas
svm._objective = obj
svm._compute_separator(K)
svm._K = K
cself.mention('Recomputing classes...')
p_confs = svm.predict(bs.pos_instances)
pos_selectors = bs.L_n + np.array([l + np.argmax(p_confs[l:u])
for l, u in slices(bs.pos_groups)])
new_selectors = np.hstack([neg_selectors, pos_selectors])
if selectors is None:
sel_diff = len(new_selectors)
else:
sel_diff = np.nonzero(new_selectors - selectors)[0].size
cself.mention('Selector differences: %d' % sel_diff)
if sel_diff == 0:
return None, svm
elif sel_diff > 5:
# Clear results to avoid a
# bad starting point in
# the next iteration
qp.clear_results()
cself.mention('Updating QP...')
indices = (new_selectors,)
K = K_all[indices].T[indices].T
D = spdiag(classes)
qp.update_H(D * K * D)
return {'svm': svm, 'selectors': new_selectors,
'instances': bs.instances[indices], 'K': K}, None
cccp = MISVMCCCP(verbose=self.verbose, svm=None, selectors=None,
instances=intial_instances, K=K, max_iters=self.max_iters)
svm = cccp.solve()
if svm is not None:
obj = float(svm._objective)
if obj < best_obj:
best_svm = svm
best_obj = obj
if best_svm is not None:
self._X = best_svm._X
self._y = best_svm._y
self._alphas = best_svm._alphas
self._objective = best_svm._objective
self._compute_separator(best_svm._K)
def update_H(self, H):
self.P = cvxmat(H)
def update_ub(self, ub):
self.h = cvxmat(vstack([-self.lb, ub]))
# Old results no longer valid
self.last_results = None