def test_small_qp_without_constraints(self):
# Arrange
A = np.array([[1, 0], [0, 1]], dtype=float)
b = np.array([1, -1], dtype=float).reshape((2, ))
lb = -1
ub = 1
gsmo_solver = GSMO(A=2 * A, b=b, bounds=(lb, ub), step_size=1)
x = cp.Variable(A.shape[0])
G = np.zeros((2 * A.shape[0], A.shape[0]))
h = np.zeros((2 * A.shape[0]))
cond_idx = 0
for i in range(A.shape[0]):
G[cond_idx, i] = 1
h[cond_idx] = ub
G[cond_idx + 1, i] = -1
h[cond_idx + 1] = -lb
cond_idx += 2
prob = cp.Problem(cp.Minimize(cp.quad_form(x, A) + b.T @ x),
[G @ x <= h])
prob.solve()
print("\n#### CVXPY #### (QP-no-c)")
print(x.value)
# Act
gsmo_solver.solve()
print("#### SMO #### (QP-no-c)")
print(gsmo_solver.x)
# Assert
np.testing.assert_almost_equal(gsmo_solver.x, x.value, 5)
def test_init_small_svm(self):
# Arrange
pwd = os.path.dirname(os.path.abspath(__file__))
test_data_file = os.path.join(pwd, "small_svm_problem_data.csv")
data = pd.read_csv(test_data_file, delimiter=',')
print(data)
A = np.zeros((data.shape[0], data.shape[0]))
points = data[['X', 'Y']]
y = data['Label']
for i in range(A.shape[0]):
for j in range(A.shape[0]):
A[i, j] = y.iloc[i] * y.iloc[j] * points.iloc[i].dot(
points.iloc[j])
A = (-0.5) * A
b = np.ones((1, A.shape[0]))
C = y.to_numpy()
d = 0
# Act
gsmo_solver = GSMO(A, b, C, d, bounds=(0, 100))
# Assert
# Cx = d
result = C.dot(gsmo_solver.x)
np.testing.assert_almost_equal(d, result)
def test_init_x_valueError(self):
# Arrange
C = np.array([[-1, 1, 1], [-1, 1, 1]])
d = np.array([2, 3])
# Act
with self.assertRaises(ValueError):
GSMO(A=np.zeros((3, 3)), b=np.zeros((3, 1)), C=C, d=d, bounds=(0, 5))
def test_init_x_2d(self):
# Arrange
C = np.array([[-1, 1, -1], [2, 0, 3]])
d = np.array([3, 1])
# Act
gsmo_solver = GSMO(A=np.zeros((3, 3)), b=np.zeros((3, 1)), C=C, d=d, bounds=(0, 5))
# Assert
# Cx = d
result = C.dot(gsmo_solver.x)
np.testing.assert_almost_equal(d, result)
def test_init_x_1d(self):
# Arrange
C = np.array([-1, 1, -1])
d = 3
# Act
gsmo_solver = GSMO(A=np.zeros((3, 3)), b=np.zeros((3, 1)), C=C, d=d, bounds=(0, 5))
# Assert
# Cx = d
result = C.dot(gsmo_solver.x)
self.assertAlmostEqual(d, result)
def test_solve_small_svm(self):
# Arrange
pwd = os.path.dirname(os.path.abspath(__file__))
test_data_file = os.path.join(pwd, "small_svm_problem_data.csv")
data = pd.read_csv(test_data_file, delimiter=',')
print(data)
A = np.zeros((data.shape[0], data.shape[0]))
points = data[['X', 'Y']]
y = data['Label']
for i in range(A.shape[0]):
for j in range(A.shape[0]):
A[i, j] = y.iloc[i] * y.iloc[j] * points.iloc[i].dot(points.iloc[j])
A = (0.5) * A
b = -np.ones((A.shape[0],))
C = y.to_numpy()
C_t = C.reshape((1, C.shape[0]))
d = 0
gsmo_solver = GSMO(A, b, C_t, d, bounds=(0, 10), step_size=0.1)
fun = lambda x, H, f: x.transpose().dot(H).dot(x) + f.transpose().dot(x)
bnds = tuple([(0, 1) for i in range(A.shape[0])])
constr = ({'type': 'eq', 'args': C_t, 'fun': lambda x, c: c.transpose().dot(x)})
res = minimize(fun, np.ones((A.shape[0],)), args=(A, b), bounds=bnds, constraints=constr)
clf = SVC(C=1, kernel='linear')
clf.fit(points, y)
# Act
print("#### SMO ####")
gsmo_solver.solve()
print(gsmo_solver.x.round(3))
print("\n#### MINIMIZE ####")
print(res.x)
print("\n#### SVC ####")
print(clf.dual_coef_)
print(clf.support_)
plt.scatter(points['X'], points['Y'], c=y)
plt.scatter(points['X'].iloc[clf.support_], points['Y'].iloc[clf.support_], c='r')
plt.show()
# Assert
# Cx = d
result = C.dot(gsmo_solver.x)
np.testing.assert_almost_equal(d, result)
np.testing.assert_almost_equal(gsmo_solver.x, res.x)
def test_small_qp_without_constraints(self):
# Arrange
A = np.array([[1, 0], [0, 1]])
b = np.array([1, -1]).reshape((2,))
gsmo_solver = GSMO(A=A, b=b, bounds=(None, None), step_size=0.01)
fun = lambda x, H, f: x.transpose().dot(H).dot(x) + f.transpose().dot(x)
# bnds = tuple([(0, 1) for i in range(A.shape[0])])
res = minimize(fun, np.ones((A.shape[0],)), args=(A, b))
print("\n#### MINIMIZE ####")
print(res.x)
# Act
print("#### SMO ####")
gsmo_solver.solve()
print(gsmo_solver.x.round(3))
# Assert
np.testing.assert_almost_equal(gsmo_solver.x, res.x)
def test_dim4_qp_with_multiple_constraints(self):
# Arrange
A = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
dtype=float)
b = np.array([1, -1, 2, -2], dtype=float).reshape((4, ))
C = np.array([[1, 1, 0, 0], [0, 1, 1, 1], [1, 0, 0, 0]], dtype=float)
d = np.array([26.5, 40, 26.5])
lb = 0
ub = 100
gsmo_solver = GSMO(A=A,
b=b,
C=C,
d=d,
bounds=(lb, ub),
step_size=1,
epsilon=1e-14)
x = cp.Variable(A.shape[0])
G = np.zeros((2 * A.shape[0], A.shape[0]))
h = np.zeros((2 * A.shape[0]))
cond_idx = 0
for i in range(A.shape[0]):
G[cond_idx, i] = 1
h[cond_idx] = ub
G[cond_idx + 1, i] = -1
h[cond_idx + 1] = -lb
cond_idx += 2
prob = cp.Problem(cp.Minimize(cp.quad_form(x, A) + b.T @ x),
[G @ x <= h, C @ x == d])
prob.solve()
print("\n#### CVXPY ####")
print(x.value)
# Act
print("#### SMO ####")
gsmo_solver.solve()
print(gsmo_solver.x)
# Assert
np.testing.assert_almost_equal(gsmo_solver.x, x.value, 3)
def test_solve_small_svm(self):
# Arrange
pwd = os.path.dirname(os.path.abspath(__file__))
test_data_file = os.path.join(pwd, "small_svm_problem_data.csv")
data = pd.read_csv(test_data_file, delimiter=',')
print(data)
A = np.zeros((data.shape[0], data.shape[0]), dtype=float)
points = data[['X', 'Y']]
DataScaled = points.to_numpy()
# print(f'Huhu: {DataScaled.mean(axis=0)}')
# print(f'Huhu: {DataScaled.std(axis=0,ddof=1)}')
# DataScaled = sklearn.preprocessing.scale(DataScaled) # this is surprisingly wrong
# DataScaled = stats.zscore(DataScaled) # this also not
DataScaled = (DataScaled - DataScaled.mean(axis=0))
DataScaled /= np.std(DataScaled, axis=0, ddof=1)
# print(f'Huhu: {DataScaled}')
y = data['Label']
for i in range(A.shape[0]):
for j in range(A.shape[0]):
A[i, j] = y.iloc[i] * y.iloc[j] * DataScaled[i, :].dot(
DataScaled[j, :])
A = (0.5) * A
b = -np.ones((A.shape[0], ), dtype=float)
C = y.to_numpy().astype(dtype=float)
C_t = C.reshape((1, C.shape[0]))
d = np.zeros((1, ), dtype=float)
lb = 0
ub = 1
gsmo_solver = GSMO(A,
b,
C_t,
d,
bounds=(lb, ub),
step_size=1,
epsilon=1e-14)
clf = SVC(C=1, kernel='linear')
clf.fit(DataScaled, y)
# Act
print("#### SMO ####")
gsmo_solver.solve()
print(f'x:{gsmo_solver.x}')
Qsmo = gsmo_solver.x.transpose().dot(A.dot(
gsmo_solver.x)) + b.transpose().dot(gsmo_solver.x)
print(f'Q-smo:{Qsmo}')
w, b_gsmo = self.__calcWandB(gsmo_solver.x, DataScaled, y)
print(f'GSMO w = {w} and b = {b_gsmo}')
print(f'GSMO max margin {1 / np.linalg.norm(w)}')
for i in range(DataScaled.shape[0]):
print(f'pred: {i}: {w.dot(DataScaled[i, :] - b_gsmo)}')
G = np.zeros((2 * A.shape[0], A.shape[0]))
h = np.zeros((2 * A.shape[0]))
cond_idx = 0
for i in range(A.shape[0]):
G[cond_idx, i] = 1
h[cond_idx] = ub
G[cond_idx + 1, i] = -1
h[cond_idx + 1] = -lb
cond_idx += 2
x = cp.Variable(A.shape[0])
b = -b
prob = cp.Problem(cp.Minimize(cp.quad_form(x, A) - b.T @ x),
[G @ x <= h, C @ x == d])
prob.solve()
print("\n#### CVXPY ####")
print(x.value)
Qcvx = x.value.transpose().dot(A.dot(x.value)) + b.transpose().dot(
x.value)
print(f'Q-cvx:{Qcvx}')
w_cvx, b_cvx = self.__calcWandB(x.value, DataScaled, y)
print(f'CVX w = {w_cvx} and b = {b_cvx}')
print(f'CVX max margin {1 / np.linalg.norm(w_cvx)}')
for i in range(DataScaled.shape[0]):
print(f'pred: {i}: {w_cvx.dot(DataScaled[i, :] - b_cvx)}')
print("\n#### SVC ####")
print(clf.dual_coef_)
print(clf.support_)
print(clf.coef_)
plt.scatter(points['X'], points['Y'], c=y)
plt.scatter(points['X'].iloc[clf.support_],
points['Y'].iloc[clf.support_],
c='r')
plt.show()
) # original 7 - the solution for 4.5 and 7 is correct in the way that the constraints are fulfilled - is the lsqr to strict? - no the other solver solved the inequality instead of equality constraints
# Matlab quadprog will give [2.5 4.5] -- all solution are valid but the cvx is optimal (costs)
H = np.array([[5, 8, 1], [8, 13, 1], [1, 1, 2]])
f = np.array([-16, -25, 4])
C2 = np.array([[0, 0, 0], [0, 0, 0]])
d2 = np.array([0, 0])
#A = np.array([[1, 0], [0, 1]], dtype=float)
#b = np.array([1, -1], dtype=float).reshape((2,))
#lb = -1
#ub = 1
#oSMO = GSMO(A=A, b=b, bounds=(lb, ub), step_size=0.1)
#oSMO = GSMO(A,b) # bounds eigentlich pro variable nicht global
#oSMO = GSMO(A,b,C,d,(0,1e10)) # bounds eigentlich pro variable nicht global
#oSMO = GSMO(A,b,C,d,(-1,10)) # bounds eigentlich pro variable nicht global
oSMO = GSMO(A, b, C, d,
(-0.3, 0.3)) # bounds eigentlich pro variable nicht global
#oSMO = GSMO(A,b,None,0,(-1,1)) # bounds eigentlich pro variable nicht global -- case simon without constraints (to check)
#oSMO = GSMO(H,f,C2,d2,(0,10))
oSMO.solve()
print(oSMO.x)
if A is not None:
Qsmo = oSMO.x.transpose().dot(A.dot(oSMO.x)) + b.transpose().dot(oSMO.x)
print(f'{Qsmo}')
if C is not None:
print(f'{C.dot(oSMO.x)}')