def miniball(X, L=None):
r""" Compute the smallest enclosing sphere
TODO: Use implementation of https://github.com/hbf/miniball
Parameters
----------
X: array-like (M,m)
Points to enclose in a ball
L: optional, (m,m)
Lipschitz-like weighting metric
Returns
-------
x: np.array(m)
Center of circle
r: float
radius of circle
"""
X = np.array(X)
M, m = X.shape
if L is None:
L = np.eye(m)
x = cp.Variable(m)
ones = np.ones((1, M))
obj = cp.mixed_norm((L @ (cp.reshape(x, (m, 1)) @ ones - X.T)).T, 2,
np.inf)
prob = cp.Problem(cp.Minimize(obj))
prob.solve()
return x.value, obj.value
def test_mixed_norm(self):
"""Test mixed norm.
"""
y = Variable((5, 5))
obj = Minimize(cp.mixed_norm(y, "inf", 1))
prob = Problem(obj, [y == np.ones((5, 5))])
result = prob.solve()
self.assertAlmostEqual(result, 5)
def norm_l1linf(mat):
r"""The :math:`\ell_1 \otimes \ell_\infty` nuclear norm for matrices.
Sum of the :math:`\ell_\infty` norms of the rows of the matrix.
Synonymous with the matrix norm :math:`\Vert \cdot \Vert_{\infty,1}`.
"""
return cvxpy.mixed_norm(mat, np.Inf, 1)
def norm_linfl1(mat):
r"""The :math:`\ell_\infty \otimes \ell_1` nuclear norm for matrices.
Sum of the :math:`\ell_\infty` norms of the columns of the matrix.
Synonymous with the matrix norm :math:`\Vert \cdot \Vert_{\infty,1}`
performed on the transpose of the matrix.
"""
return cvxpy.mixed_norm(mat.T, np.Inf, 1)
def costL12(K, X, S, lam):
# compute log loss
loss = 0.
for q in S:
i, j, k = q
# should this be Mt^T * K??
Mt = (2. * np.outer(X[i], X[j]) - 2. * np.outer(X[i], X[k]) -
np.outer(X[j], X[j]) + np.outer(X[k], X[k]))
score = cvx.trace(Mt.T * K)
loss = loss + cvx.logistic(-score)
# regularize with the 1,2 norm
loss = loss / len(S) + lam * cvx.mixed_norm(K, 2, 1)
return loss
def _cq_center_cvxpy(Y,
L,
q=10,
xhat=None,
solver_opts={'warm_start': True},
domain=None):
xhat_value = xhat
xhat = cp.Variable(L.shape[1])
if xhat_value is not None:
xhat.value = np.copy(xhat_value)
# This is the objective we want to solve, but
# all the reductions make this formulation too
# expensive to use.
# obj = cp.sum([cp.norm(L*xhat - y)**q for y in Y])
# Instead we formulate the objective using only
# matrix operations
# L @ xhat
#Lxhat = cp.reshape(xhat.__rmatmul__(L), (L.shape[0],1))
# outer product so copied over all points
#LXhat = Lxhat.T.__rmatmul__(np.ones( (len(Y),1)))
# 2-norm error for all points
ones_vec = np.ones((Y.shape[0], 1))
obj = cp.mixed_norm(ones_vec @ cp.reshape(L @ xhat, (1, L.shape[0])) - Y,
2, q)
#norms = cp.sum((LXhat - Y)**2, axis = 1)
#obj = cp.sum(norms**(q/2.)
constraints = []
if domain is not None:
constraints += domain._build_constraints(xhat)
prob = cp.Problem(cp.Minimize(obj), constraints)
prob.solve(**solver_opts)
return np.array(xhat.value)
def dilat_lvggm_ccg_cvx_sub(S, alpha, beta, covariance_h, precision_h, max_iter_in=1000, threshold_in=1e-3, verbose=False):
'''
A cvx implementation of the Decayed-influence Latent variable Gaussian Graphial Model
The subproblem in Convex-Concave Procedure
min_{R} -log det(R) + trace(R*S_t) + alpha*||[1,0]*R*[1;0]||_1 + gamma*||[0, Theta]*R*[1;0]||_{1,2}
s.t. [0,1]*R*[0;1] = Theta^{-1}
R >= 0
S_t = [Cov(X), -Cov*B*Theta; -Theta*B'*Cov, beta I]
'''
if np.linalg.norm(S-S.T) > 1e-3:
raise ValueError("Covariance matrix should be symmetric.")
n = S.shape[0]
#n1 = covariance.shape[0]
n2 = covariance_h.shape[0]
n1 = n - n2
#if n != (n1+n2):
if n1 < 0:
raise ValueError("dimension mismatch n=%d, n1=%d, n2=%d" % (n,n1,n2))
mask = np.zeros((n,n))
mask[np.ix_(np.arange(n1), np.arange(n1))]
J1 = np.zeros((n, n1))
J1[np.arange(n1),:] = np.eye(n1)
J2 = np.zeros((n, n2))
J2[np.arange(n1,n), :] = np.eye(n2)
Q = np.zeros((n,n2))
Q[np.arange(n1,n),:] = precision_h
#Q = np.zeros((n, n1))
#Q[np.arange(n1),:] = -covariance
J1 = np.asmatrix(J1)
J2 = np.asmatrix(J2)
Q = np.asmatrix(Q)
S - np.asmatrix(S)
R = cvx.Semidef(n)
# define the SDP problem
objective = cvx.Minimize(-cvx.log_det(R) + cvx.trace(S*R) + alpha*(cvx.norm((J1.T*R*J1), 1) + beta*cvx.mixed_norm((J1.T*R*Q).T, 2, 1) ))#beta*cvx.norm( (J1.T*R*Q), 1)) )
constraints = [J2.T*R*J2 == covariance_h]
# solve the problem
problem = cvx.Problem(objective, constraints)
problem.solve(verbose = verbose)
return np.asarray(R.value)
[3,
-4]]], Constant([5.47722557])),
(lambda x: cp.norm(x, 1), tuple(), [v_np], Constant([5])),
(lambda x: cp.norm(x, 1), tuple(), [[[-1, 2], [3, -4]]], Constant([10])),
(lambda x: cp.norm(x, "inf"), tuple(), [v_np], Constant([2])),
(lambda x: cp.norm(x, "inf"), tuple(), [[[-1, 2], [3,
-4]]], Constant([4])),
(lambda x: cp.norm(x, "nuc"), tuple(), [[[2, 0], [0, 1]]], Constant([3])),
(lambda x: cp.norm(x, "nuc"), tuple(), [[[3, 4, 5], [6, 7, 8], [9, 10,
11]]],
Constant([23.173260452512931])),
(lambda x: cp.norm(x, "nuc"), tuple(), [[[3, 4, 5], [6, 7, 8]]],
Constant([14.618376738088918])),
(lambda x: cp.sum_largest(cp.abs(x), 3), tuple(), [[1, 2, 3, -4, -5]],
Constant([5 + 4 + 3])),
(lambda x: cp.mixed_norm(x, 1, 1), tuple(), [[[1, 2], [3, 4],
[5, 6]]], Constant([21])),
(lambda x: cp.mixed_norm(x, 1, 1), tuple(), [[[1, 2, 3],
[4, 5, 6]]], Constant([21])),
# (lambda x: mixed_norm(x, 2, 1), tuple(), [[[3, 1], [4, math.sqrt(3)]]],
# Constant([7])),
(lambda x: cp.mixed_norm(x, 1, 'inf'), tuple(), [[[1, 4],
[5,
6]]], Constant([10])),
(cp.pnorm, tuple(), [[1, 2, 3]], Constant([3.7416573867739413])),
(lambda x: cp.pnorm(x, 1), tuple(), [[1.1, 2, -3]], Constant([6.1])),
(lambda x: cp.pnorm(x, 2), tuple(), [[1.1, 2, -3]],
Constant([3.7696153649941531])),
(lambda x: cp.pnorm(x, 2, axis=0), (2, ), [[[1, 2], [3, 4]]],
Constant([math.sqrt(5), 5.]).T),
(lambda x: cp.pnorm(x, 2, axis=1), (2, ), [[[1, 2], [4, 5]]],
def train_network(X_f,
X_i,
X_s,
Y_i,
dY_i=None,
beta=1e-6,
M=int(5000),
fuse_xxstar=False,
abs_act=False,
sdf=True,
norm=2):
n_f, d = X_f.shape
n_i, d = X_i.shape
X_f = np.append(X_f, np.ones((n_f, 1)), axis=1)
X_i = np.append(X_i, np.ones((n_i, 1)), axis=1)
X_s = np.append(X_s, np.ones((n_i, 1)), axis=1)
d += 1
# Y_i = np.atleast_1d(Y_i.squeeze())
dual_norm = np.inf if norm == 1 else int(1 / (1 - 1 / float(norm)))
## Finite approximation of all possible sign patterns
t0 = time.time()
if n_f + 2 * n_i < 10:
D_f, D_i, D_s = enumerate_signed_patterns(X_f, X_i, X_s, fuse_xxstar)
else:
D_f, D_i, D_s = random_signed_patterns(X_f,
X_i,
X_s,
M,
fuse_xxstar=fuse_xxstar)
m1 = D_f.shape[1]
print(f'Dmat creation: {time.time()-t0}s, {m1} arrangements identified.')
# Optimal CVX
Uopt0 = cp.Variable((d, 1), value=np.random.randn(d, 1))
Uopt1 = cp.Variable((d, m1), value=np.random.randn(d, m1))
Uopt2 = cp.Variable((d, m1), value=np.random.randn(d, m1))
constraints = []
loss = 0
# Feasible points
if n_f > 0:
ux_f_1 = cp.multiply(D_f, (X_f @ Uopt1))
ux_f_2 = cp.multiply(D_f, (X_f @ Uopt2))
if abs_act:
y_f = X_f @ Uopt0 + cp.sum(ux_f_1 - ux_f_2, axis=1, keepdims=True)
deriv_f = D_f @ (Uopt1 - Uopt2).T + Uopt0.T
else:
y_f = X_f @ Uopt0 + cp.sum(cp.multiply((D_f + 1) / 2,
(X_f @ (Uopt1 - Uopt2))),
axis=1,
keepdims=True)
deriv_f = ((D_f + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T
constraints += [ux_f_1 >= 0, ux_f_2 >= 0, ux_f_1 >= 0, ux_f_2 >= 0]
Y_f_lb = np.max(Y_i - np.linalg.norm(
X_f[None, :, :] - X_i[:, None, :], axis=-1, ord=norm),
axis=0,
keepdims=True).T
if sdf:
loss_f = cp.sum(cp.abs(y_f - Y_f_lb))
else:
loss_f = cp.sum(cp.pos(y_f))
loss = loss + loss_f
lipsh_f = cp.sum(cp.neg(y_f - Y_f_lb))
else:
loss_f = cp.Variable(value=0.)
lipsh_f = cp.Variable(value=0.)
# Infeasible points
if n_i > 0:
ux_i_1 = cp.multiply(D_i, (X_i @ Uopt1))
ux_s_1 = cp.multiply(D_s, (X_s @ Uopt1))
ux_i_2 = cp.multiply(D_i, (X_i @ Uopt2))
ux_s_2 = cp.multiply(D_s, (X_s @ Uopt2))
if abs_act:
y_i = X_i @ Uopt0 + cp.sum(ux_i_1 - ux_i_2, axis=1, keepdims=True)
y_s = X_s @ Uopt0 + cp.sum(ux_s_1 - ux_s_2, axis=1, keepdims=True)
deriv_i = D_i @ (Uopt1 - Uopt2).T + Uopt0.T
deriv_s = D_s @ (Uopt1 - Uopt2).T + Uopt0.T
else:
y_i = X_i @ Uopt0 + cp.sum(cp.multiply((D_i + 1) / 2,
(X_i @ (Uopt1 - Uopt2))),
axis=1,
keepdims=True)
y_s = X_s @ Uopt0 + cp.sum(cp.multiply((D_s + 1) / 2,
(X_s @ (Uopt1 - Uopt2))),
axis=1,
keepdims=True)
deriv_i = ((D_i + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T
deriv_s = ((D_s + 1) / 2) @ (Uopt1 - Uopt2).T + Uopt0.T
constraints += [
ux_i_1 >= 0,
ux_s_1 >= 0,
ux_i_2 >= 0,
ux_s_2 >= 0,
]
loss_i = cp.sum(cp.abs(Y_i - y_i))
loss_s = cp.sum(cp.abs(y_s))
if dY_i is not None:
loss_di = cp.sum(cp.abs(deriv_i[:, :-1] - dY_i))
loss_ds = cp.sum(cp.abs(deriv_s[:, :-1] - dY_i))
else:
loss_di = cp.Variable(value=0., nonneg=True)
loss_ds = cp.Variable(value=0., nonneg=True)
loss = loss + loss_i + loss_s + loss_ds + loss_di
lipsh_i = cp.sum(cp.neg(Y_i - y_i))
else:
loss_i = cp.Variable(value=0.)
loss_s = cp.Variable(value=0.)
lipsh_i = cp.Variable(value=0.)
lipsh_s = cp.Variable(value=0.)
loss_di = cp.Variable(value=0.)
loss_ds = cp.Variable(value=0.)
# Regularization
groupnorm_reg = cp.norm(Uopt0) + cp.mixed_norm(
Uopt1.T, 2, 1) + cp.mixed_norm(Uopt2.T, 2, 1)
if norm == 2:
reg_nrm = cp.norm(Uopt0[:-1]) + cp.mixed_norm(
Uopt1[:-1].T, 2, 1) + cp.mixed_norm(Uopt2[:-1].T, 2, 1)
elif norm == 1:
reg_nrm = cp.Variable(nonneg=True)
for i in range(d - 1):
for s in [-1, 1]:
constraints += [
cp.sum(cp.pos(Uopt1[i] * s) + cp.pos(-Uopt2[i] * s)) <=
reg_nrm
]
reg = reg_nrm + groupnorm_reg / 100.
# Solution
prob = cp.Problem(cp.Minimize(100 * (loss + beta * reg)), constraints)
t0 = time.time()
options = {} #dict(mosek_params = {'MSK_DPAR_BASIS_TOL_X':1e-8})
prob.solve(verbose=True, **options)
# prob.solve(solver=cp.SCS, verbose=True)
print(
f'Status: {prob.status}, \n '
f'Value: {prob.value :.2E}, \n '
f'loss_f: {loss_f.value :.2E}, \n '
f'loss_i: {loss_i.value :.2E}, \n '
f'loss_s: {loss_s.value :.2E}, \n '
f'loss_di: {loss_di.value :.2E}, \n '
f'loss_ds: {loss_ds.value :.2E}, \n '
f'Reg: {reg.value : .2E}, \n '
# # f'lipsh_f: {lipsh_f.value :.2E}, \n '
# # f'lipsh_i: {lipsh_i.value :.2E}, \n '
f'Time: {time.time()-t0 :.2f}s')
if prob.status.lower() == 'infeasible':
st()
return None
u0, u1, u2 = torch.tensor(Uopt0.value), torch.tensor(
Uopt1.value), torch.tensor(Uopt2.value)
torch.save(
{
'u1': u1,
'u2': u2,
'u0': u0,
'D_f': torch.tensor(D_f),
'D_i': torch.tensor(D_i),
'D_s': torch.tensor(D_s)
}, 'tmp.csv')
return u0, u1, u2
def minimax_lloyd(domain,
M,
L=None,
maxiter=100,
Xhat=None,
verbose=True,
xtol=1e-5,
full=None):
r""" A fixed point iteration for a minimax design
This algorithm can be interpreted as a block coordinate descent type algorithm
for the optimal minimax experimental design on the given domain.
SD96.
"""
# Terminate early if we have a simple case
try:
return minimax_design_1d(domain, M, L=L)
except AssertionError:
pass
if full is None:
if len(domain) < 3:
_update_voronoi = _update_voronoi_full
else:
_update_voronoi = _update_voronoi_sample
elif full is True:
_update_voronoi = _update_voronoi_full
elif full is False:
_update_voronoi = _update_voronoi_sample
if L is None:
L = np.eye(len(domain))
M0 = 10 * len(domain) * M
X0 = domain.sample(M0)
if Xhat is None:
if verbose:
print(10 * '=' + " Building Coffeehouse Design " + 10 * '=')
Xhat = maximin_coffeehouse(domain, M, L, verbose=verbose)
if verbose:
print('\n' + 10 * '=' + " Building Maximin Design " + 10 * '=')
Xhat = maximin_block(domain,
M,
L=L,
maxiter=50,
verbose=verbose,
X0=Xhat)
# The Shrinkage suggested by Pro17 hasn't been demonstrated to be useful with this initialization, so we avoid it
if verbose:
print('\n' + 10 * '=' + " Building Minimax Design " + 10 * '=')
x = cp.Variable(len(domain))
c = cp.Variable(len(domain))
constraints = domain._build_constraints(x)
if verbose:
printer = IterationPrinter(it='4d', minimax='18.10e', dx='9.3e')
printer.print_header(it='iter', minimax='minimax est', dx='Δx')
V = domain.sample(M0)
Xhat_new = np.zeros_like(Xhat)
for it in range(maxiter):
# Compute new Voronoi vertices
V = _update_voronoi(domain, Xhat, V, L, M0)
D = cdist(Xhat, V, L=L)
d = np.min(D, axis=0)
for k in range(M):
# Identify closest points to Xhat[k]
I = np.isclose(D[k], d)
# Move the Xhat[k] to the circumcenter
ones = np.ones((1, np.sum(I)))
obj = cp.mixed_norm(
(L @ (cp.reshape(x, (len(domain), 1)) @ ones - V[I].T)).T, 2,
np.inf)
prob = cp.Problem(cp.Minimize(obj), constraints)
prob.solve()
Xhat_new[k] = x.value
dx = np.max(np.sqrt(np.sum((Xhat_new - Xhat)**2, axis=1)))
if verbose:
printer.print_iter(it=it, minimax=np.max(d), dx=dx)
Xhat[:, :] = Xhat_new[:, :]
if dx < xtol:
if verbose:
print('small change in design')
break
return Xhat
def train_network(X_f, X_i, X_s, Y_i, beta, M=int(5000), fuse_xxstar=False, abs_act=False, sdf=True, norm=2):
n_f, d = X_f.shape
n_i, d = X_i.shape
X_f = np.append(X_f,np.ones((n_f,1)),axis=1)
X_i = np.append(X_i,np.ones((n_i,1)),axis=1)
X_s = np.append(X_s,np.ones((n_i,1)),axis=1)
d += 1
Y_i = np.atleast_1d(Y_i.squeeze())
dual_norm = np.inf if norm==1 else int(1/(1-1/float(norm)))
## Finite approximation of all possible sign patterns
t0 = time.time()
if n_f + 2*n_i < 15:
D_f, D_i, D_s = enumerate_signed_patterns(X_f, X_i, X_s, fuse_xxstar)
else:
D_f, D_i, D_s = random_signed_patterns(X_f, X_i, X_s, M, fuse_xxstar=fuse_xxstar)
m1 = D_f.shape[1]
print(f'Dmat creation: {time.time()-t0}s, {m1} arrangements identified.')
# Optimal CVX
Uopt1=cp.Variable((d,m1), value=np.random.randn(d,m1))
Uopt2=cp.Variable((d,m1), value=np.random.randn(d,m1))
constraints = []
loss = 0
# Feasible points
if n_f > 0:
ux_f_1 = cp.multiply(D_f,(X_f @ Uopt1))
ux_f_2 = cp.multiply(D_f,(X_f @ Uopt2))
y_f = cp.sum(ux_f_1 - ux_f_2,axis=1) if abs_act \
else cp.sum(cp.multiply((D_f+1)/2,(X_f @ (Uopt1 - Uopt2))),axis=1)
constraints += [
ux_f_1>=0,
ux_f_2>=0
]
Y_f_lb = np.max(Y_i[:, None] - np.linalg.norm(X_f[None, :, :] - X_i[:, None, :], axis=-1,
ord=dual_norm), axis=0)
if sdf:
loss_f = cp.sum(cp.abs(y_f - Y_f_lb))
else:
loss_f = cp.sum(cp.pos(y_f))
loss = loss + loss_f
lipsh_f = cp.sum(cp.neg(y_f - Y_f_lb))
else:
loss_f = cp.Variable(value=0.)
lipsh_f = cp.Variable(value=0.)
# Infeasible points
if n_i > 0:
ux_i_1 = cp.multiply(D_i,(X_i @ Uopt1))
ux_s_1 = cp.multiply(D_s,(X_s @ Uopt1))
ux_i_2 = cp.multiply(D_i,(X_i @ Uopt2))
ux_s_2 = cp.multiply(D_s,(X_s @ Uopt2))
if abs_act:
y_i = cp.sum(ux_i_1 - ux_i_2,axis=1)
y_s = cp.sum(ux_s_1 - ux_s_2,axis=1)
if sdf:
if norm == 2:
constraints += [cp.sum(cp.square((Uopt1[:2] - Uopt2[:2]) @ D_i.T), axis=0) <= 1]
elif norm == 1:
constraints += [cp.max(cp.abs((Uopt1[:2] - Uopt2[:2]) @ D_i.T), axis=0) <= 1]
else:
y_i = cp.sum(cp.multiply((D_i+1)/2,(X_i @ (Uopt1 - Uopt2))),axis=1)
y_s = cp.sum(cp.multiply((D_s+1)/2,(X_s @ (Uopt1 - Uopt2))),axis=1)
if sdf:
if norm == 2:
constraints += [cp.sum(cp.square((Uopt1[:2] - Uopt2[:2]) @ ((D_i+1)/2).T), axis=0) <= 1]
elif norm == 1:
constraints += [cp.max(cp.abs((Uopt1[:2] - Uopt2[:2]) @ ((D_i+1)/2).T), axis=0) <= 1]
constraints += [
ux_i_1>=0,
ux_s_1>=0,
ux_i_2>=0,
ux_s_2>=0,
]
loss_i = cp.sum(cp.abs(Y_i - y_i))
loss_s = cp.sum(cp.abs(y_s))
loss = loss + loss_i + loss_s
lipsh_i = cp.sum(cp.neg(Y_i - y_i))
else:
loss_i = cp.Variable(value=0.)
loss_s = cp.Variable(value=0.)
lipsh_i = cp.Variable(value=0.)
lipsh_s = cp.Variable(value=0.)
# Regularization
if norm == 2:
reg = cp.mixed_norm(Uopt1[:-1].T,2,1) + cp.mixed_norm(Uopt2[:-1].T,2,1)
elif norm == 1:
reg = cp.Variable(nonneg=True)
for i in range(d-1):
for s in [-1,1]:
constraints += [cp.sum(cp.pos(Uopt1[i] * s) + cp.pos(-Uopt2[i] * s)) <= reg]
# Solution
prob=cp.Problem(cp.Minimize(100*(loss + beta * reg)),constraints)
t0 = time.time()
options = dict(mosek_params = {'MSK_DPAR_BASIS_TOL_X':1e-8})
prob.solve(solver=cp.MOSEK, verbose=True, **options)
# prob.solve(solver=cp.SCS)
print(f'Status: {prob.status}, \n '
f'Value: {prob.value :.2E}, \n '
f'loss_f: {loss_f.value :.2E}, \n '
f'loss_i: {loss_i.value :.2E}, \n '
f'loss_s: {loss_s.value :.2E}, \n '
f'Reg: {reg.value : .2E}, \n '
f'lipsh_f: {lipsh_f.value :.2E}, \n '
f'lipsh_i: {lipsh_i.value :.2E}, \n '
f'Time: {time.time()-t0 :.2f}s')
if prob.status.lower() == 'infeasible':
st()
return None
u1, u2 = torch.tensor(Uopt1.value), torch.tensor(Uopt2.value)
st()
return u1, u2
