def test_mlfbf(self):
"""
Test the MLFBF solver with arbitrarily selected functions.
"""
x = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
params = {'solver': solver, 'verbosity': 'NONE'}
def x0():
return np.zeros(len(x))
# L2-norm prox and dummy prox.
f = functions.dummy()
f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x)
g = functions.norm_l2(lambda_=0.5)
h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Same test, but with callable L
solver = solvers.mlfbf(L=lambda x: np.dot(L, x),
Lt=lambda y: np.dot(L.T, y),
d0=np.dot(L, x0()),
step=max_step / 2.)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity check
self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_mlfbf(self):
"""
Test the MLFBF solver with arbitrarily selected functions.
"""
x = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
params = {'solver': solver, 'verbosity': 'NONE'}
def x0(): return np.zeros(len(x))
# L2-norm prox and dummy prox.
f = functions.dummy()
f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x)
g = functions.norm_l2(lambda_=0.5)
h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Same test, but with callable L
solver = solvers.mlfbf(L=lambda x: np.dot(L, x),
Lt=lambda y: np.dot(L.T, y),
d0=np.dot(L, x0()),
step=max_step / 2.)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity check
self.assertRaises(ValueError, solver.pre, [f, g], x0())
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.random.randn(3)
L = np.random.randn(4, 3)
sol = y
y2 = L.dot(y)
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l2(y=y2)
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step, L=L))
slvs.append(solvers.projection_based(step=step, L=L))
# Compare solutions.
niter = 1000
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': niter}
for solver in slvs:
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
# The initial value was not modified.
nptest.assert_array_equal(x0, np.zeros(len(y)))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3],
x0,
solver,
inplace=True,
**params)
else:
ret = solvers.solve([f1, f2],
x0,
solver,
inplace=True,
**params)
# The initial value was modified.
self.assertIs(ret['sol'], x0)
nptest.assert_allclose(ret['sol'], sol)
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.array([294, 390, 361])
sol = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1()
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step))
slvs.append(solvers.projection_based(step=step))
# Compare solutions.
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': 50}
niters = [50, 50]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
self.assertIs(ret['sol'], x0) # The initial value was modified.
def test_primal_dual_solver_comparison(self):
"""
Test that all primal-dual solvers return the same and correct solution.
I had to create this separate function because the primal-dual solvers
were too slow for the problem above.
"""
# Convex functions.
y = np.array([294, 390, 361])
sol = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
f1 = functions.norm_l1(y=y)
f2 = functions.norm_l1()
f3 = functions.dummy()
# Solvers.
step = 0.5 / (1 + np.linalg.norm(L, 2))
slvs = []
slvs.append(solvers.mlfbf(step=step))
slvs.append(solvers.projection_based(step=step))
# Compare solutions.
params = {'rtol': 0, 'verbosity': 'NONE', 'maxit': 50}
niters = [50, 50]
for solver, niter in zip(slvs, niters):
x0 = np.zeros(len(y))
if type(solver) is solvers.mlfbf:
ret = solvers.solve([f1, f2, f3], x0, solver, **params)
else:
ret = solvers.solve([f1, f2], x0, solver, **params)
nptest.assert_allclose(ret['sol'], sol)
self.assertEqual(ret['niter'], niter)
self.assertIs(ret['sol'], x0) # The initial value was modified.
def l2_degree_reg(X,
dist_type='sqeuclidean',
alpha=1,
s=None,
step=0.5,
w0=None,
maxit=1000,
rtol=1e-5,
retall=False,
verbosity='NONE'):
r"""
Learn graph by regularizing the l2-norm of the degrees.
This is done by solving
:math:`\tilde{W} = \underset{W \in \mathcal{W}_m}{\text{arg}\min} \,
\|W \odot Z\|_{1,1} + \alpha \|W1}\|^2 + \alpha \| W \|_{F}^{2}`, subject
to :math:`\|W\|_{1,1} = s`, where :math:`Z` is a pairwise distance matrix,
and :math:`\mathcal{W}_m`is the set of valid symmetric weighted adjacency
matrices.
Parameters
----------
X : array_like
An N-by-M data matrix of N variable observations in an M-dimensional
space. The learned graph will have N nodes.
dist_type : string
Type of pairwise distance between variables. See
:func:`spatial.distance.pdist` for the possible options.
alpha : float, optional
Regularization parameter acting on the l2-norm.
s : float, optional
The "sparsity level" of the weight matrix, as measured by its l1-norm.
step : float, optional
A number between 0 and 1 defining a stepsize value in the admissible
stepsize interval (see [Komodakis & Pesquet, 2015], Algorithm 6)
w0 : array_like, optional
Initialization of the edge weights. Must be an N(N-1)/2-dimensional
vector.
maxit : int, optional
Maximum number of iterations.
rtol : float, optional
Stopping criterion. Relative tolerance between successive updates.
retall : boolean
Return solution and problem details. See output of
:func:`pyunlocbox.solvers.solve`.
verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional
Level of verbosity of the solver. See :func:`pyunlocbox.solvers.solve`.
Returns
-------
W : array_like
Learned weighted adjacency matrix
problem : dict, optional
Information about the solution of the optimization. Only returned if
retall == True.
Notes
-----
This is the problem proposed in [Dong et al., 2015].
Examples
--------
"""
# Parse X
N = X.shape[0]
E = int(N * (N - 1.) / 2.)
z = spatial.distance.pdist(X, dist_type) # Pairwise distances
# Parse s
s = N if s is None else s
# Parse step
if (step <= 0) or (step > 1):
raise ValueError("step must be a number between 0 and 1.")
# Parse initial weights
w0 = np.zeros(z.shape) if w0 is None else w0
if (w0.shape != z.shape):
raise ValueError("w0 must be of dimension N(N-1)/2.")
# Get primal-dual linear map
one_vec = np.ones(E)
def K(w):
return np.array([2 * np.dot(one_vec, w)])
def Kt(n):
return 2 * n * one_vec
norm_K = 2 * np.sqrt(E)
# Get weight-to-degree map
S, St = utils.weight2degmap(N)
# Assemble functions in the objective
f1 = functions.func()
f1._eval = lambda w: 2 * np.dot(w, z)
f1._prox = lambda w, gamma: np.maximum(0, w - (2 * gamma * z))
f2 = functions.func()
f2._eval = lambda w: 0.
f2._prox = lambda d, gamma: s
f3 = functions.func()
f3._eval = lambda w: alpha * (2 * np.sum(w**2) + np.sum(S(w)**2))
f3._grad = lambda w: alpha * (4 * w + St(S(w)))
lipg = 2 * alpha * (N + 1)
# Rescale stepsize
stepsize = step / (1 + lipg + norm_K)
# Solve problem
solver = solvers.mlfbf(L=K, Lt=Kt, step=stepsize)
problem = solvers.solve([f1, f2, f3],
x0=w0,
solver=solver,
maxit=maxit,
rtol=rtol,
verbosity=verbosity)
# Transform weight matrix from vector form to matrix form
W = spatial.distance.squareform(problem['sol'])
if retall:
return W, problem
else:
return W
def log_degree_barrier(X,
dist_type='sqeuclidean',
alpha=1,
beta=1,
step=0.5,
w0=None,
maxit=1000,
rtol=1e-5,
retall=False,
verbosity='NONE'):
r"""
Learn graph by imposing a log barrier on the degrees
This is done by solving
:math:`\tilde{W} = \underset{W \in \mathcal{W}_m}{\text{arg}\min} \,
\|W \odot Z\|_{1,1} - \alpha 1^{T} \log{W1} + \beta \| W \|_{F}^{2}`,
where :math:`Z` is a pairwise distance matrix, and :math:`\mathcal{W}_m`
is the set of valid symmetric weighted adjacency matrices.
Parameters
----------
X : array_like
An N-by-M data matrix of N variable observations in an M-dimensional
space. The learned graph will have N nodes.
dist_type : string
Type of pairwise distance between variables. See
:func:`spatial.distance.pdist` for the possible options.
alpha : float, optional
Regularization parameter acting on the log barrier
beta : float, optional
Regularization parameter controlling the density of the graph
step : float, optional
A number between 0 and 1 defining a stepsize value in the admissible
stepsize interval (see [Komodakis & Pesquet, 2015], Algorithm 6)
w0 : array_like, optional
Initialization of the edge weights. Must be an N(N-1)/2-dimensional
vector.
maxit : int, optional
Maximum number of iterations.
rtol : float, optional
Stopping criterion. Relative tolerance between successive updates.
retall : boolean
Return solution and problem details. See output of
:func:`pyunlocbox.solvers.solve`.
verbosity : {'NONE', 'LOW', 'HIGH', 'ALL'}, optional
Level of verbosity of the solver. See :func:`pyunlocbox.solvers.solve`.
Returns
-------
W : array_like
Learned weighted adjacency matrix
problem : dict, optional
Information about the solution of the optimization. Only returned if
retall == True.
Notes
-----
This is the solver proposed in [Kalofolias, 2016] :cite:`kalofolias2016`.
Examples
--------
>>> import learn_graph as lg
>>> import networkx as nx
>>> import numpy as np
>>> import matplotlib.pyplot as plt
>>> from scipy import spatial
>>> G_gt = nx.waxman_graph(100)
>>> pos = nx.random_layout(G_gt)
>>> coords = np.array(list(pos.values()))
>>> def s1(x, y):
return np.sin((2 - x - y)**2)
>>> def s2(x, y):
return np.cos((x + y)**2)
>>> def s3(x, y):
return (x - 0.5)**2 + (y - 0.5)**3 + x - y
>>> def s4(x, y):
return np.sin(3 * ( (x - 0.5)**2 + (y - 0.5)**2 ) )
>>> X = np.array((s1(coords[:,0], coords[:,1]),
s2(coords[:,0], coords[:,1]),
s3(coords[:,0], coords[:,1]),
s4(coords[:,0], coords[:,1]))).T
>>> z = 25 * spatial.distance.pdist(X, 'sqeuclidean')
>>> W = lg.log_degree_barrier(z)
>>> W[W < np.percentile(W, 96)] = 0
>>> G_learned = nx.from_numpy_matrix(W)
>>> plt.figure(figsize=(12, 6))
>>> plt.subplot(1,2,1)
>>> nx.draw(G_gt, pos=pos)
>>> plt.title('Ground Truth')
>>> plt.subplot(1,2,2)
>>> nx.draw(G_learned, pos=pos)
>>> plt.title('Learned')
"""
# Parse X
N = X.shape[0]
z = spatial.distance.pdist(X, dist_type) # Pairwise distances
# Parse stepsize
if (step <= 0) or (step > 1):
raise ValueError("step must be a number between 0 and 1.")
# Parse initial weights
w0 = np.zeros(z.shape) if w0 is None else w0
if (w0.shape != z.shape):
raise ValueError("w0 must be of dimension N(N-1)/2.")
# Get primal-dual linear map
K, Kt = utils.weight2degmap(N)
norm_K = np.sqrt(2 * (N - 1))
# Assemble functions in the objective
f1 = functions.func()
f1._eval = lambda w: 2 * np.dot(w, z)
f1._prox = lambda w, gamma: np.maximum(0, w - (2 * gamma * z))
f2 = functions.func()
f2._eval = lambda w: -alpha * np.sum(
np.log(np.maximum(np.finfo(np.float64).eps, K(w))))
f2._prox = lambda d, gamma: np.maximum(
0, 0.5 * (d + np.sqrt(d**2 + (4 * alpha * gamma))))
f3 = functions.func()
f3._eval = lambda w: beta * np.sum(w**2)
f3._grad = lambda w: 2 * beta * w
lipg = 2 * beta
# Rescale stepsize
stepsize = step / (1 + lipg + norm_K)
# Solve problem
solver = solvers.mlfbf(L=K, Lt=Kt, step=stepsize)
problem = solvers.solve([f1, f2, f3],
x0=w0,
solver=solver,
maxit=maxit,
rtol=rtol,
verbosity=verbosity)
# Transform weight matrix from vector form to matrix form
W = spatial.distance.squareform(problem['sol'])
if retall:
return W, problem
else:
return W
def graph_pnorm_interpolation(self, gradient, P, w, labels_bin, x0=None, p=1., **kwargs):
r"""
Solve an interpolation problem via gradient p-norm minimization.
A signal :math:`x` is estimated from its measurements :math:`y = A(x)` by solving
:math:`\text{arg}\underset{z \in \mathbb{R}^n}{\min}
\| \nabla_G z \|_p^p \text{ subject to } Az = y`
via a primal-dual, forward-backward-forward algorithm.
Parameters
----------
gradient : array_like
A matrix representing the graph gradient operator
P : callable
Orthogonal projection operator mapping points in :math:`z \in \mathbb{R}^n`
onto the set satisfying :math:`A P(z) = A z`.
x0 : array_like, optional
Initial point of the iteration. Must be of dimension n.
(Default is `numpy.random.randn(n)`)
p : {1., 2.}
labels_bin : array_like
A vector that holds the binary labels.
kwargs :
Additional solver parameters, such as maximum number of iterations
(maxit), relative tolerance on the objective (rtol), and verbosity
level (verbosity). See :func:`pyunlocbox.solvers.solve` for the full
list of options.
Returns
-------
x : array_like
The solution to the optimization problem.
"""
grad = lambda z: gradient.dot(z)
div = lambda z: gradient.transpose().dot(z)
# Indicator function of the set satisfying :math:`y = A(z)`
f = functions.func()
f._eval = lambda z: 0
f._prox = lambda z, gamma: P(z, w, labels_bin)
# :math:`\ell_1` norm of the dual variable :math:`d = \nabla_G z`
g = functions.func()
g._eval = lambda z: np.sum(np.abs(grad(z)))
g._prox = lambda d, gamma: functions._soft_threshold(d, gamma)
# :math:`\ell_2` norm of the gradient (for the smooth case)
h = functions.norm_l2(A=grad, At=div)
stepsize = (0.9 / (1. + scipy.sparse.linalg.norm(gradient, ord='fro'))) ** p
solver = solvers.mlfbf(L=grad, Lt=div, step=stepsize)
if p == 1.:
problem = solvers.solve([f, g, functions.dummy()], x0=x0, solver=solver, **kwargs)
return problem['sol']
if p == 2.:
problem = solvers.solve([f, functions.dummy(), h], x0=x0, solver=solver, **kwargs)
return problem['sol']
else:
return x0
def test_mlfbf(self):
"""
Test the MLFBF solver with arbitrarily selected functions.
"""
x = [1., 1., 1.]
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
params = {'solver': solver, 'verbosity': 'NONE'}
def x0():
return np.zeros(len(x))
# L2-norm prox and dummy prox.
f = functions.dummy()
f._prox = lambda x, T: np.maximum(np.zeros(len(x)), x)
g = functions.norm_l2(lambda_=0.5)
h = functions.norm_l2(y=np.array([294, 390, 361]), lambda_=0.5)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Same test, but with callable L
solver = solvers.mlfbf(L=lambda x: np.dot(L, x),
Lt=lambda y: np.dot(L.T, y),
d0=np.dot(L, x0()),
step=max_step / 2.)
ret = solvers.solve([f, g, h], x0(), maxit=1000, rtol=0, **params)
nptest.assert_allclose(ret['sol'], x, rtol=1e-5)
# Sanity check
self.assertRaises(ValueError, solver.pre, [f, g], x0())
# Make a second test where the solution is calculated by hand
n = 10
y = np.random.rand(n) * 2
z = np.random.rand(n)
c = 1
delta = (y - z - c)**2 + 4 * (1 + y * z - z * c)
sol = 0.5 * ((y - z - c) + np.sqrt(delta))
class mlog(functions.func):
def __init__(self, z):
super().__init__()
self.z = z
def _eval(self, x):
return -np.sum(np.log(x + self.z))
def _prox(self, x, T):
delta = (x - self.z)**2 + 4 * (T + x * self.z)
sol = 0.5 * (x - self.z + np.sqrt(delta))
return sol
f = functions.norm_l1(lambda_=c)
g = mlog(z=z)
h = functions.norm_l2(lambda_=0.5, y=y)
mu = 1 + 1
step = 1 / mu / 2
solver = solvers.mlfbf(step=step)
ret = solvers.solve([f, g, h],
y.copy(),
solver,
maxit=200,
rtol=0,
verbosity="NONE")
nptest.assert_allclose(ret["sol"], sol, atol=1e-10)
# Make a final test where the function g can not be evaluate
# on the primal variables
y = np.random.rand(3)
y_2 = L.dot(y)
L = np.array([[5, 9, 3], [7, 8, 5], [4, 4, 9], [0, 1, 7]])
x0 = np.zeros(len(y))
f = functions.norm_l1(y=y)
g = functions.norm_l2(lambda_=0.5, y=y_2)
h = functions.norm_l2(y=y, lambda_=0.5)
max_step = 1 / (1 + np.linalg.norm(L, 2))
solver = solvers.mlfbf(L=L, step=max_step / 2.)
ret = solvers.solve([f, g, h], x0, solver, maxit=1000, rtol=0)
np.testing.assert_allclose(ret["sol"], y)