def test_consensus():
n = 100
k = 4
np.random.seed(0)
proxes = []
for _ in range(k):
A = np.random.randn(n, n)
b = np.random.randn(n)
x = cvx.Variable(n)
obj = cvx.Minimize(cvx.norm(A * x - b))
prob = cvx.Problem(obj)
prox = Prox(prob, {'x': x})
proxes += [prox]
admm = ADMM(proxes, rho=1)
admm.step(100)
# check that the residuals are relatively small
assert 1e-7 <= admm.infos[-1]['r'] <= 1e-4
assert 1e-7 <= admm.infos[-1]['s'] <= 1e-4
# a few more admm steps should make the residuals very small
admm.step(100)
assert admm.infos[-1]['r'] <= 1e-8
assert admm.infos[-1]['s'] <= 1e-8
def test_defaults():
prob, xvars = example()
prox = Prox(prob, xvars)
assert prox.settings == dict(eps=1e-3, max_iters=100, verbose=False)
# make sure the CySCS settings are set to the correct defaults
assert prox._work.settings['eps'] == 1e-3
assert prox._work.settings['max_iters'] == 100
assert not prox._work.settings['verbose']
def test1():
prob, xvars = example()
prox = Prox(prob, xvars)
x = prox()
# assert that info has correct keys
for key in 'status', 'iter', 'time':
assert key in prox.info
assert 'x' in x
# test that SCS settings get passed down
prox.update_settings(eps=4.5, max_iters=123)
assert prox.settings['eps'] == 4.5
assert prox.settings['max_iters'] == 123
# note: settings don't actually get passed to CySCS until we call the CySCS solve internally, i.e., compute the prox
assert prox._work.settings['eps'] != 4.5
assert prox._work.settings['max_iters'] != 123
prox()
assert prox._work.settings['eps'] == 4.5
assert prox._work.settings['max_iters'] == 123
def test1():
prob, xvars = example()
prox = Prox(prob, xvars)
x = prox()
# assert that info has correct keys
for key in 'status', 'iter', 'time':
assert key in prox.info
assert 'x' in x
# test that SCS settings get passed down
prox.update_settings(eps=4.5, max_iters=123)
assert prox.settings['eps'] == 4.5
assert prox.settings['max_iters'] == 123
# note: settings don't actually get passed to CySCS until we call the CySCS solve internally, i.e., compute the prox
assert prox._work.settings['eps'] != 4.5
assert prox._work.settings['max_iters'] != 123
prox()
assert prox._work.settings['eps'] == 4.5
assert prox._work.settings['max_iters'] == 123
def test1():
prob, xvars = example()
prox = Prox(prob, xvars)
assert prox.settings == dict(eps=1e-3, max_iters=100, verbose=False)
prox.update_settings(eps=1e-7)
assert prox.settings['eps'] == 1e-7
with pytest.raises(ValueError):
prox.update_settings(cows=17)
# won't catch the bad key here
prox.settings['cows'] = 17
# but will catch it when prox is run
with pytest.raises(ValueError):
prox()
def test1():
prob, xvars = example()
prox = Prox(prob, xvars)
assert prox.settings == dict(eps=1e-3, max_iters=100, verbose=False)
prox.update_settings(eps=1e-7)
assert prox.settings['eps'] == 1e-7
with pytest.raises(ValueError):
prox.update_settings(cows=17)
# won't catch the bad key here
prox.settings['cows'] = 17
# but will catch it when prox is run
with pytest.raises(ValueError):
prox()
def test_memory():
prob, x_vars, _ = example_rand(100, 50)
prox = Prox(prob, x_vars, verbose=False, max_iters=20, eps=1e-7)
x0 = prox()
num_checks = 100
check_iters = 100
mem = np.zeros(num_checks)
# a few iterations allows the memory variation to settle down
for _ in range(check_iters * 3):
x0 = prox(x0)
# see if memory grows with iterations
for i in range(num_checks * check_iters):
x0 = prox(x0)
if (i + 1) % check_iters == 0:
m = get_mem_MB()
mem[i // check_iters] = m
assert np.var(mem) / np.mean(mem) <= 1e-6
def admm_inner_iter(data):
(idx, orig_prob, prox, rho_val, gamma_merit, max_iter,
random_z, polish_best, seed, sigma, show_progress, neighbor_func, polish_func,
prox_polished, polish_depth, lower_bound, alpha, args, kwargs) = data
noncvx_vars = get_noncvx_vars(orig_prob)
np.random.seed(idx + seed)
random.seed(idx + seed)
# Augmented objective.
# gamma = cvx.Parameter(sign="positive")
merit_func = orig_prob.objective.args[0]
for constr in orig_prob.constraints:
merit_func += gamma_merit*get_constr_error(constr)
# Form ADMM problem.
# obj = orig_prob.objective.args[0]
# for var in noncvx_vars:
# obj += (rho_val/2)*cvx.sum_squares(var - var.z + var.u)
# prob = cvx.Problem(cvx.Minimize(obj), orig_prob.constraints)
if prox is None:
xvars = {var.id: var for var in orig_prob.variables()}
prox = Prox(orig_prob, xvars)
for var in noncvx_vars:
# var.init_z(random=random_z)
# var.init_u()
if idx == 0 or not random_z:
var.z.value = np.zeros(var.size)
elif var.z.value is not None:
var.z.value = np.random.normal(0, sigma, var.size)
var.u.value = np.zeros(var.size)
# x^k prev.
old_vars = {var.id:np.zeros(var.size) for var in orig_prob.variables()}
best_so_far = [np.inf, {v.id:np.zeros(v.size) for v in orig_prob.variables()}]
cur_merit = best_so_far[0]
# ADMM loop
for k in range(max_iter):
prev_merit = cur_merit
try:
# prob.solve(*args, **kwargs)
x0 = {}
for var in orig_prob.variables():
x0[var.id] = var.value
for var in noncvx_vars:
x0[var.id] = var.z.value.A1 - var.u.value.A1
x1 = prox(x0, rho_val)
for var in orig_prob.variables():
var.value = np.reshape(x1[var.id], var.size, order='F')
# print "post solve cost", idx, k, orig_prob.objective.value
except cvx.SolverError as e:
pass
if prox.info['status'] in ['Solved', 'Solved/Inaccurate']:
for var in noncvx_vars:
var.z.value = var.project(alpha*var.value + (1-alpha)*old_vars[var.id] + var.u.value)
# var.z.value = var.project(np.random.randn(*var.size))
# var.z.value = var.project(np.random.uniform(0, 1, size=var.size))
var.u.value += alpha*var.value + (1-alpha)*old_vars[var.id] - var.z.value
# Update previous iterate.
old_vars = {var.id: var.value for var in orig_prob.variables()}
if only_discrete(orig_prob):
if polish_depth == 0:
noncvx_vars[0].value = noncvx_vars[0].z.value
cur_merit = orig_prob.objective.value
sltn = {var.id: var.value for var in orig_prob.variables()}
elif neighbor_func is None:
cur_merit, sltn = neighbor_search(merit_func, old_vars, best_so_far,
idx, polish_depth)
else:
sltn = noncvx_vars[0].z.value.A.copy()
noncvx_vars[0].value = sltn
cur_merit = orig_prob.objective.value
for i in range(polish_depth):
prev_merit = cur_merit
cur_merit, sltn = neighbor_func(sltn, cur_merit)
if (prev_merit - cur_merit)/(prev_merit + 1) < 1e-3:
break
sltn = {noncvx_vars[0].id: sltn}
else:
if polish_func is None:
# Try to polish.
try:
polish_opt_val, status = polish(orig_prob, polish_depth, *args, **kwargs)
# print "post polish cost", idx, k, orig_prob.objective.value
except cvx.SolverError as e:
polish_opt_val = None
status = cvx.SOLVER_ERROR
# print "polish_opt_val", polish_opt_val
if status not in [cvx.OPTIMAL, cvx.OPTIMAL_INACCURATE]:
# Undo change in var.value.
for var in orig_prob.variables():
if isinstance(var, NonCvxVariable):
var.value = var.z.value
else:
var.value = old_vars[var.id]
cur_merit = merit_func.value
sltn = {v.id:v.value for v in orig_prob.variables()}
else:
sltn = {}
for var in orig_prob.variables():
sltn[var.id] = var.value
for var in noncvx_vars:
sltn[var.id] = var.z.value
prev_merit = np.inf
for i in range(polish_depth):
cur_merit, sltn = polish_func(sltn)
if (prev_merit - cur_merit)/(prev_merit + 1) < 1e-5:
break
prev_merit = cur_merit
if show_progress and idx == 0:
print("objective", idx, k, cur_merit, best_so_far[0])
if cur_merit < best_so_far[0]:
best_so_far[0] = cur_merit
best_so_far[1] = sltn
# # Restore variable values.
# for var in noncvx_vars:
# var.value = var.z.value
# Termination conditions.
if best_so_far[0] - lower_bound <= 1e-4:# or \
# abs(cur_merit - prev_merit) + abs(cur_merit - prev_merit)/(prev_merit + 1) < 1e-4:
return best_so_far
else:
print(prox.info['status'])
break
return best_so_far
def admm(self, rho=None, max_iter=50, restarts=5, alpha=1.8,
random=False, sigma=1.0, gamma=1e6, polish_best=True,
num_procs=None, parallel=True, seed=1, show_progress=False,
prox_polished=False, polish_depth=5,
neighbor_func=None, polish_func=None,
*args, **kwargs):
# rho is a list of values, one for each restart.
if rho is None:
rho = [np.random.uniform() for i in range(restarts)]
else:
assert len(rho) == restarts
# num_procs is the number of processors to launch.
if num_procs is None:
num_procs = multiprocessing.cpu_count()
# Construct the relaxation.
if type(self.objective) == cvx.Minimize:
rel_obj = self.objective
else:
rel_obj = -self.objective
rel_constr = self.constraints
for var in get_noncvx_vars(self):
rel_constr += var.relax()
rel_prob = cvx.Problem(rel_obj, rel_constr)
# HACK skip this.
# lower_bound = rel_prob.solve(*args, **kwargs)
lower_bound = -np.inf
if show_progress:
print("lower bound =", lower_bound)
# Algorithm.
if parallel:
pool = multiprocessing.Pool(num_procs)
tmp_prob = cvx.Problem(rel_prob.objective, rel_prob.constraints)
best_per_rho = pool.map(admm_inner_iter,
[(idx, tmp_prob, None, rho_val, gamma, max_iter,
random, polish_best, seed, sigma, show_progress, neighbor_func, polish_func,
prox_polished, polish_depth, lower_bound, alpha, args, kwargs) for idx, rho_val in enumerate(rho)])
pool.close()
pool.join()
else:
xvars = {var.id: var for var in rel_prob.variables()}
prox = Prox(rel_prob, xvars)
best_per_rho = list(map(admm_inner_iter,
[(idx, rel_prob, prox, rho_val, gamma, max_iter,
random, polish_best, seed, sigma, show_progress, neighbor_func, polish_func,
prox_polished, polish_depth, lower_bound, alpha, args, kwargs) for idx, rho_val in enumerate(rho)]))
# Merge best so far.
argmin = min([(val[0], idx) for idx, val in enumerate(best_per_rho)])[1]
best_so_far = best_per_rho[argmin]
#print "best found", best_so_far[0]
# Unpack result.
for var in self.variables():
var.value = best_so_far[1][var.id]
residual = cvx.Constant(0)
for constr in self.constraints:
residual += get_constr_error(constr)
return self.objective.value, residual.value
def admm_inner_iter(data):
(
idx,
orig_prob,
prox,
rho_val,
gamma_merit,
max_iter,
random_z,
polish_best,
seed,
sigma,
show_progress,
neighbor_func,
polish_func,
prox_polished,
polish_depth,
lower_bound,
alpha,
args,
kwargs,
) = data
noncvx_vars = get_noncvx_vars(orig_prob)
np.random.seed(idx + seed)
random.seed(idx + seed)
# Augmented objective.
# gamma = cvx.Parameter(nonneg=True)
merit_func = orig_prob.objective.args[0]
for constr in orig_prob.constraints:
merit_func += gamma_merit * get_constr_error(constr)
# Form ADMM problem.
# obj = orig_prob.objective.args[0]
# for var in noncvx_vars:
# obj += (rho_val/2)*cvx.sum_squares(var - var.z + var.u)
# prob = cvx.Problem(cvx.Minimize(obj), orig_prob.constraints)
if prox is None:
xvars = {var.id: var for var in orig_prob.variables()}
prox = Prox(orig_prob, xvars)
for var in noncvx_vars:
# var.init_z(random=random_z)
# var.init_u()
if idx == 0 or not random_z:
var.z.value = np.zeros(var.shape)
elif var.z.value is not None:
var.z.value = np.random.normal(0, sigma, var.shape)
var.u.value = np.zeros(var.shape)
# x^k prev.
old_vars = {var.id: np.zeros(var.shape) for var in orig_prob.variables()}
best_so_far = [np.inf, {v.id: np.zeros(v.shape) for v in orig_prob.variables()}]
cur_merit = best_so_far[0]
# ADMM loop
for k in range(max_iter):
prev_merit = cur_merit
try:
# prob.solve(*args, **kwargs)
x0 = {}
for var in orig_prob.variables():
x0[var.id] = var.value
for var in noncvx_vars:
x0[var.id] = (
np.asarray(var.z.value, order="F").ravel()
- np.asarray(var.u.value, order="F").ravel()
)
x1 = prox(x0, rho_val)
for var in orig_prob.variables():
# Bypass the projection step when going through the setter, and instead
# set the private value directly. Should probably revisit to see if some
# longer-term approach we want to take; it's obviously a bit uncomfortable
# to bypass the setter.
var._value = np.reshape(x1[var.id], var.shape, order="F")
except cp.SolverError:
pass
if prox.info["status"] in ["Solved", "Solved/Inaccurate"]:
for var in noncvx_vars:
var.z.value = var.project(
alpha * var.value + (1 - alpha) * old_vars[var.id] + var.u.value
)
var.u.value += (
alpha * var.value + (1 - alpha) * old_vars[var.id] - var.z.value
)
# Update previous iterate.
old_vars = {var.id: var.value for var in orig_prob.variables()}
if only_discrete(orig_prob):
if polish_depth == 0:
noncvx_vars[0].value = noncvx_vars[0].z.value
cur_merit = orig_prob.objective.value
sltn = {var.id: var.value for var in orig_prob.variables()}
elif neighbor_func is None:
cur_merit, sltn = neighbor_search(
merit_func, old_vars, best_so_far, idx, polish_depth
)
else:
sltn = noncvx_vars[0].z.value.copy()
noncvx_vars[0].value = sltn
cur_merit = orig_prob.objective.value
for i in range(polish_depth):
prev_merit = cur_merit
cur_merit, sltn = neighbor_func(sltn, cur_merit)
if (prev_merit - cur_merit) / (prev_merit + 1) < 1e-3:
break
sltn = {noncvx_vars[0].id: sltn}
else:
if polish_func is None:
# Try to polish.
try:
polish_opt_val, status = polish(
orig_prob, polish_depth, *args, **kwargs
)
# print "post polish cost", idx, k, orig_prob.objective.value
except cp.SolverError as e:
polish_opt_val = None
status = cp.SOLVER_ERROR
# print "polish_opt_val", polish_opt_val
if status not in [cp.OPTIMAL, cp.OPTIMAL_INACCURATE]:
# Undo change in var.value.
for var in orig_prob.variables():
if isinstance(var, NonCvxVariable):
var.value = var.z.value
else:
var.value = old_vars[var.id]
cur_merit = merit_func.value
sltn = {v.id: v.value for v in orig_prob.variables()}
else:
sltn = {}
for var in orig_prob.variables():
sltn[var.id] = var.value
for var in noncvx_vars:
sltn[var.id] = var.z.value
prev_merit = np.inf
for i in range(polish_depth):
cur_merit, sltn = polish_func(sltn)
if (prev_merit - cur_merit) / (prev_merit + 1) < 1e-5:
break
prev_merit = cur_merit
if show_progress and idx == 0:
print("objective", idx, k, cur_merit, best_so_far[0])
if cur_merit < best_so_far[0]:
best_so_far[0] = cur_merit
best_so_far[1] = sltn
# # Restore variable values.
# for var in noncvx_vars:
# var.value = var.z.value
# Termination conditions.
if best_so_far[0] - lower_bound <= 1e-4: # or \
# abs(cur_merit - prev_merit) + abs(cur_merit - prev_merit)/(prev_merit + 1) < 1e-4:
return best_so_far
else:
print(prox.info["status"])
break
return best_so_far