def run(self, function, w):
"""Apply the algorithm to minimise function, starting at the positions
of the vectors in the list w.
Parameters
----------
function : MultiblockFunction
The function to minimise.
w : list of numpy arrays
Each element of the list is the parameter vector corresponding to a
block.
"""
# Not ok until the end.
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
# Initialise info variables. Info variables have the prefix "_".
if self.info_requested(Info.time):
_t = []
if self.info_requested(Info.func_val):
_f = []
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
w_old = [0] * len(w)
it = 0
while True:
for i in range(len(w)):
# Wrap a function around the ith block:
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
if hasattr(function, "at_point"):
def new_at_point(self, w):
return function.at_point(self.w[:self.index] + [w] +
self.w[self.index + 1:])
import types
func.at_point = types.MethodType(new_at_point, func)
w_old[i] = w[i]
self.algorithm.reset()
w[i] = self.algorithm.run(func, w_old[i])
# Store info from algorithm:
if self.info_requested(Info.time):
time = self.algorithm.info_get(Info.time)
_t.extend(time)
if self.info_requested(Info.func_val):
func_val = self.algorithm.info_get(Info.func_val)
_f.extend(func_val)
# Update iteration counts.
self.num_iter += self.algorithm.num_iter
# Test global stopping criterion.
all_converged = True
for i in range(len(w)):
# Wrap a function around the ith block.
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# Test if converged for block i.
if maths.norm(w[i] - w_old[i]) > self.eps:
all_converged = False
break
# Converged in all blocks!
if all_converged:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
# Stop after maximum number of iterations.
if self.num_iter >= self.max_iter:
break
it += 1
# Store information.
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, _f)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return w
def run(self, function, w):
# self.info.clear()
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
if self.info_requested(Info.time):
t = []
if self.info_requested(Info.fvalue):
f = []
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
# print "len(w):", len(w)
# print "max_iter:", self.max_iter
num_iter = [0] * len(w)
for it in range(1, self.outer_iter + 1):
all_converged = True
for i in range(len(w)):
# print "it: %d, i: %d" % (it, i)
if function.has_nesterov_function(i):
# print "Block %d has a Nesterov function!" % (i,)
func = mb_losses.MultiblockNesterovFunctionWrapper(
function, w, i)
algorithm = self.conesta
else:
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
algorithm = self.fista
# self.alg_info.clear()
# self.algorithm.set_params(max_iter=self.max_iter - num_iter[i])
# w[i] = self.algorithm.run(func, w_old[i])
if i == 1:
pass
w[i] = algorithm.run(func, w[i])
if algorithm.info_requested(Info.num_iter):
num_iter[i] += algorithm.info_get(Info.num_iter)
if algorithm.info_requested(Info.time):
tval = algorithm.info_get(Info.time)
if algorithm.info_requested(Info.fvalue):
fval = algorithm.info_get(Info.fvalue)
if self.info_requested(Info.time):
t = t + tval
if self.info_requested(Info.fvalue):
f = f + fval
# print "l0 :", maths.norm0(w[i]), \
# ", l1 :", maths.norm1(w[i]), \
# ", l2²:", maths.norm(w[i]) ** 2.0
# print "f:", fval[-1]
for i in range(len(w)):
# Take one ISTA step for use in the stopping criterion.
step = function.step(w, i)
w_tilde = function.prox(
w[:i] + [w[i] - step * function.grad(w, i)] + w[i + 1:], i,
step)
# func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# step2 = func.step(w[i])
# w_tilde2 = func.prox(w[i] - step2 * func.grad(w[i]), step2)
#
# print "diff:", maths.norm(w_tilde - w_tilde2)
# print "err:", maths.norm(w[i] - w_tilde) * (1.0 / step)
if (1.0 / step) * maths.norm(w[i] - w_tilde) > self.eps:
all_converged = False
break
if all_converged:
# print "All converged!"
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
# # If all blocks have used max_iter iterations, stop.
# if np.all(np.asarray(num_iter) >= self.max_iter):
# break
# it += 1
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, t)
if self.info_requested(Info.fvalue):
self.info_set(Info.fvalue, f)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return w
def run(self, function, w):
"""Apply the algorithm to minimise function, starting at the positions
of the vectors in the list w.
Parameters
----------
function : MultiblockFunction
The function to minimise.
w : list of numpy arrays
Each element of the list is the parameter vector corresponding to a
block.
"""
# Not ok until the end.
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
# Initialise info variables. Info variables have the prefix "_".
if self.info_requested(Info.time):
_t = []
if self.info_requested(Info.func_val):
_f = []
if self.info_requested(Info.other):
_other = dict()
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
w_old = [0] * len(w)
it = 0
while True:
for i in range(len(w)):
if self.info_requested(Info.time):
tm = utils.time()
# Wrap a function around the ith block:
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
if hasattr(function, "at_point"):
def new_at_point(self, w):
return self.function.at_point(self.w[:self.index] +
[w] +
self.w[self.index + 1:])
func.at_point = types.MethodType(new_at_point, func)
w_old[i] = w[i]
self.algorithm.reset()
func.at_point(w_old[i])
w[i] = self.algorithm.run(func, w_old[i])
# Store info from algorithm:
if self.info_requested(Info.time):
# time = self.algorithm.info_get(Info.time)
time = utils.time() - tm
_t.append(time)
if self.info_requested(Info.func_val):
# func_val = self.algorithm.info_get(Info.func_val)
func.at_point(w[i])
# func_val = func.f(w[i])
func_val = function.f(w)
_f.append(func_val)
if len(_f) > 2 and _f[-2] < _f[-1]:
print("w00t!! "
+ str(function.f(w_old))
+ " "
+ str(function.f(w)))
if self.info_requested(Info.other):
nfo = self.algorithm.info_get()
for key in nfo.keys():
# if key == Info.time:
# continue
# if key == Info.func_val:
# continue
if key not in _other:
_other[key] = list()
value = nfo[key]
_other[key].append(value)
# Update iteration counts.
self.num_iter += self.algorithm.num_iter
# Test global stopping criterion.
all_converged = True
for i in range(len(w)):
# Wrap a function around the ith block.
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# Test if converged for block i.
if maths.norm(w[i] - w_old[i]) > self.eps:
all_converged = False
break
# Converged in all blocks!
if all_converged:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
# Stop after maximum number of iterations.
if self.num_iter >= self.max_iter:
break
it += 1
# Store information.
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, _f)
if self.info_requested(Info.other):
self.info_set(Info.other, _other)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return w
def run(self, function, w):
# Not ok until the end.
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
# Initialise info variables. Info variables have the prefix "_".
if self.info_requested(Info.time):
_t = []
if self.info_requested(Info.func_val):
_f = []
if self.info_requested(Info.smooth_func_val):
_fmu = []
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
FISTA = True
if FISTA:
exp = 4.0 + consts.FLOAT_EPSILON
else:
exp = 2.0 + consts.FLOAT_EPSILON
block_iter = [1] * len(w)
it = 0
while True:
for i in range(len(w)):
# print "it: %d, i: %d" % (it, i)
# if True:
# pass
# Wrap a function around the ith block.
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# Run FISTA.
w_old = w[i]
for k in range(
1,
max(self.min_iter + 1,
self.max_iter - self.num_iter + 1)):
if self.info_requested(Info.time):
time = utils.time_wall()
if FISTA:
# Take an interpolated step.
z = w[i] + ((k - 2.0) / (k + 1.0)) * (w[i] - w_old)
else:
z = w[i]
# Compute the step.
step = func.step(z)
# Compute inexact precision.
eps = max(consts.FLOAT_EPSILON, 1.0 / (block_iter[i]**exp))
# eps = consts.TOLERANCE
w_old = w[i]
# Take a FISTA step.
w[i] = func.prox(z - step * func.grad(z),
factor=step,
eps=eps)
# Store info variables.
if self.info_requested(Info.time):
_t.append(utils.time_wall() - time)
if self.info_requested(Info.func_val):
_f.append(function.f(w))
if self.info_requested(Info.smooth_func_val):
_fmu.append(function.fmu(w))
# Update iteration counts.
self.num_iter += 1
block_iter[i] += 1
# print i, function.fmu(w), step, \
# (1.0 / step) * maths.norm(w[i] - z), self.eps, \
# k, self.num_iter, self.max_iter
# Test stopping criterion.
if maths.norm(w[i] - z) < step * self.eps \
and k >= self.min_iter:
break
# Test global stopping criterion.
all_converged = True
for i in range(len(w)):
# Wrap a function around the ith block.
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# Compute the step.
step = func.step(w[i])
# Compute inexact precision.
eps = max(consts.FLOAT_EPSILON, 1.0 / (block_iter[i]**exp))
# eps = consts.TOLERANCE
# Take one ISTA step for use in the stopping criterion.
w_tilde = func.prox(w[i] - step * func.grad(w[i]),
factor=step,
eps=eps)
# Test if converged for block i.
if maths.norm(w[i] - w_tilde) > step * self.eps:
all_converged = False
break
# Converged in all blocks!
if all_converged:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
# Stop after maximum number of iterations.
if self.num_iter >= self.max_iter:
break
it += 1
# Store information.
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, _f)
if self.info_requested(Info.smooth_func_val):
self.info_set(Info.smooth_func_val, _fmu)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return w
def run(self, function, w):
# Not ok until the end.
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
# Initialise info variables. Info variables have the prefix "_".
if self.info_requested(Info.time):
_t = []
if self.info_requested(Info.func_val):
_f = []
if self.info_requested(Info.smooth_func_val):
_fmu = []
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
for i in range(len(w)): # For each block.
# Store info variables.
if self.info_requested(Info.time):
time = utils.time_wall()
if self.info_requested(Info.func_val):
_f.append(function.f(w))
if self.info_requested(Info.smooth_func_val):
_fmu.append(function.fmu(w))
if self.info_requested(Info.time):
_t.append(utils.time_wall() - time)
exp = 2.0 + consts.FLOAT_EPSILON
block_iter = [1] * len(w)
it = 0
while True:
for i in range(len(w)): # Loop over the blocks
# Wrap a function around the ith block.
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# Run ISTA.
for k in range(1, max(self.min_iter + 1,
self.max_iter - self.num_iter + 1)):
if self.info_requested(Info.time):
time = utils.time_wall()
# Compute the step.
step = func.step(w[i])
# Compute inexact precision.
eps = max(consts.FLOAT_EPSILON,
1.0 / (block_iter[i] ** exp))
w_old = w[i]
# Take an ISTA step.
w[i] = func.prox(w[i] - step * func.grad(w[i]),
factor=step, eps=eps)
# Store info variables.
if self.info_requested(Info.time):
_t.append(utils.time_wall() - time)
if self.info_requested(Info.func_val):
_f.append(function.f(w))
if self.info_requested(Info.smooth_func_val):
_fmu.append(function.fmu(w))
# Update iteration counts.
self.num_iter += 1
block_iter[i] += 1
# Test stopping criterion.
if maths.norm(w[i] - w_old) < step * self.eps \
and k >= self.min_iter:
break
# Test global stopping criterion.
all_converged = True
for i in range(len(w)):
# Wrap a function around the ith block.
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# Compute the step.
step = func.step(w[i])
# Compute inexact precision.
eps = max(consts.FLOAT_EPSILON,
1.0 / (block_iter[i] ** exp))
# Take one ISTA step for use in the stopping criterion.
w_tilde = func.prox(w[i] - step * func.grad(w[i]),
factor=step, eps=eps)
# TODO: Use this step?
# Test if converged for block i.
if maths.norm(w[i] - w_tilde) > step * self.eps:
all_converged = False
break
# Converged in all blocks!
if all_converged:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
# Stop after maximum number of iterations.
if self.num_iter >= self.max_iter:
break
it += 1
# Store information.
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, _f)
if self.info_requested(Info.smooth_func_val):
self.info_set(Info.smooth_func_val, _fmu)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return w