def run(self, function, w):
"""Apply the algorithm to minimise function, starting at the positions
of the vectors in the list w.
Parameters
----------
function : MultiblockFunction
The function to minimise.
w : list of numpy arrays
Each element of the list is the parameter vector corresponding to a
block.
"""
# Not ok until the end.
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
# Initialise info variables. Info variables have the prefix "_".
if self.info_requested(Info.time):
_t = []
if self.info_requested(Info.func_val):
_f = []
if self.info_requested(Info.other):
_other = dict()
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
w_old = [0] * len(w)
it = 0
while True:
for i in range(len(w)):
if self.info_requested(Info.time):
tm = utils.time()
# Wrap a function around the ith block:
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
if hasattr(function, "at_point"):
def new_at_point(self, w):
return self.function.at_point(self.w[:self.index] +
[w] +
self.w[self.index + 1:])
func.at_point = types.MethodType(new_at_point, func)
w_old[i] = w[i]
self.algorithm.reset()
func.at_point(w_old[i])
w[i] = self.algorithm.run(func, w_old[i])
# Store info from algorithm:
if self.info_requested(Info.time):
# time = self.algorithm.info_get(Info.time)
time = utils.time() - tm
_t.append(time)
if self.info_requested(Info.func_val):
# func_val = self.algorithm.info_get(Info.func_val)
func.at_point(w[i])
# func_val = func.f(w[i])
func_val = function.f(w)
_f.append(func_val)
if len(_f) > 2 and _f[-2] < _f[-1]:
print("w00t!! "
+ str(function.f(w_old))
+ " "
+ str(function.f(w)))
if self.info_requested(Info.other):
nfo = self.algorithm.info_get()
for key in nfo.keys():
# if key == Info.time:
# continue
# if key == Info.func_val:
# continue
if key not in _other:
_other[key] = list()
value = nfo[key]
_other[key].append(value)
# Update iteration counts.
self.num_iter += self.algorithm.num_iter
# Test global stopping criterion.
all_converged = True
for i in range(len(w)):
# Wrap a function around the ith block.
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# Test if converged for block i.
if maths.norm(w[i] - w_old[i]) > self.eps:
all_converged = False
break
# Converged in all blocks!
if all_converged:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
# Stop after maximum number of iterations.
if self.num_iter >= self.max_iter:
break
it += 1
# Store information.
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, _f)
if self.info_requested(Info.other):
self.info_set(Info.other, _other)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return w
def run(self, function, beta):
# Copy the allowed info keys for FISTA.
fista_info = list()
for nfo in self.info_copy():
if nfo in FISTA.INFO_PROVIDED:
fista_info.append(nfo)
# Create the inner algorithm.
algorithm = FISTA(info=fista_info, eps=self.eps,
max_iter=self.max_iter, min_iter=self.min_iter)
# Not ok until the end.
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
# Time the init computation.
if self.info_requested(Info.time):
init_time = utils.time()
# Estimate the initial precision, eps, and the smoothing parameter mu.
gM = function.eps_max(1.0) # gamma * M
if maths.norm(beta) > consts.TOLERANCE:
mu = function.estimate_mu(beta)
eps = mu * gM
else:
eps = self._approximate_eps(function, beta)
mu = eps / gM
function.set_mu(mu)
# Initialise info variables. Info variables have the suffix "_".
if self.info_requested(Info.time):
t_ = []
init_time = utils.time() - init_time
if self.info_requested(Info.fvalue) \
or self.info_requested(Info.func_val):
f_ = []
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
if self.info_requested(Info.mu):
mu_ = []
i = 0 # Iteration counter.
while True:
converged = False
# Give current parameters to the algorithm.
algorithm.set_params(eps=eps,
max_iter=self.max_iter - self.num_iter)
# Run FISTA.
beta_new = algorithm.run(function, beta)
# Update global iteration count.
self.num_iter += algorithm.num_iter
# Get info from algorithm.
if Info.time in algorithm.info and \
self.info_requested(Info.time):
t_ += algorithm.info_get(Info.time)
if i == 0: # Add init time to first iteration.
t_[0] += init_time
if Info.func_val in algorithm.info \
and self.info_requested(Info.func_val):
f_ += algorithm.info_get(Info.func_val)
elif Info.fvalue in algorithm.info \
and self.info_requested(Info.fvalue):
f_ += algorithm.info_get(Info.fvalue)
if self.info_requested(Info.mu):
mu_ += [mu] * algorithm.num_iter
# Unless this is a simulation, you want the algorithm to stop when
# it has converged.
if not self.simulation:
# Stopping criterion.
step = function.step(beta_new)
if maths.norm(beta_new - beta) < step * self.eps:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
converged = True
beta = beta_new
if self.callback is not None:
self.callback(locals())
if self.info_requested(Info.verbose):
print("StaticCONESTA ite: %i, eps: %g, mu: %g" % (i, eps, mu))
# All combined stopping criteria.
if (converged or self.num_iter >= self.max_iter) \
and self.num_iter >= self.min_iter:
break
# Update the precision eps.
eps = self.tau * eps
# Compute and update mu.
mu = max(self.mu_min, eps / gM)
function.set_mu(mu)
i = i + 1
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.continuations):
self.info_set(Info.continuations, i + 1)
if self.info_requested(Info.time):
self.info_set(Info.time, t_)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, f_)
if self.info_requested(Info.fvalue):
self.info_set(Info.fvalue, f_)
if self.info_requested(Info.mu):
self.info_set(Info.mu, mu_)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return beta
def run(self, X, y, alpha=None):
"""Find the best separating margin for the samples in X.
Parameters
----------
X : ndarray
The matrix with samples to separate.
y : array_like
The class belongings for the samples in X. Values must be -1
or 1.
alpha : numpy array
A start vector for the lagrange multipliers. Default is to use a
zero vector.
"""
X, y = check_arrays(X, check_array_in(y, [-1, 1]))
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, False)
if self.info_requested(utils.Info.time):
_t = time()
if self.info_requested(utils.Info.func_val):
self._f = []
n, p = X.shape
# Set up error cache
self._E = np.zeros(n)
self._Evalid = np.zeros(n, dtype=np.bool)
# Bias (intercept/threshold)
self.bias = 0.0
# Lagrange multipliers
if alpha is None:
self.alpha = np.zeros((n, 1))
else:
self.alpha = alpha
numChanged = 0
examineAll = True
while numChanged > 0 or examineAll:
numChanged = 0
if examineAll:
for i in xrange(n):
numChanged += self._examineSample(i, X, y)
else:
for i in xrange(n):
if self.alpha[i, 0] > 0.0 and self.alpha[i, 0] < self.C:
numChanged += self._examineSample(i, X, y)
if examineAll:
examineAll = False
elif numChanged == 0:
examineAll = True
if self.info_requested(utils.Info.time):
self.info_set(utils.Info.time, time() - _t)
if self.info_requested(utils.Info.func_val):
self.info_set(utils.Info.func_val, self._f)
del self._f # Remove for future runs
if self.info_requested(utils.Info.ok):
self.info_set(utils.Info.ok, True)
return self._compute_w(X, y)
def run(self, function, w):
"""Apply the algorithm to minimise function, starting at the positions
of the vectors in the list w.
Parameters
----------
function : MultiblockFunction
The function to minimise.
w : list of numpy arrays
Each element of the list is the parameter vector corresponding to a
block.
"""
# Not ok until the end.
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
# Initialise info variables. Info variables have the prefix "_".
if self.info_requested(Info.time):
_t = []
if self.info_requested(Info.func_val):
_f = []
if self.info_requested(Info.other):
_other = dict()
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
w_old = [0] * len(w)
it = 0
while True:
for i in range(len(w)):
if self.info_requested(Info.time):
tm = utils.time()
# Wrap a function around the ith block:
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
if hasattr(function, "at_point"):
def new_at_point(self, w):
return self.function.at_point(self.w[:self.index] +
[w] +
self.w[self.index + 1:])
func.at_point = types.MethodType(new_at_point, func)
w_old[i] = w[i]
self.algorithm.reset()
func.at_point(w_old[i])
w[i] = self.algorithm.run(func, w_old[i])
# Store info from algorithm:
if self.info_requested(Info.time):
# time = self.algorithm.info_get(Info.time)
time = utils.time() - tm
_t.append(time)
if self.info_requested(Info.func_val):
# func_val = self.algorithm.info_get(Info.func_val)
func.at_point(w[i])
# func_val = func.f(w[i])
func_val = function.f(w)
_f.append(func_val)
if len(_f) > 2 and _f[-2] < _f[-1]:
print("w00t!! "
+ str(function.f(w_old))
+ " "
+ str(function.f(w)))
if self.info_requested(Info.other):
nfo = self.algorithm.info_get()
for key in nfo.keys():
# if key == Info.time:
# continue
# if key == Info.func_val:
# continue
if key not in _other:
_other[key] = list()
value = nfo[key]
_other[key].append(value)
# Update iteration counts.
self.num_iter += self.algorithm.num_iter
# Test global stopping criterion.
all_converged = True
for i in range(len(w)):
# Wrap a function around the ith block.
func = mb_losses.MultiblockFunctionWrapper(function, w, i)
# Test if converged for block i.
if maths.norm(w[i] - w_old[i]) > self.eps:
all_converged = False
break
# Converged in all blocks!
if all_converged:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
# Stop after maximum number of iterations.
if self.num_iter >= self.max_iter:
break
it += 1
# Store information.
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, _f)
if self.info_requested(Info.other):
self.info_set(Info.other, _other)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return w
def run(self, majoriser, x):
"""Run the optimiser using the majoriser function starting at the given
point.
Parameters
----------
majoriser : MajoriserFunction
The function that majorises self.function.
x : array_like
The point at which to start the minimisation process.
"""
# x = check_arrays(x)
# x = multiblock_array(x)
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
if self.info_requested(Info.time):
_t = []
if self.info_requested(Info.func_val):
_f = []
if self.info_requested(Info.smooth_func_val):
_fmu = []
if self.info_requested(Info.other):
_other = dict()
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
for it in range(1, self.max_iter + 1):
if self.info_requested(Info.time):
tm = utils.time()
# if isinstance(x, list):
# for i in range(len(x)):
#
# xold = x
# if isinstance(majoriser, properties.MajoriserFunction):
# maj_f = majoriser(self.function, xold)
# else:
# majoriser.at_point(xold)
# maj_f = majoriser
#
# func = mb_losses.MultiblockFunctionWrapper(maj_f, xold, i)
#
# x[i] = self.algorithm.run(func, xold[i])
# else:
xold = x
if isinstance(majoriser, properties.MajoriserFunction):
maj_f = majoriser(self.function, xold)
else:
majoriser.at_point(xold)
maj_f = majoriser
x = self.algorithm.run(maj_f, xold)
if self.info_requested(Info.time):
_t.append(utils.time() - tm)
if self.info_requested(Info.func_val) \
or self.info_requested(Info.smooth_func_val):
if isinstance(majoriser, properties.MajoriserFunction):
maj_f = majoriser(self.function, xold)
else:
majoriser.at_point(xold)
maj_f = majoriser
if self.info_requested(Info.func_val):
_f.append(maj_f.f(x))
if self.info_requested(Info.smooth_func_val):
if hasattr(maj_f, "fmu"):
_fmu.append(maj_f.fmu(x))
if self.info_requested(Info.other):
nfo = self.algorithm.info_get()
for key in nfo.keys():
if key not in _other:
_other[key] = list()
value = nfo[key]
_other[key].append(value)
if self.callback is not None:
self.callback(locals())
if isinstance(x, list):
val = utils.list_op((x, xold),
lambda new, old: np.linalg.norm(new - old),
aggregator=np.max)
else:
val = np.linalg.norm(x - xold)
if it >= self.min_iter and val < self.eps:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
self.num_iter = it
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, _f)
if self.info_requested(Info.smooth_func_val):
self.info_set(Info.smooth_func_val, _fmu)
if self.info_requested(Info.other):
self.info_set(Info.other, _other)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return x
def run(self, X, y, alpha=None):
"""Find the best separating margin for the samples in X.
Parameters
----------
X : ndarray
The matrix with samples to separate.
y : array_like
The class belongings for the samples in X. Values must be -1
or 1.
alpha : numpy array
A start vector for the lagrange multipliers. Default is to use a
zero vector.
"""
X, y = check_arrays(X, check_array_in(y, [-1, 1]))
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
if self.info_requested(Info.time):
_t = utils.time()
if self.info_requested(Info.func_val):
self._f = []
n, p = X.shape
# Set up error cache
self._E = np.zeros(n)
self._Evalid = np.zeros(n, dtype=np.bool)
# Bias (intercept/threshold)
self.bias = 0.0
# Lagrange multipliers
if alpha is None:
self.alpha = np.zeros((n, 1))
else:
self.alpha = alpha
numChanged = 0
examineAll = True
while numChanged > 0 or examineAll:
numChanged = 0
if examineAll:
for i in range(n):
numChanged += self._examineSample(i, X, y)
else:
for i in range(n):
if self.alpha[i, 0] > 0.0 and self.alpha[i, 0] < self.C:
numChanged += self._examineSample(i, X, y)
if examineAll:
examineAll = False
elif numChanged == 0:
examineAll = True
if self.info_requested(Info.time):
self.info_set(Info.time, utils.time() - _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, self._f)
del self._f # Remove for future runs
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return self._compute_w(X, y)
def run(self, majoriser, x):
"""Run the optimiser using the majoriser function starting at the given
point.
Parameters
----------
majoriser : MajoriserFunction
The function that majorises self.function.
x : array_like
The point at which to start the minimisation process.
"""
# x = check_arrays(x)
# x = multiblock_array(x)
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
if self.info_requested(Info.time):
_t = []
if self.info_requested(Info.func_val):
_f = []
if self.info_requested(Info.smooth_func_val):
_fmu = []
if self.info_requested(Info.other):
_other = dict()
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
for it in range(1, self.max_iter + 1):
if self.info_requested(Info.time):
tm = utils.time()
# if isinstance(x, list):
# for i in range(len(x)):
#
# xold = x
# if isinstance(majoriser, properties.MajoriserFunction):
# maj_f = majoriser(self.function, xold)
# else:
# majoriser.at_point(xold)
# maj_f = majoriser
#
# func = mb_losses.MultiblockFunctionWrapper(maj_f, xold, i)
#
# x[i] = self.algorithm.run(func, xold[i])
# else:
xold = x
if isinstance(majoriser, properties.MajoriserFunction):
maj_f = majoriser(self.function, xold)
else:
majoriser.at_point(xold)
maj_f = majoriser
x = self.algorithm.run(maj_f, xold)
if self.info_requested(Info.time):
_t.append(utils.time() - tm)
if self.info_requested(Info.func_val) \
or self.info_requested(Info.smooth_func_val):
if isinstance(majoriser, properties.MajoriserFunction):
maj_f = majoriser(self.function, xold)
else:
majoriser.at_point(xold)
maj_f = majoriser
if self.info_requested(Info.func_val):
_f.append(maj_f.f(x))
if self.info_requested(Info.smooth_func_val):
if hasattr(maj_f, "fmu"):
_fmu.append(maj_f.fmu(x))
if self.info_requested(Info.other):
nfo = self.algorithm.info_get()
for key in nfo.keys():
if key not in _other:
_other[key] = list()
value = nfo[key]
_other[key].append(value)
if self.callback is not None:
self.callback(locals())
if isinstance(x, list):
val = utils.list_op((x, xold),
lambda new, old: np.linalg.norm(new - old),
aggregator=np.max)
else:
val = np.linalg.norm(x - xold)
if it >= self.min_iter and val < self.eps:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
break
self.num_iter = it
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.time):
self.info_set(Info.time, _t)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, _f)
if self.info_requested(Info.smooth_func_val):
self.info_set(Info.smooth_func_val, _fmu)
if self.info_requested(Info.other):
self.info_set(Info.other, _other)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return x
def run(self, function, beta):
# Copy the allowed info keys for FISTA.
fista_info = list()
for nfo in self.info_copy():
if nfo in FISTA.INFO_PROVIDED:
fista_info.append(nfo)
# Create the inner algorithm.
algorithm = FISTA(info=fista_info, eps=self.eps,
max_iter=self.max_iter, min_iter=self.min_iter)
# Not ok until the end.
if self.info_requested(Info.ok):
self.info_set(Info.ok, False)
# Time the init computation.
if self.info_requested(Info.time):
init_time = utils.time()
# Estimate the initial precision, eps, and the smoothing parameter mu.
gM = function.eps_max(1.0) # gamma * M
if maths.norm(beta) > consts.TOLERANCE:
mu = function.estimate_mu(beta)
eps = mu * gM
else:
eps = self._approximate_eps(function, beta)
mu = eps / gM
function.set_mu(mu)
# Initialise info variables. Info variables have the suffix "_".
if self.info_requested(Info.time):
t_ = []
init_time = utils.time() - init_time
if self.info_requested(Info.fvalue) \
or self.info_requested(Info.func_val):
f_ = []
if self.info_requested(Info.converged):
self.info_set(Info.converged, False)
if self.info_requested(Info.mu):
mu_ = []
i = 0 # Iteration counter.
while True:
converged = False
# Give current parameters to the algorithm.
algorithm.set_params(eps=eps,
max_iter=self.max_iter - self.num_iter)
# Run FISTA.
beta_new = algorithm.run(function, beta)
# Update global iteration count.
self.num_iter += algorithm.num_iter
# Get info from algorithm.
if Info.time in algorithm.info and \
self.info_requested(Info.time):
t_ += algorithm.info_get(Info.time)
if i == 0: # Add init time to first iteration.
t_[0] += init_time
if Info.func_val in algorithm.info \
and self.info_requested(Info.func_val):
f_ += algorithm.info_get(Info.func_val)
elif Info.fvalue in algorithm.info \
and self.info_requested(Info.fvalue):
f_ += algorithm.info_get(Info.fvalue)
if self.info_requested(Info.mu):
mu_ += [mu] * algorithm.num_iter
# Unless this is a simulation, you want the algorithm to stop when
# it has converged.
if not self.simulation:
# Stopping criterion.
step = function.step(beta_new)
if maths.norm(beta_new - beta) < step * self.eps:
if self.info_requested(Info.converged):
self.info_set(Info.converged, True)
converged = True
beta = beta_new
if self.callback is not None:
self.callback(locals())
if self.info_requested(Info.verbose):
print("StaticCONESTA ite: %i, eps: %g, mu: %g" % (i, eps, mu))
# All combined stopping criteria.
if (converged or self.num_iter >= self.max_iter) \
and self.num_iter >= self.min_iter:
break
# Update the precision eps.
eps = self.tau * eps
# Compute and update mu.
mu = max(self.mu_min, eps / gM)
function.set_mu(mu)
i = i + 1
if self.info_requested(Info.num_iter):
self.info_set(Info.num_iter, self.num_iter)
if self.info_requested(Info.continuations):
self.info_set(Info.continuations, i + 1)
if self.info_requested(Info.time):
self.info_set(Info.time, t_)
if self.info_requested(Info.func_val):
self.info_set(Info.func_val, f_)
if self.info_requested(Info.fvalue):
self.info_set(Info.fvalue, f_)
if self.info_requested(Info.mu):
self.info_set(Info.mu, mu_)
if self.info_requested(Info.ok):
self.info_set(Info.ok, True)
return beta
