def _update_with_backtracking(self, d, c, gradient_d, gradient_c,
step_d, step_c):
x = self.X
d_0 = d
c_0 = c
error = 0.5 * np.linalg.norm(x - c_0.dot(d_0))**2
sigma = 0.9
for i in range(1, 1000):
# compute new matrices
d_1 = self.dict_penalty. \
apply_prox_operator(d_0 - step_d * gradient_d, gamma=step_d)
d_1 = non_negative_projection(d_1, self.non_negativity, 'dict')
difference_d = np.linalg.norm(d_1 - d_0)
c_1 = self.coeff_penalty. \
apply_prox_operator(c_0 - step_c * gradient_c, gamma=step_c)
c_1 = non_negative_projection(c_1, self.non_negativity, 'coeff')
difference_c = np.linalg.norm(c_1 - c_0)
first_part = self.objective_function_value(d=d_1, c=c_1)
second_part = (error +
self.dict_penalty.value(d_1) +
self.coeff_penalty.value(c_1) +
(difference_c*gradient_c).trace() +
(difference_d*gradient_d).trace() +
step_c*sigma**i * difference_c +
step_d*sigma**i * difference_d)
if first_part <= second_part:
break
return d_1, c_1
def _simple_update(self, d, c, gradient_d, gradient_c, step_d, step_c):
d = self.dict_penalty. \
apply_prox_operator(d - step_d * gradient_d, gamma=step_d)
d = non_negative_projection(d, self.non_negativity, 'dict')
c = self.coeff_penalty. \
apply_prox_operator(c - step_c * gradient_c, gamma=step_c)
c = non_negative_projection(c, self.non_negativity, 'coeff')
return d, c
def _proximal_gradient_minimization(self,
random_state,
backtracking=0,
n_iter=20000):
x = self.X
d = self.D
n, p = x.shape
k, _ = d.shape
c = non_negative_projection(
random_state.rand(n, k) * 10 - 5, self.non_negativity)
gamma = 1.1
step = _step_lipschitz(d, gamma)
iters = 20000
epsilon = 1e-4
objective = self.objective_function_value(d=d, c=c)
c_old = c
logging.debug("Starting optimization")
for i in range(iters):
gradient = (c.dot(d) - x).dot(d.T)
if backtracking:
c = self._update_with_backtracking(x, d, c, gradient, step)
else:
c = self._simple_update(c, gradient, step)
new_objective = self.objective_function_value(d=d, c=c)
difference_objective = new_objective - objective
objective = new_objective
difference_c = np.linalg.norm(c - c_old)
c_old = c
step = _step_lipschitz(d, gamma)
logging.debug("Iteration: %i" % (i))
logging.debug("Objective function: %f" % (objective))
logging.debug("Difference between new objective and "
"previous one: %f" % (difference_objective))
logging.debug("Difference between previous and new coefficients: "
"%f" % (difference_c))
logging.debug("\n")
assert ((not np.isnan(difference_objective))
and (not np.isinf(difference_objective))
and abs(difference_objective) < 1e+20)
if (abs(difference_objective) <= epsilon
and difference_c <= epsilon):
break
return c
def _update_with_backtracking(self, x, d, c, gradient, step):
c_0 = c
error = 0.5 * np.linalg.norm(x - c_0.dot(d))**2
sigma = 0.9
for i in range(1, 1000):
# compute new matrices
c_1 = self.penalty. \
apply_prox_operator(c_0 - step * gradient, gamma=step)
c_1 = non_negative_projection(c_1, self.non_negativity)
difference_c = np.linalg.norm(c_1 - c_0)
first_part = self.objective_function_value(d=d, c=c_1)
second_part = (error + self.coeff_penalty.value(c_1) +
(difference_c * gradient).trace() +
step * sigma**i * difference_c)
if first_part <= second_part:
break
return c_1
def _simple_update(self, c, gradient, step):
c = self.penalty. \
apply_prox_operator(c - step * gradient, gamma=step)
c = non_negative_projection(c, self.non_negativity)
return c
def _alternating_proximal_gradient_minimization(self, random_state,
n_iter=20000,
backtracking=0):
x = self.X
n, p = x.shape
d = non_negative_projection(random_state.rand(self.k, p)*10-5,
self.non_negativity, 'dict')
c = non_negative_projection(random_state.rand(n, self.k)*10-5,
self.non_negativity, 'coeff')
gamma_c = 1.1
gamma_d = gamma_c
step_d, step_c = _step_lipschitz(d, c,
gamma_d=gamma_d, gamma_c=gamma_c)
epsilon = 1e-4
objective = self.objective_function_value(d=d, c=c)
d_old = d
c_old = c
logging.debug("Starting optimization")
for i in range(n_iter):
gradient_d = c.T.dot(c.dot(d) - x)
gradient_c = (c.dot(d) - x).dot(d.T)
if backtracking:
d, c = self._update_with_backtracking(d, c, gradient_d,
gradient_c, step_d,
step_c)
else:
d, c = self._simple_update(d, c, gradient_d, gradient_c,
step_d, step_c)
new_objective = self.objective_function_value(d=d, c=c)
difference_objective = new_objective - objective
objective = new_objective
difference_d = np.linalg.norm(d - d_old)
difference_c = np.linalg.norm(c - c_old)
d_old = d
c_old = c
step_d, step_c = _step_lipschitz(d, c,
gamma_c=gamma_c, gamma_d=gamma_d)
logging.debug("Iteration: "+str(i))
logging.debug("Objective function: "+str(objective))
logging.debug("Difference between new objective and "
"previous one: "+str(difference_objective))
logging.debug("Difference between previous and new dictionary: " +
str(difference_d))
logging.debug("Difference between previous and new coefficients:" +
str(difference_c)+"`\n\n")
assert ((not np.isnan(difference_objective)) and
(not np.isinf(difference_objective)) and
(abs(difference_objective) < 1e+20))
if (abs(difference_objective) <= epsilon and
difference_d <= epsilon and
difference_c <= epsilon):
break
return d, c