def force_mi(D, X, Z, unused_data, eta, max_tries=100):
# force mutual incoherence within a dictionary
n_atoms = D.shape[1]
G = np.abs(np.dot(D.T, D))
np.fill_diagonal(G, 0)
for atom_idx1 in range(n_atoms):
atom_idx2 = np.argmax(G[atom_idx1, :])
# the maximum coherence
mcoh = G[atom_idx1, atom_idx2]
if mcoh < eta:
print "less than the eta"
continue
# choose one of the two to replace
# should we choose the one least used?
if norm(Z[atom_idx1, :]) > norm(Z[atom_idx2, :]):
c_atom = atom_idx1
else:
c_atom = atom_idx2
# new_atom = None
cnt = 0
available_data = unused_data[:]
min_idx = None
min_coh = mcoh
while mcoh > eta:
# replace the coherent atom
if cnt > max_tries:
break
# no datapoint available to be used as atom
if len(available_data) == 0:
return D
_idx = np.random.choice(available_data, size=1)
if len(_idx) == 0:
return D, unused_data
idx = _idx[0]
new_atom = X[:, idx]
new_atom = normalize(new_atom)
available_data.remove(idx)
g = np.abs(np.dot(D.T, new_atom))
mcoh = np.max(g)
if mcoh < min_coh or min_coh is None:
min_coh = mcoh
min_idx = idx
cnt += 1
D[:, c_atom] = X[:, min_idx]
D[:, c_atom] = normalize(D[:, c_atom])
unused_data.remove(min_idx)
return D, unused_data
def force_mi(D, X, Z, unused_data, eta, max_tries=100):
# force mutual incoherence within a dictionary
n_atoms = D.shape[1]
G = np.abs(np.dot(D.T, D))
np.fill_diagonal(G, 0)
for atom_idx1 in range(n_atoms):
atom_idx2 = np.argmax(G[atom_idx1, :])
# the maximum coherence
mcoh = G[atom_idx1, atom_idx2]
if mcoh < eta:
print "less than the eta"
continue
# choose one of the two to replace
# should we choose the one least used?
if norm(Z[atom_idx1, :]) > norm(Z[atom_idx2, :]):
c_atom = atom_idx1
else:
c_atom = atom_idx2
# new_atom = None
cnt = 0
available_data = unused_data[:]
min_idx = None
min_coh = mcoh
while mcoh > eta:
# replace the coherent atom
if cnt > max_tries:
break
# no datapoint available to be used as atom
if len(available_data) == 0:
return D
_idx = np.random.choice(available_data, size=1)
if len(_idx) == 0:
return D, unused_data
idx = _idx[0]
new_atom = X[:, idx]
new_atom = normalize(new_atom)
available_data.remove(idx)
g = np.abs(np.dot(D.T, new_atom))
mcoh = np.max(g)
if mcoh < min_coh or min_coh is None:
min_coh = mcoh
min_idx = idx
cnt += 1
D[:, c_atom] = X[:, min_idx]
D[:, c_atom] = normalize(D[:, c_atom])
unused_data.remove(min_idx)
return D, unused_data
def _group_omp(x, D, Gram, alpha, groups=None, n_groups=None, tol=None):
# TODO: also use a tolerance parameter
_, n_atoms = D.shape
# the dict indexes of the groups
# this datapoint uses
Gx = np.array([]).astype(int)
z = np.zeros(n_atoms)
# the residual
r = np.copy(x)
i = 0
if n_groups is not None:
tol = 1e-10
def cont_criterion():
not_reached_sparsity = i < n_groups
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) > tol
while (cont_criterion()):
# find the group of atoms that correlates
# the most with the residual
if i == 0:
group_scores = [norm(alpha[group]) for group in groups]
else:
group_scores = [norm(np.dot(D[:, group].T, r)) for group in groups]
g = np.argmax(group_scores)
if g in Gx or norm(r) < 1e-10:
# group already selected
break
Gx = np.append(Gx, g)
# solve the Least Squares problem
# to find the coefs z
idx = np.array([k for g_idx in Gx for k in groups[g_idx]])
G = Gram[idx, :][:, idx]
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
z[idx] = np.dot(np.dot(G_inv, D[:, idx].T), x)
approx = np.dot(D[:, idx], z[idx])
r = x - approx
i += 1
return z
def _group_omp(x, D, Gram, alpha, groups=None, n_groups=None, tol=None):
# TODO: also use a tolerance parameter
_, n_atoms = D.shape
# the dict indexes of the groups
# this datapoint uses
Gx = np.array([]).astype(int)
z = np.zeros(n_atoms)
# the residual
r = np.copy(x)
i = 0
if n_groups is not None:
tol = 1e-10
def cont_criterion():
not_reached_sparsity = i < n_groups
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) > tol
while (cont_criterion()):
# find the group of atoms that correlates
# the most with the residual
if i == 0:
group_scores = [norm(alpha[group]) for group in groups]
else:
group_scores = [norm(np.dot(D[:, group].T, r)) for group in groups]
g = np.argmax(group_scores)
if g in Gx or norm(r) < 1e-10:
# group already selected
break
Gx = np.append(Gx, g)
# solve the Least Squares problem
# to find the coefs z
idx = np.array([k for g_idx in Gx for k in groups[g_idx]])
G = Gram[idx, :][:, idx]
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
z[idx] = np.dot(np.dot(G_inv, D[:, idx].T), x)
approx = np.dot(D[:, idx], z[idx])
r = x - approx
i += 1
return z
def nn_ksvd(Y, D, X, n_cycles=1, verbose=True):
# the non-negative variant
n_atoms = D.shape[1]
n_features, n_samples = Y.shape
unused_atoms = []
R = Y - fast_dot(D, X)
for k in range(n_atoms):
if verbose:
sys.stdout.write("\r" + "k-svd..." + ":%3.2f%%" %
((k / float(n_atoms)) * 100))
sys.stdout.flush()
# find all the datapoints that use the kth atom
omega_k = X[k, :] != 0
if not np.any(omega_k):
unused_atoms.append(k)
continue
# the residual due to all the other atoms but k
Rk = R[:, omega_k] + np.outer(D[:, k], X[k, omega_k])
try:
U, S, V = randomized_svd(Rk,
n_components=1,
n_iter=50,
flip_sign=False)
except:
warnings.warn('SVD error')
continue
d = U[:, 0]
x = V[0, :] * S[0]
# projection to the constraint set
d[d < 0] = 0
x[x < 0] = 0
dTd = np.dot(d, d)
xTx = np.dot(x, x)
if dTd <= np.finfo('float').eps or xTx <= np.finfo('float').eps:
continue
for j in range(n_cycles):
d = np.dot(Rk, x) / np.dot(x, x)
d[d < 0] = 0
x = np.dot(d.T, Rk) / np.dot(d, d)
x[x < 0] = 0
_norm = norm(d)
d = d / _norm
x = x * _norm
D[:, k] = d
X[k, omega_k] = x
# update the residual
R[:, omega_k] = Rk - np.outer(D[:, k], X[k, omega_k])
print ""
return D, X, unused_atoms
def nn_omp(X, D, n_nonzero_coefs=None, tol=None):
""" The Non Negative OMP algorithm of
'On the Uniqueness of Nonnegative Sparse Solutions to Underdetermined Systems of Equations'"""
n_samples = X.shape[1]
n_atoms = D.shape[1]
Z = np.zeros((n_atoms, n_samples))
_norm = np.sum(D**2, axis=0)
for i in range(n_samples):
x = X[:, i]
r = x
z = np.zeros(n_atoms)
Dx = np.array([]).astype(int)
j = 0
if n_nonzero_coefs is not None:
tol = 1e-20
def cont_criterion():
not_reached_sparsity = j < n_nonzero_coefs
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) > tol
while (cont_criterion()):
a = np.dot(D.T, r)
a[a < 0] = 0
e = (norm(r)**2) - (a**2) / _norm
k = np.argmin(e)
Dx = np.append(Dx, k)
z_est = nnls(D[:, Dx], x)[0]
r = x - np.dot(D[:, Dx], z_est)
j += 1
if j != 0:
z[Dx] = z_est
Z[:, i] = z
return Z
def nn_omp(X, D, n_nonzero_coefs=None, tol=None):
""" The Non Negative OMP algorithm of
'On the Uniqueness of Nonnegative Sparse Solutions to Underdetermined Systems of Equations'"""
n_samples = X.shape[1]
n_atoms = D.shape[1]
Z = np.zeros((n_atoms, n_samples))
_norm = np.sum(D ** 2, axis=0)
for i in range(n_samples):
x = X[:, i]
r = x
z = np.zeros(n_atoms)
Dx = np.array([]).astype(int)
j = 0
if n_nonzero_coefs is not None:
tol = 1e-20
def cont_criterion():
not_reached_sparsity = j < n_nonzero_coefs
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) > tol
while (cont_criterion()):
a = np.dot(D.T, r)
a[a < 0] = 0
e = (norm(r) ** 2) - (a ** 2) / _norm
k = np.argmin(e)
Dx = np.append(Dx, k)
z_est = nnls(D[:, Dx], x)[0]
r = x - np.dot(D[:, Dx], z_est)
j += 1
if j != 0:
z[Dx] = z_est
Z[:, i] = z
return Z
def nn_ksvd(Y, D, X, n_cycles=1, verbose=True):
# the non-negative variant
n_atoms = D.shape[1]
n_features, n_samples = Y.shape
unused_atoms = []
R = Y - fast_dot(D, X)
for k in range(n_atoms):
if verbose:
sys.stdout.write("\r" + "k-svd..." + ":%3.2f%%" % ((k / float(n_atoms)) * 100))
sys.stdout.flush()
# find all the datapoints that use the kth atom
omega_k = X[k, :] != 0
if not np.any(omega_k):
unused_atoms.append(k)
continue
# the residual due to all the other atoms but k
Rk = R[:, omega_k] + np.outer(D[:, k], X[k, omega_k])
try:
U, S, V = randomized_svd(Rk, n_components=1, n_iter=50, flip_sign=False)
except:
warnings.warn('SVD error')
continue
d = U[:, 0]
x = V[0, :] * S[0]
# projection to the constraint set
d[d < 0] = 0
x[x < 0] = 0
dTd = np.dot(d, d)
xTx = np.dot(x, x)
if dTd <= np.finfo('float').eps or xTx <= np.finfo('float').eps:
continue
for j in range(n_cycles):
d = np.dot(Rk, x) / np.dot(x, x)
d[d < 0] = 0
x = np.dot(d.T, Rk) / np.dot(d, d)
x[x < 0] = 0
_norm = norm(d)
d = d / _norm
x = x * _norm
D[:, k] = d
X[k, omega_k] = x
# update the residual
R[:, omega_k] = Rk - np.outer(D[:, k], X[k, omega_k])
print ""
return D, X, unused_atoms
def _omp(x, D, Gram, alpha, n_nonzero_coefs=None, tol=None):
_, n_atoms = D.shape
# the dict indexes of the atoms this datapoint uses
Dx = np.array([]).astype(int)
z = np.zeros(n_atoms)
# the residual
r = np.copy(x)
i = 0
if n_nonzero_coefs is not None:
tol = 1e-10
def cont_criterion():
not_reached_sparsity = i < n_nonzero_coefs
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) >= tol
while (cont_criterion()):
# find the atom that correlates the
# most with the residual
k = np.argmax(np.abs(alpha))
if k in Dx:
break
Dx = np.append(Dx, k)
# solve the Least Squares problem
# to find the coefs z
DI = D[:, Dx]
G = Gram[Dx, :][:, Dx]
G = np.atleast_2d(G)
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
z[Dx] = np.dot(G_inv, np.dot(D.T, x)[Dx])
r = x - np.dot(D[:, Dx], z[Dx])
alpha = np.dot(D.T, r)
i += 1
return z
def _omp(x, D, Gram, alpha, n_nonzero_coefs=None, tol=None):
_, n_atoms = D.shape
# the dict indexes of the atoms this datapoint uses
Dx = np.array([]).astype(int)
z = np.zeros(n_atoms)
# the residual
r = np.copy(x)
i = 0
if n_nonzero_coefs is not None:
tol = 1e-10
def cont_criterion():
not_reached_sparsity = i < n_nonzero_coefs
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) >= tol
while (cont_criterion()):
# find the atom that correlates the
# most with the residual
k = np.argmax(np.abs(alpha))
if k in Dx:
break
Dx = np.append(Dx, k)
# solve the Least Squares problem to find the coefs z
G = Gram[Dx, :][:, Dx]
G = np.atleast_2d(G)
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
z[Dx] = np.dot(G_inv, np.dot(D.T, x)[Dx])
r = x - np.dot(D[:, Dx], z[Dx])
alpha = np.dot(D.T, r)
i += 1
return z
def llc(X, D, knn=5):
# the sparse coder introduced in
# "Locality-constrained Linear Coding for Image Classification"
n_samples = X.shape[1]
n_atoms = D.shape[1]
# has the distance of
# each sample to each atom
dist = np.zeros((n_samples, n_atoms))
# calculate the distances
for i in range(n_samples):
for j in range(n_atoms):
dist[i, j] = norm(X[:, i] - D[:, j])
# has the indices of the atoms
# that are nearest neighbour to each sample
knn_idx = np.zeros((n_samples, knn)).astype(int)
for i in xrange(n_samples):
knn_idx[i, :] = np.argsort(dist[i, :])[:knn]
# the sparse coding matrix
Z = np.zeros((n_atoms, n_samples))
II = np.eye(knn)
beta = 1e-4
b = np.ones(knn)
for i in xrange(n_samples):
idx = knn_idx[i, :]
z = D.T[idx, :] - repmat(X.T[i, :], knn, 1)
C = np.dot(z, z.T)
C = C + II * beta * np.trace(C)
# solve the linear system C*c=b
c = solve(C, b)
# enforce the constraint on the sparse codes
# such that sum(c)=1
c = c / float(np.sum(c))
Z[idx, i] = c
return Z
def _somp(X_g, D, Gram, n_nonzero_coefs=None):
n_atoms = D.shape[1]
n_group_samples = X_g.shape[1]
Z = np.zeros((n_atoms, n_group_samples))
Dx = np.array([])
Dx = Dx.astype(int)
R = X_g
if n_nonzero_coefs is not None:
tol = 1e-20
def cont_criterion():
not_reached_sparsity = i < n_nonzero_coefs
return (not_reached_sparsity and frobenius_squared(R) > tol)
else:
cont_criterion = lambda: frobenius_squared(R) > tol
i = 0
while (cont_criterion()):
A = fast_dot(D.T, R)
j = np.argmax([norm(A[k, :]) for k in range(n_atoms)])
Dx = np.append(Dx, j)
G = Gram[Dx, :][:, Dx]
G = np.atleast_2d(G)
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
Z[Dx, :] = fast_dot(fast_dot(inv(G_inv), D[:, Dx].T), X_g)
R = X_g - fast_dot(D, Z)
i += 1
return Z
def _somp(X_g, D, Gram, n_nonzero_coefs=None):
n_atoms = D.shape[1]
n_group_samples = X_g.shape[1]
Z = np.zeros((n_atoms, n_group_samples))
Dx = np.array([])
Dx = Dx.astype(int)
R = X_g
if n_nonzero_coefs is not None:
tol = 1e-20
def cont_criterion():
not_reached_sparsity = i < n_nonzero_coefs
return (not_reached_sparsity and frobenius_squared(R) > tol)
else:
cont_criterion = lambda: frobenius_squared(R) > tol
i = 0
while (cont_criterion()):
A = fast_dot(D.T, R)
j = np.argmax([norm(A[k, :]) for k in range(n_atoms)])
Dx = np.append(Dx, j)
G = Gram[Dx, :][:, Dx]
G = np.atleast_2d(G)
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
Z[Dx, :] = fast_dot(fast_dot(inv(G_inv), D[:, Dx].T), X_g)
R = X_g - fast_dot(D, Z)
i += 1
return Z
def llc(X, D, knn=5):
# the sparse coder introduced in
# "Locality-constrained Linear Coding for Image Classification"
n_samples = X.shape[1]
n_atoms = D.shape[1]
# has the distance of
# each sample to each atom
dist = np.zeros((n_samples, n_atoms))
# calculate the distances
for i in range(n_samples):
for j in range(n_atoms):
dist[i, j] = norm(X[:, i] - D[:, j])
# has the indices of the atoms
# that are nearest neighbour to each sample
knn_idx = np.zeros((n_samples, knn)).astype(int)
for i in xrange(n_samples):
knn_idx[i, :] = np.argsort(dist[i, :])[:knn]
# the sparse coding matrix
Z = np.zeros((n_atoms, n_samples))
II = np.eye(knn)
beta = 1e-4
b = np.ones(knn)
for i in xrange(n_samples):
idx = knn_idx[i, :]
z = D.T[idx, :] - repmat(X.T[i, :], knn, 1)
C = np.dot(z, z.T)
C = C + II * beta * np.trace(C)
# solve the linear system C*c=b
c = solve(C, b)
# enforce the constraint on the sparse codes
# such that sum(c)=1
c = c / float(np.sum(c))
Z[idx, i] = c
return Z
def cont_criterion():
not_reached_sparsity = j < n_nonzero_coefs
return (not_reached_sparsity and norm(r) > tol)
def cont_criterion():
not_reached_sparsity = i < n_groups
return (not_reached_sparsity and norm(r) > tol)
def _sparse_group_omp(x,
D,
Gram,
alpha,
groups=None,
n_groups=None,
n_nonzero_coefs=None):
_, n_atoms = D.shape
# the dict indexes of the groups
# this datapoint uses
Gx = np.array([])
Gx = Gx.astype(int)
z = np.zeros(n_atoms)
# the residual
r = np.copy(x)
i = 0
if n_groups is not None:
tol = 1e-10
def cont_criterion():
not_reached_sparsity = i < n_groups
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) > tol
# first select the groups
for i in range(n_groups):
# find the group of atoms that correlates the
# most with the residual
if i == 0:
group_scores = [norm(alpha[group]) for group in groups]
else:
group_scores = [norm(np.dot(D[:, group].T, r)) for group in groups]
g = np.argmax(group_scores)
if g in Gx or norm(r) < 1e-10:
# group already selected
break
Gx = np.append(Gx, g)
# solve the Least Squares problem
# to find the coefs z
idx = np.array([k for g_idx in Gx for k in groups[g_idx]])
G = Gram[idx, :][:, idx]
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
z[idx] = np.dot(np.dot(G_inv, D[:, idx].T), x)
approx = np.dot(D[:, idx], z[idx])
r = x - approx
i += 1
# apply OMP using only the atoms of the groups selected
Dx = np.array([])
Dx = Dx.astype(int)
# the atom indices selected from the previous step
idx = np.array([k for g_idx in Gx for k in groups[g_idx]])
Dsel = D[:, idx]
Gram = fast_dot(Dsel.T, Dsel)
z = np.zeros(len(idx))
z_final = np.zeros(n_atoms)
# the residual
r = np.copy(x)
i = 0
if n_nonzero_coefs is not None:
tol = 1e-20
def cont_criterion():
not_reached_sparsity = i < n_nonzero_coefs
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) > tol
while (cont_criterion()):
# find the atom that correlates the
# most with the residual
k = np.argmax(np.abs(np.dot(Dsel.T, r)))
if k in Dx:
break
Dx = np.append(Dx, k)
# solve the Least Squares problem
# to find the coefs z
DI = Dsel[:, Dx]
G = Gram[Dx, :][:, Dx]
G = np.atleast_2d(G)
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
z[Dx] = np.dot(G_inv, np.dot(Dsel.T, x)[Dx])
z_final[idx[Dx]] = z[Dx]
r = x - np.dot(Dsel[:, Dx], z[Dx])
i += 1
return z_final
def _sparse_group_omp(x, D, Gram, alpha, groups=None, n_groups=None, n_nonzero_coefs=None):
_, n_atoms = D.shape
# the dict indexes of the groups
# this datapoint uses
Gx = np.array([])
Gx = Gx.astype(int)
z = np.zeros(n_atoms)
# the residual
r = np.copy(x)
i = 0
if n_groups is not None:
tol = 1e-10
def cont_criterion():
not_reached_sparsity = i < n_groups
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) > tol
# first select the groups
for i in range(n_groups):
# find the group of atoms that correlates the
# most with the residual
if i == 0:
group_scores = [norm(alpha[group]) for group in groups]
else:
group_scores = [norm(np.dot(D[:, group].T, r)) for group in groups]
g = np.argmax(group_scores)
if g in Gx or norm(r) < 1e-10:
# group already selected
break
Gx = np.append(Gx, g)
# solve the Least Squares problem
# to find the coefs z
idx = np.array([k for g_idx in Gx for k in groups[g_idx]])
G = Gram[idx, :][:, idx]
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
z[idx] = np.dot(np.dot(G_inv, D[:, idx].T), x)
approx = np.dot(D[:, idx], z[idx])
r = x - approx
i += 1
# apply OMP using only the atoms of the groups selected
Dx = np.array([])
Dx = Dx.astype(int)
# the atom indices selected from the previous step
idx = np.array([k for g_idx in Gx for k in groups[g_idx]])
Dsel = D[:, idx]
Gram = fast_dot(Dsel.T, Dsel)
z = np.zeros(len(idx))
z_final = np.zeros(n_atoms)
# the residual
r = np.copy(x)
i = 0
if n_nonzero_coefs is not None:
tol = 1e-20
def cont_criterion():
not_reached_sparsity = i < n_nonzero_coefs
return (not_reached_sparsity and norm(r) > tol)
else:
cont_criterion = lambda: norm(r) > tol
while (cont_criterion()):
# find the atom that correlates the
# most with the residual
k = np.argmax(np.abs(np.dot(Dsel.T, r)))
if k in Dx:
break
Dx = np.append(Dx, k)
# solve the Least Squares problem
# to find the coefs z
DI = Dsel[:, Dx]
G = Gram[Dx, :][:, Dx]
G = np.atleast_2d(G)
try:
G_inv = inv(G)
except LinAlgError:
print gram_singular_msg
break
z[Dx] = np.dot(G_inv, np.dot(Dsel.T, x)[Dx])
z_final[idx[Dx]] = z[Dx]
r = x - np.dot(Dsel[:, Dx], z[Dx])
i += 1
return z_final
def cont_criterion():
not_reached_sparsity = i < n_groups
return (not_reached_sparsity and norm(r) > tol)
def cont_criterion():
not_reached_sparsity = j < n_nonzero_coefs
return (not_reached_sparsity and norm(r) > tol)
