def force_mi(D, X, Z, unused_data, eta, max_tries=100): # force mutual incoherence within a dictionary n_atoms = D.shape[1] G = np.abs(np.dot(D.T, D)) np.fill_diagonal(G, 0) for atom_idx1 in range(n_atoms): atom_idx2 = np.argmax(G[atom_idx1, :]) # the maximum coherence mcoh = G[atom_idx1, atom_idx2] if mcoh < eta: print "less than the eta" continue # choose one of the two to replace # should we choose the one least used? if norm(Z[atom_idx1, :]) > norm(Z[atom_idx2, :]): c_atom = atom_idx1 else: c_atom = atom_idx2 # new_atom = None cnt = 0 available_data = unused_data[:] min_idx = None min_coh = mcoh while mcoh > eta: # replace the coherent atom if cnt > max_tries: break # no datapoint available to be used as atom if len(available_data) == 0: return D _idx = np.random.choice(available_data, size=1) if len(_idx) == 0: return D, unused_data idx = _idx[0] new_atom = X[:, idx] new_atom = normalize(new_atom) available_data.remove(idx) g = np.abs(np.dot(D.T, new_atom)) mcoh = np.max(g) if mcoh < min_coh or min_coh is None: min_coh = mcoh min_idx = idx cnt += 1 D[:, c_atom] = X[:, min_idx] D[:, c_atom] = normalize(D[:, c_atom]) unused_data.remove(min_idx) return D, unused_data
def force_mi(D, X, Z, unused_data, eta, max_tries=100): # force mutual incoherence within a dictionary n_atoms = D.shape[1] G = np.abs(np.dot(D.T, D)) np.fill_diagonal(G, 0) for atom_idx1 in range(n_atoms): atom_idx2 = np.argmax(G[atom_idx1, :]) # the maximum coherence mcoh = G[atom_idx1, atom_idx2] if mcoh < eta: print "less than the eta" continue # choose one of the two to replace # should we choose the one least used? if norm(Z[atom_idx1, :]) > norm(Z[atom_idx2, :]): c_atom = atom_idx1 else: c_atom = atom_idx2 # new_atom = None cnt = 0 available_data = unused_data[:] min_idx = None min_coh = mcoh while mcoh > eta: # replace the coherent atom if cnt > max_tries: break # no datapoint available to be used as atom if len(available_data) == 0: return D _idx = np.random.choice(available_data, size=1) if len(_idx) == 0: return D, unused_data idx = _idx[0] new_atom = X[:, idx] new_atom = normalize(new_atom) available_data.remove(idx) g = np.abs(np.dot(D.T, new_atom)) mcoh = np.max(g) if mcoh < min_coh or min_coh is None: min_coh = mcoh min_idx = idx cnt += 1 D[:, c_atom] = X[:, min_idx] D[:, c_atom] = normalize(D[:, c_atom]) unused_data.remove(min_idx) return D, unused_data
def _group_omp(x, D, Gram, alpha, groups=None, n_groups=None, tol=None): # TODO: also use a tolerance parameter _, n_atoms = D.shape # the dict indexes of the groups # this datapoint uses Gx = np.array([]).astype(int) z = np.zeros(n_atoms) # the residual r = np.copy(x) i = 0 if n_groups is not None: tol = 1e-10 def cont_criterion(): not_reached_sparsity = i < n_groups return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) > tol while (cont_criterion()): # find the group of atoms that correlates # the most with the residual if i == 0: group_scores = [norm(alpha[group]) for group in groups] else: group_scores = [norm(np.dot(D[:, group].T, r)) for group in groups] g = np.argmax(group_scores) if g in Gx or norm(r) < 1e-10: # group already selected break Gx = np.append(Gx, g) # solve the Least Squares problem # to find the coefs z idx = np.array([k for g_idx in Gx for k in groups[g_idx]]) G = Gram[idx, :][:, idx] try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break z[idx] = np.dot(np.dot(G_inv, D[:, idx].T), x) approx = np.dot(D[:, idx], z[idx]) r = x - approx i += 1 return z
def _group_omp(x, D, Gram, alpha, groups=None, n_groups=None, tol=None): # TODO: also use a tolerance parameter _, n_atoms = D.shape # the dict indexes of the groups # this datapoint uses Gx = np.array([]).astype(int) z = np.zeros(n_atoms) # the residual r = np.copy(x) i = 0 if n_groups is not None: tol = 1e-10 def cont_criterion(): not_reached_sparsity = i < n_groups return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) > tol while (cont_criterion()): # find the group of atoms that correlates # the most with the residual if i == 0: group_scores = [norm(alpha[group]) for group in groups] else: group_scores = [norm(np.dot(D[:, group].T, r)) for group in groups] g = np.argmax(group_scores) if g in Gx or norm(r) < 1e-10: # group already selected break Gx = np.append(Gx, g) # solve the Least Squares problem # to find the coefs z idx = np.array([k for g_idx in Gx for k in groups[g_idx]]) G = Gram[idx, :][:, idx] try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break z[idx] = np.dot(np.dot(G_inv, D[:, idx].T), x) approx = np.dot(D[:, idx], z[idx]) r = x - approx i += 1 return z
def nn_ksvd(Y, D, X, n_cycles=1, verbose=True): # the non-negative variant n_atoms = D.shape[1] n_features, n_samples = Y.shape unused_atoms = [] R = Y - fast_dot(D, X) for k in range(n_atoms): if verbose: sys.stdout.write("\r" + "k-svd..." + ":%3.2f%%" % ((k / float(n_atoms)) * 100)) sys.stdout.flush() # find all the datapoints that use the kth atom omega_k = X[k, :] != 0 if not np.any(omega_k): unused_atoms.append(k) continue # the residual due to all the other atoms but k Rk = R[:, omega_k] + np.outer(D[:, k], X[k, omega_k]) try: U, S, V = randomized_svd(Rk, n_components=1, n_iter=50, flip_sign=False) except: warnings.warn('SVD error') continue d = U[:, 0] x = V[0, :] * S[0] # projection to the constraint set d[d < 0] = 0 x[x < 0] = 0 dTd = np.dot(d, d) xTx = np.dot(x, x) if dTd <= np.finfo('float').eps or xTx <= np.finfo('float').eps: continue for j in range(n_cycles): d = np.dot(Rk, x) / np.dot(x, x) d[d < 0] = 0 x = np.dot(d.T, Rk) / np.dot(d, d) x[x < 0] = 0 _norm = norm(d) d = d / _norm x = x * _norm D[:, k] = d X[k, omega_k] = x # update the residual R[:, omega_k] = Rk - np.outer(D[:, k], X[k, omega_k]) print "" return D, X, unused_atoms
def nn_omp(X, D, n_nonzero_coefs=None, tol=None): """ The Non Negative OMP algorithm of 'On the Uniqueness of Nonnegative Sparse Solutions to Underdetermined Systems of Equations'""" n_samples = X.shape[1] n_atoms = D.shape[1] Z = np.zeros((n_atoms, n_samples)) _norm = np.sum(D**2, axis=0) for i in range(n_samples): x = X[:, i] r = x z = np.zeros(n_atoms) Dx = np.array([]).astype(int) j = 0 if n_nonzero_coefs is not None: tol = 1e-20 def cont_criterion(): not_reached_sparsity = j < n_nonzero_coefs return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) > tol while (cont_criterion()): a = np.dot(D.T, r) a[a < 0] = 0 e = (norm(r)**2) - (a**2) / _norm k = np.argmin(e) Dx = np.append(Dx, k) z_est = nnls(D[:, Dx], x)[0] r = x - np.dot(D[:, Dx], z_est) j += 1 if j != 0: z[Dx] = z_est Z[:, i] = z return Z
def nn_omp(X, D, n_nonzero_coefs=None, tol=None): """ The Non Negative OMP algorithm of 'On the Uniqueness of Nonnegative Sparse Solutions to Underdetermined Systems of Equations'""" n_samples = X.shape[1] n_atoms = D.shape[1] Z = np.zeros((n_atoms, n_samples)) _norm = np.sum(D ** 2, axis=0) for i in range(n_samples): x = X[:, i] r = x z = np.zeros(n_atoms) Dx = np.array([]).astype(int) j = 0 if n_nonzero_coefs is not None: tol = 1e-20 def cont_criterion(): not_reached_sparsity = j < n_nonzero_coefs return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) > tol while (cont_criterion()): a = np.dot(D.T, r) a[a < 0] = 0 e = (norm(r) ** 2) - (a ** 2) / _norm k = np.argmin(e) Dx = np.append(Dx, k) z_est = nnls(D[:, Dx], x)[0] r = x - np.dot(D[:, Dx], z_est) j += 1 if j != 0: z[Dx] = z_est Z[:, i] = z return Z
def nn_ksvd(Y, D, X, n_cycles=1, verbose=True): # the non-negative variant n_atoms = D.shape[1] n_features, n_samples = Y.shape unused_atoms = [] R = Y - fast_dot(D, X) for k in range(n_atoms): if verbose: sys.stdout.write("\r" + "k-svd..." + ":%3.2f%%" % ((k / float(n_atoms)) * 100)) sys.stdout.flush() # find all the datapoints that use the kth atom omega_k = X[k, :] != 0 if not np.any(omega_k): unused_atoms.append(k) continue # the residual due to all the other atoms but k Rk = R[:, omega_k] + np.outer(D[:, k], X[k, omega_k]) try: U, S, V = randomized_svd(Rk, n_components=1, n_iter=50, flip_sign=False) except: warnings.warn('SVD error') continue d = U[:, 0] x = V[0, :] * S[0] # projection to the constraint set d[d < 0] = 0 x[x < 0] = 0 dTd = np.dot(d, d) xTx = np.dot(x, x) if dTd <= np.finfo('float').eps or xTx <= np.finfo('float').eps: continue for j in range(n_cycles): d = np.dot(Rk, x) / np.dot(x, x) d[d < 0] = 0 x = np.dot(d.T, Rk) / np.dot(d, d) x[x < 0] = 0 _norm = norm(d) d = d / _norm x = x * _norm D[:, k] = d X[k, omega_k] = x # update the residual R[:, omega_k] = Rk - np.outer(D[:, k], X[k, omega_k]) print "" return D, X, unused_atoms
def _omp(x, D, Gram, alpha, n_nonzero_coefs=None, tol=None): _, n_atoms = D.shape # the dict indexes of the atoms this datapoint uses Dx = np.array([]).astype(int) z = np.zeros(n_atoms) # the residual r = np.copy(x) i = 0 if n_nonzero_coefs is not None: tol = 1e-10 def cont_criterion(): not_reached_sparsity = i < n_nonzero_coefs return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) >= tol while (cont_criterion()): # find the atom that correlates the # most with the residual k = np.argmax(np.abs(alpha)) if k in Dx: break Dx = np.append(Dx, k) # solve the Least Squares problem # to find the coefs z DI = D[:, Dx] G = Gram[Dx, :][:, Dx] G = np.atleast_2d(G) try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break z[Dx] = np.dot(G_inv, np.dot(D.T, x)[Dx]) r = x - np.dot(D[:, Dx], z[Dx]) alpha = np.dot(D.T, r) i += 1 return z
def _omp(x, D, Gram, alpha, n_nonzero_coefs=None, tol=None): _, n_atoms = D.shape # the dict indexes of the atoms this datapoint uses Dx = np.array([]).astype(int) z = np.zeros(n_atoms) # the residual r = np.copy(x) i = 0 if n_nonzero_coefs is not None: tol = 1e-10 def cont_criterion(): not_reached_sparsity = i < n_nonzero_coefs return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) >= tol while (cont_criterion()): # find the atom that correlates the # most with the residual k = np.argmax(np.abs(alpha)) if k in Dx: break Dx = np.append(Dx, k) # solve the Least Squares problem to find the coefs z G = Gram[Dx, :][:, Dx] G = np.atleast_2d(G) try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break z[Dx] = np.dot(G_inv, np.dot(D.T, x)[Dx]) r = x - np.dot(D[:, Dx], z[Dx]) alpha = np.dot(D.T, r) i += 1 return z
def llc(X, D, knn=5): # the sparse coder introduced in # "Locality-constrained Linear Coding for Image Classification" n_samples = X.shape[1] n_atoms = D.shape[1] # has the distance of # each sample to each atom dist = np.zeros((n_samples, n_atoms)) # calculate the distances for i in range(n_samples): for j in range(n_atoms): dist[i, j] = norm(X[:, i] - D[:, j]) # has the indices of the atoms # that are nearest neighbour to each sample knn_idx = np.zeros((n_samples, knn)).astype(int) for i in xrange(n_samples): knn_idx[i, :] = np.argsort(dist[i, :])[:knn] # the sparse coding matrix Z = np.zeros((n_atoms, n_samples)) II = np.eye(knn) beta = 1e-4 b = np.ones(knn) for i in xrange(n_samples): idx = knn_idx[i, :] z = D.T[idx, :] - repmat(X.T[i, :], knn, 1) C = np.dot(z, z.T) C = C + II * beta * np.trace(C) # solve the linear system C*c=b c = solve(C, b) # enforce the constraint on the sparse codes # such that sum(c)=1 c = c / float(np.sum(c)) Z[idx, i] = c return Z
def _somp(X_g, D, Gram, n_nonzero_coefs=None): n_atoms = D.shape[1] n_group_samples = X_g.shape[1] Z = np.zeros((n_atoms, n_group_samples)) Dx = np.array([]) Dx = Dx.astype(int) R = X_g if n_nonzero_coefs is not None: tol = 1e-20 def cont_criterion(): not_reached_sparsity = i < n_nonzero_coefs return (not_reached_sparsity and frobenius_squared(R) > tol) else: cont_criterion = lambda: frobenius_squared(R) > tol i = 0 while (cont_criterion()): A = fast_dot(D.T, R) j = np.argmax([norm(A[k, :]) for k in range(n_atoms)]) Dx = np.append(Dx, j) G = Gram[Dx, :][:, Dx] G = np.atleast_2d(G) try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break Z[Dx, :] = fast_dot(fast_dot(inv(G_inv), D[:, Dx].T), X_g) R = X_g - fast_dot(D, Z) i += 1 return Z
def _somp(X_g, D, Gram, n_nonzero_coefs=None): n_atoms = D.shape[1] n_group_samples = X_g.shape[1] Z = np.zeros((n_atoms, n_group_samples)) Dx = np.array([]) Dx = Dx.astype(int) R = X_g if n_nonzero_coefs is not None: tol = 1e-20 def cont_criterion(): not_reached_sparsity = i < n_nonzero_coefs return (not_reached_sparsity and frobenius_squared(R) > tol) else: cont_criterion = lambda: frobenius_squared(R) > tol i = 0 while (cont_criterion()): A = fast_dot(D.T, R) j = np.argmax([norm(A[k, :]) for k in range(n_atoms)]) Dx = np.append(Dx, j) G = Gram[Dx, :][:, Dx] G = np.atleast_2d(G) try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break Z[Dx, :] = fast_dot(fast_dot(inv(G_inv), D[:, Dx].T), X_g) R = X_g - fast_dot(D, Z) i += 1 return Z
def llc(X, D, knn=5): # the sparse coder introduced in # "Locality-constrained Linear Coding for Image Classification" n_samples = X.shape[1] n_atoms = D.shape[1] # has the distance of # each sample to each atom dist = np.zeros((n_samples, n_atoms)) # calculate the distances for i in range(n_samples): for j in range(n_atoms): dist[i, j] = norm(X[:, i] - D[:, j]) # has the indices of the atoms # that are nearest neighbour to each sample knn_idx = np.zeros((n_samples, knn)).astype(int) for i in xrange(n_samples): knn_idx[i, :] = np.argsort(dist[i, :])[:knn] # the sparse coding matrix Z = np.zeros((n_atoms, n_samples)) II = np.eye(knn) beta = 1e-4 b = np.ones(knn) for i in xrange(n_samples): idx = knn_idx[i, :] z = D.T[idx, :] - repmat(X.T[i, :], knn, 1) C = np.dot(z, z.T) C = C + II * beta * np.trace(C) # solve the linear system C*c=b c = solve(C, b) # enforce the constraint on the sparse codes # such that sum(c)=1 c = c / float(np.sum(c)) Z[idx, i] = c return Z
def cont_criterion(): not_reached_sparsity = j < n_nonzero_coefs return (not_reached_sparsity and norm(r) > tol)
def cont_criterion(): not_reached_sparsity = i < n_groups return (not_reached_sparsity and norm(r) > tol)
def _sparse_group_omp(x, D, Gram, alpha, groups=None, n_groups=None, n_nonzero_coefs=None): _, n_atoms = D.shape # the dict indexes of the groups # this datapoint uses Gx = np.array([]) Gx = Gx.astype(int) z = np.zeros(n_atoms) # the residual r = np.copy(x) i = 0 if n_groups is not None: tol = 1e-10 def cont_criterion(): not_reached_sparsity = i < n_groups return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) > tol # first select the groups for i in range(n_groups): # find the group of atoms that correlates the # most with the residual if i == 0: group_scores = [norm(alpha[group]) for group in groups] else: group_scores = [norm(np.dot(D[:, group].T, r)) for group in groups] g = np.argmax(group_scores) if g in Gx or norm(r) < 1e-10: # group already selected break Gx = np.append(Gx, g) # solve the Least Squares problem # to find the coefs z idx = np.array([k for g_idx in Gx for k in groups[g_idx]]) G = Gram[idx, :][:, idx] try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break z[idx] = np.dot(np.dot(G_inv, D[:, idx].T), x) approx = np.dot(D[:, idx], z[idx]) r = x - approx i += 1 # apply OMP using only the atoms of the groups selected Dx = np.array([]) Dx = Dx.astype(int) # the atom indices selected from the previous step idx = np.array([k for g_idx in Gx for k in groups[g_idx]]) Dsel = D[:, idx] Gram = fast_dot(Dsel.T, Dsel) z = np.zeros(len(idx)) z_final = np.zeros(n_atoms) # the residual r = np.copy(x) i = 0 if n_nonzero_coefs is not None: tol = 1e-20 def cont_criterion(): not_reached_sparsity = i < n_nonzero_coefs return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) > tol while (cont_criterion()): # find the atom that correlates the # most with the residual k = np.argmax(np.abs(np.dot(Dsel.T, r))) if k in Dx: break Dx = np.append(Dx, k) # solve the Least Squares problem # to find the coefs z DI = Dsel[:, Dx] G = Gram[Dx, :][:, Dx] G = np.atleast_2d(G) try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break z[Dx] = np.dot(G_inv, np.dot(Dsel.T, x)[Dx]) z_final[idx[Dx]] = z[Dx] r = x - np.dot(Dsel[:, Dx], z[Dx]) i += 1 return z_final
def _sparse_group_omp(x, D, Gram, alpha, groups=None, n_groups=None, n_nonzero_coefs=None): _, n_atoms = D.shape # the dict indexes of the groups # this datapoint uses Gx = np.array([]) Gx = Gx.astype(int) z = np.zeros(n_atoms) # the residual r = np.copy(x) i = 0 if n_groups is not None: tol = 1e-10 def cont_criterion(): not_reached_sparsity = i < n_groups return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) > tol # first select the groups for i in range(n_groups): # find the group of atoms that correlates the # most with the residual if i == 0: group_scores = [norm(alpha[group]) for group in groups] else: group_scores = [norm(np.dot(D[:, group].T, r)) for group in groups] g = np.argmax(group_scores) if g in Gx or norm(r) < 1e-10: # group already selected break Gx = np.append(Gx, g) # solve the Least Squares problem # to find the coefs z idx = np.array([k for g_idx in Gx for k in groups[g_idx]]) G = Gram[idx, :][:, idx] try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break z[idx] = np.dot(np.dot(G_inv, D[:, idx].T), x) approx = np.dot(D[:, idx], z[idx]) r = x - approx i += 1 # apply OMP using only the atoms of the groups selected Dx = np.array([]) Dx = Dx.astype(int) # the atom indices selected from the previous step idx = np.array([k for g_idx in Gx for k in groups[g_idx]]) Dsel = D[:, idx] Gram = fast_dot(Dsel.T, Dsel) z = np.zeros(len(idx)) z_final = np.zeros(n_atoms) # the residual r = np.copy(x) i = 0 if n_nonzero_coefs is not None: tol = 1e-20 def cont_criterion(): not_reached_sparsity = i < n_nonzero_coefs return (not_reached_sparsity and norm(r) > tol) else: cont_criterion = lambda: norm(r) > tol while (cont_criterion()): # find the atom that correlates the # most with the residual k = np.argmax(np.abs(np.dot(Dsel.T, r))) if k in Dx: break Dx = np.append(Dx, k) # solve the Least Squares problem # to find the coefs z DI = Dsel[:, Dx] G = Gram[Dx, :][:, Dx] G = np.atleast_2d(G) try: G_inv = inv(G) except LinAlgError: print gram_singular_msg break z[Dx] = np.dot(G_inv, np.dot(Dsel.T, x)[Dx]) z_final[idx[Dx]] = z[Dx] r = x - np.dot(Dsel[:, Dx], z[Dx]) i += 1 return z_final
def cont_criterion(): not_reached_sparsity = i < n_groups return (not_reached_sparsity and norm(r) > tol)
def cont_criterion(): not_reached_sparsity = j < n_nonzero_coefs return (not_reached_sparsity and norm(r) > tol)