def test_ztz(valid_shape, atom_shape): tol = .5 reg = .1 n_atoms = 7 n_channels = 5 random_state = None sig_shape = tuple([ (size_valid_ax + size_atom_ax - 1) for size_atom_ax, size_valid_ax in zip(atom_shape, valid_shape)]) rng = check_random_state(random_state) X = rng.randn(n_channels, *sig_shape) D = rng.randn(n_atoms, n_channels, *atom_shape) D /= np.sqrt(np.sum(D * D, axis=(1, 2), keepdims=True)) z_hat, ztz, ztX, *_ = dicod(X, D, reg, tol=tol, n_jobs=N_WORKERS, return_ztz=True, verbose=VERBOSE) ztz_full = compute_ztz(z_hat, atom_shape) assert np.allclose(ztz_full, ztz) ztX_full = compute_ztX(z_hat, X) assert np.allclose(ztX_full, ztX)
def test_distributed_sparse_encoder(): rng = check_random_state(42) n_atoms = 10 n_channels = 3 n_times_atom = 10 n_times = 10 * n_times_atom reg = 5e-1 params = dict(tol=1e-2, n_seg='auto', timing=False, timeout=None, verbose=100, strategy='greedy', max_iter=100000, soft_lock='border', z_positive=True, return_ztz=False, freeze_support=False, warm_start=False, random_state=27) X = rng.randn(n_channels, n_times) D = rng.randn(n_atoms, n_channels, n_times_atom) sum_axis = tuple(range(1, D.ndim)) D /= np.sqrt(np.sum(D * D, axis=sum_axis, keepdims=True)) DtD = compute_DtD(D) encoder = DistributedSparseEncoder(n_workers=2) encoder.init_workers(X, D, reg, params, DtD=DtD) with pytest.raises(ValueError, match=r"pre-computed value DtD"): encoder.set_worker_D(D) encoder.process_z_hat() z_hat = encoder.get_z_hat() # Check that distributed computations are correct for cost and sufficient # statistics cost_distrib = encoder.get_cost() cost = compute_objective(X, z_hat, D, reg) assert np.allclose(cost, cost_distrib) ztz_distrib, ztX_distrib = encoder.get_sufficient_statistics() ztz = compute_ztz(z_hat, (n_times_atom, )) ztX = compute_ztX(z_hat, X) assert np.allclose(ztz, ztz_distrib) assert np.allclose(ztX, ztX_distrib)
def compute_sufficient_statistics(self): _, _, *atom_support = self.D.shape z_slice = (Ellipsis, ) + tuple([ slice(start, end) for start, end in self.local_segments.inner_bounds ]) X_slice = (Ellipsis, ) + tuple([ slice(start, end + size_atom_ax - 1) for (start, end), size_atom_ax in zip( self.local_segments.inner_bounds, atom_support) ]) ztX = compute_ztX(self.z_hat[z_slice], self.X_worker[X_slice]) padding_shape = self.workers_segments.get_padding_to_overlap(self.rank) ztz = compute_ztz(self.z_hat, atom_support, padding_shape=padding_shape) return np.array(ztz, dtype='d'), np.array(ztX, dtype='d')
def test_ztz(valid_shape, atom_shape, sparsity): n_atoms = 7 n_channels = 5 random_state = None rng = check_random_state(random_state) z = rng.randn(n_atoms, *valid_shape) z *= rng.rand(*z.shape) < sparsity D = rng.randn(n_atoms, n_channels, *atom_shape) ztz = compute_ztz(z, atom_shape) grad = np.sum( [[[fftconvolve(ztz_k0_k, d_kp, mode='valid') for d_kp in d_k] for ztz_k0_k, d_k in zip(ztz_k0, D)] for ztz_k0 in ztz], axis=1) cost = np.dot(D.ravel(), grad.ravel()) X_hat = reconstruct(z, D) assert np.isclose(cost, np.dot(X_hat.ravel(), X_hat.ravel()))
def test_ztz(valid_support, atom_support): tol = .5 reg = .1 n_atoms = 7 n_channels = 5 random_state = None sig_support = get_full_support(valid_support, atom_support) rng = check_random_state(random_state) X = rng.randn(n_channels, *sig_support) D = rng.randn(n_atoms, n_channels, *atom_support) D /= np.sqrt(np.sum(D * D, axis=(1, 2), keepdims=True)) z_hat, ztz, ztX, *_ = dicod(X, D, reg, tol=tol, n_workers=N_WORKERS, return_ztz=True, verbose=VERBOSE) ztz_full = compute_ztz(z_hat, atom_support) assert np.allclose(ztz_full, ztz) ztX_full = compute_ztX(z_hat, X) assert np.allclose(ztX_full, ztX)
def coordinate_descent(X_i, D, reg, z0=None, DtD=None, n_seg='auto', strategy='greedy', tol=1e-5, max_iter=100000, timeout=None, z_positive=False, freeze_support=False, return_ztz=False, timing=False, random_state=None, verbose=0): """Coordinate Descent Algorithm for 2D convolutional sparse coding. Parameters ---------- X_i : ndarray, shape (n_channels, *sig_support) Image to encode on the dictionary D D : ndarray, shape (n_atoms, n_channels, *atom_support) Current dictionary for the sparse coding reg : float Regularization parameter z0 : ndarray, shape (n_atoms, *valid_support) or None Warm start value for z_hat. If not present, z_hat is initialized to 0. DtD : ndarray, shape (n_atoms, n_atoms, 2 * valid_support - 1) or None Warm start value for DtD. If not present, it is computed on init. n_seg : int or 'auto' Number of segments to use for each dimension. If set to 'auto' use segments of twice the size of the dictionary. strategy : str in {strategies} Coordinate selection scheme for the coordinate descent. If set to 'greedy'|'gs-r', the coordinate with the largest value for dz_opt is selected. If set to 'random, the coordinate is chosen uniformly on the segment. If set to 'gs-q', the value that reduce the most the cost function is selected. In this case, dE must holds the value of this cost reduction. tol : float Tolerance for the minimal update size in this algorithm. max_iter : int Maximal number of iteration run by this algorithm. z_positive : boolean If set to true, the activations are constrained to be positive. freeze_support : boolean If set to True, only update the coefficient that are non-zero in z0. return_ztz : boolean If True, returns the constants ztz and ztX, used to compute D-updates. timing : boolean If set to True, log the cost and timing information. random_state : None or int or RandomState current random state to seed the random number generator. verbose : int Verbosity level of the algorithm. Return ------ z_hat : ndarray, shape (n_atoms, *valid_support) Activation associated to X_i for the given dictionary D """ n_channels, *sig_support = X_i.shape n_atoms, n_channels, *atom_support = D.shape valid_support = get_valid_support(sig_support, atom_support) if strategy not in STRATEGIES: raise ValueError("'The coordinate selection strategy should be in " "{}. Got '{}'.".format(STRATEGIES, strategy)) # compute sizes for the segments for LGCD. Auto gives segments of size # twice the support of the atoms. if n_seg == 'auto': n_seg = np.array(valid_support) // (2 * np.array(atom_support) - 1) n_seg = tuple(np.maximum(1, n_seg)) segments = Segmentation(n_seg, signal_support=valid_support) # Pre-compute constants for maintaining the auxillary variable beta and # compute the coordinate update values. constants = {} constants['norm_atoms'] = compute_norm_atoms(D) if DtD is None: constants['DtD'] = compute_DtD(D) else: constants['DtD'] = DtD # Initialization of the algorithm variables i_seg = -1 accumulator = 0 if z0 is None: z_hat = np.zeros((n_atoms,) + valid_support) else: z_hat = np.copy(z0) n_coordinates = z_hat.size # Get a random number genator from the given random_state rng = check_random_state(random_state) order = None if strategy in ['cyclic', 'cyclic-r', 'random']: order = get_order_iterator(z_hat.shape, strategy=strategy, random_state=rng) t_start_init = time.time() return_dE = strategy == "gs-q" beta, dz_opt, dE = _init_beta(X_i, D, reg, z_i=z0, constants=constants, z_positive=z_positive, return_dE=return_dE) if strategy == "gs-q": raise NotImplementedError("This is still WIP") if freeze_support: freezed_support = z0 == 0 dz_opt[freezed_support] = 0 else: freezed_support = None p_obj, next_log_iter = [], 1 t_init = time.time() - t_start_init if timing: p_obj.append((0, t_init, 0, compute_objective(X_i, z_hat, D, reg))) n_coordinate_updates = 0 t_run = 0 t_select_coord, t_update_coord = [], [] t_start = time.time() if timeout is not None: deadline = t_start + timeout else: deadline = None for ii in range(max_iter): if ii % 1000 == 0 and verbose > 0: print("\r[LGCD:PROGRESS] {:.0f}s - {:7.2%} iterations" .format(t_run, ii / max_iter), end='', flush=True) i_seg = segments.increment_seg(i_seg) if segments.is_active_segment(i_seg): t_start_selection = time.time() k0, pt0, dz = _select_coordinate(dz_opt, dE, segments, i_seg, strategy=strategy, order=order) selection_duration = time.time() - t_start_selection t_select_coord.append(selection_duration) t_run += selection_duration else: dz = 0 accumulator = max(abs(dz), accumulator) # Update the selected coordinate and beta, only if the update is # greater than the convergence tolerance. if abs(dz) > tol: t_start_update = time.time() # update the current solution estimate and beta beta, dz_opt, dE = coordinate_update( k0, pt0, dz, beta=beta, dz_opt=dz_opt, dE=dE, z_hat=z_hat, D=D, reg=reg, constants=constants, z_positive=z_positive, freezed_support=freezed_support) touched_segs = segments.get_touched_segments( pt=pt0, radius=atom_support) n_changed_status = segments.set_active_segments(touched_segs) # Logging of the time and the cost function if necessary update_duration = time.time() - t_start_update n_coordinate_updates += 1 t_run += update_duration t_update_coord.append(update_duration) if timing and ii + 1 >= next_log_iter: p_obj.append((ii + 1, t_run, np.sum(t_select_coord), compute_objective(X_i, z_hat, D, reg))) next_log_iter = next_log_iter * 1.3 # If debug flag CHECK_ACTIVE_SEGMENTS is set, check that all # inactive segments should be inactive if flags.CHECK_ACTIVE_SEGMENTS and n_changed_status: segments.test_active_segment(dz_opt, tol) elif strategy in ["greedy", 'gs-r']: segments.set_inactive_segments(i_seg) # check stopping criterion if _check_convergence(segments, tol, ii, dz_opt, n_coordinates, strategy, accumulator=accumulator): assert np.all(abs(dz_opt) <= tol) if verbose > 0: print("\r[LGCD:INFO] converged in {} iterations ({} updates)" .format(ii + 1, n_coordinate_updates)) break # Check is we reach the timeout if deadline is not None and time.time() >= deadline: if verbose > 0: print("\r[LGCD:INFO] Reached timeout. Done {} iterations " "({} updates). Max of |dz|={}." .format(ii + 1, n_coordinate_updates, abs(dz_opt).max())) break else: if verbose > 0: print("\r[LGCD:INFO] Reached max_iter. Done {} coordinate " "updates. Max of |dz|={}." .format(n_coordinate_updates, abs(dz_opt).max())) print(f"\r[LGCD:{strategy}] " f"t_select={np.mean(t_select_coord):.3e}s " f"t_update={np.mean(t_update_coord):.3e}s" ) runtime = time.time() - t_start if verbose > 0: print("\r[LGCD:INFO] done in {:.3f}s ({:.3f}s)" .format(runtime, t_run)) ztz, ztX = None, None if return_ztz: ztz = compute_ztz(z_hat, atom_support) ztX = compute_ztX(z_hat, X_i) p_obj.append([n_coordinate_updates, t_run, compute_objective(X_i, z_hat, D, reg)]) run_statistics = dict(iterations=ii + 1, runtime=runtime, t_init=t_init, t_run=t_run, n_updates=n_coordinate_updates, t_select=np.mean(t_select_coord), t_update=np.mean(t_update_coord)) return z_hat, ztz, ztX, p_obj, run_statistics
def coordinate_descent(X_i, D, reg, z0=None, n_seg='auto', strategy='greedy', tol=1e-5, max_iter=100000, timeout=None, z_positive=False, freeze_support=False, return_ztz=False, timing=False, random_state=None, verbose=0): """Coordinate Descent Algorithm for 2D convolutional sparse coding. Parameters ---------- X_i : ndarray, shape (n_channels, *sig_shape) Image to encode on the dictionary D z_i : ndarray, shape (n_atoms, *valid_shape) Warm start value for z_hat D : ndarray, shape (n_atoms, n_channels, *atom_shape) Current dictionary for the sparse coding reg : float Regularization parameter n_seg : int or { 'auto' } Number of segments to use for each dimension. If set to 'auto' use segments of twice the size of the dictionary. tol : float Tolerance for the minimal update size in this algorithm. strategy : str in { 'greedy' | 'random' | 'gs-r' | 'gs-q' } Coordinate selection scheme for the coordinate descent. If set to 'greedy'|'gs-r', the coordinate with the largest value for dz_opt is selected. If set to 'random, the coordinate is chosen uniformly on the segment. If set to 'gs-q', the value that reduce the most the cost function is selected. In this case, dE must holds the value of this cost reduction. max_iter : int Maximal number of iteration run by this algorithm. z_positive : boolean If set to true, the activations are constrained to be positive. freeze_support : boolean If set to True, only update the coefficient that are non-zero in z0. timing : boolean If set to True, log the cost and timing information. random_state : None or int or RandomState current random state to seed the random number generator. verbose : int Verbosity level of the algorithm. Return ------ z_hat : ndarray, shape (n_atoms, *valid_shape) Activation associated to X_i for the given dictionary D """ n_channels, *sig_shape = X_i.shape n_atoms, n_channels, *atom_shape = D.shape valid_shape = tuple([ size_ax - size_atom_ax + 1 for size_ax, size_atom_ax in zip(sig_shape, atom_shape) ]) # compute sizes for the segments for LGCD if n_seg == 'auto': n_seg = [] for axis_size, atom_size in zip(valid_shape, atom_shape): n_seg.append(max(axis_size // (2 * atom_size - 1), 1)) segments = Segmentation(n_seg, signal_shape=valid_shape) # Pre-compute some quantities constants = {} constants['norm_atoms'] = compute_norm_atoms(D) constants['DtD'] = compute_DtD(D) # Initialization of the algorithm variables i_seg = -1 p_obj, next_cost = [], 1 accumulator = 0 if z0 is None: z_hat = np.zeros((n_atoms, ) + valid_shape) else: z_hat = np.copy(z0) n_coordinates = z_hat.size t_update = 0 t_start_update = time.time() return_dE = strategy == "gs-q" beta, dz_opt, dE = _init_beta(X_i, D, reg, z_i=z0, constants=constants, z_positive=z_positive, return_dE=return_dE) if strategy == "gs-q": raise NotImplementedError("This is still WIP") if freeze_support: freezed_support = z0 == 0 dz_opt[freezed_support] = 0 else: freezed_support = None t_start = time.time() n_coordinate_updates = 0 if timeout is not None: deadline = t_start + timeout else: deadline = None for ii in range(max_iter): if ii % 1000 == 0 and verbose > 0: print("\r[LGCD:PROGRESS] {:.0f}s - {:7.2%} iterations".format( t_update, ii / max_iter), end='', flush=True) i_seg = segments.increment_seg(i_seg) if segments.is_active_segment(i_seg): k0, pt0, dz = _select_coordinate(dz_opt, dE, segments, i_seg, strategy=strategy, random_state=random_state) else: k0, pt0, dz = None, None, 0 accumulator = max(abs(dz), accumulator) # Update the selected coordinate and beta, only if the update is # greater than the convergence tolerance. if abs(dz) > tol: n_coordinate_updates += 1 # update beta beta, dz_opt, dE = coordinate_update( k0, pt0, dz, beta=beta, dz_opt=dz_opt, dE=dE, z_hat=z_hat, D=D, reg=reg, constants=constants, z_positive=z_positive, freezed_support=freezed_support) touched_segs = segments.get_touched_segments(pt=pt0, radius=atom_shape) n_changed_status = segments.set_active_segments(touched_segs) if flags.CHECK_ACTIVE_SEGMENTS and n_changed_status: segments.test_active_segment(dz_opt, tol) t_update += time.time() - t_start_update if timing: if ii >= next_cost: p_obj.append( (ii, t_update, compute_objective(X_i, z_hat, D, reg))) next_cost = next_cost * 2 t_start_update = time.time() elif strategy in ["greedy", 'gs-r']: segments.set_inactive_segments(i_seg) # check stopping criterion if _check_convergence(segments, tol, ii, dz_opt, n_coordinates, strategy, accumulator=accumulator): assert np.all(abs(dz_opt) <= tol) if verbose > 0: print("\r[LGCD:INFO] converged after {} iterations".format(ii + 1)) break # Check is we reach the timeout if deadline is not None and time.time() >= deadline: if verbose > 0: print("\r[LGCD:INFO] Reached timeout. Done {} coordinate " "updates. Max of |dz|={}.".format( n_coordinate_updates, abs(dz_opt).max())) break else: if verbose > 0: print("\r[LGCD:INFO] Reached max_iter. Done {} coordinate " "updates. Max of |dz|={}.".format(n_coordinate_updates, abs(dz_opt).max())) runtime = time.time() - t_start if verbose > 0: print("\r[LGCD:INFO] done in {:.3}s".format(runtime)) ztz, ztX = None, None if return_ztz: ztz = compute_ztz(z_hat, atom_shape) ztX = compute_ztX(z_hat, X_i) p_obj.append([n_coordinate_updates, t_update, None]) return z_hat, ztz, ztX, p_obj, None