def run_without_soft_lock(n_atoms=25,
atom_support=(12, 12),
reg=.01,
tol=5e-2,
n_workers=100,
random_state=60):
rng = np.random.RandomState(random_state)
X = get_mandril()
D_init = init_dictionary(X, n_atoms, atom_support, random_state=rng)
lmbd_max = get_lambda_max(X, D_init).max()
reg_ = reg * lmbd_max
z_hat, *_ = dicod(X,
D_init,
reg_,
max_iter=1000000,
n_workers=n_workers,
tol=tol,
strategy='greedy',
verbose=1,
soft_lock='none',
z_positive=False,
timing=False)
pobj = compute_objective(X, z_hat, D_init, reg_)
z_hat = np.clip(z_hat, -1e3, 1e3)
print("[DICOD] final cost : {}".format(pobj))
X_hat = reconstruct(z_hat, D_init)
X_hat = np.clip(X_hat, 0, 1)
return X_hat, pobj
def test_init_beta():
n_atoms = 5
n_channels = 2
height, width = 31, 37
height_atom, width_atom = 11, 13
height_valid = height - height_atom + 1
width_valid = width - width_atom + 1
rng = np.random.RandomState(42)
X = rng.randn(n_channels, height, width)
D = rng.randn(n_atoms, n_channels, height_atom, width_atom)
D /= np.sqrt(np.sum(D * D, axis=(1, 2, 3), keepdims=True))
# z = np.zeros((n_atoms, height_valid, width_valid))
z = rng.randn(n_atoms, height_valid, width_valid)
lmbd = 1
beta, dz_opt, dE = _init_beta(X, D, lmbd, z_i=z)
assert beta.shape == z.shape
assert dz_opt.shape == z.shape
for _ in range(50):
k = rng.randint(n_atoms)
h = rng.randint(height_valid)
w = rng.randint(width_valid)
# Check that the optimal value is independent of the current value
z_old = z[k, h, w]
z[k, h, w] = rng.randn()
beta_new, *_ = _init_beta(X, D, lmbd, z_i=z)
assert np.isclose(beta_new[k, h, w], beta[k, h, w])
# Check that the chosen value is optimal
z[k, h, w] = z_old + dz_opt[k, h, w]
c0 = compute_objective(X, z, D, lmbd)
eps = 1e-5
z[k, h, w] -= 3.5 * eps
for _ in range(5):
z[k, h, w] += eps
assert c0 <= compute_objective(X, z, D, lmbd)
z[k, h, w] = z_old
def test_distributed_sparse_encoder():
rng = check_random_state(42)
n_atoms = 10
n_channels = 3
n_times_atom = 10
n_times = 10 * n_times_atom
reg = 5e-1
params = dict(tol=1e-2,
n_seg='auto',
timing=False,
timeout=None,
verbose=100,
strategy='greedy',
max_iter=100000,
soft_lock='border',
z_positive=True,
return_ztz=False,
freeze_support=False,
warm_start=False,
random_state=27)
X = rng.randn(n_channels, n_times)
D = rng.randn(n_atoms, n_channels, n_times_atom)
sum_axis = tuple(range(1, D.ndim))
D /= np.sqrt(np.sum(D * D, axis=sum_axis, keepdims=True))
DtD = compute_DtD(D)
encoder = DistributedSparseEncoder(n_workers=2)
encoder.init_workers(X, D, reg, params, DtD=DtD)
with pytest.raises(ValueError, match=r"pre-computed value DtD"):
encoder.set_worker_D(D)
encoder.process_z_hat()
z_hat = encoder.get_z_hat()
# Check that distributed computations are correct for cost and sufficient
# statistics
cost_distrib = encoder.get_cost()
cost = compute_objective(X, z_hat, D, reg)
assert np.allclose(cost, cost_distrib)
ztz_distrib, ztX_distrib = encoder.get_sufficient_statistics()
ztz = compute_ztz(z_hat, (n_times_atom, ))
ztX = compute_ztX(z_hat, X)
assert np.allclose(ztz, ztz_distrib)
assert np.allclose(ztX, ztX_distrib)
def test_cost(valid_support, atom_support):
tol = .5
reg = 0
n_atoms = 7
n_channels = 5
random_state = None
sig_support = get_full_support(valid_support, atom_support)
rng = check_random_state(random_state)
D = rng.randn(n_atoms, n_channels, *atom_support)
D /= np.sqrt(np.sum(D * D, axis=(1, 2), keepdims=True))
z = rng.randn(n_atoms, *valid_support)
z *= rng.rand(n_atoms, *valid_support) > .5
X = rng.randn(n_channels, *sig_support)
z_hat, *_, pobj, _ = dicod(X, D, reg, z0=z, tol=tol, n_workers=N_WORKERS,
max_iter=1000, freeze_support=True,
verbose=VERBOSE)
cost = pobj[-1][2]
assert np.isclose(cost, compute_objective(X, z_hat, D, reg))
def coordinate_descent(X_i, D, reg, z0=None, DtD=None, n_seg='auto',
strategy='greedy', tol=1e-5, max_iter=100000,
timeout=None, z_positive=False, freeze_support=False,
return_ztz=False, timing=False,
random_state=None, verbose=0):
"""Coordinate Descent Algorithm for 2D convolutional sparse coding.
Parameters
----------
X_i : ndarray, shape (n_channels, *sig_support)
Image to encode on the dictionary D
D : ndarray, shape (n_atoms, n_channels, *atom_support)
Current dictionary for the sparse coding
reg : float
Regularization parameter
z0 : ndarray, shape (n_atoms, *valid_support) or None
Warm start value for z_hat. If not present, z_hat is initialized to 0.
DtD : ndarray, shape (n_atoms, n_atoms, 2 * valid_support - 1) or None
Warm start value for DtD. If not present, it is computed on init.
n_seg : int or 'auto'
Number of segments to use for each dimension. If set to 'auto' use
segments of twice the size of the dictionary.
strategy : str in {strategies}
Coordinate selection scheme for the coordinate descent. If set to
'greedy'|'gs-r', the coordinate with the largest value for dz_opt is
selected. If set to 'random, the coordinate is chosen uniformly on the
segment. If set to 'gs-q', the value that reduce the most the cost
function is selected. In this case, dE must holds the value of this
cost reduction.
tol : float
Tolerance for the minimal update size in this algorithm.
max_iter : int
Maximal number of iteration run by this algorithm.
z_positive : boolean
If set to true, the activations are constrained to be positive.
freeze_support : boolean
If set to True, only update the coefficient that are non-zero in z0.
return_ztz : boolean
If True, returns the constants ztz and ztX, used to compute D-updates.
timing : boolean
If set to True, log the cost and timing information.
random_state : None or int or RandomState
current random state to seed the random number generator.
verbose : int
Verbosity level of the algorithm.
Return
------
z_hat : ndarray, shape (n_atoms, *valid_support)
Activation associated to X_i for the given dictionary D
"""
n_channels, *sig_support = X_i.shape
n_atoms, n_channels, *atom_support = D.shape
valid_support = get_valid_support(sig_support, atom_support)
if strategy not in STRATEGIES:
raise ValueError("'The coordinate selection strategy should be in "
"{}. Got '{}'.".format(STRATEGIES, strategy))
# compute sizes for the segments for LGCD. Auto gives segments of size
# twice the support of the atoms.
if n_seg == 'auto':
n_seg = np.array(valid_support) // (2 * np.array(atom_support) - 1)
n_seg = tuple(np.maximum(1, n_seg))
segments = Segmentation(n_seg, signal_support=valid_support)
# Pre-compute constants for maintaining the auxillary variable beta and
# compute the coordinate update values.
constants = {}
constants['norm_atoms'] = compute_norm_atoms(D)
if DtD is None:
constants['DtD'] = compute_DtD(D)
else:
constants['DtD'] = DtD
# Initialization of the algorithm variables
i_seg = -1
accumulator = 0
if z0 is None:
z_hat = np.zeros((n_atoms,) + valid_support)
else:
z_hat = np.copy(z0)
n_coordinates = z_hat.size
# Get a random number genator from the given random_state
rng = check_random_state(random_state)
order = None
if strategy in ['cyclic', 'cyclic-r', 'random']:
order = get_order_iterator(z_hat.shape, strategy=strategy,
random_state=rng)
t_start_init = time.time()
return_dE = strategy == "gs-q"
beta, dz_opt, dE = _init_beta(X_i, D, reg, z_i=z0, constants=constants,
z_positive=z_positive, return_dE=return_dE)
if strategy == "gs-q":
raise NotImplementedError("This is still WIP")
if freeze_support:
freezed_support = z0 == 0
dz_opt[freezed_support] = 0
else:
freezed_support = None
p_obj, next_log_iter = [], 1
t_init = time.time() - t_start_init
if timing:
p_obj.append((0, t_init, 0, compute_objective(X_i, z_hat, D, reg)))
n_coordinate_updates = 0
t_run = 0
t_select_coord, t_update_coord = [], []
t_start = time.time()
if timeout is not None:
deadline = t_start + timeout
else:
deadline = None
for ii in range(max_iter):
if ii % 1000 == 0 and verbose > 0:
print("\r[LGCD:PROGRESS] {:.0f}s - {:7.2%} iterations"
.format(t_run, ii / max_iter), end='', flush=True)
i_seg = segments.increment_seg(i_seg)
if segments.is_active_segment(i_seg):
t_start_selection = time.time()
k0, pt0, dz = _select_coordinate(dz_opt, dE, segments, i_seg,
strategy=strategy, order=order)
selection_duration = time.time() - t_start_selection
t_select_coord.append(selection_duration)
t_run += selection_duration
else:
dz = 0
accumulator = max(abs(dz), accumulator)
# Update the selected coordinate and beta, only if the update is
# greater than the convergence tolerance.
if abs(dz) > tol:
t_start_update = time.time()
# update the current solution estimate and beta
beta, dz_opt, dE = coordinate_update(
k0, pt0, dz, beta=beta, dz_opt=dz_opt, dE=dE, z_hat=z_hat, D=D,
reg=reg, constants=constants, z_positive=z_positive,
freezed_support=freezed_support)
touched_segs = segments.get_touched_segments(
pt=pt0, radius=atom_support)
n_changed_status = segments.set_active_segments(touched_segs)
# Logging of the time and the cost function if necessary
update_duration = time.time() - t_start_update
n_coordinate_updates += 1
t_run += update_duration
t_update_coord.append(update_duration)
if timing and ii + 1 >= next_log_iter:
p_obj.append((ii + 1, t_run, np.sum(t_select_coord),
compute_objective(X_i, z_hat, D, reg)))
next_log_iter = next_log_iter * 1.3
# If debug flag CHECK_ACTIVE_SEGMENTS is set, check that all
# inactive segments should be inactive
if flags.CHECK_ACTIVE_SEGMENTS and n_changed_status:
segments.test_active_segment(dz_opt, tol)
elif strategy in ["greedy", 'gs-r']:
segments.set_inactive_segments(i_seg)
# check stopping criterion
if _check_convergence(segments, tol, ii, dz_opt, n_coordinates,
strategy, accumulator=accumulator):
assert np.all(abs(dz_opt) <= tol)
if verbose > 0:
print("\r[LGCD:INFO] converged in {} iterations ({} updates)"
.format(ii + 1, n_coordinate_updates))
break
# Check is we reach the timeout
if deadline is not None and time.time() >= deadline:
if verbose > 0:
print("\r[LGCD:INFO] Reached timeout. Done {} iterations "
"({} updates). Max of |dz|={}."
.format(ii + 1, n_coordinate_updates, abs(dz_opt).max()))
break
else:
if verbose > 0:
print("\r[LGCD:INFO] Reached max_iter. Done {} coordinate "
"updates. Max of |dz|={}."
.format(n_coordinate_updates, abs(dz_opt).max()))
print(f"\r[LGCD:{strategy}] "
f"t_select={np.mean(t_select_coord):.3e}s "
f"t_update={np.mean(t_update_coord):.3e}s"
)
runtime = time.time() - t_start
if verbose > 0:
print("\r[LGCD:INFO] done in {:.3f}s ({:.3f}s)"
.format(runtime, t_run))
ztz, ztX = None, None
if return_ztz:
ztz = compute_ztz(z_hat, atom_support)
ztX = compute_ztX(z_hat, X_i)
p_obj.append([n_coordinate_updates, t_run,
compute_objective(X_i, z_hat, D, reg)])
run_statistics = dict(iterations=ii + 1, runtime=runtime, t_init=t_init,
t_run=t_run, n_updates=n_coordinate_updates,
t_select=np.mean(t_select_coord),
t_update=np.mean(t_update_coord))
return z_hat, ztz, ztX, p_obj, run_statistics
def coordinate_descent(X_i,
D,
reg,
z0=None,
n_seg='auto',
strategy='greedy',
tol=1e-5,
max_iter=100000,
timeout=None,
z_positive=False,
freeze_support=False,
return_ztz=False,
timing=False,
random_state=None,
verbose=0):
"""Coordinate Descent Algorithm for 2D convolutional sparse coding.
Parameters
----------
X_i : ndarray, shape (n_channels, *sig_shape)
Image to encode on the dictionary D
z_i : ndarray, shape (n_atoms, *valid_shape)
Warm start value for z_hat
D : ndarray, shape (n_atoms, n_channels, *atom_shape)
Current dictionary for the sparse coding
reg : float
Regularization parameter
n_seg : int or { 'auto' }
Number of segments to use for each dimension. If set to 'auto' use
segments of twice the size of the dictionary.
tol : float
Tolerance for the minimal update size in this algorithm.
strategy : str in { 'greedy' | 'random' | 'gs-r' | 'gs-q' }
Coordinate selection scheme for the coordinate descent. If set to
'greedy'|'gs-r', the coordinate with the largest value for dz_opt is
selected. If set to 'random, the coordinate is chosen uniformly on the
segment. If set to 'gs-q', the value that reduce the most the cost
function is selected. In this case, dE must holds the value of this
cost reduction.
max_iter : int
Maximal number of iteration run by this algorithm.
z_positive : boolean
If set to true, the activations are constrained to be positive.
freeze_support : boolean
If set to True, only update the coefficient that are non-zero in z0.
timing : boolean
If set to True, log the cost and timing information.
random_state : None or int or RandomState
current random state to seed the random number generator.
verbose : int
Verbosity level of the algorithm.
Return
------
z_hat : ndarray, shape (n_atoms, *valid_shape)
Activation associated to X_i for the given dictionary D
"""
n_channels, *sig_shape = X_i.shape
n_atoms, n_channels, *atom_shape = D.shape
valid_shape = tuple([
size_ax - size_atom_ax + 1
for size_ax, size_atom_ax in zip(sig_shape, atom_shape)
])
# compute sizes for the segments for LGCD
if n_seg == 'auto':
n_seg = []
for axis_size, atom_size in zip(valid_shape, atom_shape):
n_seg.append(max(axis_size // (2 * atom_size - 1), 1))
segments = Segmentation(n_seg, signal_shape=valid_shape)
# Pre-compute some quantities
constants = {}
constants['norm_atoms'] = compute_norm_atoms(D)
constants['DtD'] = compute_DtD(D)
# Initialization of the algorithm variables
i_seg = -1
p_obj, next_cost = [], 1
accumulator = 0
if z0 is None:
z_hat = np.zeros((n_atoms, ) + valid_shape)
else:
z_hat = np.copy(z0)
n_coordinates = z_hat.size
t_update = 0
t_start_update = time.time()
return_dE = strategy == "gs-q"
beta, dz_opt, dE = _init_beta(X_i,
D,
reg,
z_i=z0,
constants=constants,
z_positive=z_positive,
return_dE=return_dE)
if strategy == "gs-q":
raise NotImplementedError("This is still WIP")
if freeze_support:
freezed_support = z0 == 0
dz_opt[freezed_support] = 0
else:
freezed_support = None
t_start = time.time()
n_coordinate_updates = 0
if timeout is not None:
deadline = t_start + timeout
else:
deadline = None
for ii in range(max_iter):
if ii % 1000 == 0 and verbose > 0:
print("\r[LGCD:PROGRESS] {:.0f}s - {:7.2%} iterations".format(
t_update, ii / max_iter),
end='',
flush=True)
i_seg = segments.increment_seg(i_seg)
if segments.is_active_segment(i_seg):
k0, pt0, dz = _select_coordinate(dz_opt,
dE,
segments,
i_seg,
strategy=strategy,
random_state=random_state)
else:
k0, pt0, dz = None, None, 0
accumulator = max(abs(dz), accumulator)
# Update the selected coordinate and beta, only if the update is
# greater than the convergence tolerance.
if abs(dz) > tol:
n_coordinate_updates += 1
# update beta
beta, dz_opt, dE = coordinate_update(
k0,
pt0,
dz,
beta=beta,
dz_opt=dz_opt,
dE=dE,
z_hat=z_hat,
D=D,
reg=reg,
constants=constants,
z_positive=z_positive,
freezed_support=freezed_support)
touched_segs = segments.get_touched_segments(pt=pt0,
radius=atom_shape)
n_changed_status = segments.set_active_segments(touched_segs)
if flags.CHECK_ACTIVE_SEGMENTS and n_changed_status:
segments.test_active_segment(dz_opt, tol)
t_update += time.time() - t_start_update
if timing:
if ii >= next_cost:
p_obj.append(
(ii, t_update, compute_objective(X_i, z_hat, D, reg)))
next_cost = next_cost * 2
t_start_update = time.time()
elif strategy in ["greedy", 'gs-r']:
segments.set_inactive_segments(i_seg)
# check stopping criterion
if _check_convergence(segments,
tol,
ii,
dz_opt,
n_coordinates,
strategy,
accumulator=accumulator):
assert np.all(abs(dz_opt) <= tol)
if verbose > 0:
print("\r[LGCD:INFO] converged after {} iterations".format(ii +
1))
break
# Check is we reach the timeout
if deadline is not None and time.time() >= deadline:
if verbose > 0:
print("\r[LGCD:INFO] Reached timeout. Done {} coordinate "
"updates. Max of |dz|={}.".format(
n_coordinate_updates,
abs(dz_opt).max()))
break
else:
if verbose > 0:
print("\r[LGCD:INFO] Reached max_iter. Done {} coordinate "
"updates. Max of |dz|={}.".format(n_coordinate_updates,
abs(dz_opt).max()))
runtime = time.time() - t_start
if verbose > 0:
print("\r[LGCD:INFO] done in {:.3}s".format(runtime))
ztz, ztX = None, None
if return_ztz:
ztz = compute_ztz(z_hat, atom_shape)
ztX = compute_ztX(z_hat, X_i)
p_obj.append([n_coordinate_updates, t_update, None])
return z_hat, ztz, ztX, p_obj, None