def get_steps(n_dim, n_atoms, n_samples, n_test, n_layers, reg, rng, max_iter):
lista_kwargs = dict(
n_layers=n_layers,
max_iter=max_iter)
x, D, z = make_coding(n_samples=n_samples + n_test, n_atoms=n_atoms,
n_dim=n_dim, random_state=rng)
x_test = x[n_samples:]
x = x[:n_samples]
c_star = Lista(D, 1000).score(x_test, reg)
L = np.linalg.norm(D, ord=2) ** 2 # Lipschitz constant
network = Lista(D, **lista_kwargs,
parametrization='step', per_layer='oneshot')
init_score = network.score(x_test, reg)
print(init_score)
network.fit(x, reg)
print(network.score(x_test, reg))
steps = network.get_parameters(name='step_size')
z_ = network.transform(x, reg)
supports = np.unique(z_ != 0, axis=0)
S_pca = []
for support in supports:
D_s = D[support]
G_s = D_s.T.dot(D_s)
S_pca.append(np.linalg.eigvalsh(G_s)[-1])
L_s = np.max(S_pca)
return steps, L, L_s, S_pca
def get_steps(n_dim, n_atoms, n_samples, n_test, n_layers, reg, rng, max_iter):
lista_kwargs = dict(n_layers=n_layers, max_iter=max_iter)
x, D, z = make_coding(n_samples=n_samples + n_test,
n_atoms=n_atoms,
n_dim=n_dim,
random_state=rng)
x_test = x[n_samples:]
x = x[:n_samples]
c_star = Lista(D, 1000).score(x_test, reg)
L = np.linalg.norm(D, ord=2)**2 # Lipschitz constant
network = Lista(D,
**lista_kwargs,
parametrization='step',
per_layer='oneshot')
init_score = network.score(x_test, reg)
print(init_score)
network.fit(x, reg)
print(network.score(x_test, reg))
steps = network.get_parameters(name='step_size')
L_s = np.zeros((n_layers, n_samples))
for layer in range(1, n_layers + 1):
z_ = network.transform(x, reg, output_layer=layer)
supports = z_ != 0
S_pca = []
for support in supports:
idx = np.where(support)[0]
D_s = D[idx]
G_s = D_s.T.dot(D_s)
S_pca.append(np.linalg.eigvalsh(G_s)[-1])
L_s[layer - 1] = np.array(S_pca)
return steps, L, L_s, S_pca
def compute_loss(n_sample, parametrization):
x_train = x[:n_sample]
lista = Lista(D,
n_layers,
parametrization=parametrization,
max_iter=max_iter,
per_layer=training,
verbose=1)
lista.fit(x_train, reg)
sc = lista.score(x_test, reg)
print(n_sample, parametrization, sc)
return sc
def run_one(parametrization, data, reg, n_layer, max_iter, n_samples, n_test,
n_atoms, n_dim, per_layer, random_state):
# try to avoid having dangling memory
torch.cuda.empty_cache()
# Stread the computations on the different GPU. This strategy
# might fail and some GPU might be overloaded if some workers are
# re-spawned.
if N_GPU == 0:
device = None
else:
pid = os.getpid()
device = f"cuda:{pid % N_GPU}"
tag = f"[{parametrization} - {n_layer}]"
current_time = time.time() - START
msg = f"\r{tag} started at T={current_time:.0f} sec (device={device})"
print(colorify(msg, BLUE))
if data == "images":
x, D, _ = make_image_coding(n_samples=n_samples + n_test,
n_atoms=n_atoms,
normalize=True,
random_state=random_state)
elif data == "simulations":
x, D, _ = make_coding(n_samples=n_samples + n_test,
n_atoms=n_atoms,
n_dim=n_dim,
normalize=True,
random_state=random_state)
x_test = x[n_samples:]
x = x[:n_samples]
network = Lista(D,
n_layers=n_layer,
parametrization=parametrization,
max_iter=max_iter,
per_layer=per_layer,
device=device,
name=parametrization)
network.fit(x, reg)
loss = network.score(x_test, reg)
training_loss = network.training_loss_
duration = time.time() - START - current_time
msg = (f"\r{tag} done in {duration:.0f} sec "
f"at T={current_time:.0f} sec")
print(colorify(msg, GREEN))
return (parametrization, data, reg, n_layer, max_iter, n_samples, n_test,
n_atoms, n_dim, random_state, loss, training_loss)
def test_save(parametrization, learn_th):
n_dim = 20
n_atoms = 10
n_samples = 1000
random_state = 42
reg = .1
n_layers = 4
if "step" in parametrization and not learn_th:
pytest.skip(msg="For parametrization 'step' and 'coupled_step', "
"learn_th need to be set to True.")
# Generate a problem
x, D, z = make_coding(n_samples=n_samples,
n_atoms=n_atoms,
n_dim=n_dim,
random_state=random_state)
lista = Lista(D,
n_layers,
parametrization=parametrization,
learn_th=learn_th,
max_iter=15)
lista.fit(x, reg)
parameters = lista.export_parameters()
lista_ = Lista(D,
n_layers,
parametrization=parametrization,
learn_th=learn_th,
initial_parameters=parameters)
parameters_ = lista_.export_parameters()
assert np.all([
np.allclose(pl[k], pl_[k]) for pl, pl_ in zip(parameters, parameters_)
for k in pl
])
cost_lista = lista.score(x, reg)
cost_lista_ = lista_.score(x, reg)
assert np.allclose(cost_lista, cost_lista_)
z_lista = lista.transform(x, reg)
z_lista_ = lista_.transform(x, reg)
atol = abs(z_lista).max() * 1e-6
assert np.allclose(z_lista, z_lista_, atol=atol)
def get_params(n_samples, n_atoms, n_dim, rng, max_iter, training, n_layers):
x, D, z = make_coding(n_samples=n_samples,
n_atoms=n_atoms,
n_dim=n_dim,
random_state=rng)
lista = Lista(D,
n_layers,
parametrization='coupled',
max_iter=max_iter,
per_layer=training,
verbose=1)
lista.fit(x, reg)
W_list = lista.get_parameters('W_coupled')
thresholds = lista.get_parameters('threshold')
return D, W_list, thresholds
x_hat = z_hat.dot(D)
path.append(z_hat.dot(D))
z0 = np.zeros((1, n_atoms))
z0[0] = 1
z_hat = lista.transform(x, reg, z0=z0, output_layer=i_layer)
x_hat2 = z_hat.dot(D)
path2.append(z_hat.dot(D))
path = np.array(path)
path2 = np.array(path2)
plt.plot(path[:, :n_display, 0], path[:, :n_display, 1], 'C1')
plt.scatter(x_hat[:n_display, 0], x_hat[:n_display, 1], c='C1')
plt.plot(path2[:, :n_display, 0], path2[:, :n_display, 1], 'C4')
plt.scatter(x_hat2[:n_display, 0], x_hat2[:n_display, 1], c='C4')
lista = Lista(D, n_layers, parametrization="hessian", max_iter=10000)
lista.fit(x_train, reg)
cmap = plt.get_cmap('viridis')
path = []
for i_layer in range(1, n_layers + 1, 1):
z_hat = lista.transform(x, reg, output_layer=i_layer)
x_hat = z_hat.dot(D)
path.append(z_hat.dot(D))
# plt.scatter(x_hat[:n_display, 0], x_hat[:n_display, 1],
# c=np.array([cmap(i_layer / n_layers)]))
path = np.array(path)
plt.plot(path[:, :n_display, 0], path[:, :n_display, 1], 'C2')
plt.scatter(x_hat[:n_display, 0], x_hat[:n_display, 1], c='C2')
for k, dk in enumerate(D):
plt.arrow(0, 0, z_hat[0][k] * dk[0], z_hat[0][k] * dk[1], color="r")
def get_trained_lista(D, x, reg, n_layers, max_iter):
lista = Lista(D, n_layers, max_iter=max_iter)
lista.fit(x, reg)
return lista
saved_model = {}
for parametrization in ['alista', 'hessian', 'coupled']:
lista = Lista(D,
n_layers,
parametrization=parametrization,
max_iter=5000,
device=device)
z_hat_test = lista.transform(x_test, reg)
cost_test = cost_lasso(z_hat_test, D, x_test, reg)
print(
format_cost.format("Un-trained[{}]".format(parametrization),
"test", cost_test))
# Train and evaluate the network
lista.fit(x, reg, tol=0)
z_hat_test = lista.transform(x_test, reg)
cost_test = cost_lasso(z_hat_test, D, x_test, reg)
print(
format_cost.format("Trained[{}]".format(parametrization), "test",
cost_test))
saved_model[parametrization] = lista
z_hat = lista.transform(x, reg)
plt.semilogy(lista.training_loss_ - c_star, label=parametrization)
plt.legend()
# Save the figure if it is not displayed
if mpl.get_backend() == 'agg':
plt.savefig("output.pdf", dpi=300)
for parametrization in ['first_step', 'step']:
for per_layer in ['oneshot', 'recursive']:
#######################################################################
# Training
#
name = "{} - {}".format(parametrization, per_layer)
print(80 * "=" + "\n" + name + "\n" + 80 * "=")
network = Lista(D,
**lista_kwargs,
name=name,
parametrization=parametrization,
per_layer=per_layer)
init_score = network.score(x_test, reg)
print(init_print.format(init_score))
network.fit(x, reg)
losses = np.array([
network.score(x_test, reg, output_layer=layer + 1)
for layer in range(n_layers)
])
final_score = network.score(x_test, reg)
print(final_print.format(final_score, init_score - final_score))
networks[name] = network
#######################################################################
# Compute maximal sparsity eigenvalue
#
L_s_list = []
for layer in range(n_layers):
z_ = network.transform(x, reg, output_layer=layer + 1)