def test_init_parameters(n, lbda, parametrization):
""" Test the gradient of z. """
rng = check_random_state(27)
x, _, _, L, _, A = synthetic_1d_dataset(n=n, s=0.5, snr=0.0, seed=rng)
n_layers = 5
# limit the number of inner layers for origtv to avoid long computations
kwargs = {}
if parametrization == 'origtv':
kwargs['n_inner_layers'] = 5
lista_1 = LearnTVAlgo(algo_type=parametrization, A=A, n_layers=n_layers,
max_iter=10, net_solver_type='one_shot', **kwargs)
lista_1.fit(x, lbda=lbda)
params = lista_1.export_parameters()
loss_lista_1 = []
for n_layer_ in range(n_layers + 1):
z_1 = lista_1.transform(x=x, lbda=lbda, output_layer=n_layer_)
loss_lista_1.append(synthesis_primal_obj(z=z_1, A=A, L=L, x=x,
lbda=lbda))
loss_lista_1 = np.array(loss_lista_1)
lista_2 = LearnTVAlgo(algo_type=parametrization, A=A, n_layers=n_layers,
initial_parameters=params, max_iter=10, **kwargs)
loss_lista_2 = []
for n_layer_ in range(n_layers + 1):
z_2 = lista_2.transform(x=x, lbda=lbda, output_layer=n_layer_)
loss_lista_2.append(synthesis_primal_obj(z=z_2, A=A, L=L, x=x,
lbda=lbda))
loss_lista_2 = np.array(loss_lista_2)
np.testing.assert_allclose(z_1, z_2)
np.testing.assert_allclose(loss_lista_1, loss_lista_2)
def test_coherence_analysis_loss(parametrization, lbda, n):
""" Test coherence regarding the loss function between learnt and fixed
algorithms. """
rng = check_random_state(None)
x, _, _, _, D, A = synthetic_1d_dataset(n=n, s=0.5, snr=0.0, seed=rng)
_, _, z = init_vuz(A, D, x)
z_ = check_tensor(z, device='cpu')
cost = analysis_primal_obj(z, A, D, x, lbda=lbda)
ltv = LearnTVAlgo(algo_type=parametrization,
A=A,
n_layers=10,
device='cpu')
cost_ref = ltv._loss_fn(x, lbda, z_)
np.testing.assert_allclose(cost_ref, cost, atol=1e-30)
def test_untrained_analysis_lista(lbda, parametrization, n):
""" Test the gradient of z. """
rng = check_random_state(None)
x, _, _, _, D, A = synthetic_1d_dataset(n=n, s=0.5, snr=0.0, seed=rng)
v0, u0, _ = init_vuz(A, D, x)
n_layers = 10
rho = 1.0
sigma = 0.5
L_D = np.linalg.norm(D.dot(D.T), ord=2)
L_A = np.linalg.norm(A.dot(A.T), ord=2)
tau = 1.0 / (L_A / 2.0 + sigma * L_D**2)
lista = LearnTVAlgo(algo_type=parametrization,
A=A,
n_layers=n_layers,
device='cpu')
loss_untrained_condat = [analysis_primal_obj(u0, A, D, x, lbda)]
for n_layer_ in range(1, n_layers + 1):
z = lista.transform(x=x, lbda=lbda, output_layer=n_layer_)
loss_untrained_condat.append(analysis_primal_obj(z, A, D, x, lbda))
loss_untrained_condat = np.array(loss_untrained_condat)
v0, u0, _ = init_vuz(A, D, x, force_numpy=True)
params = dict(
grad=lambda u: analysis_primal_grad(u, A, x),
obj=lambda u: analysis_primal_obj(u, A, D, x, lbda),
prox=lambda z: pseudo_soft_th_numpy(z, lbda, 1.0 / sigma),
psi=lambda u: u.dot(D),
adj_psi=lambda v: v.dot(D.T),
v0=v0,
z0=u0,
lbda=lbda,
sigma=sigma,
tau=tau,
rho=rho,
max_iter=n_layers,
early_stopping=False,
debug=True,
verbose=0,
)
_, _, loss_condat = condatvu(**params)
np.testing.assert_allclose(loss_condat, loss_untrained_condat, atol=1e-20)
def test_coherence_training_analysis_loss(parametrization, lbda, n):
""" Test coherence regarding the loss function between learnt and fixed
algorithms. """
rng = check_random_state(None)
x, _, _, _, D, A = synthetic_1d_dataset(n=n, s=0.5, snr=0.0, seed=rng)
_, u0, _ = init_vuz(A, D, x)
train_loss = [analysis_primal_obj(u0, A, D, x, lbda)]
train_loss_ = [analysis_primal_obj(u0, A, D, x, lbda)]
for n_layers in range(1, 10):
lista = LearnTVAlgo(algo_type=parametrization,
A=A,
n_layers=n_layers,
max_iter=10)
lista.fit(x, lbda=lbda)
train_loss_.append(lista.training_loss_[-1])
u = lista.transform(x, lbda, output_layer=n_layers)
train_loss.append(analysis_primal_obj(u, A, D, x, lbda))
np.testing.assert_allclose(train_loss_, train_loss, atol=1e-30)
def test_analysis_subgrad(n, lbda):
""" Test the sub-gradient of LASSO. """
rng = check_random_state(None)
x, u, _, _, D, A = synthetic_1d_dataset(n=n, s=0.5, snr=0.0, seed=rng)
u = rng.rand(*u.shape)
n_atoms = D.shape[0]
def finite_grad(u):
def f(u):
u = u.reshape(n, n_atoms)
# the actual considered loss is not normalized but for
# convenience we want to check the sample-loss average
return analysis_primal_obj(u, A, D, x, lbda=lbda) * n
grad = approx_fprime(xk=u.ravel(), f=f, epsilon=1.0e-6)
return grad.reshape(n, n_atoms)
grad_ref = finite_grad(u)
grad_test = analysis_primal_subgrad(u, A, D, x, lbda)
np.testing.assert_allclose(grad_ref, grad_test, atol=1e-5) # bad precision
def test_synthesis_grad(n, m):
""" Test the gradient of LASSO. """
rng = check_random_state(None)
x, _, z, L, D, A = synthetic_1d_dataset(n=n, s=0.5, snr=0.0, seed=rng)
z = rng.rand(*z.shape)
n_atoms = D.shape[0]
def finite_grad(z):
def f(z):
z = z.reshape(n, n_atoms)
# the actual considered loss is not normalized but for
# convenience we want to check the sample-loss average
return synthesis_primal_obj(z, A, L, x, lbda=0.0) * n
grad = approx_fprime(xk=z.ravel(), f=f, epsilon=1e-6)
return grad.reshape(n, n_atoms)
grad_ref = finite_grad(z)
grad_test = synthesis_primal_grad(z, A, L, x)
np.testing.assert_allclose(grad_ref, grad_test, rtol=5e-2) # bad precision
def test_untrained_synthesis_lista(lbda, parametrization, n):
""" Test the gradient of z. """
rng = check_random_state(None)
x, _, _, L, D, A = synthetic_1d_dataset(n=n, s=0.5, snr=0.0, seed=rng)
_, _, z0 = init_vuz(A, D, x)
n_layers = 10
LA = L.dot(A)
step_size = 1.0 / np.linalg.norm(LA, ord=2)**2
lista = LearnTVAlgo(algo_type=parametrization,
A=A,
n_layers=n_layers,
device='cpu')
loss_untrained_lista = [
synthesis_primal_obj(z=z0, A=A, L=L, x=x, lbda=lbda)
]
for n_layer_ in range(1, n_layers + 1):
z_hat = lista.transform(x=x, lbda=lbda, output_layer=n_layer_)
loss_untrained_lista.append(
synthesis_primal_obj(z=z_hat, A=A, L=L, x=x, lbda=lbda))
loss_untrained_lista = np.array(loss_untrained_lista)
params = dict(
grad=lambda z: synthesis_primal_grad(z, A, L, x),
obj=lambda z: synthesis_primal_obj(z, A, L, x, lbda),
prox=lambda z, s: pseudo_soft_th_numpy(z, lbda, s),
x0=z0,
momentum=None,
restarting=None,
max_iter=n_layers,
step_size=step_size,
early_stopping=False,
debug=True,
verbose=0,
)
_, loss_ista = fista(**params)
np.testing.assert_allclose(loss_ista, loss_untrained_lista, atol=1e-20)
def test_coherence_init(lbda, seed):
rng = check_random_state(seed)
x, _, _, L, D, A = synthetic_1d_dataset()
v0 = None
v0, u0, z0 = init_vuz(A, D, x, v0=v0)
cost_1 = synthesis_primal_obj(z0, A, L, x, lbda)
cost_2 = analysis_primal_obj(u0, A, D, x, lbda)
cost_3 = analysis_primal_obj(v_to_u(v0, x, A, D), A, D, x, lbda)
np.testing.assert_allclose(cost_1, cost_2)
np.testing.assert_allclose(cost_1, cost_3)
v0 = rng.randn(*v0.shape)
v0, u0, z0 = init_vuz(A, D, x, v0=v0)
synthesis_primal_obj(z0, A, L, x, lbda)
cost_1 = synthesis_primal_obj(z0, A, L, x, lbda)
cost_2 = analysis_primal_obj(u0, A, D, x, lbda)
cost_3 = analysis_primal_obj(v_to_u(v0, x, A, D), A, D, x, lbda)
np.testing.assert_allclose(cost_1, cost_2)
np.testing.assert_allclose(cost_1, cost_3)
def test_analysis_dual_grad(n, lbda):
""" Test the gradient of dual analysis. """
rng = check_random_state(None)
x, _, _, _, D, A = synthetic_1d_dataset(n=n, s=0.5, snr=0.0, seed=rng)
eps = 1e-3
v_dim = D.shape[1]
v = np.clip(rng.randn(n, v_dim), -(lbda - eps), (lbda - eps))
Psi_A = np.linalg.pinv(A).dot(D)
# Finite grad v
def finite_grad(v):
def f(v):
v = v.reshape(n, v_dim)
# the actual considered loss is not normalized but for
# convenience we want to check the sample-loss average
return analysis_dual_obj(v, A, D, x, lbda, Psi_A=Psi_A) * n
grad = approx_fprime(xk=v.ravel(), f=f, epsilon=1.0e-6)
return grad.reshape(n, v_dim)
grad_ref = finite_grad(v)
grad_test = analysis_dual_grad(v, A, D, x, Psi_A=Psi_A)
np.testing.assert_allclose(grad_ref, grad_test, atol=1e-4) # bad precision
def run_experiment(max_iter,
max_iter_ref=1000,
lmbd=.1,
seed=None,
net_solver_type='recursive',
n_jobs=1,
device=None):
# Define variables
n_samples_train = 1000
n_samples_testing = 1000
n_samples = n_samples_train + n_samples_testing
n_atoms = 8
n_dim = 5
s = 0.2
snr = 0.0
# Layers that are sampled
all_n_layers = logspace_layers(n_layers=10, max_depth=40)
timestamp = datetime.now()
print(__doc__)
print('*' * 80)
print(f"Script started on: {timestamp.strftime('%Y/%m/%d %Hh%M')}")
if seed is None:
seed = np.random.randint(0, 1000)
print(f'Seed used = {seed}')
# Store meta data of the problem
meta_pb = dict(n_atoms=n_atoms,
n_dim=n_dim,
s=s,
snr=snr,
seed=seed,
n_samples_train=n_samples_train,
n_samples_testing=n_samples_testing)
# Generate data
x, _, z, L, D, A = synthetic_1d_dataset(n_atoms=n_atoms,
n_dim=n_dim,
n=n_samples,
s=s,
snr=snr,
seed=seed)
x_train = x[n_samples_testing:, :]
x_test = x[:n_samples_testing, :]
learning_parameters = dict(net_solver_type=net_solver_type,
max_iter=max_iter)
methods = {
'lista_synthesis': {
'label': 'Synthesis LISTA',
'network': CoupledIstaLASSO,
'extra_args': dict(**learning_parameters),
'style': dict(color='tab:orange', marker='*', linestyle='-')
},
'lpgd_taut': {
'label': 'Analysis LPGD - taut-string',
'network': LpgdTautString,
'extra_args': dict(**learning_parameters),
'style': dict(color='tab:red', marker='*', linestyle='-.')
},
'ista_synthesis': {
'label': 'Synthesis ISTA',
'network': IstaSynthesis,
'extra_args': dict(momentum=None),
'style': dict(color='tab:orange', marker='s', linestyle='--')
},
'fista_synthesis': {
'label': 'Synthesis FISTA',
'network': IstaSynthesis,
'extra_args': dict(momentum='fista'),
'style': dict(color='tab:orange', marker='*', linestyle='--')
},
'ista_analysis': {
'label': 'Analysis ISTA',
'network': IstaAnalysis,
'extra_args': dict(momentum=None),
'style': dict(color='tab:red', marker='s', linestyle='--')
},
'fista_analysis': {
'label': 'Analysis FISTA',
'network': IstaAnalysis,
'extra_args': dict(momentum='fista'),
'style': dict(color='tab:red', marker='*', linestyle='--')
},
# reference cost, use all_n_layers to override the computations
'reference': {
'label': 'Analysis FISTA',
'network': IstaAnalysis,
'extra_args': dict(momentum='fista'),
'style': dict(color='tab:red', marker='*', linestyle='--'),
'all_n_layers': [max_iter_ref]
}
}
# for i, learn_prox in enumerate(['none', 'global', 'per-layer']):
for i, learn_prox in enumerate(['none']):
# for n_inner_layers, marker in [(10, '*'), (50, 's'), (100, 'h'),
# (300, 'o'), (500, '>')]:
for n_inner_layers, marker in [(50, 's'), (20, '*')]:
methods[f'lpgd_lista_{learn_prox}_{n_inner_layers}'] = {
'label':
f'LPGD - LISTA[{learn_prox}-{n_inner_layers}]',
'network':
ListaTV,
'extra_args':
dict(n_inner_layers=n_inner_layers,
learn_prox=learn_prox,
**learning_parameters),
'style':
dict(color=f'C{i}', marker=marker, linestyle='-')
}
# launch all experiments
print("=" * 80)
t0 = time.time()
results = Parallel(n_jobs=n_jobs)(
delayed(run_one)(x_train,
x_test,
A,
D,
L,
lmbd=lmbd,
key=k,
network=m['network'],
extra_args=m['extra_args'],
all_n_layers=m.get('all_n_layers', all_n_layers),
device=device,
meta=meta_pb) for k, m in methods.items())
# concatenate all results as a big list. Also update style and label
# here to avoid recomputing the results when changing the style only.
log = []
for records in results:
for rec in records:
k = rec['key']
m = methods.get(k, None)
if m is None:
from copy import deepcopy
m = deepcopy(methods[k.replace('per-layer', 'none')])
m['style']['color'] = 'C1'
m['label'] = m['label'].replace('none', 'per-layer')
rec.update(style=m['style'], label=m['label'])
log.append(rec)
# Save the computations in a pickle file
df = pd.DataFrame(log)
t_tag = timestamp.strftime('%Y-%m-%d_%Hh%M')
tag = f'{t_tag}_{lmbd}_{seed}'
df.to_pickle(OUTPUT_DIR / f'{SCRIPT_NAME}_{tag}.pkl')
delta_t = time.strftime("%H h %M min %S s", time.gmtime(time.time() - t0))
print("=" * 80)
print("Script runs in: {}".format(delta_t))
n_samples_testing = n_samples - 1
n_atoms = 40
n_dim = 40
s = 0.1
snr = 0.0
all_n_layers = logspace_layers(n_layers=10, max_depth=args.max_iter)
ticks_layers = np.array([0] + all_n_layers)
seed = args.seed if args.seed is not None else np.random.randint(0, 1000)
rng = check_random_state(seed)
print(f'Seed used = {seed}') # noqa: E999
# Generate data
results = synthetic_1d_dataset(n_atoms=n_atoms,
n_dim=n_dim,
n=n_samples,
s=s,
snr=snr,
seed=seed)
x, _, z, L, D, A = results
x_train = x[n_samples_testing:, :]
x_test = x[:n_samples_testing, :]
###########################################################################
# Main experiment
methods = [
('Synthesis primal PGD', synthesis_iter_algo, 'ista', dict(),
'tab:blue', 's', 'dashed'),
('Synthesis primal APGD', synthesis_iter_algo, 'fista', dict(),
'tab:blue', 's', 'solid'),
('Analysis primal PGD', analysis_primal_iter_algo, 'ista', dict(),
