def lasso_test():
"""Runs a sample problem through the lasso function"""
(A, b, l) = dg.data_gen()
(x, hist) = lasso.lasso(A, b, l, 1.0, 1.0)
K = len(hist['objval'])
x = np.arange(K)
# Plot the objective values
fig = plt.figure()
y = hist['objval']
ax = fig.add_subplot(111)
ax.plot(x, y)
plt.xlabel('iter(k)')
plt.ylabel('f(x^k) + g(z^k)')
plt.show()
# Plot norms
fig2 = plt.figure()
bx = fig2.add_subplot(211)
bx.semilogy(x, np.maximum(hist['r_norm'], 1e-8))
bx.semilogy(x, hist['eps_pri'], 'r--')
plt.ylabel('||r||_2')
cx = fig2.add_subplot(212)
cx.semilogy(x, np.maximum(hist['s_norm'], 1e-8))
cx.semilogy(x, hist['eps_dual'], 'r--')
plt.ylabel('||s||_2')
plt.xlabel('iter(k)')
plt.show()
def main():
A = np.random.random((10, 10))
b = np.random.random((10, 1))
lambd = 1.0
rho = 1.0e-6
alpha = 1.5
x = lasso(A, b, lambd, rho, alpha)
print(x)
def aux_mp_grid(A, y, target_support, sparsity_level, beta_min, beta_max,
n_beta, beta_scaling, method, **kwargs):
""" Auxiliary method for mp_lasso_grid and mp_lars_grid since both calls are
almost equal. This is called from both methods to reduce/avoid code
duplication. """
suppress_warning = kwargs.get('suppress_warning', False)
start = timer()
# SVD of A
U, S, V = np.linalg.svd(A.dot(A.T))
max_iter = kwargs.get('max_iter', 2 * sparsity_level + 10)
if beta_scaling == 'linscale':
beta_range = np.linspace(beta_min, beta_max, n_beta)
elif beta_scaling == 'logscale':
beta_range = np.logspace(np.log10(beta_min), np.log10(beta_max),
n_beta)
results = []
coefs = np.zeros((A.shape[1], 0))
for beta in beta_range:
B_beta, y_beta = calc_B_y_beta(A, y, U, S, beta)
if method == 'lar':
# Calculate LAR path
new_coefs, elapsed_time, support = least_angle_regression(
B_beta, y_beta, sparsity_level)
elif method == 'lasso':
# Calculate Lasso path
new_coefs, elapsed_time, support = lasso(B_beta, y_beta,
target_support,
sparsity_level)
else:
raise RuntimeError("Method must be 'lar' or 'lasso'.")
coefs = np.concatenate(
(coefs, np.reshape(new_coefs, (new_coefs.shape[0], 1))), axis=1)
binary_coefs = coefs.astype('bool')
support, index = find_support_minimalSD(binary_coefs, target_support)
coefs = coefs[:, index]
elapsed_time = timer() - start
return coefs, elapsed_time, support
def Lambda_effect(A, b, u_0, erreur, Lambda_min, Lambda_max, Nb_Lambda=10):
'''
Inputs:
- A, b, u_0, erreur
- Lambda_min, Lambda_max
- Nb_Lambda : le nombre de pas
Outputs:
- Non_zero : nombre d'elements non nuls dans la solution optimale
- Solutions : les solutions optimales pour chaque Lambda
'''
step = (Lambda_max - Lambda_min) / Nb_Lambda
Non_zero = [] # Contien le nombre d'element non nulle pour un lambda donné
Solutions = []
for i in range(Nb_Lambda):
Lambda_i = Lambda_min + i * step
res = lasso(A, b, Lambda_i, u_0, erreur)
Solution = res['Sol_opt']
Count_non_zero = np.count_nonzero(Solution)
Non_zero.append(Count_non_zero)
Solutions.append(Solution)
return Non_zero, Solutions
def recover_support(A,
y,
u_real,
method,
sparsity_level,
verbose=True,
**kwargs):
""" Handler method to call the different sparse encoders. Ultimatively uses
encoder specified under method with the given data and recovers the support.
Returns a success flag, the final support, the target support, the elapsed
time and the relative error to the real solution.
Parameters
--------------
A : np.array, shape (n_measurements, n_features)
Sampling matrix
y : np.array, shape (n_measurements)
Vector of measurements.
u_real : np.array, shape (n_features)
Signal that generated the measurements y under sampling of A, ie.
A * u_real = y (+ potential signal/measurements noise).
method : python string
Sparse encoder that should be used. Should be in the
__list_of_encoders__ specified above. promp and romp do not work
currently.
sparsity_level : python Integer
Support size of the generating signal u_real.
verbose : python Boolean
Controls print-outs of this method.
kwargs : python dictionary
Keyword arguments than will be passed to the solver if necessary.
Returns
-----------------
success : True if correct support was recovered, false otherwise
support : Support recovered at the end.
target_support : Support of u_real.
elapsed_time : time spent for calculating the support/solution.
relative_error : relative l2 error between the recovered solution and u_real.
"""
target_support = np.where(u_real)[0]
if method == "lar":
result = least_angle_regression(A, y, sparsity_level, **kwargs)
elif method == "lasso":
result = lasso(A, y, target_support, sparsity_level)
elif method == 'mp_lasso_grid':
result = mp_lasso_grid(A, y, target_support, sparsity_level, **kwargs)
elif method == 'mp_lars_grid':
result = mp_lars_grid(A, y, target_support, sparsity_level, **kwargs)
elif method == 'bp':
result = basis_pursuit(A, y, verbose=verbose, **kwargs)
elif method == "mp":
result = matching_pursuit(A, y, sparsity_level)
elif method == "omp":
result = orthogonal_matching_pursuit(A, y, sparsity_level)
elif method == "iht":
result = iterative_hard_thresholding(A,
y,
sparsity_level,
verbose=verbose)
elif method == "l1iht":
result = l1_iterative_hard_thresholding(A,
y,
sparsity_level,
verbose=verbose)
elif method == "romp":
result = regularized_orthogonal_matching_pursuit(A, y, sparsity_level)
elif method == "cosamp":
result = cosamp(A, y, sparsity_level)
elif method == "sp":
result = subspace_pursuit(A, y, sparsity_level)
elif method == "enet":
result = elastic_net(A, y, target_support, sparsity_level, **kwargs)
elif method in [
"pmp", "pomp", "plar", "plasso", "plar", "piht", "promp",
"pcosamp", "psp", "penet"
]:
# Calculate preconditioned system
start_time = timer()
V_lp, y_new = get_preconditioned_system(A, y)
# Call method again without p in method name
result = recover_support(V_lp,
y_new,
u_real,
method[1:],
sparsity_level,
verbose=verbose,
**kwargs)
elapsed_time = start_time - timer() # Replace time with total time
return result[0], result[1], result[2], elapsed_time, result[4]
else:
raise RuntimeError(
"Encoder not found. Use one of {0}.".format(__list_of_encoders__))
# Postprocess solution
coefs, elapsed_time, support = result
relative_error = calc_relative_error(coefs, u_real)
success = np.array_equal(support, target_support)
if verbose:
print "Finished example with method {0} in {1} seconds.".format(
method, elapsed_time)
print "Recovered: {0} Correct: {1}".format(support, target_support)
print "Success: {0}".format(success)
print "Error: {0}".format(relative_error)
return success, support, target_support, elapsed_time, relative_error
from ridge import ridge from lasso import lasso from elastic import elastic import utilities # Load dataset. normalized = True dataset = utilities.import_CCPP(normalized) fit, _ = my_lslr(dataset, 15000, 0.1) utilities.stats(dataset, fit, 'My LSLR') fit, _ = my_ridge(dataset, 15000, 0.1, 0.1) utilities.stats(dataset, fit, 'My Ridge') fit = lslr(dataset) utilities.stats(dataset, fit, 'Library LSLR') fit = ridge(dataset) utilities.stats(dataset, fit, 'Library Ridge') # Load dataset. normalized = False dataset = utilities.import_CCPP(normalized) fit = elastic(dataset) utilities.stats(dataset, fit, 'Library Elastic') fit = lasso(dataset) utilities.stats(dataset, fit, 'Library Lasso')
scoreMat = scoreMat / np.linalg.norm(scoreMat) # Normalize the scores
#########################
# GET THE MEASURED MATRIX
#########################
y = np.zeros((m, 1)) # This is the matrix we get as measured in real life
for i in range(m):
y[i] = A[i, :].dot(scoreMat)
print("True Local Betweenness Values: ")
print(scoreMat)
print()
#####################
# SOLVE LASSO PROBLEM
#####################
x = lasso(A, y)
print("Reconstructed Local Betweenness Values: ")
print(x)
###############################
#CALCULATE RECONSTRUCTION ERROR
###############################
MSE = np.linalg.norm(scoreMat - x)
print("Relative L2-Norm Error Percentage: ", 100 * MSE, "%")
# print(np.linalg.norm(scoreMat)) # This is ofc 1
#################
#PLOTTING UTILITY
#################
g = igraph.Graph([(e.GetSrcNId(), e.GetDstNId()) for e in graph.Edges()])
def plot_evaluation(my_colors, lib_colors):
# Load dataset.
dataset = utilities.import_CCPP(True)
# Store constant variables.
y_real = np.array(dataset.iloc[:, 4].values)
std_patch = mpatches.Patch(color='darkslategray')
fits = []
losses = []
losses_per_epoch = []
titles = ['LSLR Loss Per Epoch', 'Ridge Loss Per Epoch']
patches = ['Our LSLR Loss', 'Our Ridge Loss']
colors = my_colors
epochs = 15000
# Calculate my regression fits.
fit, loss = my_lslr(dataset, epochs, 0.1)
fits.append(fit)
losses_per_epoch.append(loss)
losses.append(evaluate_fit(dataset, fits[0]))
fit, loss = my_ridge(dataset, epochs, 0.1, 0.1)
fits.append(fit)
losses_per_epoch.append(loss)
losses.append(evaluate_fit(dataset, fits[1]))
# Print stats.
utilities.stats(dataset, fits[0], 'My LSLR')
utilities.stats(dataset, fits[1], 'My Ridge')
# Setup plot.
fig = plt.figure()
ax = fig.add_subplot(111)
x_axis = np.linspace(0, epochs, len(losses_per_epoch[0]), endpoint=True)
ax.plot(x_axis, losses_per_epoch[1], color=colors[1], alpha=0.75)
ax.plot(x_axis, losses_per_epoch[0], color=colors[0], alpha=0.75)
ax.legend(
[mpatches.Patch(color=colors[0]),
mpatches.Patch(color=colors[1])], [patches[0], patches[1]])
ax.set(title='Loss Per Epoch', xlabel='Epoch', ylabel='Loss')
plt.show()
# Plot fit losses.
titles = ['Our LSLR', 'Our Ridge']
x_axis = np.linspace(0,
len(dataset.iloc[:, 4].values),
len(dataset.iloc[:, 4].values),
endpoint=True)
fig, ax = plt.subplots(1, 2)
for i in range(0, len(fits)):
mean = np.mean(losses[i])
print(titles[i] + ' Mean: ' + str(mean))
loss_patch = mpatches.Patch(color=colors[i])
ax[i].semilogy(x_axis, losses[i], color=colors[i])
ax[i].set_xlim(0, len(dataset.iloc[:, 4].values))
ax[i].axhline(y=np.std(y_real)**2,
color='darkslategray',
xmin=0,
xmax=len(dataset.iloc[:, 4].values),
linestyle='-')
ax[i].axhline(y=mean,
color='w',
xmin=0,
xmax=len(dataset.iloc[:, 4].values),
linestyle='-')
ax[i].text(1.05,
mean,
'{:.4f}'.format(mean),
va='center',
ha="left",
bbox=dict(alpha=0),
transform=ax[i].get_yaxis_transform())
ax[i].legend([loss_patch, std_patch],
[patches[i], 'Standard Deviation'])
ax[i].set(title=titles[i], xlabel='Value', ylabel='Logarithmic Loss')
plt.subplots_adjust(wspace=0.3)
plt.show()
# Reset variables.
fits = []
losses = []
titles = ['LSLR', 'Ridge', 'Lasso', 'Elastic Net']
patches = ['LSLR Loss', 'Ridge Loss', 'Lasso Loss', 'Elastic Net Loss']
colors = lib_colors
# Calculate regression fits from libraries.
fits.append(lslr(dataset))
losses.append(evaluate_fit(dataset, fits[0]))
fits.append(ridge(dataset))
losses.append(evaluate_fit(dataset, fits[1]))
fits.append(lasso(dataset))
losses.append(evaluate_fit(dataset, fits[2]))
fits.append(elastic(dataset))
losses.append(evaluate_fit(dataset, fits[3]))
# Print stats.
utilities.stats(dataset, fits[0], 'Library LSLR')
utilities.stats(dataset, fits[1], 'Library Ridge')
utilities.stats(dataset, fits[2], 'Library Lasso')
utilities.stats(dataset, fits[3], 'Library Elastic Net')
# Plot fit losses.
fig, ax = plt.subplots(2, 2)
fig.suptitle('Evaluation of Regression Libraries')
for i in range(0, len(fits)):
mean = np.mean(losses[i])
print(titles[i] + ' Mean: ' + str(mean))
loss_patch = mpatches.Patch(color=colors[i])
ax[int(i / 2) % 2, i % 2].semilogy(x_axis, losses[i], color=colors[i])
ax[int(i / 2) % 2, i % 2].set_xlim(0, len(dataset.iloc[:, 4].values))
ax[int(i / 2) % 2, i % 2].axhline(y=np.std(y_real)**2,
color='darkslategray',
xmin=0,
xmax=len(dataset.iloc[:, 4].values),
linestyle='-')
ax[int(i / 2) % 2, i % 2].axhline(y=mean,
color='w',
xmin=0,
xmax=len(dataset.iloc[:, 4].values),
linestyle='-')
ax[int(i / 2) % 2,
i % 2].text(1.05,
mean,
'{:.4f}'.format(mean),
va='center',
ha="left",
bbox=dict(alpha=0),
transform=ax[int(i / 2) % 2,
i % 2].get_yaxis_transform())
ax[int(i / 2) % 2, i % 2].legend([loss_patch, std_patch],
[patches[i], 'Standard Deviation'])
ax[int(i / 2) % 2, i % 2].set(title=titles[i],
xlabel='Value',
ylabel='Loss (Logarithmic)')
plt.subplots_adjust(wspace=0.3, hspace=0.7)
plt.show()
k = 10
wt = np.zeros((1, d))
wt[:, 0:5] = -10
wt[:, 5:k] = 10
b = 0
mu = 0
sigma = 1
noise = np.random.normal(mu, sigma, (1, n))
X = np.random.normal(0, 1, (d, n))
y = np.dot(wt, X) + b + noise
X = csr_matrix(X)
### plot the precision recall - lambda
l_syn = lasso()
lamb = max_lamb(X, y)
print(lamb)
precision_list = []
recall_list = []
lamb_list = []
for i in range(10):
l_syn.fit(X, y, lamb)
wt_pred, b = l_syn.get_param()
p, r = precision_recall(wt_pred)
precision_list.append(p)
recall_list.append(r)
lamb_list.append(lamb)
lamb = lamb / 2
print(lamb_list)
import time
from analysis import analyze
sys.path.insert(0, './ridge')
from ridge import ridge
sys.path.insert(0, './lasso')
from lasso import lasso
sys.path.insert(0, './bayesian')
from bayesian import bayesian
sys.path.insert(0, './SVR')
from svr import svr
start = time.time()
ridge(sys.argv[1],sys.argv[2])
ridge_end = time.time()
lasso(sys.argv[1],sys.argv[2])
lasso_end = time.time()
bayesian(sys.argv[1],sys.argv[2])
bayesian_end = time.time()
svr(sys.argv[1],sys.argv[2])
svr_end = time.time()
print "Ridge Results"
print "Running Time: "+str(ridge_end-start)+" seconds"
analyze("ridge_out.csv")
print "---"
print "Lasso Results"
print "Running Time: "+str(lasso_end-ridge_end)+" seconds"
analyze("lasso_out.csv")
print "---"
print "Bayesian Results"
#print 'linear regression cvn mse is ',cvn_lr
#Now we will compute cvn for LASSO with a number of lambdas
#if m==1:
if False:
lasso_train_vs_lambda = []
lasso_test_vs_lambda = []
lambdas = np.linspace(0.0000,100.0,11,endpoint=True)
n_test = int(X_aug.shape[0]/5)
X_aug_test = X_aug[:n_test,:]
Y_test = Y[:n_test,:]
X_aug_train = X_aug[n_test:,:]
Y_train = Y[n_test:,:]
print 'number of training data points = ',Y_train.shape[0]
print 'number of testing data points = ',Y_test.shape[0]
lasso_obj = lasso(X_aug_train,Y_train,True,np.ones(Y_train.shape[0]),True,True)
lasso_obj.obtain_guess_for_lambda(lambdas[0])
for l in lambdas:
converged = lasso_obj.train(l,False)
if converged:
b = np.concatenate((lasso_obj.beta,np.array([lasso_obj.beta_0])))
predicted_Y_train = np.dot(np.column_stack((X_aug_train,np.ones(X_aug_train.shape[0]))),b)
predicted_Y_test = np.dot(np.column_stack((X_aug_test,np.ones(X_aug_test.shape[0]))),b)
mse_train = mean_squared_error(predicted_Y_train.reshape((len(predicted_Y_train),1)),Y_train)
mse_test = mean_squared_error(predicted_Y_test.reshape((len(predicted_Y_test),1)),Y_test)
print 'lambda = '+str(l)+' train mse = '+str(mse_train)+' test mse = '+str(mse_test)
lasso_train_vs_lambda.append(mse_train)
lasso_test_vs_lambda.append(mse_test)
print 'converged coefs '
print lasso_obj.beta
print 'converged intercept ',lasso_obj.beta_0
coef_v_lambda = []
test_v_lambda = []
train_v_lambda = []
# In[189]:
#lambdas = np.linspace(0.000,0.02,211,endpoint=True)
lambdas = np.arange(0.0,0.02,0.0001)
print lambdas
# In[190]:
lasso_obj = lasso(X,y,False,W,True,True)
# In[191]:
lasso_obj.obtain_guess_for_lambda(lambdas[0])
# In[192]:
for l in lambdas:
converged = lasso_obj.train(l,False)
if converged:
y_pred = np.dot(X,lasso_obj.beta)
err = y-y_pred
train_rms = math.sqrt(np.dot(err,np.dot(np.diag(W),err))/float(1072))
import os
import sys
import time
import cPickle
# Scientific computing
import numpy as np
import scipy.linalg as lin
import scipy.sparse as sps
# Plotting
import matplotlib.pyplot as plt
sys.path.append('../optimization')
# Custom modules import.
import lasso
if __name__ == '__main__':
# Test lasso.
m = 10
n = 100
x = sps.random(n, 1)
A = np.random.randn(m, n)
b = (x.T.dot(A.T)).T
l = 1
niters = 20
debug = True
xhat = lasso.lasso(A, b, l, niters, debug)
v_0 = u_0
# constante de régularisation
Lambda = 0.1
# Input pour l'impact de Lambda
Lambda_min = 0
Lambda_max = 20
Nb_Lambda = 10
'''
RESOLUTION DU PROBLEME
'''
if (Nom_algo == 'LASSO'):
Resultats = lasso(A, b, Lambda, u_0, erreur)
if (Nom_algo == 'FISTA'):
Resultats = lasso_accelerated(A, b, Lambda, u_0, v_0, erreur)
#Plot resultats
Erreur_histo = Resultats['Err_hist']
Solution = Resultats['Sol_opt']
Nb_iteration = Resultats['nb_iterations']
print("Solution trouvée :", Solution)
print("Nb_iteration :", Nb_iteration)
plt.figure(figsize=(15, 7))
plt.plot(Erreur_histo)
plt.title(" Evolution de l'erreur ")
plt.xlabel('Iterations')