def test_sparse_regression():
"""
Test sparse regression
"""
A, y, x_true = generate_sparse_system()
# least squares solution
xls = np.linalg.lstsq(A, y)[0]
# helper function to test relative error
def test_error(xhat):
test_err = np.linalg.norm(xhat - x_true, 2)
naive_err = np.linalg.norm(xls - x_true, 2)
err_ratio = test_err / naive_err
assert err_ratio <= 0.01
# Proximal gradient descent and Accelerated proximal gradient descent
for algorithm in ['sgd', 'nag']:
# objective
def f_df(x):
err = A.dot(x) - y
obj = 0.5 * np.linalg.norm(err) ** 2
grad = A.T.dot(err)
return obj, grad
# optimizer
opt = GradientDescent(xls, f_df, algorithm, {'lr': 5e-3}, proxop=sparse(10.0), rho=1.0)
opt.run(maxiter=5000)
# test
test_error(opt.theta)
def train(self, l, method='normal'):
k = 10
kfolder = KFolder(self.D, k, normalize=True)
self.X_train, self.Y_train = [], []
self.X_test, self.Y_test, self.W = [], [], []
for i in range(k):
# Get data and labels at fold k
X,Y = kfolder.training(i)
# Solve for the vector of linear factors, W
if method == 'normal':
rsolver = RegressionSolver(X, Y)
Wi = rsolver.solve(l)
elif method == 'descent':
gd = GradientDescent(X, Y)
Wi = gd.linreg_stoch2()
elif method == 'logistic':
gd = GradientDescent(X, Y)
Wi = gd.logreg_stoch()
# Get the testing data
Xi,Yi = kfolder.testing(i)
# Store the results
self.X_train.append(X), self.Y_train.append(Y)
self.X_test.append(Xi), self.Y_test.append(Yi), self.W.append(Wi)
def train(self, l, method='normal'):
if method == 'normal':
rsolver = RegressionSolver(self.X_train, self.Y_train)
self.W = rsolver.solve(l)
elif method == 'descent':
gd = GradientDescent(self.X_train, self.Y_train)
self.W = gd.linreg_stoch1(r=1e-6)
def test_lowrank_matrix_approx():
"""
Test low rank matrix approximation
"""
Xobs, Xtrue = generate_lowrank_matrix()
# helper function to test relative error of given parameters
def test_error(Xhat):
test_err = np.linalg.norm(Xhat - Xtrue, 'fro')
naive_err = np.linalg.norm(Xobs - Xtrue, 'fro')
err_ratio = test_err / naive_err
assert err_ratio <= 0.5
# Proximal gradient descent and Accelerated proximal gradient descent
for algorithm in ['sgd', 'nag']:
# objective
def f_df(X):
grad = X - Xtrue
obj = 0.5 * np.linalg.norm(grad.ravel()) ** 2
return obj, grad
# optimizer
opt = GradientDescent(Xobs, f_df, algorithm, {'lr': 5e-3}, proxop=nucnorm(0.2), rho=1.0)
opt.callbacks = []
opt.run(maxiter=5000)
# test
test_error(opt.theta)
def calculate():
f = request.args.get('f')
start_point = request.args.get('start_point')
eps = request.args.get('eps')
option = request.args.get('option')
if start_point is not None:
start_point = [float(x) for x in start_point.split()]
if eps is not None:
eps = float(eps)
find_min = True
if option == 'max':
find_min = False
descent = GradientDescent(f, start_point, eps, find_min)
length = Diff.length(descent.get_start_point())
diff_by_direction = descent.calculate_diff_by_direction(length)
mess = 'not monotonically'
if Diff.is_increasing(diff_by_direction):
mess = 'monotonically increasing'
if Diff.is_decreasing(diff_by_direction):
mess = 'monotonically decreasing'
res = descent.calculate()
report = descent.get_report()
return render_template('index.html',
descent=descent,
point=' '.join(map(str, descent.get_start_point())),
length=length,
diff_by_direction=diff_by_direction,
mess=mess,
report=report,
res='; '.join(map(str, res)))
def test_quadratic_bowl():
"""
Test optimization in a quadratic bowl
"""
t = np.linspace(0, 2*np.pi, 100)
tol = 1e-3
theta_true = [np.sin(t), np.cos(t)]
theta_init = [np.cos(t), np.sin(t)]
def f_df(theta):
obj = 0.5*(theta[0]-theta_true[0])**2 + 0.5*(theta[1]-theta_true[1])**2
grad = [theta[0]-theta_true[0], theta[1]-theta_true[1]]
return np.sum(obj), grad
opt = GradientDescent(theta_init, f_df, sgd, {'lr': 1e-2})
opt.run(maxiter=1e3)
for theta in zip(opt.theta, theta_true):
assert np.linalg.norm(theta[0] - theta[1]) <= tol
def test_rosen(tol=1e-2):
"""Test minimization of the rosenbrock function"""
# check that the gradient is zeros at the optimal point
xstar = np.array([1, 1])
assert np.all(rosenbrock(xstar)[1] == 0)
# list of algorithms to test (and their parameters)
algorithms = [('sgd', {'lr': 1e-3, 'momentum': 0.1}),
('nag', {'lr': 1e-3}),
('rmsprop', {'lr': 1e-3}),
('adam', {'lr': 1e-3}),
('sag', {'nterms': 2, 'lr': 2e-3})]
# loop over algorithms
for algorithm, options in algorithms:
# initialize
opt = GradientDescent(np.zeros(2), rosenbrock, algorithm, options)
# run the optimization algorithm
opt.run(maxiter=1e4)
assert np.linalg.norm(opt.theta - xstar) <= tol
from polluted import PollutedSpambase
from evaluator import Evaluator
from descent import GradientDescent
if __name__=="__main__":
# Get data
dataset = PollutedSpambase()
train_data, train_labels = dataset.training()
test_data, test_labels = dataset.testing()
# Do Logistic Regression
gd = GradientDescent(train_data, train_labels)
# 200,000 iterations gives ~85% acc
W = gd.logreg_stoch(it=200001)
# Evaluate solution
evaluator = Evaluator([test_data], [test_labels], [W])
evaluator.accuracy()
from polluted import PollutedSpambase
from evaluator import Evaluator
from descent import GradientDescent
import sys
import argparse
if __name__=="__main__":
# Parse cmdline arguments
parser = argparse.ArgumentParser()
parser.add_argument('-m', help='Method of regularization. {lasso, ridge}')
args = parser.parse_args(sys.argv[1:])
# Get data
dataset = PollutedSpambase()
train_data, train_labels = dataset.training()
test_data, test_labels = dataset.testing()
# Do ridge regularization
gd = GradientDescent(train_data, train_labels)
W = gd.logreg_ridge_stoch(it=5000001)
# Evaluate solution
evaluator = Evaluator([test_data], [test_labels], [W])
evaluator.accuracy()