def test_sparse_regression(): """ Test sparse regression """ A, y, x_true = generate_sparse_system() # least squares solution xls = np.linalg.lstsq(A, y)[0] # helper function to test relative error def test_error(xhat): test_err = np.linalg.norm(xhat - x_true, 2) naive_err = np.linalg.norm(xls - x_true, 2) err_ratio = test_err / naive_err assert err_ratio <= 0.01 # Proximal gradient descent and Accelerated proximal gradient descent for algorithm in ['sgd', 'nag']: # objective def f_df(x): err = A.dot(x) - y obj = 0.5 * np.linalg.norm(err) ** 2 grad = A.T.dot(err) return obj, grad # optimizer opt = GradientDescent(xls, f_df, algorithm, {'lr': 5e-3}, proxop=sparse(10.0), rho=1.0) opt.run(maxiter=5000) # test test_error(opt.theta)
def train(self, l, method='normal'): k = 10 kfolder = KFolder(self.D, k, normalize=True) self.X_train, self.Y_train = [], [] self.X_test, self.Y_test, self.W = [], [], [] for i in range(k): # Get data and labels at fold k X,Y = kfolder.training(i) # Solve for the vector of linear factors, W if method == 'normal': rsolver = RegressionSolver(X, Y) Wi = rsolver.solve(l) elif method == 'descent': gd = GradientDescent(X, Y) Wi = gd.linreg_stoch2() elif method == 'logistic': gd = GradientDescent(X, Y) Wi = gd.logreg_stoch() # Get the testing data Xi,Yi = kfolder.testing(i) # Store the results self.X_train.append(X), self.Y_train.append(Y) self.X_test.append(Xi), self.Y_test.append(Yi), self.W.append(Wi)
def train(self, l, method='normal'): if method == 'normal': rsolver = RegressionSolver(self.X_train, self.Y_train) self.W = rsolver.solve(l) elif method == 'descent': gd = GradientDescent(self.X_train, self.Y_train) self.W = gd.linreg_stoch1(r=1e-6)
def test_lowrank_matrix_approx(): """ Test low rank matrix approximation """ Xobs, Xtrue = generate_lowrank_matrix() # helper function to test relative error of given parameters def test_error(Xhat): test_err = np.linalg.norm(Xhat - Xtrue, 'fro') naive_err = np.linalg.norm(Xobs - Xtrue, 'fro') err_ratio = test_err / naive_err assert err_ratio <= 0.5 # Proximal gradient descent and Accelerated proximal gradient descent for algorithm in ['sgd', 'nag']: # objective def f_df(X): grad = X - Xtrue obj = 0.5 * np.linalg.norm(grad.ravel()) ** 2 return obj, grad # optimizer opt = GradientDescent(Xobs, f_df, algorithm, {'lr': 5e-3}, proxop=nucnorm(0.2), rho=1.0) opt.callbacks = [] opt.run(maxiter=5000) # test test_error(opt.theta)
def calculate(): f = request.args.get('f') start_point = request.args.get('start_point') eps = request.args.get('eps') option = request.args.get('option') if start_point is not None: start_point = [float(x) for x in start_point.split()] if eps is not None: eps = float(eps) find_min = True if option == 'max': find_min = False descent = GradientDescent(f, start_point, eps, find_min) length = Diff.length(descent.get_start_point()) diff_by_direction = descent.calculate_diff_by_direction(length) mess = 'not monotonically' if Diff.is_increasing(diff_by_direction): mess = 'monotonically increasing' if Diff.is_decreasing(diff_by_direction): mess = 'monotonically decreasing' res = descent.calculate() report = descent.get_report() return render_template('index.html', descent=descent, point=' '.join(map(str, descent.get_start_point())), length=length, diff_by_direction=diff_by_direction, mess=mess, report=report, res='; '.join(map(str, res)))
def test_quadratic_bowl(): """ Test optimization in a quadratic bowl """ t = np.linspace(0, 2*np.pi, 100) tol = 1e-3 theta_true = [np.sin(t), np.cos(t)] theta_init = [np.cos(t), np.sin(t)] def f_df(theta): obj = 0.5*(theta[0]-theta_true[0])**2 + 0.5*(theta[1]-theta_true[1])**2 grad = [theta[0]-theta_true[0], theta[1]-theta_true[1]] return np.sum(obj), grad opt = GradientDescent(theta_init, f_df, sgd, {'lr': 1e-2}) opt.run(maxiter=1e3) for theta in zip(opt.theta, theta_true): assert np.linalg.norm(theta[0] - theta[1]) <= tol
def test_rosen(tol=1e-2): """Test minimization of the rosenbrock function""" # check that the gradient is zeros at the optimal point xstar = np.array([1, 1]) assert np.all(rosenbrock(xstar)[1] == 0) # list of algorithms to test (and their parameters) algorithms = [('sgd', {'lr': 1e-3, 'momentum': 0.1}), ('nag', {'lr': 1e-3}), ('rmsprop', {'lr': 1e-3}), ('adam', {'lr': 1e-3}), ('sag', {'nterms': 2, 'lr': 2e-3})] # loop over algorithms for algorithm, options in algorithms: # initialize opt = GradientDescent(np.zeros(2), rosenbrock, algorithm, options) # run the optimization algorithm opt.run(maxiter=1e4) assert np.linalg.norm(opt.theta - xstar) <= tol
from polluted import PollutedSpambase from evaluator import Evaluator from descent import GradientDescent if __name__=="__main__": # Get data dataset = PollutedSpambase() train_data, train_labels = dataset.training() test_data, test_labels = dataset.testing() # Do Logistic Regression gd = GradientDescent(train_data, train_labels) # 200,000 iterations gives ~85% acc W = gd.logreg_stoch(it=200001) # Evaluate solution evaluator = Evaluator([test_data], [test_labels], [W]) evaluator.accuracy()
from polluted import PollutedSpambase from evaluator import Evaluator from descent import GradientDescent import sys import argparse if __name__=="__main__": # Parse cmdline arguments parser = argparse.ArgumentParser() parser.add_argument('-m', help='Method of regularization. {lasso, ridge}') args = parser.parse_args(sys.argv[1:]) # Get data dataset = PollutedSpambase() train_data, train_labels = dataset.training() test_data, test_labels = dataset.testing() # Do ridge regularization gd = GradientDescent(train_data, train_labels) W = gd.logreg_ridge_stoch(it=5000001) # Evaluate solution evaluator = Evaluator([test_data], [test_labels], [W]) evaluator.accuracy()