def __init__(self, numComponents, data, num_iter=100, step_fn = lambda x : 9e-4/(x+1)): Y = np.random.randn(data.shape[1], numComponents) def pca_grad(w, x): x = x[np.newaxis].T return -x.dot(x.T).dot(w) def pca_objective(w, x): x = x[np.newaxis].T return np.linalg.norm(w.dot(x.dot(x.T)).dot(w.T)) loss= [] def iteration_logger(sgd): current_objective = objective(sgd.w.T) loss.append(current_objective) self.pca_grad = pca_grad self.pca_objective = pca_objective pca_loss = EmpiricalLossFn(pca_objective, pca_grad) objective = pca_loss.partial_eval_objective_full(data) sgd = AlectonStochasticGradientMethod(Y, step_fn, iteration_logger) gradients = [(pca_loss.partial_eval_gradient_streaming(x)) for x in data] sgd.train(gradients) sgd.train(gradients) sgd.train(gradients) self.loss = loss self.pca_mat = sgd.w.T
def pca_loss_test(): #This is nonconvex!!! digits = sklearn.datasets.load_digits(2) X = digits.data w_star = np.linalg.svd(X)[-1][0] def pca_grad(w, x): x = x[np.newaxis].T return -x.dot(x.T).dot(w) def pca_objective(w, x): x = x[np.newaxis].T return -w.dot(x.dot(x.T)).dot(w) pca_loss = EmpiricalLossFn(pca_objective, pca_grad) objective = pca_loss.partial_eval_objective_full(X) gradients = [pca_loss.partial_eval_gradient_streaming(x) for x in X] shuffled_gradients = np.random.permutation(gradients) w_0 = np.random.rand(*X[0].shape) w_0 = w_0 / np.linalg.norm(w_0) sgd = ShamirStochasticGradientMethod(w_0, lambda x: 1e-3 / (x + 1)) sgd.train(np.random.permutation(gradients), False) sgd.train(np.random.permutation(gradients), False) assert np.linalg.norm(sgd.w - w_star) < 0.01
def hinge_loss_test(): def objective(w, x): y = x[-1] x_ = x[:-1] return max(0, 1 - x_.dot(w) * y) def gradient_fn(w, x): y = x[-1] x_ = x[:-1] if (y * w.dot(x_) < 1): return -x_ * y else: return np.zeros(x_.shape) hinge_loss = EmpiricalLossFn(objective, gradient_fn) digits = sklearn.datasets.load_digits(2) data = digits.data labels = (digits.target * 2) - 1 X = np.hstack((data, labels[np.newaxis].T)) objective = hinge_loss.partial_eval_objective_full(X) gradients = [hinge_loss.partial_eval_gradient_streaming(x) for x in X] sgd = BasicStochasticGradientMethod(np.zeros(data[0].shape), lambda x: 1) # Do 3 passes JIC sgd.train(gradients) sgd.train(gradients) sgd.train(gradients) assert objective(sgd.w) == 0 assert sum(np.sign(data.dot(sgd.w)) == labels) == 360
def shamirPCA(): digits = sklearn.datasets.load_digits(2) X = digits.data w_star = np.linalg.svd(X)[-1][0] def pca_grad(w, x): x = x[np.newaxis].T return -x.dot(x.T).dot(w) def pca_objective(w, x): x = x[np.newaxis].T return -w.dot(x.dot(x.T)).dot(w) pca_loss = EmpiricalLossFn(pca_objective, pca_grad) objective = pca_loss.partial_eval_objective_full(X) gradients = [pca_loss.partial_eval_gradient_streaming(x) for x in X] shuffled_gradients = np.random.permutation(gradients) w_0 = np.random.rand(*X[0].shape) w_0 = w_0 / np.linalg.norm(w_0) loss_per_iter = [] power_loss_per_iter = [] def iteration_logger(sgd): print "Sgd Iteration: {0}, Loss: {1}".format(sgd._train_iter, objective(sgd.w)) loss_per_iter.append(objective(sgd.w)) sgd = ShamirStochasticGradientMethod(w_0, lambda x: 1e-3 / (x + 1), iteration_logger) sgd.train(np.random.permutation(gradients)) for i in range(len(loss_per_iter)): w_p = X.T.dot(np.random.rand(*X[:, 0].shape)) w_p = w_p / np.linalg.norm(w_p) print "Power Iteration: {0}, Loss: {1}".format(i, objective(w_p)) power_loss_per_iter.append(objective(w_p)) print power_loss_per_iter plt.figure() plt.plot(zip(loss_per_iter, power_loss_per_iter)) plt.show()
def __init__(self, numComponents, data, step_fn = lambda x: 1e-3/(x+1)): w_0 = np.random.rand(*data[0].shape) w_0 = w_0/np.linalg.norm(w_0) self.step_fn = step_fn sub = 0*w_0 pc = [] losses = [] for i in range(numComponents): def pca_grad(w, x): x = x[np.newaxis].T - sub.T return -x.dot(x.T).dot(w) def pca_objective(w, x): x = x[np.newaxis].T - sub.T return -w.dot(x.dot(x.T)).dot(w) w_star = np.linalg.svd(data)[-1][0] self.pca_grad = pca_grad self.pca_objective = pca_objective pca_loss = EmpiricalLossFn(pca_objective, pca_grad) objective = pca_loss.partial_eval_objective_full(data) target_obj = objective(w_star) loss_per_iter = [] def iteration_logger(sgd): current_objective = objective(sgd.w) loss_per_iter.append(objective(sgd.w)) sgd = ShamirStochasticGradientMethod(w_0, self.step_fn) gradients = [(pca_loss.partial_eval_gradient_streaming(x)) for x in data] sgd.train(gradients) pc.append(sgd.w) losses.append(loss_per_iter) self.pca_mat = np.array(pc) self.losses = losses print self.pca_mat.shape