def test_simple_loss_multiple_args(self): # test that args and kwargs are both handled by optimizers x0 = np.zeros(2), np.zeros(3) ans = np.arange(2.), np.arange(3.) opt = fmin_l_bfgs_b(simple_loss_multiple_args, x0) opt = fmin_l_bfgs_b(simple_loss_multiple_args, init_kwargs=dict(p=x0[0], q=x0[1])) opt = fmin_l_bfgs_b(simple_loss_multiple_args, init_args=x0[0], init_kwargs=dict(q=x0[1])) self.assertTrue(np.allclose(opt[0], ans[0])) self.assertTrue(np.allclose(opt[1], ans[1])) opt = fmin_cg(simple_loss_multiple_args, x0) opt = fmin_cg(simple_loss_multiple_args, init_kwargs=dict(p=x0[0], q=x0[1])) opt = fmin_cg(simple_loss_multiple_args, init_args=x0[0], init_kwargs=dict(q=x0[1])) self.assertTrue(np.allclose(opt[0], ans[0])) self.assertTrue(np.allclose(opt[1], ans[1])) opt = fmin_ncg(simple_loss_multiple_args, x0) opt = fmin_ncg(simple_loss_multiple_args, init_kwargs=dict(p=x0[0], q=x0[1])) opt = fmin_ncg(simple_loss_multiple_args, init_args=x0[0], init_kwargs=dict(q=x0[1])) self.assertTrue(np.allclose(opt[0], ans[0])) self.assertTrue(np.allclose(opt[1], ans[1]))
def fit(self, X, y=None): n_samples, n_dim = X.shape W = np.random.randn(n_dim, self.n_features) b = np.random.randn(self.n_features) obj_fn = self.get_objective_fn(X) self.W_, self.b_ = optimize.fmin_l_bfgs_b(obj_fn, (W, b), iprint=1, maxfun=self.n_iterations) return self
def test_svm(): rng = np.random.RandomState(1) # -- create some fake data x = rng.rand(10, 5) y = 2 * (rng.rand(10) > 0.5) - 1 l2_regularization = 1e-4 # -- loss function def loss_fn(weights, bias): margin = y * (np.dot(x, weights) + bias) loss = np.maximum(0, 1 - margin) ** 2 l2_cost = 0.5 * l2_regularization * np.dot(weights, weights) loss = np.mean(loss) + l2_cost print 'ran loss_fn(), returning', loss return loss # -- call optimizer w_0, b_0 = np.zeros(5), np.zeros(()) w, b = fmin_l_bfgs_b(loss_fn, init_args=(w_0, b_0)) final_loss = loss_fn(w, b) assert np.allclose(final_loss, 0.7229) print 'optimization successful!'
def test_subtensor(self): x0 = np.zeros(2) ans = [-3, 4] opt = fmin_l_bfgs_b(subtensor_loss, x0) self.assertTrue(np.allclose(opt, ans)) opt = fmin_cg(subtensor_loss, x0) self.assertTrue(np.allclose(opt, ans)) opt = fmin_ncg(subtensor_loss, x0) self.assertTrue(np.allclose(opt, ans))
def test_L2(self): x0 = np.zeros((5, 3)) ans = np.array([[+3.0, -1.0, -5.0], [+1.5, -0.5, -2.5], [ +0.0, 0.0, 0.0 ], [-1.5, 0.5, 2.5], [-3.0, 1.0, 5.0]]) / 10.0 opt = fmin_l_bfgs_b(l2_loss, x0) self.assertTrue(np.allclose(opt, ans)) opt = fmin_cg(l2_loss, x0) self.assertTrue(np.allclose(opt, ans)) opt = fmin_ncg(l2_loss, x0) self.assertTrue(np.allclose(opt, ans))
def test_L2(self): x0 = np.zeros((5, 3)) ans = np.array([[+3.0, -1.0, -5.0], [+1.5, -0.5, -2.5], [+0.0, 0.0, 0.0], [-1.5, 0.5, 2.5], [-3.0, 1.0, 5.0]]) / 10.0 opt = fmin_l_bfgs_b(l2_loss, x0) self.assertTrue(np.allclose(opt, ans)) opt = fmin_cg(l2_loss, x0) self.assertTrue(np.allclose(opt, ans)) opt = fmin_ncg(l2_loss, x0) self.assertTrue(np.allclose(opt, ans))
def train_svm(x, y): l2_regularization = 1e-4 print y def loss_fn(weights, bias): margin = y * (np.dot(x, weights) + bias) loss = np.maximum(0, 1 - margin) ** 2 l2_cost = 0.5 * l2_regularization * np.dot(weights, weights) loss = np.mean(loss) + l2_cost return loss n, m = x.shape w0 = np.zeros(m) b0 = np.zeros(()) w, b = fmin_l_bfgs_b(loss_fn, init_args=(w0, b0)) return w, b
def test_svm(self): rng = np.random.RandomState(1) # -- create some fake data x = rng.rand(10, 5) y = 2 * (rng.rand(10) > 0.5) - 1 l2_regularization = 1e-4 def loss_fn(weights, bias): margin = y * (np.dot(x, weights) + bias) loss = np.maximum(0, 1 - margin) ** 2 l2_cost = 0.5 * l2_regularization * np.dot(weights, weights) loss = np.mean(loss) + l2_cost print 'ran loss_fn(), returning', loss return loss w, b = fmin_l_bfgs_b(loss_fn, (np.zeros(5), np.zeros(()))) final_loss = loss_fn(w, b) assert np.allclose(final_loss, 0.7229)