def train(self, X, y, sigma=None, lam=None): # Set sigma and lambda if sigma is None: sigma = self.sigma if lam is None: lam = self.lam # Initialize kernel self.kernel = kernels.GaussianKernel(sigma=sigma) # Compute indices of nystrom centers indices = self.compute_indices_selection(y) center_selector = MyCenterSelector(indices) opt = FalkonOptions(min_cuda_iter_size_32=0, min_cuda_iter_size_64=0, keops_active="no", min_cuda_pc_size_32=0, min_cuda_pc_size_64=0) # Initialize FALKON model self.model = InCoreFalkon(kernel=self.kernel, penalty=lam, M=len(indices), center_selection=center_selector, options=opt) # Train FALKON model if self.model is not None: self.model.fit(X, y) else: print('Model is None in trainRegionClassifier function') sys.exit(0) return copy.deepcopy(self.model)
def test_classif(self, cls_data): X, Y = cls_data Xc = X.cuda() Yc = Y.cuda() kernel = kernels.GaussianKernel(2.0) torch.manual_seed(13) np.random.seed(13) def error_fn(t, p): return 100 * torch.sum(t * p <= 0).to( torch.float32) / t.shape[0], "c-err" opt = FalkonOptions(use_cpu=False, keops_active="no", debug=True) M = 500 flkc = InCoreFalkon(kernel=kernel, penalty=1e-6, M=M, seed=10, options=opt, maxiter=20, error_fn=error_fn) flkc.fit(Xc, Yc) cpreds = flkc.predict(Xc) assert cpreds.device == Xc.device err = error_fn(cpreds, Yc)[0] assert err < 5
def test_cuda_predict(self, reg_data): Xtr, Ytr, Xts, Yts = reg_data kernel = kernels.GaussianKernel(20.0) def error_fn(t, p): return torch.sqrt(torch.mean((t - p)**2)), "RMSE" opt = FalkonOptions(use_cpu=False, keops_active="no", debug=True, min_cuda_pc_size_64=1, min_cuda_iter_size_64=1) flk = Falkon(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt, error_fn=error_fn) flk.fit(Xtr, Ytr, Xts=Xts, Yts=Yts) flk.to("cuda:0") cuda_ts_preds = flk.predict(Xts.to("cuda:0")) cuda_tr_preds = flk.predict(Xtr.to("cuda:0")) assert cuda_ts_preds.device.type == "cuda" assert cuda_ts_preds.shape == (Yts.shape[0], 1) ts_err = error_fn(cuda_ts_preds.cpu(), Yts)[0] tr_err = error_fn(cuda_tr_preds.cpu(), Ytr)[0] assert tr_err < ts_err assert ts_err < 2.5
def test_compare_cuda_cpu(self, reg_data): Xtr, Ytr, Xts, Yts = reg_data kernel = kernels.GaussianKernel(20.0) def error_fn(t, p): return torch.sqrt(torch.mean((t - p)**2)).item(), "RMSE" opt_cpu = FalkonOptions(use_cpu=True, keops_active="no", debug=True) flk_cpu = Falkon(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt_cpu, maxiter=10, error_fn=error_fn) flk_cpu.fit(Xtr, Ytr, Xts=Xts, Yts=Yts) opt_gpu = FalkonOptions(use_cpu=False, keops_active="no", debug=True) flk_gpu = Falkon(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt_gpu, maxiter=10, error_fn=error_fn) flk_gpu.fit(Xtr, Ytr, Xts=Xts, Yts=Yts) np.testing.assert_allclose(flk_cpu.alpha_.numpy(), flk_gpu.alpha_.numpy())
def test_logistic_loss_derivative(): kernel = kernels.GaussianKernel(3) pred = torch.linspace(-10, 10, 10, dtype=torch.float64).requires_grad_() true = torch.bernoulli(torch.ones_like(pred, dtype=torch.float64) / 2) * 2 - 1 # -1, +1 random values log_loss = LogisticLoss(kernel) derivative_test(naive_logistic_loss, log_loss, pred, true)
def test_no_opt(self): kernel = kernels.GaussianKernel(2.0) Falkon( kernel=kernel, penalty=1e-6, M=500, center_selection='uniform', options=None, )
def test_classif(self, cls_data, cuda_usage): X, Y = cls_data if cuda_usage == "incore": X, Y = X.cuda(), Y.cuda() flk_cls = InCoreFalkon else: flk_cls = Falkon kernel = kernels.GaussianKernel(2.0) def error_fn(t, p): return 100 * torch.sum(t * p <= 0).to( torch.float32) / t.shape[0], "c-err" def weight_fn(y): weight = torch.empty_like(y) weight[y == 1] = 1 weight[y == -1] = 2 return weight opt = FalkonOptions(use_cpu=cuda_usage == "cpu_only", keops_active="no", debug=False) flk_weight = flk_cls(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt, error_fn=error_fn, weight_fn=weight_fn) flk_weight.fit(X, Y) preds_weight = flk_weight.predict(X) preds_weight_m1 = preds_weight[Y == -1] preds_weight_p1 = preds_weight[Y == 1] err_weight_m1 = error_fn(preds_weight_m1, Y[Y == -1])[0] err_weight_p1 = error_fn(preds_weight_p1, Y[Y == 1])[0] flk = flk_cls(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt, error_fn=error_fn, weight_fn=None) flk.fit(X, Y) preds = flk.predict(X) preds_m1 = preds[Y == -1] preds_p1 = preds[Y == 1] err_m1 = error_fn(preds_m1, Y[Y == -1])[0] err_p1 = error_fn(preds_p1, Y[Y == 1])[0] print( "Weighted errors: -1 (%f) +1 (%f) -- Normal errors: -1 (%f) +1 (%f)" % (err_weight_m1, err_weight_p1, err_m1, err_p1)) assert err_weight_m1 < err_m1, "Error of weighted class is higher than without weighting" assert err_weight_p1 >= err_p1, "Error of unweighted class is lower than in flk with no weights"
def test_weighted_bce_derivative(): kernel = kernels.GaussianKernel(3) pred = torch.linspace(-10, 10, 10, dtype=torch.float64).requires_grad_() true = torch.bernoulli(torch.ones_like(pred, dtype=torch.float64) / 2) # 0, +1 random values neg_weight = 0.5 wbce_loss = WeightedCrossEntropyLoss(kernel, neg_weight=neg_weight) derivative_test(lambda t, p: naive_bce(t, p, neg_weight), wbce_loss, pred, true)
def run(exp_num, dset, show_intermediate_errors: bool = False): opt = falkon.FalkonOptions( lauum_par_blk_multiplier=16, debug=True, pc_epsilon_32=1e-6, pc_epsilon_64=1e-13, compute_arch_speed=False) params = { 'seed': 12, 'kernel': kernels.GaussianKernel(3.8), 'penalty': 1e-7, 'M': 100_000, 'maxiter': 10, }
def test_fails_cpu_tensors(self, cls_data): X, Y = cls_data kernel = kernels.GaussianKernel(2.0) opt = FalkonOptions(use_cpu=False, keops_active="no", debug=True) flk = InCoreFalkon(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt) with pytest.raises(ValueError): flk.fit(X, Y) flk.fit(X.cuda(), Y.cuda()) with pytest.raises(ValueError): flk.predict(X)
def test_simple(self, data): X, Y = data kernel = kernels.GaussianKernel(3.0) loss = LogisticLoss(kernel=kernel) def error_fn(t, p): return float(100 * torch.sum(t * p <= 0)) / t.shape[0], "c-err" opt = FalkonOptions(use_cpu=True, keops_active="no", debug=True) logflk = LogisticFalkon( kernel=kernel, loss=loss, penalty_list=[1e-1, 1e-3, 1e-5, 1e-8, 1e-8, 1e-8, 1e-8, 1e-8], iter_list=[3, 3, 3, 3, 8, 8, 8, 8], M=500, seed=10, options=opt, error_fn=error_fn) logflk.fit(X, Y) preds = logflk.predict(X) err = error_fn(preds, Y)[0] assert err < 0.1
def test_classif(self, cls_data): X, Y = cls_data kernel = kernels.GaussianKernel(2.0) def error_fn(t, p): return 100 * torch.sum(t * p <= 0).to( torch.float32) / t.shape[0], "c-err" opt = FalkonOptions(use_cpu=True, keops_active="no", debug=True) flk = Falkon(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt, error_fn=error_fn) flk.fit(X, Y) preds = flk.predict(X) err = error_fn(preds, Y)[0] assert err < 5
def test_regression(self, reg_data): Xtr, Ytr, Xts, Yts = reg_data kernel = kernels.GaussianKernel(20.0) def error_fn(t, p): return torch.sqrt(torch.mean((t - p)**2)).item(), "RMSE" opt = FalkonOptions(use_cpu=True, keops_active="no", debug=True) flk = Falkon(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt, maxiter=10) flk.fit(Xtr, Ytr, Xts=Xts, Yts=Yts) assert flk.predict(Xts).shape == (Yts.shape[0], 1) ts_err = error_fn(flk.predict(Xts), Yts)[0] tr_err = error_fn(flk.predict(Xtr), Ytr)[0] assert tr_err < ts_err assert ts_err < 2.5
def test_multiclass(self, multicls_data): X, Y = multicls_data kernel = kernels.GaussianKernel(10.0) def error_fn(t, p): t = torch.argmax(t, dim=1) p = torch.argmax(p, dim=1) return torch.mean((t.reshape(-1, ) != p.reshape(-1, )).to( torch.float64)), "multic-err" opt = FalkonOptions(use_cpu=True, keops_active="no", debug=True) flk = Falkon(kernel=kernel, penalty=1e-6, M=500, seed=10, options=opt, error_fn=error_fn) flk.fit(X, Y) preds = flk.predict(X) err = error_fn(preds, Y)[0] assert err < 0.23
def test_multiple_falkons(self): num_processes = 6 num_rep = 5 max_iter = 20 num_centers = 1000 num_gpus = torch.cuda.device_count() X, Y = datasets.make_regression(50000, 2048, random_state=11) X = torch.from_numpy(X.astype(np.float32)) Y = torch.from_numpy(Y.astype(np.float32).reshape(-1, 1)) with tempfile.TemporaryDirectory() as folder: X_file = os.path.join(folder, "data_x.pkl") Y_file = os.path.join(folder, "data_y.pkl") out_files = [ os.path.join(folder, f"output_{i}.pkl") for i in range(num_processes) ] # Write data to file with open(X_file, 'wb') as fh: pickle.dump(X, fh) with open(Y_file, 'wb') as fh: pickle.dump(Y, fh) threads = [] for i in range(num_processes): t = threading.Thread(target=_runner_str, kwargs={ 'fname_X': X_file, 'fname_Y': Y_file, 'fname_out': out_files[i], 'num_rep': num_rep, 'max_iter': max_iter, 'num_centers': num_centers, 'gpu_num': i % num_gpus, }, daemon=False) threads.append(t) for t in threads: t.start() for t in threads: t.join() # Load outputs actual = [] for of in out_files: with open(of, 'rb') as fh: actual.append(pickle.load(fh)) # Expected result kernel = kernels.GaussianKernel(20.0) X, Y = X.cuda(), Y.cuda() opt = FalkonOptions(use_cpu=False, keops_active="no", debug=False, never_store_kernel=True, max_gpu_mem=1 * 2**30, cg_full_gradient_every=2) center_sel = FixedSelector(X[:num_centers]) flk = InCoreFalkon(kernel=kernel, penalty=1e-6, M=num_centers, seed=10, options=opt, maxiter=max_iter, center_selection=center_sel) flk.fit(X, Y) expected = flk.predict(X).cpu() # Compare actual vs expected wrong = 0 for i in range(len(actual)): for j in range(len(actual[i])): try: np.testing.assert_allclose(expected.numpy(), actual[i][j].numpy(), rtol=1e-7) except AssertionError: print(f"Result {j} from process {i} is incorrect") wrong += 1 assert wrong == 0, "%d results were not equal" % (wrong)