def test_comparison_with_scikit(self): import warnings warnings.filterwarnings("ignore") from lale.lib.sklearn import PCA import sklearn.datasets from lale.helpers import cross_val_score pca = PCA(n_components=3, random_state=42, svd_solver='arpack') nys = Nystroem(n_components=10, random_state=42) concat = ConcatFeatures() lr = LogisticRegression(random_state=42, C=0.1) trainable = (pca & nys) >> concat >> lr digits = sklearn.datasets.load_digits() X, y = sklearn.utils.shuffle(digits.data, digits.target, random_state=42) cv_results = cross_val_score(trainable, X, y) cv_results = ['{0:.1%}'.format(score) for score in cv_results] from sklearn.pipeline import make_pipeline, FeatureUnion from sklearn.decomposition import PCA as SklearnPCA from sklearn.kernel_approximation import Nystroem as SklearnNystroem from sklearn.linear_model import LogisticRegression as SklearnLR from sklearn.model_selection import cross_val_score union = FeatureUnion([("pca", SklearnPCA(n_components=3, random_state=42, svd_solver='arpack')), ("nys", SklearnNystroem(n_components=10, random_state=42))]) lr = SklearnLR(random_state=42, C=0.1) pipeline = make_pipeline(union, lr) scikit_cv_results = cross_val_score(pipeline, X, y, cv = 5) scikit_cv_results = ['{0:.1%}'.format(score) for score in scikit_cv_results] self.assertEqual(cv_results, scikit_cv_results) warnings.resetwarnings()
def test_invalid_input(self): from sklearn.linear_model import LogisticRegression as SklearnLR scikit_lr = SklearnLR() from lale.helpers import to_graphviz with self.assertRaises(TypeError): to_graphviz(scikit_lr)
def dont_test_with_gridsearchcv2_auto(self): from sklearn.datasets import load_iris from sklearn.metrics import accuracy_score, make_scorer from sklearn.model_selection import GridSearchCV lr = LogisticRegression(random_state=42) pca = PCA(random_state=42, svd_solver="arpack") trainable = pca >> lr from sklearn.pipeline import Pipeline scikit_pipeline = Pipeline([ (pca.name(), PCA(random_state=42, svd_solver="arpack")), (lr.name(), LogisticRegression(random_state=42)), ]) all_parameters = get_grid_search_parameter_grids(trainable, num_samples=1) # otherwise the test takes too long parameters = random.sample(all_parameters, 2) with warnings.catch_warnings(): warnings.simplefilter("ignore") clf = GridSearchCV(scikit_pipeline, parameters, cv=2, scoring=make_scorer(accuracy_score)) iris = load_iris() clf.fit(iris.data, iris.target) predicted = clf.predict(iris.data) accuracy_with_lale_operators = accuracy_score( iris.target, predicted) from sklearn.decomposition import PCA as SklearnPCA from sklearn.linear_model import LogisticRegression as SklearnLR from sklearn.pipeline import Pipeline scikit_pipeline = Pipeline([ (pca.name(), SklearnPCA(random_state=42, svd_solver="arpack")), (lr.name(), SklearnLR(random_state=42)), ]) with warnings.catch_warnings(): warnings.simplefilter("ignore") clf = GridSearchCV(scikit_pipeline, parameters, cv=2, scoring=make_scorer(accuracy_score)) iris = load_iris() clf.fit(iris.data, iris.target) predicted = clf.predict(iris.data) accuracy_with_scikit_operators = accuracy_score( iris.target, predicted) self.assertEqual(accuracy_with_lale_operators, accuracy_with_scikit_operators)
def test_compare_with_sklearn(self): tfm = PCA() clf = LogisticRegression(LogisticRegression.solver.lbfgs, LogisticRegression.multi_class.auto) trainable = lale.operators.make_pipeline(tfm, clf) digits = sklearn.datasets.load_digits() trained = trainable.fit(digits.data, digits.target) predicted = trained.predict(digits.data) from sklearn.decomposition import PCA as SklearnPCA from sklearn.linear_model import LogisticRegression as SklearnLR sklearn_pipeline = sklearn.pipeline.make_pipeline( SklearnPCA(), SklearnLR(solver="lbfgs", multi_class="auto")) sklearn_pipeline.fit(digits.data, digits.target) predicted_sklearn = sklearn_pipeline.predict(digits.data) lale_score = accuracy_score(digits.target, predicted) scikit_score = accuracy_score(digits.target, predicted_sklearn) self.assertEqual(lale_score, scikit_score)
def setUpClass(cls) -> None: pl.seed_everything(0) cls.n_features = 10 cls.n_params = 2 * cls.n_features cls.model = LinearRegression(cls.n_features) gpus = 1 if torch.cuda.is_available() else 0 trainer = pl.Trainer(gpus=gpus, max_epochs=10) # trainer.fit(cls.model) print(tuple(cls.model.parameters())) use_sklearn = True if use_sklearn: train_dataset = DummyDataset(cls.n_features) clf = SklearnLR() clf.fit(train_dataset.data, train_dataset.targets) with torch.no_grad(): cls.model.linear.weight = torch.nn.Parameter( torch.tensor([clf.coef_], dtype=torch.float)) cls.model.linear.bias = torch.nn.Parameter( torch.tensor([clf.intercept_], dtype=torch.float)) cls.train_loader = cls.model.train_dataloader(batch_size=40000) # Setup test point data cls.test_idx = 8 cls.x_test = torch.tensor([cls.model.test_set.data[[cls.test_idx]]], dtype=torch.float) cls.y_test = torch.tensor([cls.model.test_set.targets[[cls.test_idx]]], dtype=torch.float) # Compute estimated IVHP cls.gpu = 1 if torch.cuda.is_available() else -1 # Compute anc flatten grad grads = grad_z(cls.x_test, cls.y_test, cls.model, gpu=cls.gpu) flat_grads = parameters_to_vector(grads) print("Grads:") print(flat_grads) # Make model functional params, names = make_functional(cls.model) # Make params regular Tensors instead of nn.Parameter params = tuple(p.detach().requires_grad_() for p in params) flat_params = parameters_to_vector(params) # Initialize Hessian h = torch.zeros([flat_params.shape[0], flat_params.shape[0]]) # Compute real IHVP for x_train, y_train in cls.train_loader: if cls.gpu >= 0: x_train, y_train = x_train.cuda(), y_train.cuda() def f(flat_params_): split_params = tensor_to_tuple(flat_params_, params) load_weights(cls.model, names, split_params) out = cls.model(x_train) loss = calc_loss(out, y_train) return loss batch_h = hessian(f, flat_params, strict=True) with torch.no_grad(): h += batch_h / float(len(cls.train_loader)) print("Hessian:") print(h) complete_x_train = cls.train_loader.dataset.data real_hessian = complete_x_train.T @ complete_x_train / complete_x_train.shape[ 0] * 2 print(real_hessian) print(np.linalg.norm(real_hessian - h.cpu().numpy()[:10, :10])) np.save("hessian_pytorch.npy", h.cpu().numpy()) # Make the model back `nn` with torch.no_grad(): load_weights(cls.model, names, params, as_params=True) inv_h = torch.inverse(h) print("Inverse Hessian") print(inv_h) cls.real_ihvp = inv_h @ flat_grads print("Real IHVP") print(cls.real_ihvp)