def test_rsrbm_pipeline(): print( "========== Test the pipeline of " "ReplicatedSoftmaxRBM followed by k-nearest-neighbors (kNN) ==========" ) np.random.seed(random_seed()) from sklearn.pipeline import Pipeline from sklearn.neighbors import KNeighborsClassifier (x_train, y_train), (x_test, y_test) = demo.load_20newsgroups() estimators = [('rbm', ReplicatedSoftmaxRBM(num_hidden=15, num_visible=5000, batch_size=32, num_epochs=4, learning_rate=0.001, learning_rate_hidden=0.00001, momentum_method='sudden', weight_cost=2e-4, random_state=random_seed(), verbose=0)), ('knn', KNeighborsClassifier(n_neighbors=1))] clf = Pipeline(estimators) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_mlp_check_grad(): eps = 1e-6 num_data = 10 num_features = 4 num_classes = 3 x = np.random.rand(num_data, num_features) y = np.random.randint(0, num_classes, num_data) model = MLP(model_name="checkgrad_MLP_softmax", task='classification', hidden_units_list=(5, ), reg_lambda=0.0, random_state=random_seed()) assert model.check_grad(x, y) < eps model = MLP(model_name="checkgrad_MLP_softmax", task='classification', hidden_units_list=(5, ), reg_lambda=0.01, random_state=random_seed()) assert model.check_grad(x, y) < eps model = MLP(model_name="checkgrad_MLP_softmax", task='classification', hidden_units_list=(5, ), reg_lambda=0.1, random_state=random_seed()) assert model.check_grad(x, y) < eps
def test_bbrbm_logpartition(): print( "========== Test Computing log-partition function of BernoulliBernoulliRBM ==========" ) np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() model = BernoulliBernoulliRBM(num_hidden=15, num_visible=784, batch_size=100, num_epochs=4, momentum_method='sudden', weight_cost=2e-4, metrics=['recon_err'], random_state=random_seed(), verbose=1) model.fit(x_train) exact_logpart = model.get_logpartition(method='exact') print("Exact log-partition function = %.4f" % exact_logpart) test_exact_loglik = model.get_loglik(x_test, method='exact').mean() print("Exact log-likelihood of testing data = %.4f" % test_exact_loglik) test_csl_loglik = model.get_loglik(x_test, method='csl', num_hidden_samples=100, num_steps=100).mean() print("CSL log-likelihood of testing data = %.4f" % test_csl_loglik)
def test_glm_softmax(): print("========== Test GLM for multiclass classification ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) clf = GLM(model_name="GLM_softmax", link='softmax', loss='softmax', random_state=random_seed()) print("Use {} optimizer".format(clf.optimizer)) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) clf = GLM(model_name="GLM_softmax", optimizer='sgd', link='softmax', loss='softmax', random_state=random_seed()) print("Use {} optimizer".format(clf.optimizer)) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_fogd_softmax(): print("========== Test FOGD for multiclass classification ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) clf = FOGD(model_name="FOGD_hinge", D=100, lbd=0.0, gamma=0.5, loss='hinge', random_state=random_seed()) clf.fit(x_train, y_train) print("Mistake rate = %.4f" % clf.mistake) # offline prediction print("Offline prediction") y_train_pred = clf.predict(x_train) y_test_pred = clf.predict(x_test) train_err = 1 - metrics.accuracy_score(y_train, y_train_pred) test_err = 1 - metrics.accuracy_score(y_test, y_test_pred) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_bbrbm_csl(): print("========== Test Conservative Sampling-based Likelihood (CSL) " "of BernoulliBernoulliRBM ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() model = BernoulliBernoulliRBM(num_hidden=15, num_visible=784, batch_size=100, num_epochs=4, momentum_method='sudden', weight_cost=2e-4, metrics=['recon_err'], random_state=random_seed(), verbose=1) model.fit(x_train) train_csl = model.get_loglik(x_train, method='csl', num_hidden_samples=100, num_steps=10).mean() print("Training log-likelihood computed using CSL = %.4f" % train_csl) test_csl = model.get_loglik(x_test, method='csl', num_hidden_samples=100, num_steps=10).mean() print("Testing log-likelihood computed using CSL = %.4f" % test_csl)
def test_cgan_mnist(show_figure=False, block_figure_on_end=False): print("========== Test CGAN on MNIST data ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() x_train = x_train.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = CGAN( model_name="CGAN_MNIST", num_z=10, # set to 100 for a full run z_prior=Uniform1D(low=-1.0, high=1.0), img_size=(28, 28, 1), batch_size=64, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=2, # set to 32 for a full run num_dis_feature_maps=2, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=1, # set to 100 for a full run random_state=random_seed(), verbose=1) model.fit(x_train, y_train)
def test_glm_regression(): print("========== Test GLM for regression ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_housing() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) clf = GLM( model_name="GLM_regression", task='regression', link='linear', # link function loss='quadratic', # loss function l2_penalty=0.0, # ridge regularization l1_penalty=0.0, # Lasso regularization l1_smooth=1E-5, # smoothing for Lasso regularization l1_method='pseudo_huber', # approximation method for L1-norm random_state=random_seed()) clf.fit(x_train, y_train) train_err = -clf.score(x_train, y_train) test_err = -clf.score(x_test, y_test) print("Training MSE = %.4f" % train_err) print("Testing MSE = %.4f" % test_err)
def test_dualsgd_softmax(): print("========== Test DualSGD for multiclass classification ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) clf = DualSGD(model_name="DualSGD_hinge", k=20, D=200, gamma=1.0, lbd=3.3593684387335183e-05, loss='hinge', maintain='k-merging', max_budget_size=100, random_state=random_seed()) clf.fit(x_train, y_train) print("Mistake rate = %.4f" % clf.mistake) print("Budget size = %d" % clf.budget_size) # offline prediction print("Offline prediction") y_train_pred = clf.predict(x_train) y_test_pred = clf.predict(x_test) train_err = 1 - metrics.accuracy_score(y_train, y_train_pred) test_err = 1 - metrics.accuracy_score(y_test, y_test_pred) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_dualsgd_regression(): print("========== Test DualSGD for Regression ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_housing() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) est = DualSGD(model_name="DualSGD_eps_insensitive", k=20, D=200, gamma=1.0, eps=0.001, lbd=0.00128, loss='eps_insensitive', maintain='k-merging', max_budget_size=100, random_state=random_seed()) est.fit(x_train, y_train) print("Mistake rate = %.4f" % est.mistake) print("Budget size = %d" % est.budget_size) # offline prediction print("Offline prediction") y_train_pred = est.predict(x_train) y_test_pred = est.predict(x_test) train_err = metrics.mean_squared_error(y_train, y_train_pred) test_err = metrics.mean_squared_error(y_test, y_test_pred) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_save_load(): print("========== Test save, load tensorflow models ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) clf = TensorFlowGLM(model_name="iris_TensorFlowGLM_softmax", link='softmax', loss='softmax', num_epochs=5, random_state=random_seed()) clf.fit(x_train, y_train) print("After training:") train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) save_file_path = clf.save() clf1 = TensorFlowModel.load_model(save_file_path) print("After save and load:") train_err1 = 1.0 - clf1.score(x_train, y_train) test_err1 = 1.0 - clf1.score(x_test, y_test) print("Training error = %.4f" % train_err1) print("Testing error = %.4f" % test_err1) assert abs(train_err - train_err1) < 1e-6 assert abs(test_err - test_err1) < 1e-6
def test_node2vec_wiki_pos(show_figure=False, block_figure_on_end=False): graph, labels, walks = demo.load_wikipos() y = labels x = np.array(range(len(labels))) model = Node2Vec( model_name='Node2Vec', emb_size=16, window=3, num_walks=2, walk_length=10, p=1.0, q=1.0, num_workers=4, directed=False, ) model.fit(x, y, walks=walks) train_idx, test_idx = next( ShuffleSplit(n_splits=1, test_size=0.1, random_state=random_seed()).split(x)) x_train, y_train = model.transform(x[train_idx]), y[train_idx] x_test, y_test = model.transform(x[test_idx]), y[test_idx] clf = GLM(model_name="GLM_multilogit", task='multilabel', link='logit', loss='multilogit', random_state=random_seed()) clf.fit(x_train, y_train) train_f1 = clf.score(x_train, y_train) test_f1 = clf.score(x_test, y_test) print("Training weighted-F1-macro = %.4f" % train_f1) print("Testing weighted-F1-macro = %.4f" % test_f1)
def test_bbrbm_pipeline(): print( "========== Test the pipeline of " "BernoulliBernoulliRBM followed by k-nearest-neighbors (kNN) ==========" ) np.random.seed(random_seed()) from sklearn.pipeline import Pipeline from sklearn.neighbors import KNeighborsClassifier (x_train, y_train), (x_test, y_test) = demo.load_mnist() estimators = [('rbm', BernoulliBernoulliRBM(num_hidden=15, num_visible=784, batch_size=100, num_epochs=4, momentum_method='sudden', weight_cost=2e-4, random_state=random_seed(), verbose=0)), ('knn', KNeighborsClassifier(n_neighbors=1))] clf = Pipeline(estimators) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_dfm_save_and_load(show_figure=False, block_figure_on_end=False): print("========== Test Save and Load functions of DFM on MNIST data ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() x_train = x_train.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = DFM(model_name="DFM_MNIST_SaveLoad", num_z=10, # set to 100 for a full run img_size=(28, 28, 1), batch_size=32, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run alpha=0.03 / 10, # 0.03 / 1024 noise_std=1.0, num_dfm_layers=1, # 2 num_dfm_hidden=10, # 1024 metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=4, # set to 100 for a full run random_state=random_seed(), verbose=1) model.fit(x_train) save_file_path = model.save() model1 = TensorFlowModel.load_model(save_file_path) model1.num_epochs = 10 model1.fit(x_train)
def test_rrf_cv_gridsearch(): print( "========== Tune parameters for RRF including cross-validation ==========" ) np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) x = np.vstack([x_train, x_test, x_test]) y = np.concatenate([y_train, y_test, y_test]) params = {'gamma': [0.5, 1.0], 'learning_rate': [0.01, 0.05, 0.1]} ps = PredefinedSplit(test_fold=[-1] * x_train.shape[0] + [-1] * x_test.shape[0] + [1] * x_test.shape[0]) early_stopping = EarlyStopping(monitor='val_err', patience=2) filepath = os.path.join( model_dir(), "male/RRF/search/mnist_{epoch:04d}_{val_err:.6f}.pkl") checkpoint = ModelCheckpoint(filepath, mode='min', monitor='val_err', verbose=0, save_best_only=True) clf = RRF(model_name="RRF_hinge", D=100, lbd=0.01, gamma=0.125, mode='batch', loss='hinge', num_epochs=10, learning_rate=0.001, learning_rate_gamma=0.001, metrics=['loss', 'err'], callbacks=[early_stopping, checkpoint], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], catch_exception=True, random_state=random_seed()) gs = GridSearchCV(clf, params, cv=ps, n_jobs=-1, refit=False, verbose=True) gs.fit(x, y) print("Best error {} @ params {}".format(1 - gs.best_score_, gs.best_params_)) best_clf = clone(clf).set_params(**gs.best_params_) best_clf.fit(np.vstack([x_train, x_test]), np.concatenate([y_train, y_test])) train_err = 1.0 - best_clf.score(x_train, y_train) test_err = 1.0 - best_clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) assert abs(test_err - (1.0 - gs.best_score_)) < 1e-4
def test_kmm_cv_gridsearch(): print( "========== Tune parameters for KMM including cross-validation ==========" ) np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) x = np.vstack([x_train, x_test, x_test]) y = np.concatenate([y_train, y_test, y_test]) params = {'gamma': [0.5, 1.0], 'num_kernels': [1, 2, 4]} ps = PredefinedSplit(test_fold=[-1] * x_train.shape[0] + [-1] * x_test.shape[0] + [1] * x_test.shape[0]) early_stopping = EarlyStopping(monitor='val_loss', patience=2) clf = KMM(model_name="KMM_hinge", D=20, lbd=0.0, gamma=0.1, mode='batch', loss='hinge', num_kernels=4, batch_size=100, temperature=1.0, num_epochs=10, num_nested_epochs=1, learning_rate=0.1, learning_rate_mu=0.0, learning_rate_gamma=0.1, learning_rate_alpha=0.1, metrics=['loss', 'err'], callbacks=[early_stopping], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], catch_exception=True, random_state=random_seed()) gs = GridSearchCV(clf, params, cv=ps, n_jobs=-1, refit=False, verbose=True) gs.fit(x, y) print("Best error {} @ params {}".format(1 - gs.best_score_, gs.best_params_)) best_clf = clone(clf).set_params(**gs.best_params_) best_clf.fit(np.vstack([x_train, x_test]), np.concatenate([y_train, y_test])) train_err = 1.0 - best_clf.score(x_train, y_train) test_err = 1.0 - best_clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) assert abs(test_err - (1 - gs.best_score_)) < 1e-4
def test_wgan_cifar10(show_figure=False, block_figure_on_end=False): print("========== Test WGAN on CIFAR10 data ==========") np.random.seed(random_seed()) num_data = 128 (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train[:num_data].astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. root_dir = os.path.join(model_dir(), "male/WGAN/CIFAR10") loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[os.path.join(root_dir, "loss/loss_{epoch:04d}.png"), os.path.join(root_dir, "loss/loss_{epoch:04d}.pdf")], monitor=[{'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, filepath=os.path.join(root_dir, "samples/samples_{epoch:04d}.png"), monitor=[{'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = WGAN(model_name="WGAN_CIFAR10", num_z=10, # set to 100 for a full run z_prior=Uniform1D(low=-1.0, high=1.0), img_size=(32, 32, 3), batch_size=16, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=4, # set to 100 for a full run log_path=os.path.join(root_dir, "logs"), random_state=random_seed(), verbose=1) model.fit(x_train)
def test_kmm_softmax(): print("========== Test KMM for multiclass classification ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) clf = KMM(model_name="KMM_hinge", D=4, lbd=0.01, gamma=0.01, mode='batch', loss='hinge', num_kernels=4, batch_size=100, temperature=0.1, num_epochs=10, num_nested_epochs=1, learning_rate=0.001, learning_rate_mu=0.001, learning_rate_gamma=0.001, learning_rate_alpha=0.001, random_state=random_seed()) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) clf = KMM(model_name="KMM_hinge", D=100, lbd=0.0, gamma=0.01, mode='online', loss='hinge', num_kernels=4, batch_size=100, temperature=0.1, num_nested_epochs=1, learning_rate=0.001, learning_rate_mu=0.001, learning_rate_gamma=0.001, learning_rate_alpha=0.001, random_state=random_seed(), verbose=1) clf.fit(x_train, y_train) print("Mistake rate = %.4f" % clf.mistake)
def run_one_candidate(candidate_params): num_runs = 3 np.random.seed(random_seed()) mistake_rate_avg = 0 train_time_avg = 0 for ri in range(num_runs): print('----------------------------------') print('Run #{0}:'.format(ri + 1)) data_name = dataset n_features = 300 train_file_name = os.path.join(data_dir(), data_name + '_train.libsvm') test_file_name = os.path.join(data_dir(), data_name + '_test.libsvm') if not os.path.exists(train_file_name): raise Exception('File not found') if not os.path.exists(test_file_name): raise Exception('File not found') x_train, y_train = load_svmlight_file(train_file_name, n_features=n_features) x_test, y_test = load_svmlight_file(test_file_name, n_features=n_features) x_train = x_train.toarray() x_test = x_test.toarray() x_total = np.vstack((x_train, x_test)) y_total = np.concatenate((y_train, y_test)) print('Num total samples: {}'.format(x_total.shape[0])) clf = OnlineDualSVRG( # regular_param=0.01, learning_rate_scale=0.8, # gamma=2.0, rf_dim=4000, num_epochs=1, freq_update_full_model=100, oracle=oracle, core_max=100, coverage_radius=0.9, loss_func=loss_func, smooth_hinge_theta=0.5, smooth_hinge_tau=0.5, random_state=random_seed(), **candidate_params, ) print('Running ...') clf.fit(x_total, y_total) print('Mistake rate: {0:.2f}%, Training time: {1} seconds'.format(clf.mistake_rate * 100, int(clf.train_time))) mistake_rate_avg += clf.mistake_rate train_time_avg += clf.train_time return mistake_rate_avg / num_runs, train_time_avg / num_runs, candidate_params
def test_weighted_gan_cifar10(show_figure=False, block_figure_on_end=False): print("========== Test Weighted-GAN on CIFAR10 data ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = WeightedGAN(model_name="WeightedGAN_CIFAR10", alpha=0.5, num_z=10, # set to 100 for a full run img_size=(32, 32, 3), batch_size=32, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=4, # set to 100 for a full run random_state=random_seed(), verbose=1) model.fit(x_train)
def test_srbm_gridsearch(): print("========== Tune parameters for Supervised RBM ==========") from sklearn.model_selection import PredefinedSplit from sklearn.model_selection import GridSearchCV np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() x = np.vstack([x_train, x_test, x_test]) y = np.concatenate([y_train, y_test, y_test]) early_stopping = EarlyStopping(monitor='val_loss', patience=2, verbose=1) params = { 'batch_size': [64, 100], 'learning_rate': [0.1, 0.01], 'weight_cost': [0.001, 0.0001] } model = SupervisedRBM(num_hidden=15, num_visible=784, batch_size=100, num_epochs=4, learning_rate=0.01, momentum_method='sudden', weight_cost=0.0, inference_engine='variational_inference', approx_method='first_order', metrics=['loss'], callbacks=[early_stopping], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=0) ps = PredefinedSplit(test_fold=[-1] * x_train.shape[0] + [-1] * x_test.shape[0] + [1] * x_test.shape[0]) gs = GridSearchCV(model, params, cv=ps, n_jobs=-1, refit=False, verbose=True) gs.fit(x, y) print("Best error {} @ params {}".format(1.0 - gs.best_score_, gs.best_params_))
def test_continue_training(): print("========== Test continue training pytorch models ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() x_train = x_train.astype(np.float32) y_train = y_train.astype(np.uint8) x_test = x_test.astype(np.float32) y_test = y_test.astype(np.uint8) print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) num_epochs = 5 clf = PyTorchMLP(model_name='PyTorchMLP', arch='MLPv1', num_epochs=4, batch_size=10, metrics=['loss', 'err'], random_state=random_seed(), verbose=1) clf.fit(x_train, y_train) print("After training for {0:d} epochs".format(num_epochs)) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) clf.num_epochs = 10 print("Set number of epoch to {0:d}, then continue training...".format( clf.num_epochs)) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) save_file_path = clf.save() clf1 = PyTorchModel.load_model(save_file_path) clf1.num_epochs = 15 print("Save, load, set number of epoch to {0:d}, " "then continue training...".format(clf1.num_epochs)) clf1.fit(x_train, y_train) train_err = 1.0 - clf1.score(x_train, y_train) test_err = 1.0 - clf1.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_kmm_cv(): print("========== Test cross-validation for KMM ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) early_stopping = EarlyStopping(monitor='val_err', patience=2, verbose=1) filepath = os.path.join(model_dir(), "male/KMM/iris_{epoch:04d}_{val_err:.6f}.pkl") checkpoint = ModelCheckpoint(filepath, mode='min', monitor='val_err', verbose=0, save_best_only=True) clf = KMM(model_name="KMM_hinge", D=20, lbd=0.0, gamma=0.1, mode='batch', loss='hinge', num_kernels=3, batch_size=100, temperature=1.0, num_epochs=10, num_nested_epochs=1, learning_rate=0.1, learning_rate_mu=0.0, learning_rate_gamma=0.1, learning_rate_alpha=0.1, metrics=['loss', 'err'], callbacks=[early_stopping, checkpoint], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=1) clf.fit(x, y) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def check_grad(): print("========== Check gradients ==========") np.random.seed(random_seed()) # <editor-fold desc="Binary classification"> eps = 1e-6 num_data = 10 num_features = 5 num_classes = 2 x = np.random.rand(num_data, num_features) y = np.random.randint(0, num_classes, num_data) print(np.unique(y)) model = SVRG( regular_param=0.01, learning_rate_scale=1.0, gamma=10, rf_dim=400, num_epochs=2, cache_size=6, freq_update_full_model=10, oracle=SVRG.COVERAGE, core_max=10, coverage_radius=100.0, loss_func=SVRG.LOGISTIC, smooth_hinge_theta=0.5, smooth_hinge_tau=0.5, freq_calc_metrics=20, ) model.fit(x, y) for n in range(num_data): assert model.check_grad(x[n, :], y[n]) < eps
def test_sgd_visualization_2d(show=False, block_figure_on_end=False): (x_train, y_train), (_, _) = demo.load_synthetic_2d() display = Display(freq=10, dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[{'metrics': ['predict'], 'title': "Learning losses", 'xlabel': "X1", 'ylabel': "X2", 'grid_size': 10, 'marker_size': 10, 'left': None, 'right': None, 'top': None, 'bottom': None }] ) learner = KSGD(lbd=0.0001, eps=0.001, gamma=30, kernel='gaussian', loss='hinge', batch_size=1, callbacks=[display], avg_weight=False, random_state=random_seed()) learner.fit(x_train, y_train) print("Training error = %.4f" % (1 - learner.score(x_train, y_train)))
def test_early_stopping(): np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) early_stopping = EarlyStopping(monitor='val_err', patience=2, verbose=1) optz = SGD(learning_rate=0.01) clf = GLM(model_name="early_stopping_callback", link='softmax', loss='softmax', optimizer=optz, num_epochs=20, batch_size=10, task='classification', metrics=['loss', 'err'], callbacks=[early_stopping], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=1) clf.fit(x, y) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Model has been stopped at epoch #{0:d}".format(clf.epoch)) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) print("Continue training...") clf.fit(x, y) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Model has been stopped at epoch #{0:d}".format(clf.epoch)) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) print("Disable early stopping and continue training to the end...") clf.callbacks = [] clf.fit(x, y) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_srbm_load_to_continue_training(): print( "========== Test load to continue training Supervised RBM ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_pima() model = SupervisedRBM(num_hidden=5, num_visible=x_train.shape[1], batch_size=10, num_epochs=2, learning_rate=0.1, momentum_method='sudden', weight_cost=0.0, inference_engine='variational_inference', approx_method='second_order', metrics=['recon_err', 'loss', 'err'], random_state=random_seed(), verbose=1) model.fit(x_train, y_train) print("After training for {0:d} epochs".format(model.num_epochs)) train_err = 1.0 - model.score(x_train, y_train) test_err = 1.0 - model.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) save_file_path = model.save() model1 = Model.load_model(save_file_path) print("After save and load:") train_err1 = 1.0 - model1.score(x_train, y_train) test_err1 = 1.0 - model1.score(x_test, y_test) print("Training error = %.4f" % train_err1) print("Testing error = %.4f" % test_err1) assert abs(train_err - train_err1) < 1e-6 assert abs(test_err - test_err1) < 1e-6 model.num_epochs = 4 model.fit(x_train, y_train) print("Set number of epoch to {0:d}, then continue training...".format( model.num_epochs)) train_err = 1.0 - model.score(x_train, y_train) test_err = 1.0 - model.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_glm_logit(): print("========== Test GLM for binary classification ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_pima() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) clf = GLM(model_name="GLM_logit", l1_penalty=0.0, l2_penalty=0.0, random_state=random_seed()) print("Use {} optimizer".format(clf.optimizer)) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) clf = GLM(model_name="GLM_logit", optimizer='sgd', l1_penalty=0.0, l2_penalty=0.0, random_state=random_seed()) print("Use {} optimizer".format(clf.optimizer)) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) optz = SGD(learning_rate=0.01, momentum=0.9, nesterov=True) clf = GLM(model_name="GLM_logit", optimizer=optz, l1_penalty=0.0, l2_penalty=0.0, random_state=random_seed()) print("Use {} optimizer".format(clf.optimizer)) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_logsumeone(): np.random.seed(random_seed()) from male.utils.func_utils import logsumone assert (np.abs(logsumone(10.0) - 10.000045398899218) < 1e-6) x = np.random.rand(2, 3) r = np.array([[1.30170729, 1.0712702, 1.00492985], [1.14584669, 1.10508236, 0.94158162]]) assert np.all(np.abs(logsumone(x) - r) < 1e-2)
def test_continue_training(): print("========== Test continue training the models ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) num_epochs = 5 clf = GLM(model_name="iris_glm_softmax", link='softmax', loss='softmax', optimizer='sgd', batch_size=10, num_epochs=num_epochs, random_state=random_seed(), verbose=1) clf.fit(x_train, y_train) print("After training for {0:d} epochs".format(num_epochs)) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) clf.num_epochs = 10 print("Set number of epoch to {0:d}, then continue training...".format( clf.num_epochs)) clf.fit(x_train, y_train) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) save_file_path = clf.save() clf1 = Model.load_model(save_file_path) clf1.num_epochs = 15 print("Save, load, set number of epoch to {0:d}, " "then continue training...".format(clf.num_epochs)) clf1.fit(x_train, y_train) train_err = 1.0 - clf1.score(x_train, y_train) test_err = 1.0 - clf1.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)