def test_cgan_mnist(show_figure=False, block_figure_on_end=False): print("========== Test CGAN on MNIST data ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() x_train = x_train.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = CGAN( model_name="CGAN_MNIST", num_z=10, # set to 100 for a full run z_prior=Uniform1D(low=-1.0, high=1.0), img_size=(28, 28, 1), batch_size=64, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=2, # set to 32 for a full run num_dis_feature_maps=2, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=1, # set to 100 for a full run random_state=random_seed(), verbose=1) model.fit(x_train, y_train)
def test_dfm_save_and_load(show_figure=False, block_figure_on_end=False): print("========== Test Save and Load functions of DFM on MNIST data ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() x_train = x_train.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = DFM(model_name="DFM_MNIST_SaveLoad", num_z=10, # set to 100 for a full run img_size=(28, 28, 1), batch_size=32, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run alpha=0.03 / 10, # 0.03 / 1024 noise_std=1.0, num_dfm_layers=1, # 2 num_dfm_hidden=10, # 1024 metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=4, # set to 100 for a full run random_state=random_seed(), verbose=1) model.fit(x_train) save_file_path = model.save() model1 = TensorFlowModel.load_model(save_file_path) model1.num_epochs = 10 model1.fit(x_train)
def test_svrg_s_visualization_2d(show=False, block_figure_on_end=False): (x_train, y_train), (_, _) = demo.load_synthetic_2d() print('num_samples: {}'.format(x_train.shape[0])) predict_display = Display( freq=10, dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[{'metrics': ['predict'], 'title': "Visualization", 'xlabel': "X1", 'ylabel': "X2", 'grid_size': 100, 'marker_size': 10, 'left': None, 'right': None, 'top': None, 'bottom': None }] ) loss_display = Display( freq=1, dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[{'metrics': ['train_loss', 'obj_func'], 'type': 'line', 'title': "Learning losses", 'xlabel': "data points", 'ylabel': "loss", }] ) learner = SVRG( regular_param=0.01, learning_rate_scale=1.0, gamma=10, rf_dim=400, num_epochs=2, cache_size=6, freq_update_full_model=10, oracle=SVRG.COVERAGE, core_max=10, coverage_radius=100.0, loss_func=SVRG.LOGISTIC, smooth_hinge_theta=0.5, smooth_hinge_tau=0.5, callbacks=[loss_display, predict_display], metrics=['train_loss', 'obj_func'], freq_calc_metrics=50, random_state=random_seed()) learner.fit(x_train, y_train) y_train_pred = learner.predict(x_train) print("Training error = %.4f" % (1 - metrics.accuracy_score(y_train, y_train_pred)))
def test_visualization_2d( create_obj_func, show=False, block_figure_on_end=False, freq_predict_display=5, show_loss_display=False, grid_size=100, marker_size=10): file_name = os.path.join(data_dir(), 'demo/synthetic_2D_data_train.libsvm') x_train, y_train = load_svmlight_file(file_name) x_train = x_train.toarray() print('num_samples: {}'.format(x_train.shape[0])) predict_display = Display( freq=freq_predict_display, dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[{'metrics': ['predict'], 'title': "Visualization", 'xlabel': "X1", 'ylabel': "X2", 'grid_size': grid_size, 'marker_size': marker_size, 'left': None, 'right': None, 'top': None, 'bottom': None }] ) loss_display = Display( freq=1, dpi=72, show=show, block_on_end=block_figure_on_end, monitor=[{'metrics': ['train_loss'], 'type': 'line', 'title': "Learning losses", 'xlabel': "data points", 'ylabel': "loss", }] ) callbacks = [predict_display] if show_loss_display: callbacks.append(loss_display) users_params = { 'callbacks': callbacks, 'metrics': ['train_loss'], } learner = create_obj_func(users_params) learner.fit(x_train, y_train) y_train_pred = learner.predict(x_train) print("Training error = %.4f" % (1 - metrics.accuracy_score(y_train, y_train_pred)))
def test_wgan_cifar10(show_figure=False, block_figure_on_end=False): print("========== Test WGAN on CIFAR10 data ==========") np.random.seed(random_seed()) num_data = 128 (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train[:num_data].astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. root_dir = os.path.join(model_dir(), "male/WGAN/CIFAR10") loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[os.path.join(root_dir, "loss/loss_{epoch:04d}.png"), os.path.join(root_dir, "loss/loss_{epoch:04d}.pdf")], monitor=[{'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, filepath=os.path.join(root_dir, "samples/samples_{epoch:04d}.png"), monitor=[{'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = WGAN(model_name="WGAN_CIFAR10", num_z=10, # set to 100 for a full run z_prior=Uniform1D(low=-1.0, high=1.0), img_size=(32, 32, 3), batch_size=16, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=4, # set to 100 for a full run log_path=os.path.join(root_dir, "logs"), random_state=random_seed(), verbose=1) model.fit(x_train)
def test_sgd_visualization_2d(block_figure_on_end=False): (x_train, y_train), (_, _) = demo.load_synthetic_2d() predict_display = Display( freq=1, dpi='auto', block_on_end=block_figure_on_end, monitor=[{'metrics': ['predict'], 'title': "Visualization", 'xlabel': "X1", 'ylabel': "X2", 'grid_size': 100, 'marker_size': 10, 'left': None, 'right': None, 'top': None, 'bottom': None }] ) display = Display( freq=1, dpi='auto', block_on_end=block_figure_on_end, monitor=[{'metrics': ['mistake_rate'], 'type': 'line', 'title': "Mistake Rate", 'xlabel': "data points", 'ylabel': "Error", }] ) learner = OnlineDualSVRG( regular_param=0.01, learning_rate_scale=1.0, gamma=10, rf_dim=400, cache_size=6, freq_update_full_model=10, oracle='budget', core_max=10, coverage_radius=0.5, loss_func='hinge', smooth_hinge_theta=0.5, smooth_hinge_tau=0.5, callbacks=[display], metrics=['mistake_rate'], random_state=random_seed()) learner.fit(x_train, y_train) y_train_pred = learner.predict(x_train) print("Training error = %.4f" % (1 - metrics.accuracy_score(y_train, y_train_pred)))
def test_weighted_gan_cifar10(show_figure=False, block_figure_on_end=False): print("========== Test Weighted-GAN on CIFAR10 data ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = WeightedGAN(model_name="WeightedGAN_CIFAR10", alpha=0.5, num_z=10, # set to 100 for a full run img_size=(32, 32, 3), batch_size=32, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=4, # set to 100 for a full run random_state=random_seed(), verbose=1) model.fit(x_train)
def test_sgd_visualization_2d(show=False, block_figure_on_end=False): (x_train, y_train), (_, _) = demo.load_synthetic_2d() display = Display(freq=10, dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[{'metrics': ['predict'], 'title': "Learning losses", 'xlabel': "X1", 'ylabel': "X2", 'grid_size': 10, 'marker_size': 10, 'left': None, 'right': None, 'top': None, 'bottom': None }] ) learner = KSGD(lbd=0.0001, eps=0.001, gamma=30, kernel='gaussian', loss='hinge', batch_size=1, callbacks=[display], avg_weight=False, random_state=random_seed()) learner.fit(x_train, y_train) print("Training error = %.4f" % (1 - learner.score(x_train, y_train)))
def test_word2vec_text8(show_figure=False, block_figure_on_end=False): print("========== Test Word2Vec on Text8 - 1% data ==========") txt = demo.load_text8_1pct() loss_display = Display(layout=(1, 1), dpi='auto', freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['loss'], 'type': 'line', 'labels': ["Loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) emb_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['embedding'], 'title': "Word Embeddings", 'type': 'img', 'num_words': 10, # set to 500 for full run }, ]) model = Word2Vec(num_epochs=4, # set to 100 for full run metrics=['loss'], callbacks=[loss_display, emb_display], verbose=1) # model.fit(txt[:250]) # the first 42 words model.fit(txt[:300]) # the first 51 words
def test_svmguide1(show_figure=False, block_figure_on_end=False): print("========== Test SVRG on svmguide1 dataset ==========") data_name = 'svmguide1' (x_train, y_train), (x_test, y_test) = demo.load_svmguide1() loss_display = Display( freq=1, dpi='auto', show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['train_loss', 'obj_func'], 'type': 'line', 'title': "Learning losses", 'xlabel': "X1", 'ylabel': "X2", }] ) print(x_train.shape) learner = SVRG( mode='online', regular_param=0.01, learning_rate_scale=0.7, gamma=2.0, rf_dim=400, num_epochs=1, cache_size=50, freq_update_full_model=100, oracle=SVRG.COVERAGE, core_max=100, coverage_radius=0.9, loss_func=SVRG.LOGISTIC, smooth_hinge_theta=0.5, smooth_hinge_tau=0.5, callbacks=[loss_display], metrics=['train_loss', 'obj_func'], freq_calc_metrics=300, random_state=random_seed() ) learner.fit(x_train, y_train) y_train_pred = learner.predict(x_train) y_test_pred = learner.predict(x_test) print("Dataset: {}".format(data_name)) print("Training error = %.4f" % (1 - metrics.accuracy_score(y_train, y_train_pred))) print("Test error = %.4f" % (1 - metrics.accuracy_score(y_test, y_test_pred)))
def test_gkm_budget_visualization_2d(show=False, block_figure_on_end=False): (x_train, y_train), (_, _) = demo.load_synthetic_2d_semi() print('num_samples: {}'.format(x_train.shape[0])) predict_display = Display( freq=100, dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[{'metrics': ['predict'], 'title': "Visualization", 'xlabel': "X1", 'ylabel': "X2", 'grid_size': 20, 'marker_size': 10, 'left': None, 'right': None, 'top': None, 'bottom': None }] ) learner = BudgetedGKM( model_name="GKM_BUDGET", mode='batch', unlabel=-1, trade_off_1=1, trade_off_2=2, gamma=500.0, loss_func=BudgetedGKM.S_HINGE, smooth_hinge_theta=0.5, smooth_hinge_tau=0.5, insensitive_epsilon=0.001, unlabel_loss_func_degree=1, sim_func=None, sim_params=0.01, num_epochs=1, callbacks=[predict_display], ) learner.fit(x_train, y_train) y_train_pred = learner.predict(x_train) print("Training error = %.4f" % (1 - metrics.accuracy_score( y_train[y_train != -128], y_train_pred[y_train != -128])))
def test_svmguide1(): print("========== Test IGKOL on svmguide1 dataset ==========") data_name = 'svmguide1' (x_train, y_train), (x_test, y_test) = demo.load_svmguide1() display = Display( freq=1, dpi='auto', block_on_end=True, monitor=[{'metrics': ['mistake_rate'], 'type': 'line', 'title': "Mistake Rate", 'xlabel': "data points", 'ylabel': "Error", }] ) learner = OnlineDualSVRG( regular_param=0.01, learning_rate_scale=1.0, gamma=2.0, rf_dim=400, cache_size=50, freq_update_full_model=10, oracle='budget', core_max=100, coverage_radius=0.5, loss_func='hinge', smooth_hinge_theta=0.5, smooth_hinge_tau=0.5, callbacks=[display], metrics=['mistake_rate'], random_state=random_seed()) learner.fit(x_train, y_train) print('OFFLINE Testing') y_train_pred = learner.predict(x_train) y_test_pred = learner.predict(x_test) print("Dataset: {}".format(data_name)) print("Training error = %.4f" % (1 - metrics.accuracy_score(y_train, y_train_pred))) print("Test error = %.4f" % (1 - metrics.accuracy_score(y_test, y_test_pred)))
def test_wgan_gp_mnist(show_figure=False, block_figure_on_end=False): print("========== Test WGAN-GP on MNIST data ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() x_train = x_train.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. root_dir = os.path.join(model_dir(), "male/WGAN-GP/MNIST") loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[os.path.join(root_dir, "loss/loss_{epoch:04d}.png"), os.path.join(root_dir, "loss/loss_{epoch:04d}.pdf")], monitor=[{'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[{'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = WGAN_GP(model_name="WGAN_GP_MNIST_z_uniform", num_z=10, # set to 100 for a full run z_prior=Uniform1D(low=-1.0, high=1.0), img_size=(28, 28, 1), batch_size=16, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=4, # set to 100 for a full run # summary_freq=1, # uncomment this for a full run random_state=random_seed(), log_path=os.path.join(root_dir, "logs"), verbose=1) model.fit(x_train) model = WGAN_GP(model_name="WGAN_GP_MNIST_z_Gaussian", num_z=10, # set to 100 for a full run z_prior=Gaussian1D(mu=0.0, sigma=1.0), img_size=(28, 28, 1), batch_size=32, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=4, # set to 100 for a full run # summary_freq=1, # uncomment this for a full run random_state=random_seed(), log_path=os.path.join(root_dir, "logs"), verbose=1) model.fit(x_train)
def test_pytorch_mlp_v1(show=False, block_figure_on_end=False): print("========== Test PytorchMLPv1 ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() idx_train = np.random.permutation(x_train.shape[0]) x_train = x_train[idx_train].astype(np.float32) y_train = y_train[idx_train].astype(np.uint8) print("Number of training samples = {}".format(x_train.shape[0])) idx_test = np.random.permutation(x_test.shape[0]) x_test = x_test[idx_test].astype(np.float32) y_test = y_test[idx_test].astype(np.uint8) print("Number of testing samples = {}".format(x_test.shape[0])) x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) err_display = Display(title="Error curves", dpi='auto', layout=(1, 1), freq=1, show=show, block_on_end=block_figure_on_end, monitor=[{ 'metrics': ['err', 'val_err'], 'type': 'line', 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }]) loss_display = Display( title="Learning curves", dpi='auto', layout=(3, 1), freq=1, show=show, block_on_end=block_figure_on_end, filepath=[ os.path.join(model_dir(), "male/PyTorchMLP/" "loss/loss_{epoch:04d}.png"), os.path.join(model_dir(), "male/PyTorchMLP/" "loss/loss_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['loss', 'val_loss'], 'type': 'line', 'labels': ["training loss", "validation loss"], 'title': "Learning losses", 'xlabel': "epoch", 'xlabel_params': { 'fontsize': 50 }, 'ylabel': "loss", }, { 'metrics': ['err', 'val_err'], 'type': 'line', 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['err'], 'type': 'line', 'labels': ["training error"], 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, ]) weight_display = Display(title="Filters", dpi='auto', layout=(1, 1), figsize=(6, 15), freq=1, show=show, block_on_end=block_figure_on_end, filepath=os.path.join( model_dir(), "male/PyTorchMLP/" "weights/weights_{epoch:04d}.png"), monitor=[ { 'metrics': ['weights'], 'title': "Learned weights", 'type': 'img', 'tile_shape': (5, 2), }, ]) clf = PyTorchMLP(model_name='PyTorchMLP', arch='MLPv1', num_epochs=4, batch_size=100, metrics=['loss', 'err'], callbacks=[loss_display, err_display, weight_display], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=1) clf.fit(x, y) print("Training error = %.4f" % (1.0 - clf.score(x_train, y_train))) print("Testing error = %.4f" % (1.0 - clf.score(x_test, y_test)))
def test_glm_save_load(show=False, block_figure_on_end=False): print("========== Test Save and Load functions for GLM ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) early_stopping = EarlyStopping(monitor='val_err', patience=5, verbose=1) filepath = os.path.join(model_dir(), "male/GLM/iris_{epoch:04d}_{val_err:.6f}.pkl") checkpoint = ModelCheckpoint(filepath, mode='min', monitor='val_err', verbose=0, save_best_only=True) loss_display = Display(title="Learning curves", dpi='auto', layout=(3, 1), freq=1, show=show, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['loss', 'val_loss'], 'type': 'line', 'labels': ["training loss", "validation loss"], 'title': "Learning losses", 'xlabel': "epoch", 'ylabel': "loss", }, { 'metrics': ['err', 'val_err'], 'type': 'line', 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['err'], 'type': 'line', 'labels': ["training error"], 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, ]) weight_display = Display(title="Filters", dpi='auto', layout=(1, 1), figsize=(6, 15), freq=1, show=show, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['weights'], 'title': "Learned weights", 'type': 'img', 'disp_dim': (2, 2), 'tile_shape': (3, 1), }, ]) clf = GLM( model_name="GLM_softmax_cv", link='softmax', loss='softmax', optimizer='sgd', num_epochs=4, batch_size=10, task='classification', metrics=['loss', 'err'], callbacks=[early_stopping, checkpoint, loss_display, weight_display], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=1) clf.fit(x, y) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err) save_file_path = os.path.join(model_dir(), "male/GLM/saved_model.pkl") clf.save(file_path=save_file_path) clf1 = Model.load_model(save_file_path) clf1.num_epochs = 10 clf1.fit(x, y) train_err = 1.0 - clf1.score(x_train, y_train) test_err = 1.0 - clf1.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_gank_logit_cifar10_inception_score(show_figure=False, block_figure_on_end=False): print( "========== Test GANK-Logit with Inception Score on CIFAR10 data ==========" ) np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. filepath = os.path.join( model_dir(), "male/GANK/Logit/cifar10/checkpoints/" "{epoch:04d}_{inception_score:.6f}.ckpt") checkpoint = ModelCheckpoint(filepath, mode='max', monitor='inception_score', verbose=0, save_best_only=True) loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join( model_dir(), "male/GANK/Logit/cifar10/" "loss/loss_{epoch:04d}.png"), os.path.join( model_dir(), "male/GANK/Logit/cifar10/" "loss/loss_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) inception_score_display = Display( layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join( model_dir(), "male/GANK/Logit/cifar10/inception_score/" "inception_score_{epoch:04d}.png"), os.path.join( model_dir(), "male/GANK/Logit/cifar10/inception_score/" "inception_score_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['inception_score'], 'type': 'line', 'labels': ["Inception Score"], 'title': "Scores", 'xlabel': "epoch", 'ylabel': "score", }, ], ) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, filepath=os.path.join( model_dir(), "male/GANK/Logit/cifar10/samples/" "samples_{epoch:04d}.png"), monitor=[ { 'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = GANK( model_name="GANK-Logit_CIFAR10", num_random_features=50, # set 1000 for a full run gamma_init=0.01, loss='logit', num_z=10, # set to 100 for a full run img_size=(32, 32, 3), batch_size=32, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss', 'inception_score'], # callbacks=[loss_display, inception_score_display, sample_display, checkpoint], callbacks=[checkpoint], num_epochs=4, # set to 500 for a full run inception_score_freq=1, random_state=random_seed(), verbose=1) model.fit(x_train)
def test_efrbm_mnist(visible_layer_type='binary', hidden_layer_type='continuous', show_figure=False, block_figure_on_end=False): (x_train, y_train), (x_test, y_test) = demo.load_mnist() num_train = 1000 num_test = 500 x_train = x_train[:num_train] y_train = y_train[:num_train] x_test = x_test[:num_test] y_test = y_test[:num_test] x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) learning_display = Display( title="Learning curves", dpi='auto', layout=(1, 2), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[{ 'metrics': ['recon_err', 'val_recon_err'], 'type': 'line', 'labels': ["training recon error", "validation recon error"], 'title': "Reconstruction Errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['free_energy', 'val_free_energy'], 'type': 'line', 'title': "Free Energies", 'xlabel': "epoch", 'ylabel': "energy", }]) gen_display = Display(title="Generated data", dpi='auto', layout=(1, 1), figsize=(8, 8), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['generated_data'], 'title': "Generated data", 'type': 'img', 'num_filters': 100, 'disp_dim': (28, 28), 'tile_shape': (10, 10), }, ]) recon_display = Display(title="Reconstructed data", dpi='auto', layout=(1, 1), figsize=(8, 8), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['reconstruction'], 'title': "Reconstructed data", 'type': 'img', 'data': x_train, 'num_filters': 100, 'disp_dim': (28, 28), 'tile_shape': (10, 10), }, ]) suf_stat_dim_vis = SUFFICIENT_STATISTICS_DIM[visible_layer_type] suf_stat_dim_hid = SUFFICIENT_STATISTICS_DIM[hidden_layer_type] w_init = 0.1 learning_rate = 0.1, Gaussian_layer_trainable_sigmal2 = True if visible_layer_type == 'continous' or hidden_layer_type == 'continuous': Gaussian_layer_trainable_sigmal2 = False w_init = 0.001 learning_rate = 0.01 model = EFRBM( suf_stat_dim_vis=suf_stat_dim_vis, visible_layer_type=visible_layer_type, suf_stat_dim_hid=suf_stat_dim_hid, hidden_layer_type=hidden_layer_type, Gaussian_layer_trainable_sigmal2=Gaussian_layer_trainable_sigmal2, num_hidden=15, num_visible=784, batch_size=100, num_epochs=5, momentum_method='sudden', weight_cost=2e-4, w_init=w_init, learning_rate=learning_rate, metrics=['recon_err', 'free_energy'], callbacks=[learning_display, recon_display], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=1) model.fit(x) print("Running Logistic Regression...") x_train1 = model.transform(x_train) x_test1 = model.transform(x_test) clf = LogisticRegression() clf.fit(x_train1, y_train) y_test_pred = clf.predict(x_test1) print("Error = %.4f" % (1 - accuracy_score(y_test, y_test_pred)))
def test_kmm_syndata2(show=False, block_figure_on_end=False): print("========== Test KMM on Synthetic 2D data ==========") np.random.seed(random_seed()) M = 1000 # number of samples D = 250 # number of random features (random feature dimension) x = 4 * np.random.randn(M) w = np.random.randn(2 * D) # weights (\beta in BaNK paper) # <editor-fold desc="Example of 3 kernels with \mu different from 0"> z = np.argmax(np.random.multinomial(1, [0.5, 0.25, 0.25], size=D), axis=1) o = np.hstack([ np.pi / 2 + 0.5 * np.random.randn(D, 1), 0.25 * np.random.randn(D, 1), np.pi + (1 / 3) * np.random.randn(D, 1) ]) omega = o[range(D), z] t = np.linspace(-7, 7, M) kt = (0.5 * np.exp(-0.125 * t * t) * np.cos(np.pi * t / 2) + 0.25 * np.exp(-(1 / 32) * t * t) + 0.25 * np.exp(-(1 / 18) * t * t) * np.cos(2 * np.pi * t / 2)) # </editor-fold> # <editor-fold desc="Example of 3 kernels with \mu = 0"> # z = np.argmax(np.random.multinomial(1, [0.5, 0.25, 0.25], size=D), axis=1) # o = np.hstack([0.5 * np.random.randn(D, 1), 0.25 * np.random.randn(D, 1), # (1 / 3) * np.random.randn(D, 1)]) # omega = o[range(D), z] # t = np.linspace(-7, 7, M) # kt = (0.5 * np.exp(-0.125 * t * t) + 0.25 * np.exp(-(1 / 32) * t * t) # + 0.25 * np.exp(-(1 / 18) * t * t)) # </editor-fold> phi = np.hstack([ np.cos(np.outer(x, omega)) / np.sqrt(D), np.sin(np.outer(x, omega)) / np.sqrt(D) ]) y = phi.dot(w) + np.random.randn(M) xx = t yy = np.zeros(xx.shape) phi_xx = np.hstack([ np.cos(np.outer(xx, omega)) / np.sqrt(D), np.sin(np.outer(xx, omega)) / np.sqrt(D) ]) phi_yy = np.hstack([ np.cos(np.outer(yy, omega)) / np.sqrt(D), np.sin(np.outer(yy, omega)) / np.sqrt(D) ]) approx_kt = np.sum(phi_xx * phi_yy, axis=1) loss_display = Display(layout=(2, 1), dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['err'], 'type': 'line', 'labels': ["training error"], 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['mu', 'gamma'], 'type': 'line', 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, ]) display = Display(layout=(1, 1), dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['syndata'], 'type': 'line', 'labels': ["Synthetic data"], 'title': "Synthetic data", 'xlabel': "t", 'ylabel': "k", 't': t, 'kt': kt, 'approx_kt': approx_kt, }, ]) # <editor-fold desc="1 kernel"> # c = KMM(model_name="syndata2_kmm_l2", # save_dir=os.path.join(HOME, "rmodel/pycs"), # mode='batch', loss='l2', num_kernels=1, num_epochs=50, # batch_size=100, lbd=0.0, learning_rate=0.1, learning_rate_mu=0.05, # learning_rate_gamma=0.001, learning_rate_alpha=0.01, # temperature=1.0, decay_rate=0.95, D=D, gamma=0.5, random_state=random_seed()) # c.fit(x, y) # print("pi = {}".format(c.get_pi())) # print("mu = {}".format(c.mu_.T)) # print("sigma = {}".format(np.exp(c.gamma_.T))) # print("alpha = {}".format(c.alpha_)) # # phi_xxx = c.get_phi(xx[:, np.newaxis]) # phi_yyy = c.get_phi(yy[:, np.newaxis]) # approx_ktt = np.sum(phi_xxx * phi_yyy, axis=1) # plt.plot(t, approx_ktt, 'g:', linewidth=3, label='KMM-1') # </editor-fold> # <editor-fold desc="2 kernels"> # c = KMM(model_name="syndata2_kmm_l2", # save_dir=os.path.join(HOME, "rmodel/pycs"), # mode='batch', loss='l2', num_kernels=2, num_epochs=50, # batch_size=100, lbd=0.0, learning_rate=0.1, learning_rate_mu=0.05, # learning_rate_gamma=0.001, learning_rate_alpha=0.01, # temperature=1.0, decay_rate=0.95, D=D, gamma=0.5, random_state=random_seed()) # c.fit(x, y) # print("pi = {}".format(c.get_pi())) # print("mu = {}".format(c.mu_.T)) # print("sigma = {}".format(np.exp(c.gamma_.T))) # print("alpha = {}".format(c.alpha_)) # # phi_xxx = c.get_phi(xx[:, np.newaxis]) # phi_yyy = c.get_phi(yy[:, np.newaxis]) # approx_ktt = np.sum(phi_xxx * phi_yyy, axis=1) # plt.plot(t, approx_ktt, 'y--', linewidth=3, label='KMM-2') # </editor-fold> # 3 kernels c = KMM(model_name="syndata2_kmm_l2", D=D, lbd=0.0, loss='l2', gamma=(0.1, 1.0, 2.0), mode='batch', num_kernels=3, num_epochs=10, batch_size=10, temperature=0.05, alternative_update=True, num_nested_epochs=0, adam_update=False, learning_rate=0.001, learning_rate_mu=0.01, learning_rate_gamma=0.01, learning_rate_alpha=0.01, metrics=['loss', 'err'], callbacks=[display, loss_display], random_state=random_seed()) c.fit(x, y) print("pi = {}".format(c._get_pi())) print("mu = {}".format(c.mu.T)) print("sigma = {}".format(np.exp(c.gamma_.T))) print("alpha = {}".format(c.alpha))
def test_nrbm_general(show_figure=False, block_figure_on_end=False): print("========== Test NonnegativeRBM in General ==========") np.random.seed(random_seed()) from sklearn.metrics import accuracy_score from sklearn.neighbors import KNeighborsClassifier (x_train, y_train), (x_test, y_test) = demo.load_mnist() x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) learning_display = Display( title="Learning curves", dpi='auto', layout=(2, 2), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['recon_err', 'val_recon_err'], 'type': 'line', 'labels': ["training recon error", "validation recon error"], 'title': "Reconstruction Errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['free_energy', 'val_free_energy'], 'type': 'line', 'title': "Free Energies", 'xlabel': "epoch", 'ylabel': "energy", }, { 'metrics': ['recon_loglik', 'val_recon_loglik'], 'type': 'line', 'labels': ["training recon loglik", "validation recon loglik"], 'title': "Reconstruction Loglikelihoods", 'xlabel': "epoch", 'ylabel': "loglik", }, { 'metrics': ['recon_loglik', 'val_recon_loglik'], 'type': 'line', 'labels': ["training recon loglik", "validation recon loglik"], 'title': "Reconstruction Loglikelihoods", 'xlabel': "epoch", 'ylabel': "loglik", }, # {'metrics': ['loglik_csl', 'val_loglik_csl'], # 'type': 'line', # 'labels': ["training loglik (CSL)", "validation loglik (CSL)"], # 'title': "Loglikelihoods using CSL", # 'xlabel': "epoch", # 'ylabel': "loglik", # }, ]) filter_display = Display(title="Receptive Fields", dpi='auto', layout=(1, 1), figsize=(8, 8), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['filters'], 'title': "Receptive Fields", 'type': 'img', 'num_filters': 15, 'disp_dim': (28, 28), 'tile_shape': (3, 5), }, ]) model = NonnegativeRBM( num_hidden=15, num_visible=784, batch_size=100, num_epochs=4, momentum_method='sudden', weight_cost=2e-4, nonnegative_type='barrier', nonnegative_cost=0.001, random_state=6789, metrics=['recon_err', 'free_energy', 'recon_loglik'], callbacks=[learning_display, filter_display], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], verbose=1) model.fit(x) print("Train free energy = %.4f" % model.get_free_energy(x_train).mean()) print("Test free energy = %.4f" % model.get_free_energy(x_test).mean()) print("Train reconstruction likelihood = %.4f" % model.get_reconstruction_loglik(x_train).mean()) print("Test reconstruction likelihood = %.4f" % model.get_reconstruction_loglik(x_test).mean()) x_train1 = model.transform(x_train) x_test1 = model.transform(x_test) clf = KNeighborsClassifier(n_neighbors=1) clf.fit(x_train1, y_train) print("Error = %.4f" % (1 - accuracy_score(y_test, clf.predict(x_test1))))
def test_bbrbm_pcd(show_figure=False, block_figure_on_end=False): print("========== Test BernoulliBernoulliRBM using " "Persistent Contrastive Divergence ==========") np.random.seed(random_seed()) from sklearn.metrics import accuracy_score from sklearn.neighbors import KNeighborsClassifier (x_train, y_train), (x_test, y_test) = demo.load_mnist() x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) learning_display = Display( title="Learning curves", dpi='auto', layout=(2, 2), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['recon_err', 'val_recon_err'], 'type': 'line', 'labels': ["training recon error", "validation recon error"], 'title': "Reconstruction Errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['free_energy', 'val_free_energy'], 'type': 'line', 'title': "Free Energies", 'xlabel': "epoch", 'ylabel': "energy", }, { 'metrics': ['recon_loglik', 'val_recon_loglik'], 'type': 'line', 'labels': ["training recon loglik", "validation recon loglik"], 'title': "Reconstruction Loglikelihoods", 'xlabel': "epoch", 'ylabel': "loglik", }, { 'metrics': ['recon_loglik', 'val_recon_loglik'], 'type': 'line', 'labels': ["training recon loglik", "validation recon loglik"], 'title': "Reconstruction Loglikelihoods", 'xlabel': "epoch", 'ylabel': "loglik", }, # {'metrics': ['loglik_csl', 'val_loglik_csl'], # 'type': 'line', # 'labels': ["training loglik (CSL)", "validation loglik (CSL)"], # 'title': "Loglikelihoods using CSL", # 'xlabel': "epoch", # 'ylabel': "loglik", # }, ]) filter_display = Display(title="Receptive Fields", dpi='auto', layout=(1, 1), figsize=(8, 8), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['filters'], 'title': "Receptive Fields", 'type': 'img', 'num_filters': 15, 'disp_dim': (28, 28), 'tile_shape': (3, 5), }, ]) hidden_display = Display(title="Hidden Activations", dpi='auto', layout=(1, 1), figsize=(8, 8), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['hidden_activations'], 'title': "Hidden Activations", 'type': 'img', 'data': x_train[:1000], }, ]) model = BernoulliBernoulliRBM( num_hidden=15, num_visible=784, batch_size=100, num_epochs=4, learning_method='pcd', num_pcd=5, num_chains=10, momentum_method='sudden', weight_cost=2e-4, metrics=['recon_err', 'free_energy', 'recon_loglik'], callbacks=[learning_display, filter_display, hidden_display], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=1) model.fit(x) train_free_energy = model.get_free_energy(x_train).mean() test_free_energy = model.get_free_energy(x_test).mean() print("Train free energy = %.4f" % train_free_energy) print("Test free energy = %.4f" % test_free_energy) train_recon_loglik = model.get_reconstruction_loglik(x_train).mean() test_recon_loglik = model.get_reconstruction_loglik(x_test).mean() print("Train reconstruction likelihood = %.4f" % train_recon_loglik) print("Test reconstruction likelihood = %.4f" % test_recon_loglik) x_train1 = model.transform(x_train) x_test1 = model.transform(x_test) clf = KNeighborsClassifier(n_neighbors=1) clf.fit(x_train1, y_train) test_pred_err = 1.0 - accuracy_score(y_test, clf.predict(x_test1)) print("BBRBM->kNN: test error = %.4f" % test_pred_err)
def test_kmm_syn2d(show=False, block_figure_on_end=False): print("========== Test KMM on 2D data ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_synthetic_2d() idx_train, idx_test = next( iter( StratifiedShuffleSplit(n_splits=1, test_size=40, random_state=random_seed()).split( x_train, y_train))) x0 = x_train[idx_train] y0 = y_train[idx_train] x1 = x_train[idx_test] y1 = y_train[idx_test] x = np.vstack([x0, x1]) y = np.concatenate([y0, y1]) early_stopping = EarlyStopping(monitor='val_loss', patience=2, verbose=1) filepath = os.path.join( model_dir(), "male/KMM/syn2d_data_{epoch:04d}_{val_err:.6f}.pkl") checkpoint = ModelCheckpoint(filepath, mode='min', monitor='val_err', verbose=0, save_best_only=True) display = Display(layout=(3, 1), dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['loss', 'val_loss'], 'type': 'line', 'labels': ["training loss", "validation loss"], 'title': "Learning losses", 'xlabel': "epoch", 'ylabel': "loss", }, { 'metrics': ['err', 'val_err'], 'type': 'line', 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['err'], 'type': 'line', 'labels': ["training error"], 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, ]) clf = KMM(model_name="KMM_hinge", D=10, lbd=0.0, gamma=0.5, mode='batch', loss='hinge', num_kernels=4, batch_size=4, temperature=0.1, num_epochs=10, num_nested_epochs=0, learning_rate=0.001, learning_rate_mu=0.0, learning_rate_gamma=0.001, learning_rate_alpha=0.001, metrics=['loss', 'err'], callbacks=[display, early_stopping, checkpoint], cv=[-1] * x0.shape[0] + [0] * x1.shape[0], random_state=random_seed()) clf.fit(x, y) train_err = 1.0 - clf.score(x_train, y_train) print("Training error = %.4f" % train_err) if block_figure_on_end: # save predictions y_test_pred = clf.predict(x_test) x_test[x_test == 0] = 1e-4 dump_svmlight_file(x_test, y_test_pred, os.path.join( data_dir(), "demo/synthetic_2D_data_test_predict.libsvm"), zero_based=False)
def test_wgan_gp_cifar10_fid(show_figure=False, block_figure_on_end=False): print("========== Test WGAN-GP with FID on CIFAR10 data ==========") np.random.seed(random_seed()) num_data = 128 (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train[:num_data].astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. root_dir = os.path.join(model_dir(), "male/WGAN-GP/CIFAR10") checkpoints = ModelCheckpoint( os.path.join(root_dir, "checkpoints/{epoch:04d}_{FID:.6f}.ckpt"), mode='min', monitor='FID', verbose=1, save_best_only=True) loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[os.path.join(root_dir, "loss/loss_{epoch:04d}.png"), os.path.join(root_dir, "loss/loss_{epoch:04d}.pdf")], monitor=[{'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) fid_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[os.path.join(root_dir, "FID/" "FID_{epoch:04d}.png"), os.path.join(root_dir, "FID/" "FID_{epoch:04d}.pdf")], monitor=[{'metrics': ['FID'], 'type': 'line', 'labels': ["FID"], 'title': "Scores", 'xlabel': "epoch", 'ylabel': "score", }, ], ) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, filepath=os.path.join(root_dir, "samples/" "samples_{epoch:04d}.png"), monitor=[{'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = WGAN_GP(model_name="WGAN_GP_CIFAR10", num_z=10, # set to 100 for a full run img_size=(32, 32, 3), batch_size=16, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss', 'FID', 'FID_100points'], callbacks=[loss_display, fid_display, sample_display, checkpoints], num_epochs=4, # set to 100 for a full run inception_metrics=[FID(data="cifar10"), FID(name="FID_100points", data=x_train[:100])], inception_metrics_freq=1, # summary_freq=1, # uncomment this for a full run log_path=os.path.join(root_dir, "logs"), random_state=random_seed(), verbose=1) model.fit(x_train)
def test_dcgan_cifar10_inception_metric(show_figure=False, block_figure_on_end=False): print( "========== Test DCGAN with Inception Score and FID on CIFAR10 data ==========" ) np.random.seed(random_seed()) num_data = 128 (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train[:num_data].astype(np.float32).reshape([-1, 32, 32, 3 ]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. root_dir = os.path.join(model_dir(), "male/DCGAN/cifar10") checkpoints_is = ModelCheckpoint(os.path.join( root_dir, "checkpoints_is/the_best_is.ckpt"), mode='max', monitor='inception_score', verbose=1, save_best_only=True) checkpoints_fid = ModelCheckpoint(os.path.join( root_dir, "checkpoints_fid/the_best_fid.ckpt"), mode='min', monitor='FID', verbose=1, save_best_only=True) loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join(root_dir, "loss/loss_{epoch:04d}.png"), os.path.join(root_dir, "loss/loss_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) inception_score_display = Display( layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join(root_dir, "inception_score/" "inception_score_{epoch:04d}.png"), os.path.join(root_dir, "inception_score/" "inception_score_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['inception_score'], 'type': 'line', 'labels': ["Inception Score"], 'title': "Scores", 'xlabel': "epoch", 'ylabel': "score", }, ], ) fid_display = Display( layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join(root_dir, "FID/" "FID_{epoch:04d}.png"), os.path.join(root_dir, "FID/" "FID_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['FID'], 'type': 'line', 'labels': ["FID"], 'title': "Scores", 'xlabel': "epoch", 'ylabel': "score", }, ], ) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, filepath=os.path.join( root_dir, "samples/samples_{epoch:04d}.png"), monitor=[ { 'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = DCGAN( model_name="DCGAN_CIFAR10", num_z=10, # set to 100 for a full run img_size=(32, 32, 3), batch_size=16, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=[ 'd_loss', 'g_loss', 'inception_score', 'inception_score_std', 'FID' ], callbacks=[ loss_display, inception_score_display, fid_display, sample_display, checkpoints_is, checkpoints_fid ], num_epochs=4, # set to 100 for a full run inception_metrics=[InceptionScore(), FID(data="cifar10")], inception_metrics_freq=1, # summary_freq=1, # uncomment this for a full run log_path=os.path.join(root_dir, "logs"), random_state=random_seed(), verbose=1) model.fit(x_train) filepath = os.path.join(root_dir, "checkpoints_fid/the_best_fid.ckpt") print("Reloading the latest model at: {}".format(filepath)) model1 = TensorFlowModel.load_model(filepath) model1.inception_metrics = InceptionMetricList( [InceptionScore(), FID(data="cifar10")]) model1.num_epochs = 6 model1.fit(x_train) print("Done!")
def test_mggan2d_gmm2d(show_figure=False, block_figure_on_end=False): loss_display = Display( layout=(4, 1), dpi='auto', title='Loss', freq=1, # set to 1000 for a full run show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, { 'metrics': ['loglik'], 'type': 'line', 'labels': ["Log-likelihood"], 'title': "Evaluation", 'xlabel': "epoch", 'ylabel': "loglik", }, { 'metrics': ['d_bin_loss', 'd_mul_loss'], 'type': 'line', 'labels': ["d_bin_loss", "d_mul_loss"], 'title': "Discriminator Losses", 'xlabel': "epoch", 'ylabel': "loss", }, { 'metrics': ['g_bin_loss', 'g_mul_loss'], 'type': 'line', 'labels': ["g_bin_loss", "g_mul_loss"], 'title': "Generator Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ], ) scatter_display = Display( layout=(1, 1), figsize=(12, 12), dpi='auto', filepath=[ os.path.join(model_dir(), "male/MGGAN2D/samples/" "scatter_{epoch:04d}.png"), os.path.join(model_dir(), "male/MGGAN2D/samples/" "scatter_{epoch:04d}.pdf") ], freq=1, # set to 1000 for a full run title='Scatter', show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['scatter'], 'type': 'scatter', 'title': "Data scatter generated by MGGAN2D", 'num_samples': 512, 'xlim': (-3, 3), 'ylim': (-3, 3), }, ]) num_mixtures = 8 radius = 2.0 std = 0.02 import numpy as np thetas = np.linspace(0, 2 * np.pi, num_mixtures + 1)[:num_mixtures] xs, ys = radius * np.sin(thetas), radius * np.cos(thetas) model = MGGAN2D( data=GMM(mix_coeffs=[1 / num_mixtures] * num_mixtures, mean=list(zip(xs, ys)), cov=[[std, std]] * num_mixtures), beta=0.05, # weight for g_bin_loss num_z=4, # set to 128 for a full run generator=Gaussian1D(mu=0.0, sigma=1.0), num_gens=4, d_batch_size=4, # set to 512 for full run g_batch_size=4, # set to 128 for full run num_epochs=4, # set to 25000 for a full run d_hidden_size=4, # set to 128 for a full run g_hidden_size=4, # set to 128 for a full run loglik_freq=1, generator_learning_rate=0.0002, discriminator_learning_rate=0.0002, summary_freq=250, metrics=[ 'd_bin_loss', 'd_mul_loss', 'd_loss', 'g_bin_loss', 'g_mul_loss', 'g_loss', 'loglik' ], callbacks=[loss_display, scatter_display], random_state=random_seed(), verbose=1) model.fit()
def test_wgan_gp_resnet_cifar10(show_figure=False, block_figure_on_end=False): print("========== Test WGAN-GP-ResNet on CIFAR10 data ==========") np.random.seed(random_seed()) num_data = 128 (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train[:num_data].astype(np.float32).reshape([-1, 32, 32, 3 ]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. root_dir = os.path.join(model_dir(), "male/WGAN-GP-ResNet/CIFAR10") loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join(root_dir, "loss/loss_{epoch:04d}.png"), os.path.join(root_dir, "loss/loss_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = WGAN_GP_ResNet( model_name="WGAN_GP_ResNet_CIFAR10", num_z=10, # set to 100 for a full run img_size=(32, 32, 3), batch_size=16, # set to 64 for a full run g_blocks=('up', 'up', 'up'), d_blocks=('down', 'down', None, None), num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=2, # set to 100 for a full run random_state=random_seed(), log_path=os.path.join(root_dir, "logs"), verbose=1) model.fit(x_train) print("Saving model...") save_file_path = model.save(os.path.join(root_dir, "checkpoints/ckpt")) print("Reloading model...") model1 = TensorFlowModel.load_model(save_file_path) model1.num_epochs = 4 model1.fit(x_train) print("Done!")
def test_wgan_gp_resnet_cifar10_inception_metric(show_figure=False, block_figure_on_end=False): print( "========== Test WGAN-GP-ResNet with Inception Score and FID on CIFAR10 data ==========" ) np.random.seed(random_seed()) num_data = 128 (x_train, y_train), (x_test, y_test) = demo.load_cifar10() x_train = x_train[:num_data].astype(np.float32).reshape([-1, 32, 32, 3 ]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 32, 32, 3]) / 0.5 - 1. # uncomment for full run ''' import pickle from male.configs import data_dir tmp = pickle.load(open(os.path.join(data_dir(), "cifar10/cifar10_train.pkl"), "rb")) x_train = tmp['data'].astype(np.float32).reshape( [-1, 32, 32, 3]) / 127.5 - 1. ''' root_dir = os.path.join(model_dir(), "male/WGAN-GP-ResNet/CIFAR10") checkpoints_is = ModelCheckpoint(os.path.join( root_dir, "checkpoints_is/the_best_is.ckpt"), mode='max', monitor='inception_score', verbose=1, save_best_only=True) checkpoints_fid = ModelCheckpoint(os.path.join( root_dir, "checkpoints_fid/the_best_fid.ckpt"), mode='min', monitor='FID', verbose=1, save_best_only=True) loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join(root_dir, "loss/loss_{epoch:04d}.png"), os.path.join(root_dir, "loss/loss_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) inception_score_display = Display( layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join(root_dir, "inception_score/" "inception_score_{epoch:04d}.png"), os.path.join(root_dir, "inception_score/" "inception_score_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['inception_score'], 'type': 'line', 'labels': ["Inception Score"], 'title': "Scores", 'xlabel': "epoch", 'ylabel': "score", }, ], ) fid_display = Display( layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, filepath=[ os.path.join(root_dir, "FID/" "FID_{epoch:04d}.png"), os.path.join(root_dir, "FID/" "FID_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['FID'], 'type': 'line', 'labels': ["FID"], 'title': "Scores", 'xlabel': "epoch", 'ylabel': "score", }, ], ) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, filepath=os.path.join( root_dir, "samples/samples_{epoch:04d}.png"), monitor=[ { 'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = WGAN_GP_ResNet( model_name="WGAN_GP_ResNet_CIFAR10", num_z=8, # set to 128 for a full run img_size=(32, 32, 3), batch_size=64, # set to 64 for a full run g_blocks=('up', 'up', 'up'), d_blocks=('down', 'down', None, None), num_gen_feature_maps=8, # set to 128 for a full run num_dis_feature_maps=8, # set to 128 for a full run metrics=[ 'd_loss', 'g_loss', 'inception_score', 'inception_score_std', 'FID' ], callbacks=[ loss_display, inception_score_display, fid_display, sample_display, checkpoints_is, checkpoints_fid ], num_epochs=2, # set to 500 for a full run inception_metrics=[InceptionScore(), FID(data='cifar10')], inception_metrics_freq=1, num_inception_samples=100, # set to 50000 for a full run # summary_freq=1, # uncomment this for a full run log_path=os.path.join(root_dir, 'logs'), random_state=random_seed(), verbose=1) model.fit(x_train) filepath = os.path.join(root_dir, 'checkpoints_fid/the_best_fid.ckpt') print('Reloading the latest model at: {}'.format(filepath)) model1 = TensorFlowModel.load_model(filepath) model1.inception_metrics = InceptionMetricList( [InceptionScore(), FID(data='cifar10')]) model1.num_epochs = 4 model1.fit(x_train) print('Done!')
def test_dcgan_save_and_load(show_figure=False, block_figure_on_end=False): print( "========== Test Save and Load functions of DCGAN on MNIST data ==========" ) np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() x_train = x_train.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. x_test = x_test.astype(np.float32).reshape([-1, 28, 28, 1]) / 0.5 - 1. root_dir = os.path.join(model_dir(), "male/DCGAN/MNIST") loss_display = Display(layout=(1, 1), dpi='auto', show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['d_loss', 'g_loss'], 'type': 'line', 'labels': ["discriminator loss", "generator loss"], 'title': "Losses", 'xlabel': "epoch", 'ylabel': "loss", }, ]) sample_display = Display(layout=(1, 1), dpi='auto', figsize=(10, 10), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['x_samples'], 'title': "Generated data", 'type': 'img', 'num_samples': 100, 'tile_shape': (10, 10), }, ]) model = DCGAN( model_name="DCGAN_MNIST_SaveLoad", num_z=10, # set to 100 for a full run img_size=(28, 28, 1), batch_size=16, # set to 64 for a full run num_conv_layers=3, # set to 3 for a full run num_gen_feature_maps=4, # set to 32 for a full run num_dis_feature_maps=4, # set to 32 for a full run metrics=['d_loss', 'g_loss'], callbacks=[loss_display, sample_display], num_epochs=2, # set to 100 for a full run log_path=os.path.join(root_dir, "logs"), random_state=random_seed(), verbose=1) model.fit(x_train) print("Saving model...") save_file_path = model.save(os.path.join(root_dir, "checkpoints/ckpt")) print("Reloading model...") model1 = TensorFlowModel.load_model(save_file_path) model1.num_epochs = 4 model1.fit(x_train) print("Done!")
def test_kmm_cv_disp(show=False, block_figure_on_end=False): print("========== Test cross-validation for KMM with Display ==========") np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_iris() print("Number of training samples = {}".format(x_train.shape[0])) print("Number of testing samples = {}".format(x_test.shape[0])) x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) early_stopping = EarlyStopping(monitor='val_err', patience=2, verbose=1) filepath = os.path.join(model_dir(), "male/KMM/iris_{epoch:04d}_{val_err:.6f}.pkl") checkpoint = ModelCheckpoint(filepath, mode='min', monitor='val_err', verbose=0, save_best_only=True) display = Display(layout=(3, 1), dpi='auto', show=show, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['loss', 'val_loss'], 'type': 'line', 'labels': ["training loss", "validation loss"], 'title': "Learning losses", 'xlabel': "epoch", 'ylabel': "loss", }, { 'metrics': ['err', 'val_err'], 'type': 'line', 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['err'], 'type': 'line', 'labels': ["training error"], 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, ]) clf = KMM(model_name="KMM_hinge", D=20, lbd=0.0, gamma=0.1, mode='batch', loss='hinge', num_kernels=3, batch_size=100, temperature=1.0, num_epochs=10, num_nested_epochs=1, learning_rate=0.1, learning_rate_mu=0.0, learning_rate_gamma=0.1, learning_rate_alpha=0.1, metrics=['loss', 'err'], callbacks=[display, early_stopping, checkpoint], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=1) clf.fit(x, y) train_err = 1.0 - clf.score(x_train, y_train) test_err = 1.0 - clf.score(x_test, y_test) print("Training error = %.4f" % train_err) print("Testing error = %.4f" % test_err)
def test_pytorch_convnet_v1(show=False, block_figure_on_end=False): print("========== Test PytorchConvNetv1 ==========") np.random.seed(random_seed()) err_display = Display(title="Error curves", dpi='auto', layout=(1, 1), freq=1, show=show, block_on_end=block_figure_on_end, monitor=[{ 'metrics': ['err'], 'type': 'line', 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }]) loss_display = Display( title="Learning curves", dpi='auto', layout=(3, 1), freq=1, show=show, block_on_end=block_figure_on_end, filepath=[ os.path.join(model_dir(), "male/PyTorchConvNet/" "loss/loss_{epoch:04d}.png"), os.path.join(model_dir(), "male/PyTorchConvNet/" "loss/loss_{epoch:04d}.pdf") ], monitor=[ { 'metrics': ['loss'], 'type': 'line', 'labels': ["training loss"], 'title': "Learning losses", 'xlabel': "epoch", 'xlabel_params': { 'fontsize': 50 }, 'ylabel': "loss", }, { 'metrics': ['err'], 'type': 'line', 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['err'], 'type': 'line', 'labels': ["training error"], 'title': "Learning errors", 'xlabel': "epoch", 'ylabel': "error", }, ]) weight_display = Display(title="Filters", dpi='auto', layout=(1, 1), figsize=(6, 15), freq=1, show=show, block_on_end=block_figure_on_end, filepath=os.path.join( model_dir(), "male/PyTorchConvNet/" "weights/weights_{epoch:04d}.png"), monitor=[ { 'metrics': ['weights'], 'title': "Learned weights", 'type': 'img', 'tile_shape': (5, 2), }, ]) # Construct the model. clf = PyTorchConvNet(model_name='PyTorchConvNet', num_epochs=1, batch_size=64, metrics=['loss', 'err'], callbacks=[loss_display, err_display], random_state=random_seed(), verbose=1) print('Show some of the training images.') if False: clf.num_epochs = 0 # Build the network as well as data loaders. clf.fit() import torchvision def imshow(img): img = img / 2 + 0.5 # unnormalize npimg = img.numpy() plt.imshow(np.transpose(npimg, (1, 2, 0))) plt.show(block=True) # get some random training images dataiter = iter(clf.trainloader) images, labels = dataiter.next() # show images imshow(torchvision.utils.make_grid(images)) # print labels print(' '.join('%5s' % clf.classes[labels[j]] for j in range(4))) clf.num_epochs = 1 clf.fit() print('Testing accuracy = %.4f' % (100 * clf.acc_test())) save_file_path = clf.save() clf = PyTorchModel.load_model(save_file_path) clf.num_epochs = 4 clf.fit() print('Testing accuracy = %.4f' % (100 * clf.acc_test()))
def test_ssrbm_classification(show_figure=False, block_figure_on_end=False): print("========== Test Semi-Supervised RBM for Classification ==========") num_labeled_data = 1000 from sklearn.metrics import accuracy_score from sklearn.neighbors import KNeighborsClassifier np.random.seed(random_seed()) (x_train, y_train), (x_test, y_test) = demo.load_mnist() # remove some labels idx_train, idx_test = next( iter( StratifiedShuffleSplit(n_splits=1, test_size=num_labeled_data, random_state=random_seed()).split( x_train, y_train))) y_train[idx_train] = 10**8 x = np.vstack([x_train, x_test]) y = np.concatenate([y_train, y_test]) learning_display = Display( title="Learning curves", dpi='auto', layout=(3, 1), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['recon_err', 'val_recon_err'], 'type': 'line', 'labels': ["training recon error", "validation recon error"], 'title': "Reconstruction Errors", 'xlabel': "epoch", 'ylabel': "error", }, { 'metrics': ['loss', 'val_loss'], 'type': 'line', 'labels': ["training loss", "validation loss"], 'title': "Learning Losses", 'xlabel': "epoch", 'ylabel': "loss", }, { 'metrics': ['err', 'val_err'], 'type': 'line', 'labels': ["training error", "validation error"], 'title': "Prediction Errors", 'xlabel': "epoch", 'ylabel': "error", }, # {'metrics': ['loglik_csl', 'val_loglik_csl'], # 'type': 'line', # 'labels': ["training loglik (CSL)", "validation loglik (CSL)"], # 'title': "Loglikelihoods using CSL", # 'xlabel': "epoch", # 'ylabel': "loglik", # }, ]) filter_display = Display(title="Receptive Fields", dpi='auto', layout=(1, 1), figsize=(8, 8), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['filters'], 'title': "Receptive Fields", 'type': 'img', 'num_filters': 15, 'disp_dim': (28, 28), 'tile_shape': (3, 5), }, ]) hidden_display = Display(title="Hidden Activations", dpi='auto', layout=(1, 1), figsize=(8, 8), freq=1, show=show_figure, block_on_end=block_figure_on_end, monitor=[ { 'metrics': ['hidden_activations'], 'title': "Hidden Activations", 'type': 'img', 'data': x_train[:100], }, ]) early_stopping = EarlyStopping(monitor='val_loss', patience=2, verbose=1) filepath = os.path.join(model_dir(), "male/ssRBM/mnist_{epoch:04d}_{val_loss:.6f}.pkl") checkpoint = ModelCheckpoint(filepath, mode='min', monitor='val_loss', verbose=0, save_best_only=True) model = SemiSupervisedRBM(num_hidden=15, num_visible=784, batch_size=100, num_epochs=4, learning_rate=0.1, w_init=0.1, momentum_method='sudden', weight_cost=0.0, inference_engine='variational_inference', approx_method='first_order', metrics=['recon_err', 'loss', 'err'], callbacks=[ filter_display, learning_display, hidden_display, early_stopping, checkpoint ], cv=[-1] * x_train.shape[0] + [0] * x_test.shape[0], random_state=random_seed(), verbose=1) model.fit(x, y) print("Test reconstruction error = %.4f" % model.get_reconstruction_error(x_test).mean()) print("=========== Predicted by Semi-Supervised RBM ============") print("Train accuracy = {0:>1.4f}\tTest accuracy = {1:>1.4f}".format( accuracy_score(y_train, model.predict(x_train)), accuracy_score(y_test, model.predict(x_test)))) x_train1 = model.transform(x_train) x_test1 = model.transform(x_test) clf = KNeighborsClassifier(n_neighbors=4) clf.fit(x_train1, y_train) print("=========== Predicted by kNN ============") print("Train accuracy = {0:>1.4f}\tTest accuracy = {1:>1.4f}".format( accuracy_score(y_train, clf.predict(x_train1)), accuracy_score(y_test, clf.predict(x_test1))))