def pca_mlp(X_train, X_test, y_train, y_test, val_images=None): pca = randomized_PCA(X_train) X_train_pca = pca.transform(X_train) X_test_pca = pca.transform(X_test) param_grid = {'l2decay': [0.01, 0.1, 0.5, 0.8, 1.0, 1.5] } clf = MLPClassifier(batch_size=129, loss='cross_entropy', output_layer='softmax', l2decay=0.1) clf.fit(X_train_pca, y_train, max_epochs=100) y_pred = clf.predict(X_test_pca) show_metrics(y_test, y_pred)
class MultilayerPerceptron(Learn): def __init__(self, **kwargs): Learn.__init__(self, **kwargs) self.algo =MLPClassifier() def _train_routine(self, train_X, train_Y): return self.algo.fit(train_X, train_Y) def predict(self, test_data=[]): return self.algo.predict(test_data) def set_parameters(self, parameters): for key, value in parameters.iteritems(): setattr(self.algo, key, value) Learn.set_parameters(self, parameters)
def __init__(self): super(Classifier, self).__init__() model = DAGCN self.gnn = model(latent_dim=cmd_args.latent_dim, output_dim=cmd_args.out_dim, num_node_feats=cmd_args.feat_dim, num_edge_feats=0, multi_h_emb_weight=cmd_args.multi_h_emb_weight, max_k=cmd_args.max_k, dropout=cmd_args.dropout, max_block=cmd_args.max_block, reg=cmd_args.reg) out_dim = cmd_args.out_dim if out_dim == 0: out_dim = cmd_args.latent_dim * cmd_args.multi_h_emb_weight if cmd_args.gm == 'attention': out_dim = cmd_args.latent_dim * cmd_args.multi_h_emb_weight self.mlp = MLPClassifier(input_size=out_dim, hidden_size=cmd_args.hidden, num_class=cmd_args.num_class)
def __init__(self): super(Classifier, self).__init__() model = AttentionEmbedMeanField self.s2v = model(latent_dim=cmd_args.latent_dim, output_dim=cmd_args.out_dim, num_node_feats=cmd_args.feat_dim, num_edge_feats=0, multi_h_emb_weight=cmd_args.multi_h_emb_weight, max_k=cmd_args.max_k, dropout=cmd_args.dropout, max_block=cmd_args.max_block, reg=cmd_args.reg) out_dim = cmd_args.out_dim if out_dim == 0: out_dim = cmd_args.latent_dim * cmd_args.multi_h_emb_weight if cmd_args.gm == 'attention': #Note: outdim is 0 by most of case, may not need out_dim = cmd_args.latent_dim * cmd_args.multi_h_emb_weight # multi-head=3 # self.mlp = MLPClassifier1layer(input_size=out_dim, num_class=cmd_args.num_class) # else: self.mlp = MLPClassifier(input_size=out_dim, hidden_size=cmd_args.hidden, num_class=cmd_args.num_class)
def iris(): print("-------------iris----------------") mat = Arff("../data/perceptron/iris.arff", label_count=3) y = mat.data[:,-1] # print(y) lb = preprocessing.LabelBinarizer() lb.fit(y) y = lb.transform(y) # split it # data, labels, tData, tLabels = _shuffle_split(mat.data[:, :-1], y, .25) data, tData, labels, tLabels = train_test_split(mat.data[:, :-1], y, test_size=.25) MLPClass = MLPClassifier([2*np.shape(data)[1]], lr=0.1, shuffle=True, one_hot=True) MLPClass.fit(data, labels, momentum=0.5, percent_verify=.25) np.savetxt("Iris_eval.csv", MLPClass.stupidData[1:], header=reduce(MLPClass.stupidData[0]), delimiter=',') accuracy = MLPClass.score(tData, tLabels) print("Test Accuracy = [{:.2f}]".format(accuracy))
def sci_kit(): print("------------------------sci-kit learn------------------") mat = Arff("./tic-tac-toe.arff", label_count=1) y = mat.data[:,-1] lb = preprocessing.LabelBinarizer() lb.fit(y) y = lb.transform(y) data, tData, labels, tLabels = train_test_split(mat.data[:, :-1], y, test_size=.25) # going to randomly search learn_rate, number of nodes, number of hidden layers, and momentum for dummy_iterator in range(16): # 4 features times 4 features MLPClass = None real_one = None learn_rate = np.random.uniform(0.001, 10) number_of_nodes = int(abs(round(np.random.normal(2*np.shape(data)[1], round(2*np.shape(data)[1] - 0.6))))) hidden_layers = np.random.randint(1, 4) momentum = abs(np.random.normal(0, 0.1)) nodes = [1 if number_of_nodes == 0 else number_of_nodes] * hidden_layers print("learn rate", learn_rate, "layers", nodes, "momentum", momentum) MLPClass = MLPClassifier([2*np.shape(data)[1]], lr=0.1, shuffle=True, one_hot=True) MLPClass.fit(data, labels, momentum=0.5, percent_verify=.25) real_one = Skl_Classifier(nodes, momentum=momentum, learning_rate_init=learn_rate, activation='logistic', early_stopping=True, validation_fraction=.25) print("x") real_one.fit(data, np.reshape(labels, (-1,))) real_accuracy = real_one.score(tData, np.reshape(tLabels, (-1,))) accuracy = MLPClass.score(tData, tLabels) print(accuracy, "vs", real_accuracy)
def basic(): print("-------------basic---------------") x = np.array([[0,0],[0,1], [1, 1]]) y = np.array([[1],[0], [0]]) l1 = np.array([[1, 1], [1, 1], [1, 1]]) l2 = np.array([[1], [1], [1]]) w = [l1, l2] tx = np.array([[1, 0]]) # second has a zero ty = np.array([[1]]) # print(x) # print(y) pc = None try: pc = MLPClassifier([2], 1, shuffle=False, deterministic=10) # print(pc) print(pc.fit(x, y, w, 1, percent_verify=0)) print("fake score", pc.score(tx, ty)) # print(pc.fit(x, y).score(np.array([[0,0,0],[0,1,0],[1,0,0],[1,1,0]]), np.array([[0],[0],[1],[1]]))) # print(pc) except Exception as e: print("AN EXCEPTION OCCURRED!!!---------\n\r"*2, pc) raise e
def __init__(self): super(Classifier, self).__init__() if cmd_args.gm == 'mean_field': model = EmbedMeanField elif cmd_args.gm == 'loopy_bp': model = EmbedLoopyBP else: print('unknown gm %s' % cmd_args.gm) sys.exit() self.s2v = model(latent_dim=cmd_args.latent_dim, output_dim=cmd_args.out_dim, num_node_feats=cmd_args.feat_dim, num_edge_feats=0, max_lv=cmd_args.max_lv) out_dim = cmd_args.out_dim if out_dim == 0: out_dim = cmd_args.latent_dim self.mlp = MLPClassifier(input_size=out_dim, hidden_size=cmd_args.hidden, num_class=cmd_args.num_class)
def evaluate(): print("------------eval-------------------") mat = Arff("../data/perceptron/evaluation/data_banknote_authentication.arff", label_count=1) data = mat.data[:, 0:-1] labels = mat.data[:, -1].reshape(-1, 1) # print("data\n", data) MLPClass = MLPClassifier([2*np.shape(data)[1]], lr=0.1, shuffle=False, deterministic=10) MLPClass.fit(data, labels, momentum=0.5, percent_verify=0, standard_weight=0) Accuracy = MLPClass.score(data, labels) # print(MLPClass) # print("Final Weights =", MLPClass.get_weights()) # print(MLPClass) print(MLPClass.csv_print())
def debug(): print("------------arff-------------------") mat = Arff("../data/perceptron/debug/linsep2nonorigin.arff", label_count=1) data = mat.data[:, 0:-1] labels = mat.data[:, -1].reshape(-1, 1) # print("data\n", data) MLPClass = MLPClassifier([2*np.shape(data)[1]], lr=0.1, shuffle=False, deterministic=10) MLPClass.fit(data, labels, momentum=0.5, percent_verify=0, standard_weight=0) Accuracy = MLPClass.score(data, labels) # print(MLPClass) retrieved_weights = MLPClass.get_weights() for layer in range(len(retrieved_weights)): np.savetxt("linsep_weights_eval_" + str(layer) + ".csv", retrieved_weights[layer], delimiter=',')
def nn(X_train, y_train, X_test, y_test, conf_matrix=False): y_unique = len(np.unique(y_train)) model = MLPClassifier(dim_in=X_train.shape[0], dim_out=y_unique, dims_hid=ARG_DIMS_HID, act_hid=ARG_ACTIVATION, dist=ARG_WEIGHT_INIT, dist_scale=ARG_WEIGHT_SCALE) trainCEs, trainREs = model.train(X_train, y_train, alpha=ARG_ALPHA, epochs=ARG_EPOCHS, trace=False, alpha_drop_each=ARG_ALPHA_DROP_EACH, alpha_drop_rate=ARG_ALPHA_DROP_RATE) trainCE, trainRE = model.test(X_train, y_train) testCE, testRE = model.test(X_test, y_test) _, y_pred = model.predict(X_test) if conf_matrix: # export confussion matrix cm = confusion_matrix(y_test, y_pred, labels=list(char_to_int.values())) print(cm) file_path = os.path.join('.', 'output', 'confusion_matrix.csv') with open(file_path, 'w') as f: for k in char_to_int.keys(): f.write('\t{}'.format(k)) f.write('\n') for k, row in zip(char_to_int.keys(), range(len(char_to_int.keys()))): f.write(k) for i in cm[row]: f.write('\t{}'.format(i)) f.write('\n') plot_table(file_path, (6, 1)) #plot_dots(X_train, None, None, X_test, y_test, y_pred, save_path=os.path.join('.', 'output', 'predict.png')) # predicted #plot_dots(X_train, None, None, X_test, y_test, None, save_path=os.path.join('.', 'output', 'test.png')) # reality #plot_both_errors(trainCEs, trainREs, testCE, testRE, save_path=os.path.join('.', 'output', 'errors.png')) return trainCE, testCE
def main(): def reset(): nonlocal train_x, train_y, test_x, test_y, n_classes nonlocal train_samples, test_samples, train_accuracy, train_loss nonlocal test_accuracy, test_loss, train_accuracy_goal nonlocal test_accuracy_goal train_x, train_y, test_x, test_y, n_classes = [], [], [], [], 0 train_samples = 0 test_samples = 0 train_accuracy = 0. train_loss = np.inf test_accuracy = 0. test_loss = np.inf train_accuracy_goal = 1. test_accuracy_goal = 1. def show_confusion_matrix_ex(n_classes, y_predicted, y_true, s): train_y_typename = train_y.dtype.type.__name__ if 'float' in train_y_typename or 'float' in train_y_typename: show_confusion_matrix(n_classes, y_true, y_predicted, s) else: show_confusion_matrix(n_classes, neural_network.to_class_index(y_true), neural_network.to_class_index(y_predicted), s) def do_command(x, y, sample, sample_type: str): n_samples = x.shape[0] print(f"{sample_type.capitalize()} sample {sample + 1}\nX:", x[sample].tolist(), "\nY:", y[sample]) try: plt.imshow(x[sample]) plt.show(block=False) visualization = True except TypeError: print("This data cannot be visualized on a graph") plt.close('all') visualization = False if command == 'predict' and 0 <= sample < n_samples: if train_accuracy > 0. and train_loss < np.inf: x_sample = np.array([x[sample]]) print("Predicted Y:", neural_network.predict_classes(x_sample)[0]) while visualization and enter_bool("Experiment?"): print("1. Pixel Removal", "2. Adding noise", "3. Pixel Removal & Adding noise", "4. Black row/col", "5. White row/col", "6. Image modification", sep='\n') v = enter_number("#>", 1, 6, int) xt = neural_network.prepared_data.prepare_x(x_sample) if v == 1: pixel_percent = enter_number( "Pixel removal percentage (0-1):", 0, 1, float) r = Experiments.prepare_x_for_experiment_1( xt, pixel_percent) elif v == 2: pixel_percent = enter_number("Pixel percentage (0-1):", 0, 1, float) noise_factor = enter_number("Noise factor (0-1):", 0, 1, float) r = Experiments.prepare_x_for_experiment_2( xt, pixel_percent, noise_factor) elif v == 3: pap = enter_number("Pixel addition percentage (0-1):", 0, 1, float) prp = enter_number("Pixel removal percentage (0-1):", 0, 1, float) nf = enter_number("Noise factor (0-1):", 0, 1, float) r = Experiments.prepare_x_for_experiment_3( xt, pap, prp, nf) elif v == 4: xtm = xt.reshape((xt.shape[0], *x_sample.shape[1:])) row = enter_number(f"Row (0/1-{x_sample.shape[1]}):", 0, x_sample.shape[1], int) - 1 col = enter_number(f"Col (0/1-{x_sample.shape[2]}):", 0, x_sample.shape[2], int) - 1 r = Experiments.prepare_x_for_experiment_4( xtm, row, col).reshape(xt.shape) elif v == 5: xtm = xt.reshape((xt.shape[0], *x_sample.shape[1:])) row = enter_number(f"Row (0/1-{x_sample.shape[1]}):", 0, x_sample.shape[1], int) - 1 col = enter_number(f"Col (0/1-{x_sample.shape[2]}):", 0, x_sample.shape[2], int) - 1 r = Experiments.prepare_x_for_experiment_5( xtm, row, col).reshape(xt.shape) elif v == 6: bold = enter_number(f"Bold (0-1):", 0., 1., float) italic = enter_number(f"Italic (0-1):", 0., 1., float) underline = enter_number(f"Underline (0-1):", 0., 1., float) r = Experiments.prepare_x_for_experiment_8( xt, bold, italic, underline) else: return None print( "Predicted Y':", neural_network.predict_classes( r, _input_is_prepared=True)[0]) plt.figure() plt.imshow(r[0].reshape(x_sample.shape[1:])) plt.show(block=False) else: print("Train accuracy <= 0 or train loss == infinity") def is_renderable(x): try: plt.imshow(x) visualization = True except TypeError: visualization = False plt.close('all') return visualization def do_experiment(x, y, n_classes_exp): if train_accuracy > 0. and train_loss < np.inf and x.shape[ 0] > 0 and is_renderable(x[0]): print("1. Pixel Removal", "2. Adding noise", "3. Pixel Removal & Adding noise", "4. Black row/col", "5. White row/col", "6. Image modification", sep='\n') v = enter_number("#>", 1, 6, int) xt = neural_network.prepared_data.prepare_x(x) if v == 1: pixel_percent = enter_number("Pixel removal percentage (0-1):", 0, 1, float) r = Experiments.prepare_x_for_experiment_1(xt, pixel_percent) elif v == 2: pixel_percent = enter_number("Pixel percentage (0-1):", 0, 1, float) noise_factor = enter_number("Noise factor (0-1):", 0, 1, float) r = Experiments.prepare_x_for_experiment_2( xt, pixel_percent, noise_factor) elif v == 3: pap = enter_number("Pixel addition percentage (0-1):", 0, 1, float) prp = enter_number("Pixel removal percentage (0-1):", 0, 1, float) nf = enter_number("Noise factor (0-1):", 0, 1, float) r = Experiments.prepare_x_for_experiment_3(xt, pap, prp, nf) elif v == 4: xtm = xt.reshape((xt.shape[0], *x.shape[1:])) row = enter_number(f"Row (0/1-{x.shape[1]}):", 0, x.shape[1], int) - 1 col = enter_number(f"Col (0/1-{x.shape[2]}):", 0, x.shape[2], int) - 1 r = Experiments.prepare_x_for_experiment_4( xtm, row, col).reshape(xt.shape) elif v == 5: xtm = xt.reshape((xt.shape[0], *x.shape[1:])) row = enter_number(f"Row (0/1-{x.shape[1]}):", 0, x.shape[1], int) - 1 col = enter_number(f"Col (0/1-{x.shape[2]}):", 0, x.shape[2], int) - 1 r = Experiments.prepare_x_for_experiment_5( xtm, row, col).reshape(xt.shape) elif v == 6: bold = enter_number(f"Bold (0-1):", 0., 1., float) italic = enter_number(f"Italic (0-1):", 0., 1., float) underline = enter_number(f"Underline (0-1):", 0., 1., float) r = Experiments.prepare_x_for_experiment_8( xt, bold, italic, underline) else: return None losses_exp, correct_answers_exp = neural_network.evaluate( r, y, _input_is_prepared=True) print("EXPERIMENT RESULTS") print("Loss:", losses_exp.mean()) print("Accuracy (%):", correct_answers_exp * 100 / x.shape[0]) show_confusion_matrix_ex( n_classes_exp, y, neural_network.predict_classes(r, _input_is_prepared=True), 'Experiment. Confusion matrix') else: print("Train accuracy <= 0 or train loss == infinity") plt.rcParams['image.cmap'] = 'Greys' train_x, train_y, test_x, test_y, n_classes = [], [], [], [], 0 clear_command = 'cls' if 'windows' in platform.system().lower( ) else 'clear' chosen_dataset = '' features = 0 train_samples = 0 test_samples = 0 neural_network = None train_accuracy = 0. train_loss = np.inf test_accuracy = 0. test_loss = np.inf train_accuracy_goal = 1. test_accuracy_goal = 1. parameters = 0 cur_is_renderable = False output_classes_filename = 'output_classes.txt' best_choice = frozenset({'train_accuracy'}) while True: os.system(clear_command) if n_classes == 0: print("Menu") print("1. Load data from CSV-file/files") print("2. Load MNIST") print("3. Load EMNIST Digits") print("4. Load EMNIST Letters") print("5. Load EMNIST Balanced") print("6. Load Wine") print("7. Load Iris") print("0. Exit") menu_item = input("#>") if menu_item == '0': exit(0) if menu_item == '1': filename = input("Filename (train data):") train_x, train_y, n_classes = load_csv(filename) best_choice = frozenset({'train_accuracy'}) if n_classes > 0: test_x, test_y, _ = load_csv( input("Filename (test data) or empty string:")) best_choice = frozenset({'test_accuracy'}) chosen_dataset = filename elif menu_item == '2': train_x, train_y, test_x, test_y, n_classes = load_mnist() best_choice = frozenset({'test_accuracy'}) chosen_dataset = 'MNIST' elif menu_item == '3': train_x, train_y, test_x, test_y, n_classes = load_emnist_digits( ) best_choice = frozenset({'test_accuracy'}) chosen_dataset = 'EMNIST Digits' elif menu_item == '4': train_x, train_y, test_x, test_y, n_classes = load_emnist_letters( ) best_choice = frozenset({'test_accuracy'}) chosen_dataset = 'EMNIST Letters' elif menu_item == '5': train_x, train_y, test_x, test_y, n_classes = load_emnist_balanced( ) best_choice = frozenset({'test_accuracy'}) chosen_dataset = 'EMNIST Balanced' elif menu_item == '6': train_x, train_y, n_classes = load_wine() best_choice = frozenset({'train_accuracy'}) chosen_dataset = 'Wine' elif menu_item == '7': train_x, train_y, n_classes = load_iris() best_choice = frozenset({'train_accuracy'}) chosen_dataset = 'Iris' train_samples = min(len(train_x), len(train_y)) if train_samples > 0: test_samples = min(len(test_x), len(test_y)) features = 1 if train_x.ndim == 1 else np.product( train_x.shape[1:]) b_settings_file = enter_bool("Get settings from file?") if b_settings_file: filename = input("Filename:") try: with open(filename, "r") as fp: dictionary = json.load(fp) if len(dictionary) > 0: if dictionary[ 'classifier'] == 'mlp_classifier': neural_network = MLPClassifier( n_inputs=0, optimizer=None, loss_function=None) elif dictionary[ 'classifier'] == 'cp_classifier': neural_network = CPClassifier( n_inputs=0, kohonen_neurons=0, grossberg_neurons=0, loss_function=None, distance_function=None, kohonen_kernel_initializer=None, grossberg_kernel_initializer=None) else: input("Invalid classifier name") reset() continue neural_network.set_parameters(dictionary) if (features != neural_network.n_inputs or isinstance(neural_network, MLPClassifier) and n_classes != neural_network.output_units): input("Invalid classifier for data") reset() continue b_eval = enter_bool( 'Calculate the error and accuracy of the selected neural network?' ) if b_eval: losses, correct_answers = neural_network.evaluate( train_x, train_y) train_loss = losses.mean() train_accuracy = correct_answers * 100 / train_samples if test_samples > 0: losses, correct_answers = neural_network.evaluate( test_x, test_y) test_loss = losses.mean() test_accuracy = correct_answers * 100 / test_samples cur_is_renderable = is_renderable(train_x[0]) parameters = neural_network.count_parameters() continue except OSError as error: print(error) reset() input("Go to menu>>") continue except ValueError as error: print(f"Invalid data in file. {error}") reset() input("Go to menu>>") continue except BaseException: traceback.print_exc() reset() input("Go to menu>>") continue print("Neural networks:\n1. MLPClassifier\n2. CPClassifier") neural_network_type = enter_number("#>", 1, 2, int) loss = enter_loss_function(neural_network_type == 1) print(neural_network_type.__repr__()) if neural_network_type == 1: optimizer = enter_optimizer() neural_network = MLPClassifier(n_inputs=features, optimizer=optimizer, loss_function=loss) n_layers = enter_number( "Enter number of hidden layers (1 - 20):", 1, 20, int) for i_layer in range(1, n_layers + 1): print(i_layer, "layer") print( "Type of layer:\n1. Dense\n2. Relu\n3. LeakyRelu\n4. Swish\n5. Dropout" ) layer_type = enter_number("#>", 1, 2, int) if layer_type == 1: if i_layer != n_layers: units = enter_number( "Enter units (1 - 10000):", 1, 10000, int) else: units = n_classes print("Units:", n_classes) activation = enter_activation_function() kernel_initializer = enter_initializer( "Enter kernel initializer") bias_initializer = enter_initializer( "Enter bias initializer") neural_network.add( Dense(output_units=units, activation=activation, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer)) elif layer_type == 2: neural_network.add( ReluLayer(output_units=neural_network. layers[-1].units)) elif layer_type == 3: negative_slope = enter_number( 'Negative slope', 0, 1, float) neural_network.add( LeakyReluLayer(negative_slope=negative_slope, output_units=neural_network. layers[-1].units)) elif layer_type == 4: neural_network.add( SwishLayer(output_units=neural_network. layers[-1].units)) elif layer_type == 5: dropout = enter_number( 'Enter dropout coefficient:', 0, 0.99, float) neural_network.add(Dropout(dropout)) elif neural_network_type == 2: kohonen_neurons = enter_number( 'Kohonen neurons (1-1000000):', 1, 1000000, int) distance_function = enter_distance_function() kohonen_kernel_initializer = enter_initializer( "Enter kohonen kernel initializer") grossberg_kernel_initializer = enter_initializer( "Enter grossberg kernel initializer") neural_network = CPClassifier( n_inputs=features, kohonen_neurons=kohonen_neurons, grossberg_neurons=n_classes, loss_function=loss, distance_function=distance_function, kohonen_kernel_initializer=kohonen_kernel_initializer, grossberg_kernel_initializer= grossberg_kernel_initializer) cur_is_renderable = is_renderable(train_x[0]) parameters = neural_network.count_parameters() print("Neural network is ready") elif train_samples > 0 and n_classes > 0: print(neural_network) print("Dataset:", chosen_dataset, '[enter "close/export" to close/export this dataset]') print( "Train samples:", train_samples, '[enter "train show/predict n" to show/predict n train sample]' ) if test_samples > 0: print( "Test samples:", test_samples, '[enter "test show/predict n" to show/predict n test sample]' ) print("Features:", features) print("Classes:", n_classes) print("Trainable params:", parameters) print("Current train accuracy:", train_accuracy) print("Current train loss:", train_loss) if test_samples > 0: print("Current test accuracy:", test_accuracy) print("Current test loss:", test_loss) print("1. Train & Test") print("2. Only train") print("3. Calc accuracy and loss of train data") print("4. Calc accuracy and loss of test data") print("5. Output train and test") else: print("1. Train") print("2. Calc accuracy and loss of train data") print("3. Output train") if cur_is_renderable: print("X. Experiments") menu_item = input("#>").lower() menu_item_split = menu_item.split() if len(menu_item_split) == 3: sample_type, command, sample = menu_item_split try: sample = int(sample) - 1 except TypeError: input("Number of sample must be an integer") continue if sample < 0: input("Number of sample must be >= 1") continue if sample_type == 'train' and 0 <= sample < train_samples: do_command(train_x, train_y, sample, 'train') elif sample_type == 'test' and 0 <= sample < test_samples: do_command(test_x, test_y, sample, 'test') elif menu_item == 'close': if enter_bool("Are you sure?"): input('Dataset closed') reset() continue elif menu_item == 'export': if enter_bool("Are you sure?"): output_filename = input("Filename (train data, .csv):") if len(output_filename) > 0: n_export_samples = enter_number( f"Number of train samples to export (1-{train_samples}):", 1, train_samples, int) try: with open(output_filename, 'w') as fp: for row_x, row_y in zip( train_x[:n_export_samples].reshape( n_export_samples, np.product(train_x.shape[1:])), train_y[:n_export_samples]): fp.write(','.join(str(x) for x in row_x)) fp.write(',' + str(row_y) + '\n') print("Train data were exported to", output_filename) except BaseException: traceback.print_exc() else: print("Filename is empty") if test_samples > 0: output_filename = input("Filename (test data, .csv):") if len(output_filename) > 0: n_export_samples = enter_number( f"Number of test samples to export (1-{test_samples}):", 1, test_samples, int) try: with open(output_filename, 'w') as fp: for row_x, row_y in zip( test_x[:n_export_samples].reshape( n_export_samples, np.product(test_x.shape[1:])), test_y[:n_export_samples]): fp.write(','.join( str(x) for x in row_x)) fp.write(',' + str(row_y) + '\n') print("Test data were exported to", output_filename) except BaseException: traceback.print_exc() else: print("Filename is empty") continue elif menu_item == '1' or menu_item == '2' and test_samples > 0: has_test = (menu_item == '1' and test_samples > 0) if has_test: test_folds = enter_number( f'Enter test folds (1 - {test_samples}):', 1, test_samples, int) else: test_folds = 0 epochs = enter_number('Enter max epochs (1 - 100000):', 1, 100000, int) verbose = enter_number(f'Enter verbose (0 - {epochs}):', 0, epochs, int) if isinstance(neural_network, MLPClassifier): batch_size = enter_number( f'Enter batch size (1 - {train_samples}):', 1, train_samples, int) lr = enter_number( f'Enter learning rate (0.000001 - 1000):', 0.000001, 1000., float) else: klr = enter_number( f'Enter kohonen learning rate (0.000001 - 10):', 0.000001, 10., float) glr = enter_number( f'Enter grossberg learning rate (0.000001 - 10):', 0.000001, 10., float) optimize = enter_bool("Use optimization?") shuffle = enter_bool("Shuffle?") print("Train accuracy goal is", train_accuracy_goal) if has_test: print("Test accuracy goal is", test_accuracy_goal) if enter_bool("Edit?"): train_accuracy_goal = enter_number( "Enter train accuracy goal (0 - 1 or 2 [ignore]):", 0., 2., float) if has_test: test_accuracy_goal = enter_number( "Enter test accuracy goal (0 - 1 or 2 [ignore]):", 0., 2., float) print("Choice:", *best_choice) if enter_bool("Edit?"): choices = ['train_accuracy', 'test_accuracy' ] if has_test else ['train_accuracy'] best_choice = frozenset(t for t in choices if enter_bool(f"Choose {t}?")) print("Best choice:", *best_choice) time_start = perf_counter() try: if isinstance(neural_network, MLPClassifier): result = neural_network.fit( train_x, train_y, epochs, lr, test_x=test_x, test_y=test_y, test_folds=test_folds, verbose=verbose, batch_size=batch_size, shuffle=shuffle, train_accuracy_goal=train_accuracy_goal, test_accuracy_goal=test_accuracy_goal, best_choice=best_choice) else: result = neural_network.fit( train_x, train_y, epochs, kononen_learning_rate=klr, grossberg_learning_rate=glr, test_x=test_x, test_y=test_y, test_folds=test_folds, verbose=verbose, shuffle=shuffle, train_accuracy_goal=train_accuracy_goal, test_accuracy_goal=test_accuracy_goal, best_choice=best_choice, optimize=optimize) except BaseException: traceback.print_exc() input("Go to menu>>") continue time_stop = perf_counter() print("Time:", time_stop - time_start, "seconds") epochs_passed, train_loss_curve, train_accuracy_curve, test_loss_curve, test_accuracy_curve = result if epochs_passed >= 2: plot_loss_accuracy_curves(epochs_passed, train_loss_curve, train_accuracy_curve, test_loss_curve, test_accuracy_curve) save_params_to_file = enter_bool( "Do you want to save the parameters of the neural network to file?" ) if save_params_to_file: output_filename = input("Filename:") try: with open(output_filename, "w") as fp: json.dump(neural_network.get_parameters(), fp, cls=NpEncoder) print("Neural network parameters were saved to", output_filename) except OSError as error: print(error) except BaseException: traceback.print_exc() losses, correct_answers = neural_network.evaluate( train_x, train_y) print("TRAIN RESULTS") train_loss = losses.mean() print("Loss:", train_loss) train_accuracy = correct_answers * 100 / train_samples print("Accuracy (%):", train_accuracy) show_confusion_matrix_ex( n_classes, train_y, neural_network.predict_classes(train_x), 'Train. Confusion matrix') if has_test: losses, correct_answers = neural_network.evaluate( test_x, test_y) print("TEST RESULTS") test_loss = losses.mean() print("Loss:", test_loss) test_accuracy = correct_answers * 100 / test_samples print("Accuracy (%):", test_accuracy) show_confusion_matrix_ex( n_classes, test_y, neural_network.predict_classes(test_x), 'Test. Confusion matrix') elif menu_item == '3' and test_samples > 0 or menu_item == '2' and not test_samples > 0: losses, correct_answers = neural_network.evaluate( train_x, train_y) print("TRAIN RESULTS") train_loss = losses.mean() print("Loss:", train_loss) train_accuracy = correct_answers * 100 / train_samples print("Accuracy (%):", train_accuracy) show_confusion_matrix_ex( n_classes, train_y, neural_network.predict_classes(train_x), 'Train. Confusion matrix') elif menu_item == '4' and test_samples > 0: if train_accuracy > 0. and train_loss < np.inf: losses, correct_answers = neural_network.evaluate( test_x, test_y) print("TEST RESULTS") test_loss = losses.mean() print("Loss:", test_loss) test_accuracy = correct_answers * 100 / test_samples print("Accuracy (%):", test_accuracy) show_confusion_matrix_ex( n_classes, test_y, neural_network.predict_classes(test_x), 'Test. Confusion matrix') else: print("Train accuracy <= 0 or train loss == infinity") elif menu_item == '5' and test_samples > 0 or menu_item == '3' and not test_samples > 0: if train_accuracy > 0. and train_loss < np.inf: with open(output_classes_filename, "w") as fp: fp.write("Train data\nTrue Y:\n") fp.write(str(train_y.tolist()) + "\n") fp.write("Predicted Y:\n") fp.write( str(neural_network.predict_classes(train_x)) + "\n") if test_samples > 0: fp.write("Test data\nTrue Y:\n") fp.write(str(test_y.tolist()) + "\n") fp.write("Predicted Y:\n") fp.write( str(neural_network.predict_classes(test_x)) + "\n") print("The classification data was saved to", output_classes_filename) else: print("Train accuracy <= 0 or train loss == infinity") elif menu_item == 'x' and cur_is_renderable: if train_accuracy > 0. and train_loss < np.inf: if test_samples > 0: print( "1. Experiments on train data\n2. Experiments on test data\n3. Experiments on all data" ) b_samples = enter_number("#>", 1, 3, int) if b_samples in [1, 3]: do_experiment(train_x, train_y, n_classes) if b_samples in [2, 3]: do_experiment(test_x, test_y, n_classes) else: do_experiment(train_x, train_y, n_classes) else: print("Train accuracy <= 0 or train loss == infinity") else: reset() print("Try again...") input("Go to menu>>")
from mlp import MLPClassifier from arff import Arff import numpy as np mat = Arff("linsep2nonorigin.arff", label_count=1) data = mat.data[:, 0:-1] labels = mat.data[:, -1].reshape(-1, 1) PClass = MLPClassifier([4, 4], lr=0.1, shuffle=False, deterministic=10) PClass.fit(data, labels) Accuracy = PClass.score(data, labels) print("Accuray = [{:.2f}]".format(Accuracy)) print("Final Weights =", PClass.get_weights())
# remove vowel redundant data column data = data[:, 2:] # normalizing transformer = Normalizer().fit(data) data = transformer.transform(data) # one hot encoding enc = OneHotEncoder(handle_unknown='ignore') enc.fit(labels) labels = enc.transform(labels).toarray() # splitting data into test and training X, X_test, y, y_test = train_test_split(data, labels, test_size=0.25) instance, features = X.shape # MLP = MLPClassifier([features*2], lr=0.1, momentum=0.5, shuffle=False, deterministic=10) MLP = MLPClassifier([features * 2], lr=0.1, momentum=0.9, shuffle=True) MLP.fit(X, y) # Accuracy = MLP.score(data, labels) classifying_rate = MLP.score(X_test, y_test) MSE = MLP.getMSE(X_test, y_test) # print("testing MSE") # print(MSE) # print(MLP.epoch_counter) # print("accuracy") # print(classifying_rate) # np.savetxt("VS.csv", MLP.VS_MSE_history, delimiter=',') # np.savetxt("training.csv", MLP.training_MSE_history, delimiter=',') # np.savetxt("class_accu.csv",MLP.classification_accuracy_history,delimiter=',')
import tensorflow as tf import numpy as np from mlp import MLPClassifier # Data sets IRIS_TRAINING = "data/iris_training.csv" IRIS_TEST = "data/iris_test.csv" # Load datasets. training_set = tf.contrib.learn.datasets.base.load_csv_with_header( filename=IRIS_TRAINING, target_dtype=np.int, features_dtype=np.float32) test_set = tf.contrib.learn.datasets.base.load_csv_with_header( filename=IRIS_TEST, target_dtype=np.int, features_dtype=np.float32) # Create classifier classifier = MLPClassifier([4, 10, 10, 3], 'relu') # Fitting classifier.fit(x=training_set.data, y=training_set.target) # Evaluation accuracy_score = classifier.eval(x=test_set.data, y=test_set.target) print('Accuracy: {}'.format(accuracy_score)) new_samples = np.array([[6.4, 3.2, 4.5, 1.5], [5.8, 3.1, 5.0, 1.7]], dtype=float) y = classifier.predict(new_samples) print('Predictions: {}'.format(str(y)))
train_indices, test_indices = indices[:5000], indices[5000:] X_train, y_train = train.iloc[train_indices, 1:], train.iloc[train_indices, 0] X_test, y_test = train.iloc[test_indices, 1:], train.iloc[test_indices, 0] X_train /= 255 X_test /= 255 # %% from mlp import MLPClassifier, activation_functions, \ error_functions clf = MLPClassifier(num_iterations=7200, eta=0.05, \ batch_portion=0.3, bias=True, hidden_layers=[98, 98],\ activation_function=activation_functions.leaky_relu, \ error_function=error_functions.cross_entropy, \ moment=0.5, \ random_seed = 12369666) clf = clf.fit(X_train, y_train) # %% import joblib joblib.dump(clf, 'clf_digits.joblib') # %% print("Train accuracy:", clf.score(X_train, y_train)) # %% print("Test accuracy:", clf.score(X_test, y_test))
#debug dataset 1 print("----------DEBUG DATASET 1------") mat = Arff("datasets/linsep2nonorigin.arff") data = mat.data[:, 0:-1] labels = mat.data[:, -1].reshape(-1, 1) num_features = np.size(data, 1) num_inputs = np.size(data, 0) num_outputs = 2 hiddenLayers = [num_features, num_features * 2, num_outputs] print("LAYER INFO INPUT") print(hiddenLayers) MLP = MLPClassifier(hiddenLayers, lr=0.1, momentum=0.5, shuffle=False, validationSize=0.0, deterministic=10) trainData, trainLabels, testData, testLabels = MLP.splitTestTrainData( data, labels) print("labels", trainLabels) print("testlabels", testLabels) weights = {} weights['W1'] = np.zeros([hiddenLayers[1], hiddenLayers[0] + 1]) weights['W2'] = np.zeros([hiddenLayers[2], hiddenLayers[1] + 1]) MLP.fit(trainData, trainLabels, weights) print(MLP.get_weights())
if shuffle: np.random.shuffle(alldata) if spliting: split = int(round(.75 * alllength)) print(split) trainingset = alldata[:split] testingset = alldata[split:] else: trainingset = alldata testingset = alldata X = trainingset[:, :y_start] y = trainingset[:, y_start:] mlp = MLPClassifier(solver='adam', activation='relu', learning_rate='constant', momentum=.9, early_stopping=False) output = mlp.fit(X, y) score = mlp.score(testingset[:, :y_start], testingset[:, y_start:]) print(score) #mlp = MLPClassifier(lr=.1, momentum=0.5 # , shuffle=shuffle, hidden_layer_widths=None) #print(y) # X=[[0,0],[0,1]] # y=[[1],[0]] #initial_weights={'hidden':[[0,0,0,0,0],[0,0,0,0,0],[0,0,0,0,0],[0,0,0,0,0],[0,0,0,0,0],[0,0,0,0,0],[0,0,0,0,0],[0,0,0,0,0]] ,'output':[[0,0,0,0,0,0,0,0,0]]} #hl, ol = mlp.fit(X, y,) #if split: # X1 = testingset[:, :y_start]
def main(): def show_confusion_matrix_ex(y_predicted, y_true, s): train_y_typename = training_labels.dtype.type.__name__ if 'float' in train_y_typename or 'float' in train_y_typename: show_confusion_matrix(n_classes, y_true, y_predicted, s) else: show_confusion_matrix(n_classes, mlp.to_class_index(y_true), mlp.to_class_index(y_predicted), s) print("Starting...") training_images, training_labels, test_images, test_labels, n_classes = load_mnist( ) n_train_samples = len(training_labels) n_test_samples = len(test_labels) # Using 10% # n_iter = (training_images.shape[0] // n_classes) * 10 // 100 # training_images_new = [] # training_labels_new = [] # for i in range(n_classes): # indices = np.where(training_labels == i)[0] # np.random.shuffle(indices) # training_images_new.append(training_images[indices[:n_iter]]) # training_labels_new.append(training_labels[indices[:n_iter]]) # training_images = np.concatenate(training_images_new) # training_labels = np.concatenate(training_labels_new) # n_train_samples = len(training_labels) # n_test_samples = len(test_labels) # print("Train samples:", n_train_samples) # print("Test samples:", n_test_samples) features = 1 if training_images.ndim == 1 else np.product( training_images.shape[1:]) mlp = MLPClassifier(features, 'adadelta', 'smce') mlp.add(Dropout(0.4)) mlp.add( Dense(250, activation='linear', kernel_initializer='xnn', bias_initializer='xnn')) mlp.add(ReluLayer()) mlp.add( Dense(250, activation='linear', kernel_initializer='xnn', bias_initializer='xnn')) mlp.add(ReluLayer()) mlp.add( Dense(n_classes, activation='linear', kernel_initializer='xnn', bias_initializer='xnn')) print("Training and testing...") time_start = perf_counter() result = mlp.fit(training_images, training_labels, test_x=test_images, test_y=test_labels, test_folds=1, epochs=500, learning_rate=5, batch_size=500, shuffle=True, verbose=1, train_accuracy_goal=np.inf, best_choice=frozenset({'test_accuracy'})) time_stop = perf_counter() print("Time:", time_stop - time_start, "seconds") epochs_passed, train_loss_curve, train_accuracy_curve, test_loss_curve, test_accuracy_curve = result if epochs_passed >= 2: plot_loss_accuracy_curves(epochs_passed, train_loss_curve, train_accuracy_curve, test_loss_curve, test_accuracy_curve) tr_losses, tr_correct_answers = mlp.evaluate(training_images, training_labels) print("TRAIN RESULTS") print("Loss:", tr_losses.mean()) print("Accuracy (%):", tr_correct_answers * 100 / n_train_samples) show_confusion_matrix_ex(training_labels, mlp.predict_classes(training_images), 'Train. Confusion matrix') ts_losses, ts_correct_answers = mlp.evaluate(test_images, test_labels) print("TEST RESULTS") print("Loss:", ts_losses.mean()) print("Accuracy (%):", ts_correct_answers * 100 / n_test_samples) show_confusion_matrix_ex(test_labels, mlp.predict_classes(test_images), 'Test. Confusion matrix') save_params_to_file = enter_bool( "Do you want to save the parameters of the neural network to file?") if save_params_to_file: output_filename = input("Filename:") try: with open(output_filename, "w") as fp: json.dump(mlp.get_parameters(), fp, cls=NpEncoder) print("Neural network parameters were saved to", output_filename) except OSError as error: print(error)
import numpy as np from mlp import MLPClassifier # training data X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) y = np.array([[ 0, 1, 1, 0, ]]) y = np.transpose(y) mlp = MLPClassifier() # make object mlp.fit(X, y, W_h=[[0.5, 0.9], [0.4, 1.0]], W_o=[-1.2, 1.1], t_h=[0.8, -0.1], t_o=[0.3]) # train model with specfic weights # mlp.fit(X, y) # train model with random inits # predicting print('\n\nPrediction') print('x1\tx2\ty') for x in X: print('{}\t{}\t{}'.format(x[0], x[1], mlp.predict(x)[0]))
def vowel(): print("-------------vowel----------------") mat = Arff("../data/vowel.arff", label_count=1) y = mat.data[:,-1] lb = preprocessing.LabelBinarizer() lb.fit(y) y = lb.transform(y) # split it data, tData, labels, tLabels = train_test_split(mat.data[:, :-1], y, test_size=.25) master_window = 5 window = master_window bssf = [np.inf, 0] tolerance = 1e-4 findings = [] findings.append(["LR", "Epochs", "TestAccuracy", "MSE train", "MSE validate", "MSE test", "Best LR"]) for lr in range(-1, 5): if window <= 0: break for step in [1, 5]: entry = [] print((lr, step), end=",") learn_rate = (0.1**lr)*step MLPClass = MLPClassifier([2*np.shape(data)[1]], lr=learn_rate, shuffle=True, one_hot=True) MLPClass.fit(data, labels, momentum=0.5, percent_verify=.25) accuracy = MLPClass.score(tData, tLabels) entry.append(learn_rate) entry.append(MLPClass.getEpochCount()) entry.append(accuracy) entry.append(MLPClass._calc_l2_err(data, labels)) entry.append(MLPClass.bssf[0]) entry.append(MLPClass._calc_l2_err(tData, tLabels)) entry.append(bssf[1]) findings.append(entry) if accuracy < bssf[0] and abs(accuracy - bssf[0]) > tolerance: bssf = [accuracy, learn_rate] window = master_window else: window = window - 1 if window <= 0: break print("\n\r", findings) np.savetxt("vowel_findings_lr.csv", findings[1:], header=reduce(findings[0]), delimiter=',') lr = bssf[1] window = master_window findings = [] findings.append(["Num Nodes", "Epochs", "Train Accuracy", 'VS accuracy', 'test accuracy']) accuracy = bssf[0] doubler = 2 num_nodes = 1 bssf = [np.inf, 0] while(window > 0): num_nodes = num_nodes * doubler print("numnodes", num_nodes) MLPClass = MLPClassifier([num_nodes], lr=lr, shuffle=True, one_hot=True) MLPClass.fit(data, labels, momentum=0.5, percent_verify=.25) accuracy = MLPClass.score(tData, tLabels) entry = [] entry.append(num_nodes) entry.append(MLPClass.getEpochCount()) entry.append(MLPClass._calc_l2_err(data, labels)) entry.append(MLPClass.bssf[0]) entry.append(MLPClass._calc_l2_err(tData, tLabels)) findings.append(entry) if accuracy < bssf[0] and abs(accuracy - bssf[0]) > tolerance: bssf = [accuracy, num_nodes] window = master_window else: window = window - 1 np.savetxt("vowel_findings_hid_nodes.csv", findings[1:], header=reduce(findings[0]), delimiter=',') num_nodes = bssf[1] window = master_window findings = [] findings.append(["Momentum", "Epochs", "Train Accuracy", 'VS accuracy', 'test accuracy']) momentum = 0 bssf = [np.inf, momentum] while(window > 0): momentum = momentum + 0.05 print("momentum", momentum) MLPClass = MLPClassifier([num_nodes], lr=lr, shuffle=True, one_hot=True) MLPClass.fit(data, labels, momentum=momentum, percent_verify=.25) accuracy = MLPClass.score(tData, tLabels) entry = [] entry.append(momentum) entry.append(MLPClass.getEpochCount()) entry.append(MLPClass._calc_l2_err(data, labels)) entry.append(MLPClass.bssf[0]) entry.append(MLPClass._calc_l2_err(tData, tLabels)) findings.append(entry) if accuracy < bssf[0] and abs(accuracy - bssf[0]) > tolerance: bssf = [accuracy, momentum] window = master_window else: window = window - 1 np.savetxt("vowel_findings_momentum.csv", findings[1:], header=reduce(findings[0]), delimiter=',')
from mlp import MLPClassifier from pprint import PrettyPrinter pp = PrettyPrinter() bp = MLPClassifier(hidden_layer_widths=[1]) # bp = MLPClassifier() X = [[1, 2]] y = [[0]] bp.fit(X, y) # bp.initialize_weights(1, 1) initial_weights = bp.initial_weights # pp.pprint(initial_weights)
vis.plot_classification_result(clf, \ X_test.x, X_test.y, y_test, show = True, \ save_path = f'results/classification/{name}-{size}-result.png', \ margin = plot_margin, grid_size = plot_size) vis.plot_train_test_error(clf, \ X_train, y_train, X_test, y_test, \ save_path = f'results/classification/{name}-{size}-error.png') return score # %% circles scores = [] for i in range(10): clf = MLPClassifier(activation_function = \ activation_functions.tanh, \ error_function = error_functions.cross_entropy, \ hidden_layers = [10, 10, 10, 10], bias = True, batch_portion = 0.84, \ num_iterations = 5000, eta = 0.4155, moment = 0.5155, \ random_seed = 12369666 + i) score = process_classification_dataset('circles', clf, \ draw = True if i == 0 else False) scores.append(100 * score) print(f'circles: mean - {round(np.mean(scores), 2)}, ' + \ f'std - {round(np.std(scores), 2)}') # %% XOR scores = [] for i in range(10): clf = MLPClassifier(activation_function = \ activation_functions.tanh, \
# Neural Networks: scikit-learn mlpclassifier mlp_clf = MLPClassifier(activation='relu', algorithm='l-bfgs', alpha=1e-5, hidden_layer_sizes=(100,2), random_state=1, verbose=1) mlp_clf.fit(train_data[:,2:], train_data[:,1]) res_labels = mlp_clf.predict(test_data[:,2:]) #visual ''' X = train_data[:, 2:] y = train_data[:, 1] alpha = 0.001 mlp_clf = MLPClassifier(activationfunc='relu', tol=1e-4, hidden_layer_sizes=(10,200, 2), verbose=1, max_iter =500, lam=alpha) while mlp_clf.trainedflag is False: mlp_clf.fit(X, y) xtest = test_data[:, 2:] ytarget = test_data[:, 1] ytest = mlp_clf.predict(xtest) y_1 = abs(ytest-1) y_0 = abs(ytest) Y = y_1 <= y_0 erate = plotclaissified(X, ytarget, Y, alpha) plt.show()
# %% import pandas as pd train = pd.read_csv( "data/projekt1_test/Classification/data.circles.train.100.csv") test = pd.read_csv( "data/projekt1_test/Classification/data.circles.test.100.csv") X_train, y_train = train.iloc[:, :-1], train.cls X_test, y_test = test.iloc[:, :-1], test.cls # %% from mlp import MLPClassifier clf = MLPClassifier(num_iterations = 10000, \ random_seed = 12369666) print('Accuracy:', clf.fit(X_train, y_train, \ serialize_path='training_data.joblib')\ .score(X_test, y_test)) # %% print('Class confusion matrix:') print(clf.confusion_matrix(X_test, y_test)) # %% from mlp import Visualizer vis = Visualizer() vis.plot_train_test_error(clf, X_train, y_train, \ X_test, y_test, log_scale=True, show = True) # %% from mlp import MLPClassifier, activation_functions
def __init__(self, **kwargs): Learn.__init__(self, **kwargs) self.algo =MLPClassifier()
activation_function = activation_mapping.get(config['activation_function']) error_mapping = { 'mean_squared': error_functions.mean_squared, \ 'mean': error_functions.mean, \ 'max': error_functions.max_error, \ 'cross_entropy': error_functions.cross_entropy } error_function = error_mapping.get(config['error_function']) if config['problem_type'] == 'classification': clf = MLPClassifier(num_iterations = config['iterations'], \ bias = config['bias'], \ hidden_layers = config['hidden_layers'], \ eta = config['learning_rate'], \ moment = config['moment'], \ batch_portion = config['batch_portion'], \ random_seed = config['random_seed'], \ activation_function = activation_function, \ error_function = error_function) clf = clf.fit(X_train, y_train, \ serialize_path='training_data.joblib') print('Classification accuracy:', clf.score(X_test, y_test)) print('Class confusion matrix:') print(clf.confusion_matrix(X_test, y_test)) print('Plotting training dataset...') vis.plot_classification_dataset(X_train.iloc[:, 0], X_train.iloc[:, 1], y_train,