# Mean normalization for data mean1, mean2, mean3, mean4 = np.mean(x_train[:, 0]), np.mean( x_train[:, 1]), np.mean(x_train[:, 2]), np.mean(x_train[:, 3]) min1, max1 = x_train[:, 0].min(), x_train[:, 0].max() min2, max2 = x_train[:, 1].min(), x_train[:, 1].max() min3, max3 = x_train[:, 2].min(), x_train[:, 2].max() min4, max4 = x_train[:, 3].min(), x_train[:, 3].max() x_train[:, 0], x_train[:, 1], x_train[:, 2], x_train[:, 3] = ( x_train[:, 0] - mean1) / (max1 - min1), (x_train[:, 1] - mean2) / ( max2 - min2), (x_train[:, 2] - mean3) / (max3 - min3), ( x_train[:, 3] - mean4) / (max4 - min4) x_test[:, 0], x_test[:, 1], x_test[:, 2], x_test[:, 3] = ( x_test[:, 0] - mean1) / (max1 - min1), (x_test[:, 1] - mean2) / ( max2 - min2), (x_test[:, 2] - mean3) / (max3 - min3), ( x_test[:, 3] - mean4) / (max4 - min4) model = MLP() model.add_layer(units=8, activation='relu', input_units=4) model.add_layer(units=3, activation='softmax', input_units=8) model.fit(X=x_train, y=y_train, epochs=500, learning_rate=0.005) predictions = model.predict(X=x_test) print(model.eval(predictions, y_test)) print(predictions) plt.xlabel("Number of Epochs") plt.ylabel("Cost") plt.plot(model.epochs, model.cost) plt.show()
def wikics(method, device, num_hid, lr, dropout, num_epochs, mask, beta, n_MC, n_u, n_c, n_jobs): # Create "data" and "results" folder data_folder = osp.join(here, "data") results_folder = osp.join(here, "results") MLP_folder = osp.join(results_folder, "MLP") SAGE_folder = osp.join(results_folder, "SAGE") create_folder(data_folder) create_folder(results_folder) create_folder(MLP_folder) create_folder(SAGE_folder) # Get WikiCS dataset dataset = WikiCS(root=data_folder) data = dataset[0] X = data.x.to(device) y = data.y.to(device) edge_index = data.edge_index.to(device) # Get splits train_idx = data.train_mask[:, mask] val_idx = data.val_mask[:, mask] test_idx = data.test_mask # AWGN channel Z = lambda n, sigma: mvn.MultivariateNormal( torch.zeros(n), torch.diag(torch.ones(n) * sigma**2)) Z_ = Z(num_hid, beta) # Instantiate the model, optimizer and criterion if method == "MLP": noisy_model = MLP(X.size(-1), num_hid, dataset.num_classes, Z_, dropout, device).to(device) elif method == "SAGE": noisy_model = SAGE(X.size(-1), num_hid, dataset.num_classes, Z_, dropout, device).to(device) else: raise ValueError("Invalid method") optimizer = Adam(noisy_model.parameters(), lr=lr) criterion = F.nll_loss # Train model losses, train_accs, val_accs, test_accs, I_1, I_2 = [], [], [], [], [], [] for epoch in range(1, 1 + num_epochs): if method == "MLP": loss = train(noisy_model, X, y, train_idx, optimizer, criterion) train_acc, val_acc, test_acc = test(noisy_model, X, y, train_idx, val_idx, test_idx) u_S, c_S = get_samples(noisy_model, X[train_idx]) noisy_model.eval() z, _, _ = noisy_model(X[train_idx]) elif method == "SAGE": loss = train(noisy_model, X, y, train_idx, optimizer, criterion, edge_index) train_acc, val_acc, test_acc = test(noisy_model, X, y, train_idx, val_idx, test_idx, edge_index) u_S, c_S = get_samples(noisy_model, X, edge_index, train_idx) noisy_model.eval() z, _, _ = noisy_model(X, edge_index) z = z[train_idx] kde = KDE(n_jobs=n_jobs) I_T_1 = compute_I(u_S[0], c_S[0], Z_, n_MC, kde, device, n_u, n_c) I_T_2 = compute_I(u_S[1], c_S[1], Z_, n_MC, kde, device, n_u, n_c) print(f"Epoch: {epoch:02d}", f"Loss: {loss:.4f}", f"Train: {100 * train_acc:.2f}%", f"Valid: {100 * val_acc:.2f}%", f"Test: {100 * test_acc:.2f}%", f"I(X,T_1]) = {I_T_1:.4f}", f"I(X,T_2) = {I_T_2:.4f}") losses.append(loss) train_accs.append(train_acc) val_accs.append(val_acc) test_accs.append(test_acc) I_1.append(I_T_1) I_2.append(I_T_2) # Create a folder for every epoch to store checkpoint and t-SNE embeddings chkpt = { "epoch": epoch, "state_dict": noisy_model.state_dict(), "loss": loss, "train accuracy": 100 * train_acc, "valid accuracy": 100 * val_acc, "test accuracy": 100 * test_acc, "I(X:T_1)": I_T_1, "I(X:T_2)": I_T_2 } if method == "MLP": epoch_folder = osp.join(MLP_folder, f"{epoch}") elif method == "SAGE": epoch_folder = osp.join(SAGE_folder, f"{epoch}") create_folder(epoch_folder) torch.save(chkpt, osp.join(epoch_folder, f"{epoch}.pt")) x_coord, y_coord = zip(*TSNE().fit_transform(z.cpu().numpy())) colormap = np.array(Category20_20 + Category20b_20 + Accent8) labels = np.array([int(l) for l in data.y[train_idx]]) plt.scatter(x_coord, y_coord, c=colormap[labels]) plt.savefig(osp.join(epoch_folder, f"{epoch}.png")) plt.close() return losses, train_accs, val_accs, test_accs, I_1, I_2
p = MLP(1, 3, 1, activation_type="linear") p.train(train, trainTarget, 10001, valid_set=valid, valid_targets=validTarget) ## Bokeh plots plot = figure(title="MLP Random Sin Wave with Guassian Noise Added") ## Plot data x_dat = np.ndarray.flatten(x.T) t_dat = np.ndarray.flatten(t.T) test_dat = np.ndarray.flatten(test.T) actual_dat = np.ndarray.flatten(testTarget.T) eval_dat = np.ndarray.flatten(np.array([p.eval(val) for val in test])) ## Actual Data plot.line(x_dat, t_dat, legend="Full Input Sine Wave") plot.circle(x_dat, t_dat, legend="Full Input Sine Wave") plot.line(test_dat, actual_dat, legend="Test Set With Target Values", color="red") plot.circle(test_dat, actual_dat, legend="Test Set With Target Values", color="red") plot.line(test_dat, eval_dat, legend="Test Set with MLP Output",