# Mean normalization for data
mean1, mean2, mean3, mean4 = np.mean(x_train[:, 0]), np.mean(
x_train[:, 1]), np.mean(x_train[:, 2]), np.mean(x_train[:, 3])
min1, max1 = x_train[:, 0].min(), x_train[:, 0].max()
min2, max2 = x_train[:, 1].min(), x_train[:, 1].max()
min3, max3 = x_train[:, 2].min(), x_train[:, 2].max()
min4, max4 = x_train[:, 3].min(), x_train[:, 3].max()
x_train[:, 0], x_train[:, 1], x_train[:, 2], x_train[:, 3] = (
x_train[:, 0] - mean1) / (max1 - min1), (x_train[:, 1] - mean2) / (
max2 - min2), (x_train[:, 2] - mean3) / (max3 - min3), (
x_train[:, 3] - mean4) / (max4 - min4)
x_test[:, 0], x_test[:, 1], x_test[:, 2], x_test[:, 3] = (
x_test[:, 0] - mean1) / (max1 - min1), (x_test[:, 1] - mean2) / (
max2 - min2), (x_test[:, 2] - mean3) / (max3 - min3), (
x_test[:, 3] - mean4) / (max4 - min4)
model = MLP()
model.add_layer(units=8, activation='relu', input_units=4)
model.add_layer(units=3, activation='softmax', input_units=8)
model.fit(X=x_train, y=y_train, epochs=500, learning_rate=0.005)
predictions = model.predict(X=x_test)
print(model.eval(predictions, y_test))
print(predictions)
plt.xlabel("Number of Epochs")
plt.ylabel("Cost")
plt.plot(model.epochs, model.cost)
plt.show()
def wikics(method, device, num_hid, lr, dropout, num_epochs, mask, beta, n_MC,
n_u, n_c, n_jobs):
# Create "data" and "results" folder
data_folder = osp.join(here, "data")
results_folder = osp.join(here, "results")
MLP_folder = osp.join(results_folder, "MLP")
SAGE_folder = osp.join(results_folder, "SAGE")
create_folder(data_folder)
create_folder(results_folder)
create_folder(MLP_folder)
create_folder(SAGE_folder)
# Get WikiCS dataset
dataset = WikiCS(root=data_folder)
data = dataset[0]
X = data.x.to(device)
y = data.y.to(device)
edge_index = data.edge_index.to(device)
# Get splits
train_idx = data.train_mask[:, mask]
val_idx = data.val_mask[:, mask]
test_idx = data.test_mask
# AWGN channel
Z = lambda n, sigma: mvn.MultivariateNormal(
torch.zeros(n), torch.diag(torch.ones(n) * sigma**2))
Z_ = Z(num_hid, beta)
# Instantiate the model, optimizer and criterion
if method == "MLP":
noisy_model = MLP(X.size(-1), num_hid, dataset.num_classes, Z_,
dropout, device).to(device)
elif method == "SAGE":
noisy_model = SAGE(X.size(-1), num_hid, dataset.num_classes, Z_,
dropout, device).to(device)
else:
raise ValueError("Invalid method")
optimizer = Adam(noisy_model.parameters(), lr=lr)
criterion = F.nll_loss
# Train model
losses, train_accs, val_accs, test_accs, I_1, I_2 = [], [], [], [], [], []
for epoch in range(1, 1 + num_epochs):
if method == "MLP":
loss = train(noisy_model, X, y, train_idx, optimizer, criterion)
train_acc, val_acc, test_acc = test(noisy_model, X, y, train_idx,
val_idx, test_idx)
u_S, c_S = get_samples(noisy_model, X[train_idx])
noisy_model.eval()
z, _, _ = noisy_model(X[train_idx])
elif method == "SAGE":
loss = train(noisy_model, X, y, train_idx, optimizer, criterion,
edge_index)
train_acc, val_acc, test_acc = test(noisy_model, X, y, train_idx,
val_idx, test_idx, edge_index)
u_S, c_S = get_samples(noisy_model, X, edge_index, train_idx)
noisy_model.eval()
z, _, _ = noisy_model(X, edge_index)
z = z[train_idx]
kde = KDE(n_jobs=n_jobs)
I_T_1 = compute_I(u_S[0], c_S[0], Z_, n_MC, kde, device, n_u, n_c)
I_T_2 = compute_I(u_S[1], c_S[1], Z_, n_MC, kde, device, n_u, n_c)
print(f"Epoch: {epoch:02d}", f"Loss: {loss:.4f}",
f"Train: {100 * train_acc:.2f}%", f"Valid: {100 * val_acc:.2f}%",
f"Test: {100 * test_acc:.2f}%", f"I(X,T_1]) = {I_T_1:.4f}",
f"I(X,T_2) = {I_T_2:.4f}")
losses.append(loss)
train_accs.append(train_acc)
val_accs.append(val_acc)
test_accs.append(test_acc)
I_1.append(I_T_1)
I_2.append(I_T_2)
# Create a folder for every epoch to store checkpoint and t-SNE embeddings
chkpt = {
"epoch": epoch,
"state_dict": noisy_model.state_dict(),
"loss": loss,
"train accuracy": 100 * train_acc,
"valid accuracy": 100 * val_acc,
"test accuracy": 100 * test_acc,
"I(X:T_1)": I_T_1,
"I(X:T_2)": I_T_2
}
if method == "MLP":
epoch_folder = osp.join(MLP_folder, f"{epoch}")
elif method == "SAGE":
epoch_folder = osp.join(SAGE_folder, f"{epoch}")
create_folder(epoch_folder)
torch.save(chkpt, osp.join(epoch_folder, f"{epoch}.pt"))
x_coord, y_coord = zip(*TSNE().fit_transform(z.cpu().numpy()))
colormap = np.array(Category20_20 + Category20b_20 + Accent8)
labels = np.array([int(l) for l in data.y[train_idx]])
plt.scatter(x_coord, y_coord, c=colormap[labels])
plt.savefig(osp.join(epoch_folder, f"{epoch}.png"))
plt.close()
return losses, train_accs, val_accs, test_accs, I_1, I_2
p = MLP(1, 3, 1, activation_type="linear")
p.train(train,
trainTarget,
10001,
valid_set=valid,
valid_targets=validTarget)
## Bokeh plots
plot = figure(title="MLP Random Sin Wave with Guassian Noise Added")
## Plot data
x_dat = np.ndarray.flatten(x.T)
t_dat = np.ndarray.flatten(t.T)
test_dat = np.ndarray.flatten(test.T)
actual_dat = np.ndarray.flatten(testTarget.T)
eval_dat = np.ndarray.flatten(np.array([p.eval(val) for val in test]))
## Actual Data
plot.line(x_dat, t_dat, legend="Full Input Sine Wave")
plot.circle(x_dat, t_dat, legend="Full Input Sine Wave")
plot.line(test_dat,
actual_dat,
legend="Test Set With Target Values",
color="red")
plot.circle(test_dat,
actual_dat,
legend="Test Set With Target Values",
color="red")
plot.line(test_dat,
eval_dat,
legend="Test Set with MLP Output",
