data, target = normalize_data(data, target)
out_matrix = np.zeros((num_train_examples, num_networks))
with torch.no_grad():
print(f"Sampling {num_networks} random networks")
for network_idx in tqdm(range(num_networks)):
model = SimpleNet(depth, width)
for p in model.parameters():
p.data = torch.randn_like(p) / math.sqrt(p.shape[1])
pred = model(data).squeeze()
out_matrix[:, network_idx] = pred.numpy()
sample_mean = np.mean(out_matrix, axis=1)
sample_cov = np.cov(out_matrix)
print()
print("sample mean:\n", sample_mean)
print("sample cov:\n", sample_cov)
sigma = sanitise(torch.mm(data, data.t()) / data.shape[1])
for _ in range(depth - 1):
sigma = increment_kernel(sigma)
sigma = sigma.numpy()
diff = np.absolute(sigma - sample_cov).max()
print("analytical cov:\n", sigma)
print("max difference: ", diff)
model = ResidualNetVariancePreservingV2(depth, width, alpha)
model = model.to(device)
for p in model.parameters():
p.data = torch.randn_like(p) / math.sqrt(p.shape[1])
pred = model(data).squeeze()
out_matrix[:, network_idx] = pred
sample_mean = torch.mean(out_matrix, dim=1)
sample_cov = torch.from_numpy(np.cov(out_matrix.cpu()))
print()
print("sample mean:\n", sample_mean)
print("sample cov:\n", sample_cov)
sigma_1 = sanitise((torch.mm(data, data.t()) / data.shape[1]))
# sigma_2 = sanitise((torch.mm(data, data.t()) / data.shape[1]))
sigma_3 = sanitise((torch.mm(data, data.t()) / data.shape[1]))
for _ in range(depth - 1):
new_sigma = torch.zeros((num_train_examples, num_train_examples))
sigma = sigma_1
for i in range(num_train_examples):
for j in range(num_train_examples):
k_x_x = sigma[i][i]
k_tilde_x_tilde_x = sigma[j][j]
cross_product_term = torch.sqrt(k_x_x * k_tilde_x_tilde_x)
mean_k_term = sigma[i][j] / cross_product_term