from dnner.ensembles import Empirical, Gaussian
# Parameters
n_cols = 2000
alpha = 0.8
var_noise = 1e-8
n_runs = 10
n_rows = int(np.ceil(alpha * n_cols))
np.random.seed(42) # set random seed
mis = []
for i in range(n_runs):
# Generate random weights
w0 = np.random.randn(n_rows, n_cols) / np.sqrt(n_cols)
weights = [Empirical(w0)]
# Compute mutual information
layers = [SpikeSlab(0.2, 0, 1), Linear(var_noise)]
mi = dnner.compute_mi(layers=layers, weights=weights, verbose=0)
print("run %d, mi = %g" % (i + 1, mi))
mis.append(mi)
print("mi (empirical distribution) = %g +/- %g" % (np.mean(mis), np.std(mis)))
# Use Gaussian i.i.d. ensemble instead
weights = [Gaussian(alpha)]
mi = dnner.compute_mi(layers=layers, weights=weights, verbose=0)
print("mi (exact distribution) = %g" % (mi))
def test_Bimodal_ReLU(self):
layers = [Bimodal(0.5), ReLU(0.01)]
mi_expected = 0.693147
mi = dnner.compute_mi(layers=layers, weights=self.weights, verbose=0)
self.assertAlmostEqual(mi, mi_expected, places=6)
def test_SpikeSlab_ReLU(self):
layers = [SpikeSlab(0.2, 0, 1), ReLU(0.01)]
mi_expected = 0.799361
mi = dnner.compute_mi(layers=layers, weights=self.weights, verbose=0)
self.assertAlmostEqual(mi, mi_expected, places=6)
def test_Normal_ReLU(self):
layers = [Normal(0, 1), ReLU(0.01)]
mi_expected = 2.021801
mi = dnner.compute_mi(layers=layers, weights=self.weights, verbose=0)
self.assertAlmostEqual(mi, mi_expected, places=6)
def test_SpikeSlab_Probit(self):
layers = [SpikeSlab(0.2, 0, 1), Probit(0.01)]
mi_expected = 0.559198
mi = dnner.compute_mi(layers=layers, weights=self.weights, verbose=0)
self.assertAlmostEqual(mi, mi_expected, places=6)
def test_Normal_Probit(self):
layers = [Normal(0, 1), Probit(0.01)]
mi_expected = 0.897281
mi = dnner.compute_mi(layers=layers, weights=self.weights, verbose=0)
self.assertAlmostEqual(mi, mi_expected, places=6)
def test_SpikeSlab_Linear(self):
layers = [SpikeSlab(0.2, 0, 1), Linear(0.01)]
mi_expected = 0.890864
mi = dnner.compute_mi(layers=layers, weights=self.weights, verbose=0)
self.assertAlmostEqual(mi, mi_expected, places=6)
def test_Normal_Linear(self):
layers = [Normal(0, 1), Linear(0.01)]
mi_expected = 2.501166 # could also compare to exact
mi = dnner.compute_mi(layers=layers, weights=self.weights, verbose=0)
self.assertAlmostEqual(mi, mi_expected, places=6)