def test_MDN_weight_decay(self):
mu = 5
std = 5
X, Y = self.get_samples(mu=mu, std=std)
no_decay = MixtureDensityNetwork("mdn_no_weight_decay",
1,
1,
hidden_sizes=(32, 32),
n_centers=10,
n_training_epochs=2000,
weight_decay=0.0,
weight_normalization=False)
decay = MixtureDensityNetwork("mdn_weight_decay",
1,
1,
n_centers=10,
hidden_sizes=(32, 32),
n_training_epochs=2000,
weight_decay=1e-3,
weight_normalization=False)
no_decay.fit(X, Y)
decay.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_true = norm.pdf(y, loc=mu, scale=std)
l1_err_no_dec = np.mean(np.abs(no_decay.pdf(x, y) - p_true))
l1_err_dec = np.mean(np.abs(decay.pdf(x, y) - p_true))
self.assertLessEqual(l1_err_dec, 0.1)
self.assertLessEqual(l1_err_dec, l1_err_no_dec)
def test_MDN_l1_l2_regularization(self):
mu = 5
std = 5
X, Y = self.get_samples(mu=mu, std=std, n_samples=500)
mdn_no_reg = MixtureDensityNetwork("mdn_no_reg",
1,
1,
hidden_sizes=(32, 32),
n_centers=10,
n_training_epochs=1000,
l2_reg=0.0,
weight_normalization=False)
mdn_reg_l2 = MixtureDensityNetwork("mdn_reg_l2",
1,
1,
n_centers=10,
hidden_sizes=(32, 32),
n_training_epochs=1000,
l2_reg=1.0,
weight_normalization=False)
mdn_reg_l1 = MixtureDensityNetwork("mdn_reg_l1",
1,
1,
n_centers=10,
hidden_sizes=(32, 32),
n_training_epochs=1000,
l1_reg=1.0,
weight_normalization=False)
mdn_no_reg.fit(X, Y)
mdn_reg_l2.fit(X, Y)
mdn_reg_l1.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_true = norm.pdf(y, loc=mu, scale=std)
err_no_reg = np.mean(np.abs(mdn_no_reg.pdf(x, y) - p_true))
err_reg_l2 = np.mean(np.abs(mdn_reg_l2.pdf(x, y) - p_true))
err_reg_l1 = np.mean(np.abs(mdn_reg_l1.pdf(x, y) - p_true))
self.assertLessEqual(err_reg_l2, err_no_reg)
self.assertLessEqual(err_reg_l1, err_no_reg)
def test4_MDN_with_2d_gaussian_noise_x(self):
np.random.seed(22)
X = np.random.uniform(0, 6, size=4000)
Y = X + np.random.normal(0, 1, size=4000)
x_test_2 = np.ones(100) * 2
x_test_4 = np.ones(100) * 4
y_test = np.linspace(1, 5, num=100)
with tf.Session():
model_no_noise = MixtureDensityNetwork("mdn_no_noise_x",
1,
1,
n_centers=1,
x_noise_std=None,
y_noise_std=None)
model_no_noise.fit(X, Y)
pdf_distance_no_noise = np.mean(
np.abs(
model_no_noise.pdf(x_test_2, y_test) -
model_no_noise.pdf(x_test_4, y_test)))
model_noise = MixtureDensityNetwork("mdn_noise_x",
1,
1,
n_centers=1,
x_noise_std=2,
y_noise_std=None)
model_noise.fit(X, Y)
pdf_distance_noise = np.mean(
np.abs(
model_noise.pdf(x_test_2, y_test) -
model_noise.pdf(x_test_4, y_test)))
print("Training w/o noise - pdf distance:", pdf_distance_no_noise)
print("Training w/ noise - pdf distance", pdf_distance_noise)
self.assertGreaterEqual(pdf_distance_no_noise / pdf_distance_noise,
2.0)
def test_2_MDN_with_2d_gaussian_fit_by_crossval(self):
X, Y = self.get_samples()
param_grid = {"n_centers": [2, 10, 50]}
model = MixtureDensityNetwork()
model.fit_by_cv(X, Y, param_grid=param_grid)
y = np.arange(-1, 5, 0.5)
x = np.asarray([2 for i in range(y.shape[0])])
p_est = model.pdf(x, y)
p_true = norm.pdf(y, loc=2, scale=1)
self.assertEqual(model.get_params()["n_centers"], 10)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.2)
def testPickleUnpickleMDN(self):
X, Y = self.get_samples()
with tf.Session() as sess:
model = MixtureDensityNetwork("mdn_pickle",
2,
2,
n_training_epochs=10,
data_normalization=True,
weight_normalization=False)
model.fit(X, Y)
pdf_before = model.pdf(X, Y)
# pickle and unpickle model
dump_string = pickle.dumps(model)
tf.reset_default_graph()
with tf.Session() as sess:
model_loaded = pickle.loads(dump_string)
pdf_after = model_loaded.pdf(X, Y)
diff = np.sum(np.abs(pdf_after - pdf_before))
self.assertAlmostEqual(diff, 0, places=2)
def test_MDN_with_2d_gaussian2(self):
mu = -5
std = 2.5
X, Y = self.get_samples(mu=mu, std=std)
model = MixtureDensityNetwork("mdn2",
1,
1,
n_centers=5,
weight_normalization=True)
model.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_est = model.pdf(x, y)
p_true = norm.pdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
p_est = model.cdf(x, y)
p_true = norm.cdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_MDN_log_pdf(self):
X, Y = np.random.normal(size=(1000, 3)), np.random.normal(size=(1000,
2))
for data_norm in [True, False]:
with tf.Session() as sess:
model = MixtureDensityNetwork("mdn_logprob" + str(data_norm),
3,
2,
n_centers=1,
hidden_sizes=(8, 8),
n_training_epochs=10,
data_normalization=data_norm)
model.fit(X, Y)
x, y = np.random.normal(size=(1000,
3)), np.random.normal(size=(1000,
2))
prob = model.pdf(x, y)
log_prob = model.log_pdf(x, y)
self.assertLessEqual(np.mean(np.abs(prob - np.exp(log_prob))),
0.001)
def test_MDN_dropout(self):
mu = -8
std = 2.5
X, Y = self.get_samples(mu=mu, std=std)
dropout_model = MixtureDensityNetwork("mdn_dropout_reasonable",
1,
1,
n_centers=5,
weight_normalization=True,
dropout=0.5,
n_training_epochs=400)
dropout_model.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_est = dropout_model.pdf(x, y)
p_true = norm.pdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
p_est = dropout_model.cdf(x, y)
p_true = norm.cdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
for i, noise_std in enumerate([0.0, 0.05, 0.2]):
estimator1 = MixtureDensityNetwork('mdn%i' % i,
1,
1,
n_centers=20,
x_noise_std=noise_std,
y_noise_std=noise_std,
random_seed=SEED)
estimator1.fit(X, Y)
y = np.linspace(mu - 3 * std, mu + 3 * std, num=500)
x_cond = np.ones(500) * 0.0
p_true = model.pdf(x_cond, y)
p_est1 = estimator1.pdf(x_cond, y)
line1 = axes[i].plot(y, p_true, label='true density', color='green')
line2 = axes[i].plot(y, p_est1, label='est. density', color='red')
axes[i].set_title('noise_std = %.2f' % noise_std)
axes[0].set_xlabel('y')
axes[2].set_xlabel('y')
axes[0].set_ylabel('probability density')
fig.legend((line1[0], line2[0]), ('true', 'estimated'),
loc='lower center',
ncol=2) #loc=(0.857, 0.76))
fig.tight_layout(rect=[0.0, 0.01, 1, 1])
fig_path = os.path.join(DATA_DIR, 'plots/noise_reg/noise_reg_MDN.png')