def test_MDN_adaptive_noise(self):
adaptive_noise_fn = lambda n, d: 0.0 if n < 1000 else 5.0
X, Y = self.get_samples(mu=0, std=1, n_samples=999)
est = MixtureDensityNetwork("mdn_adaptive_noise_999",
1,
1,
n_centers=1,
y_noise_std=0.0,
x_noise_std=0.0,
hidden_sizes=(8, 8),
adaptive_noise_fn=adaptive_noise_fn,
n_training_epochs=500)
est.fit(X, Y)
std_999 = est.std_(x_cond=np.array([[0.0]]))[0]
X, Y = self.get_samples(mu=0, std=1, n_samples=1002)
est = MixtureDensityNetwork("mdn_adaptive_noise_1002",
1,
1,
n_centers=1,
y_noise_std=0.0,
x_noise_std=0.0,
hidden_sizes=(8, 8),
adaptive_noise_fn=adaptive_noise_fn,
n_training_epochs=500)
est.fit(X, Y)
std_1002 = est.std_(x_cond=np.array([[0.0]]))[0]
self.assertLess(std_999, std_1002)
self.assertGreater(std_1002, 2)
def test_MDN_weight_decay(self):
mu = 5
std = 5
X, Y = self.get_samples(mu=mu, std=std)
no_decay = MixtureDensityNetwork("mdn_no_weight_decay",
1,
1,
hidden_sizes=(32, 32),
n_centers=10,
n_training_epochs=2000,
weight_decay=0.0,
weight_normalization=False)
decay = MixtureDensityNetwork("mdn_weight_decay",
1,
1,
n_centers=10,
hidden_sizes=(32, 32),
n_training_epochs=2000,
weight_decay=1e-3,
weight_normalization=False)
no_decay.fit(X, Y)
decay.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_true = norm.pdf(y, loc=mu, scale=std)
l1_err_no_dec = np.mean(np.abs(no_decay.pdf(x, y) - p_true))
l1_err_dec = np.mean(np.abs(decay.pdf(x, y) - p_true))
self.assertLessEqual(l1_err_dec, 0.1)
self.assertLessEqual(l1_err_dec, l1_err_no_dec)
def test3_MDN_with_2d_gaussian_noise_y(self):
X, Y = self.get_samples(std=0.5)
with tf.Session():
model_no_noise = MixtureDensityNetwork("mdn_no_noise_y",
1,
1,
n_centers=1,
x_noise_std=None,
y_noise_std=None)
model_no_noise.fit(X, Y)
var_no_noise = model_no_noise.covariance(
x_cond=np.array([[2]]))[0][0][0]
model_noise = MixtureDensityNetwork("mdn_noise_y",
1,
1,
n_centers=1,
x_noise_std=None,
y_noise_std=1)
model_noise.fit(X, Y)
var_noise = model_noise.covariance(x_cond=np.array([[2]]))[0][0][0]
print("Training w/o noise:", var_no_noise)
print("Training w/ noise:", var_noise)
self.assertGreaterEqual(var_noise - var_no_noise, 0.1)
def cv_for_cde(data,
labels,
name,
std,
n_splits=5,
want_r2=False,
want_mae=False,
hidden_sizes=(16, 16)):
'''
model: must be a sklearn object with .fit and .predict methods
data: the X matrix containing the features, can be a pd.DataFrame or a np object (array or matrix)
labels: y, can be a pd.DataFrame or a np array
n_splits: number of desired folds
=> returns array of mean suqared error calculated on each fold
'''
from cde.density_estimator import MixtureDensityNetwork
input_dim = data.shape[1]
kf = KFold(n_splits=n_splits, shuffle=True)
data = np.array(data)
labels = np.array(labels)
mses = []
r2s = []
maes = []
i = 1
for train, test in kf.split(data):
model = MixtureDensityNetwork(name=name + str(i),
ndim_x=input_dim,
ndim_y=1,
n_centers=10,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=tanh,
n_training_epochs=1000,
x_noise_std=std,
y_noise_std=std)
print("Split: {}".format(i), end="\r")
X_train, X_test, y_train, y_test = data[train], data[test], labels[
train], labels[test]
model.fit(X=X_train, Y=y_train, verbose=True)
pred = model.mean_(X_test)
pred = pred.reshape((-1, 1)).flatten()
mse = sum((pred - y_test)**2) / len(test)
print('MSE: {}'.format(mse))
mses.append(mse)
r2 = r2_score(y_pred=pred, y_true=y_test)
r2s.append(r2)
maes.append(mean_absolute_error(y_pred=pred, y_true=y_test))
i = i + 1
if want_r2 and want_mae:
return (mses, r2s, maes)
elif want_r2:
return (mses, r2s)
elif want_mae:
return (mses, maes)
else:
return mses
def test_mean_mixture(self):
np.random.seed(24)
from tensorflow import set_random_seed
set_random_seed(24)
data = np.random.normal([2, 2, 7, -2], 1, size=(5000, 4))
X = data[:, 0:2]
Y = data[:, 2:4]
model = MixtureDensityNetwork("mdn_mean",
2,
2,
n_centers=3,
y_noise_std=0.1,
x_noise_std=0.1)
model.fit(X, Y)
mean_est = model.mean_(x_cond=np.array([[1.5, 2]]), n_samples=10**7)
self.assertAlmostEqual(mean_est[0][0], 7, places=0)
self.assertAlmostEqual(mean_est[0][1], -2, places=0)
def testPickleUnpickleMDN(self):
X, Y = self.get_samples()
with tf.Session() as sess:
model = MixtureDensityNetwork("mdn_pickle",
2,
2,
n_training_epochs=10,
data_normalization=True,
weight_normalization=False)
model.fit(X, Y)
pdf_before = model.pdf(X, Y)
# pickle and unpickle model
dump_string = pickle.dumps(model)
tf.reset_default_graph()
with tf.Session() as sess:
model_loaded = pickle.loads(dump_string)
pdf_after = model_loaded.pdf(X, Y)
diff = np.sum(np.abs(pdf_after - pdf_before))
self.assertAlmostEqual(diff, 0, places=2)
def test8_data_normalization(self):
np.random.seed(24)
mean = 10
std = 2
data = np.random.normal([mean, mean, mean, mean], std, size=(2000, 4))
X = data[:, 0:2]
Y = data[:, 2:4]
with tf.Session() as sess:
model = MixtureDensityNetwork("mdn_data_normalization",
2,
2,
n_centers=2,
x_noise_std=None,
y_noise_std=None,
data_normalization=True,
n_training_epochs=2000,
random_seed=22)
model.fit(X, Y)
y_mean, y_std = sess.run([model.mean_y_sym, model.std_y_sym])
print(y_mean, y_std)
cond_mean = model.mean_(Y)
mean_diff = np.abs(mean - np.mean(cond_mean))
self.assertLessEqual(mean_diff, 0.5)
cond_cov = np.mean(model.covariance(Y), axis=0)
print(cond_cov)
self.assertGreaterEqual(cond_cov[0][0], std**2 * 0.7)
self.assertLessEqual(cond_cov[0][0], std**2 * 1.3)
self.assertGreaterEqual(cond_cov[1][1], std**2 * 0.7)
self.assertLessEqual(cond_cov[1][1], std**2 * 1.3)
def test4_MDN_with_2d_gaussian_noise_x(self):
np.random.seed(22)
X = np.random.uniform(0, 6, size=4000)
Y = X + np.random.normal(0, 1, size=4000)
x_test_2 = np.ones(100) * 2
x_test_4 = np.ones(100) * 4
y_test = np.linspace(1, 5, num=100)
with tf.Session():
model_no_noise = MixtureDensityNetwork("mdn_no_noise_x",
1,
1,
n_centers=1,
x_noise_std=None,
y_noise_std=None)
model_no_noise.fit(X, Y)
pdf_distance_no_noise = np.mean(
np.abs(
model_no_noise.pdf(x_test_2, y_test) -
model_no_noise.pdf(x_test_4, y_test)))
model_noise = MixtureDensityNetwork("mdn_noise_x",
1,
1,
n_centers=1,
x_noise_std=2,
y_noise_std=None)
model_noise.fit(X, Y)
pdf_distance_noise = np.mean(
np.abs(
model_noise.pdf(x_test_2, y_test) -
model_noise.pdf(x_test_4, y_test)))
print("Training w/o noise - pdf distance:", pdf_distance_no_noise)
print("Training w/ noise - pdf distance", pdf_distance_noise)
self.assertGreaterEqual(pdf_distance_no_noise / pdf_distance_noise,
2.0)
def test_covariance_mixture(self):
np.random.seed(24)
from tensorflow import set_random_seed
set_random_seed(24)
scale = 2.0
data = np.random.normal(loc=[2, 2, 7, -2],
scale=scale,
size=(10000, 4))
X = data[:, 0:2]
Y = data[:, 2:4]
model = MixtureDensityNetwork("mdn_cov",
2,
2,
n_centers=5,
x_noise_std=0.1,
y_noise_std=0.1)
model.fit(X, Y)
cov_est = model.covariance(x_cond=np.array([[0, 1]]))
print(cov_est)
self.assertLessEqual(np.abs(cov_est[0][1][0] - 0.0), 0.2)
self.assertLessEqual(np.abs(np.sqrt(cov_est[0][0][0]) - scale), 1.0)
def test_2_MDN_with_2d_gaussian_fit_by_crossval(self):
X, Y = self.get_samples()
param_grid = {"n_centers": [2, 10, 50]}
model = MixtureDensityNetwork()
model.fit_by_cv(X, Y, param_grid=param_grid)
y = np.arange(-1, 5, 0.5)
x = np.asarray([2 for i in range(y.shape[0])])
p_est = model.pdf(x, y)
p_true = norm.pdf(y, loc=2, scale=1)
self.assertEqual(model.get_params()["n_centers"], 10)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.2)
def test_MDN_with_2d_gaussian_sampling(self):
X, Y = self.get_samples()
model = MixtureDensityNetwork("mdn_gaussian_sampling",
1,
1,
n_centers=5,
n_training_epochs=200)
model.fit(X, Y)
x_cond = np.ones(shape=(10**6, 1))
_, y_sample = model.sample(x_cond)
self.assertAlmostEqual(np.mean(y_sample),
float(model.mean_(y_sample[1])),
places=0)
self.assertAlmostEqual(np.std(y_sample),
float(model.covariance(y_sample[1])),
places=0)
def test_MDN_KMN_eval_set(self):
mu = 200
std = 23
X_train, Y_train = self.get_samples(mu=mu, std=std)
X_test, Y_test = self.get_samples(mu=mu, std=std)
X_test = X_test
model = MixtureDensityNetwork("mdn_eval_set",
1,
1,
n_centers=10,
data_normalization=True,
n_training_epochs=100)
model.fit(X_train, Y_train, eval_set=(X_test, Y_test))
model = KernelMixtureNetwork("kmn_eval_set",
1,
1,
n_centers=10,
data_normalization=True,
n_training_epochs=100)
model.fit(X_train, Y_train, eval_set=(X_test, Y_test))
def test_MDN_with_2d_gaussian2(self):
mu = -5
std = 2.5
X, Y = self.get_samples(mu=mu, std=std)
model = MixtureDensityNetwork("mdn2",
1,
1,
n_centers=5,
weight_normalization=True)
model.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_est = model.pdf(x, y)
p_true = norm.pdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
p_est = model.cdf(x, y)
p_true = norm.cdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_MDN_log_pdf(self):
X, Y = np.random.normal(size=(1000, 3)), np.random.normal(size=(1000,
2))
for data_norm in [True, False]:
with tf.Session() as sess:
model = MixtureDensityNetwork("mdn_logprob" + str(data_norm),
3,
2,
n_centers=1,
hidden_sizes=(8, 8),
n_training_epochs=10,
data_normalization=data_norm)
model.fit(X, Y)
x, y = np.random.normal(size=(1000,
3)), np.random.normal(size=(1000,
2))
prob = model.pdf(x, y)
log_prob = model.log_pdf(x, y)
self.assertLessEqual(np.mean(np.abs(prob - np.exp(log_prob))),
0.001)
def test_MDN_dropout(self):
mu = -8
std = 2.5
X, Y = self.get_samples(mu=mu, std=std)
dropout_model = MixtureDensityNetwork("mdn_dropout_reasonable",
1,
1,
n_centers=5,
weight_normalization=True,
dropout=0.5,
n_training_epochs=400)
dropout_model.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_est = dropout_model.pdf(x, y)
p_true = norm.pdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
p_est = dropout_model.cdf(x, y)
p_true = norm.cdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
std = 1
model = LinearGaussian(ndim_x=1,
mu=mu,
std=std,
mu_slope=0.002,
random_seed=SEED)
X, Y = model.simulate(n_samples=4000)
fig, axes = plt.subplots(ncols=3, nrows=1, figsize=(10, 3))
for i, noise_std in enumerate([0.0, 0.05, 0.2]):
estimator1 = MixtureDensityNetwork('mdn%i' % i,
1,
1,
n_centers=20,
x_noise_std=noise_std,
y_noise_std=noise_std,
random_seed=SEED)
estimator1.fit(X, Y)
y = np.linspace(mu - 3 * std, mu + 3 * std, num=500)
x_cond = np.ones(500) * 0.0
p_true = model.pdf(x_cond, y)
p_est1 = estimator1.pdf(x_cond, y)
line1 = axes[i].plot(y, p_true, label='true density', color='green')
line2 = axes[i].plot(y, p_est1, label='est. density', color='red')
axes[i].set_title('noise_std = %.2f' % noise_std)
def test_MDN_l1_l2_regularization(self):
mu = 5
std = 5
X, Y = self.get_samples(mu=mu, std=std, n_samples=500)
mdn_no_reg = MixtureDensityNetwork("mdn_no_reg",
1,
1,
hidden_sizes=(32, 32),
n_centers=10,
n_training_epochs=1000,
l2_reg=0.0,
weight_normalization=False)
mdn_reg_l2 = MixtureDensityNetwork("mdn_reg_l2",
1,
1,
n_centers=10,
hidden_sizes=(32, 32),
n_training_epochs=1000,
l2_reg=1.0,
weight_normalization=False)
mdn_reg_l1 = MixtureDensityNetwork("mdn_reg_l1",
1,
1,
n_centers=10,
hidden_sizes=(32, 32),
n_training_epochs=1000,
l1_reg=1.0,
weight_normalization=False)
mdn_no_reg.fit(X, Y)
mdn_reg_l2.fit(X, Y)
mdn_reg_l1.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_true = norm.pdf(y, loc=mu, scale=std)
err_no_reg = np.mean(np.abs(mdn_no_reg.pdf(x, y) - p_true))
err_reg_l2 = np.mean(np.abs(mdn_reg_l2.pdf(x, y) - p_true))
err_reg_l1 = np.mean(np.abs(mdn_reg_l1.pdf(x, y) - p_true))
self.assertLessEqual(err_reg_l2, err_no_reg)
self.assertLessEqual(err_reg_l1, err_no_reg)