def test_MDN_adaptive_noise(self): adaptive_noise_fn = lambda n, d: 0.0 if n < 1000 else 5.0 X, Y = self.get_samples(mu=0, std=1, n_samples=999) est = MixtureDensityNetwork("mdn_adaptive_noise_999", 1, 1, n_centers=1, y_noise_std=0.0, x_noise_std=0.0, hidden_sizes=(8, 8), adaptive_noise_fn=adaptive_noise_fn, n_training_epochs=500) est.fit(X, Y) std_999 = est.std_(x_cond=np.array([[0.0]]))[0] X, Y = self.get_samples(mu=0, std=1, n_samples=1002) est = MixtureDensityNetwork("mdn_adaptive_noise_1002", 1, 1, n_centers=1, y_noise_std=0.0, x_noise_std=0.0, hidden_sizes=(8, 8), adaptive_noise_fn=adaptive_noise_fn, n_training_epochs=500) est.fit(X, Y) std_1002 = est.std_(x_cond=np.array([[0.0]]))[0] self.assertLess(std_999, std_1002) self.assertGreater(std_1002, 2)
def test_MDN_weight_decay(self): mu = 5 std = 5 X, Y = self.get_samples(mu=mu, std=std) no_decay = MixtureDensityNetwork("mdn_no_weight_decay", 1, 1, hidden_sizes=(32, 32), n_centers=10, n_training_epochs=2000, weight_decay=0.0, weight_normalization=False) decay = MixtureDensityNetwork("mdn_weight_decay", 1, 1, n_centers=10, hidden_sizes=(32, 32), n_training_epochs=2000, weight_decay=1e-3, weight_normalization=False) no_decay.fit(X, Y) decay.fit(X, Y) y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20) x = np.asarray([mu for i in range(y.shape[0])]) p_true = norm.pdf(y, loc=mu, scale=std) l1_err_no_dec = np.mean(np.abs(no_decay.pdf(x, y) - p_true)) l1_err_dec = np.mean(np.abs(decay.pdf(x, y) - p_true)) self.assertLessEqual(l1_err_dec, 0.1) self.assertLessEqual(l1_err_dec, l1_err_no_dec)
def test3_MDN_with_2d_gaussian_noise_y(self): X, Y = self.get_samples(std=0.5) with tf.Session(): model_no_noise = MixtureDensityNetwork("mdn_no_noise_y", 1, 1, n_centers=1, x_noise_std=None, y_noise_std=None) model_no_noise.fit(X, Y) var_no_noise = model_no_noise.covariance( x_cond=np.array([[2]]))[0][0][0] model_noise = MixtureDensityNetwork("mdn_noise_y", 1, 1, n_centers=1, x_noise_std=None, y_noise_std=1) model_noise.fit(X, Y) var_noise = model_noise.covariance(x_cond=np.array([[2]]))[0][0][0] print("Training w/o noise:", var_no_noise) print("Training w/ noise:", var_noise) self.assertGreaterEqual(var_noise - var_no_noise, 0.1)
def test8_data_normalization(self): np.random.seed(24) mean = 10 std = 2 data = np.random.normal([mean, mean, mean, mean], std, size=(2000, 4)) X = data[:, 0:2] Y = data[:, 2:4] with tf.Session() as sess: model = MixtureDensityNetwork("mdn_data_normalization", 2, 2, n_centers=2, x_noise_std=None, y_noise_std=None, data_normalization=True, n_training_epochs=2000, random_seed=22) model.fit(X, Y) y_mean, y_std = sess.run([model.mean_y_sym, model.std_y_sym]) print(y_mean, y_std) cond_mean = model.mean_(Y) mean_diff = np.abs(mean - np.mean(cond_mean)) self.assertLessEqual(mean_diff, 0.5) cond_cov = np.mean(model.covariance(Y), axis=0) print(cond_cov) self.assertGreaterEqual(cond_cov[0][0], std**2 * 0.7) self.assertLessEqual(cond_cov[0][0], std**2 * 1.3) self.assertGreaterEqual(cond_cov[1][1], std**2 * 0.7) self.assertLessEqual(cond_cov[1][1], std**2 * 1.3)
def cv_for_cde(data, labels, name, std, n_splits=5, want_r2=False, want_mae=False, hidden_sizes=(16, 16)): ''' model: must be a sklearn object with .fit and .predict methods data: the X matrix containing the features, can be a pd.DataFrame or a np object (array or matrix) labels: y, can be a pd.DataFrame or a np array n_splits: number of desired folds => returns array of mean suqared error calculated on each fold ''' from cde.density_estimator import MixtureDensityNetwork input_dim = data.shape[1] kf = KFold(n_splits=n_splits, shuffle=True) data = np.array(data) labels = np.array(labels) mses = [] r2s = [] maes = [] i = 1 for train, test in kf.split(data): model = MixtureDensityNetwork(name=name + str(i), ndim_x=input_dim, ndim_y=1, n_centers=10, hidden_sizes=hidden_sizes, hidden_nonlinearity=tanh, n_training_epochs=1000, x_noise_std=std, y_noise_std=std) print("Split: {}".format(i), end="\r") X_train, X_test, y_train, y_test = data[train], data[test], labels[ train], labels[test] model.fit(X=X_train, Y=y_train, verbose=True) pred = model.mean_(X_test) pred = pred.reshape((-1, 1)).flatten() mse = sum((pred - y_test)**2) / len(test) print('MSE: {}'.format(mse)) mses.append(mse) r2 = r2_score(y_pred=pred, y_true=y_test) r2s.append(r2) maes.append(mean_absolute_error(y_pred=pred, y_true=y_test)) i = i + 1 if want_r2 and want_mae: return (mses, r2s, maes) elif want_r2: return (mses, r2s) elif want_mae: return (mses, maes) else: return mses
def test_MDN_with_2d_gaussian_sampling(self): X, Y = self.get_samples() model = MixtureDensityNetwork("mdn_gaussian_sampling", 1, 1, n_centers=5, n_training_epochs=200) model.fit(X, Y) x_cond = np.ones(shape=(10**6, 1)) _, y_sample = model.sample(x_cond) self.assertAlmostEqual(np.mean(y_sample), float(model.mean_(y_sample[1])), places=0) self.assertAlmostEqual(np.std(y_sample), float(model.covariance(y_sample[1])), places=0)
def test_mean_mixture(self): np.random.seed(24) from tensorflow import set_random_seed set_random_seed(24) data = np.random.normal([2, 2, 7, -2], 1, size=(5000, 4)) X = data[:, 0:2] Y = data[:, 2:4] model = MixtureDensityNetwork("mdn_mean", 2, 2, n_centers=3, y_noise_std=0.1, x_noise_std=0.1) model.fit(X, Y) mean_est = model.mean_(x_cond=np.array([[1.5, 2]]), n_samples=10**7) self.assertAlmostEqual(mean_est[0][0], 7, places=0) self.assertAlmostEqual(mean_est[0][1], -2, places=0)
def testPickleUnpickleMDN(self): X, Y = self.get_samples() with tf.Session() as sess: model = MixtureDensityNetwork("mdn_pickle", 2, 2, n_training_epochs=10, data_normalization=True, weight_normalization=False) model.fit(X, Y) pdf_before = model.pdf(X, Y) # pickle and unpickle model dump_string = pickle.dumps(model) tf.reset_default_graph() with tf.Session() as sess: model_loaded = pickle.loads(dump_string) pdf_after = model_loaded.pdf(X, Y) diff = np.sum(np.abs(pdf_after - pdf_before)) self.assertAlmostEqual(diff, 0, places=2)
def test_MDN_with_2d_gaussian2(self): mu = -5 std = 2.5 X, Y = self.get_samples(mu=mu, std=std) model = MixtureDensityNetwork("mdn2", 1, 1, n_centers=5, weight_normalization=True) model.fit(X, Y) y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20) x = np.asarray([mu for i in range(y.shape[0])]) p_est = model.pdf(x, y) p_true = norm.pdf(y, loc=mu, scale=std) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1) p_est = model.cdf(x, y) p_true = norm.cdf(y, loc=mu, scale=std) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_MDN_log_pdf(self): X, Y = np.random.normal(size=(1000, 3)), np.random.normal(size=(1000, 2)) for data_norm in [True, False]: with tf.Session() as sess: model = MixtureDensityNetwork("mdn_logprob" + str(data_norm), 3, 2, n_centers=1, hidden_sizes=(8, 8), n_training_epochs=10, data_normalization=data_norm) model.fit(X, Y) x, y = np.random.normal(size=(1000, 3)), np.random.normal(size=(1000, 2)) prob = model.pdf(x, y) log_prob = model.log_pdf(x, y) self.assertLessEqual(np.mean(np.abs(prob - np.exp(log_prob))), 0.001)
def test4_MDN_with_2d_gaussian_noise_x(self): np.random.seed(22) X = np.random.uniform(0, 6, size=4000) Y = X + np.random.normal(0, 1, size=4000) x_test_2 = np.ones(100) * 2 x_test_4 = np.ones(100) * 4 y_test = np.linspace(1, 5, num=100) with tf.Session(): model_no_noise = MixtureDensityNetwork("mdn_no_noise_x", 1, 1, n_centers=1, x_noise_std=None, y_noise_std=None) model_no_noise.fit(X, Y) pdf_distance_no_noise = np.mean( np.abs( model_no_noise.pdf(x_test_2, y_test) - model_no_noise.pdf(x_test_4, y_test))) model_noise = MixtureDensityNetwork("mdn_noise_x", 1, 1, n_centers=1, x_noise_std=2, y_noise_std=None) model_noise.fit(X, Y) pdf_distance_noise = np.mean( np.abs( model_noise.pdf(x_test_2, y_test) - model_noise.pdf(x_test_4, y_test))) print("Training w/o noise - pdf distance:", pdf_distance_no_noise) print("Training w/ noise - pdf distance", pdf_distance_noise) self.assertGreaterEqual(pdf_distance_no_noise / pdf_distance_noise, 2.0)
def test_MDN_KMN_eval_set(self): mu = 200 std = 23 X_train, Y_train = self.get_samples(mu=mu, std=std) X_test, Y_test = self.get_samples(mu=mu, std=std) X_test = X_test model = MixtureDensityNetwork("mdn_eval_set", 1, 1, n_centers=10, data_normalization=True, n_training_epochs=100) model.fit(X_train, Y_train, eval_set=(X_test, Y_test)) model = KernelMixtureNetwork("kmn_eval_set", 1, 1, n_centers=10, data_normalization=True, n_training_epochs=100) model.fit(X_train, Y_train, eval_set=(X_test, Y_test))
def test_MDN_dropout(self): mu = -8 std = 2.5 X, Y = self.get_samples(mu=mu, std=std) dropout_model = MixtureDensityNetwork("mdn_dropout_reasonable", 1, 1, n_centers=5, weight_normalization=True, dropout=0.5, n_training_epochs=400) dropout_model.fit(X, Y) y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20) x = np.asarray([mu for i in range(y.shape[0])]) p_est = dropout_model.pdf(x, y) p_true = norm.pdf(y, loc=mu, scale=std) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1) p_est = dropout_model.cdf(x, y) p_true = norm.cdf(y, loc=mu, scale=std) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_covariance_mixture(self): np.random.seed(24) from tensorflow import set_random_seed set_random_seed(24) scale = 2.0 data = np.random.normal(loc=[2, 2, 7, -2], scale=scale, size=(10000, 4)) X = data[:, 0:2] Y = data[:, 2:4] model = MixtureDensityNetwork("mdn_cov", 2, 2, n_centers=5, x_noise_std=0.1, y_noise_std=0.1) model.fit(X, Y) cov_est = model.covariance(x_cond=np.array([[0, 1]])) print(cov_est) self.assertLessEqual(np.abs(cov_est[0][1][0] - 0.0), 0.2) self.assertLessEqual(np.abs(np.sqrt(cov_est[0][0][0]) - scale), 1.0)
def test_MDN_l1_l2_regularization(self): mu = 5 std = 5 X, Y = self.get_samples(mu=mu, std=std, n_samples=500) mdn_no_reg = MixtureDensityNetwork("mdn_no_reg", 1, 1, hidden_sizes=(32, 32), n_centers=10, n_training_epochs=1000, l2_reg=0.0, weight_normalization=False) mdn_reg_l2 = MixtureDensityNetwork("mdn_reg_l2", 1, 1, n_centers=10, hidden_sizes=(32, 32), n_training_epochs=1000, l2_reg=1.0, weight_normalization=False) mdn_reg_l1 = MixtureDensityNetwork("mdn_reg_l1", 1, 1, n_centers=10, hidden_sizes=(32, 32), n_training_epochs=1000, l1_reg=1.0, weight_normalization=False) mdn_no_reg.fit(X, Y) mdn_reg_l2.fit(X, Y) mdn_reg_l1.fit(X, Y) y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20) x = np.asarray([mu for i in range(y.shape[0])]) p_true = norm.pdf(y, loc=mu, scale=std) err_no_reg = np.mean(np.abs(mdn_no_reg.pdf(x, y) - p_true)) err_reg_l2 = np.mean(np.abs(mdn_reg_l2.pdf(x, y) - p_true)) err_reg_l1 = np.mean(np.abs(mdn_reg_l1.pdf(x, y) - p_true)) self.assertLessEqual(err_reg_l2, err_no_reg) self.assertLessEqual(err_reg_l1, err_no_reg)
mu_slope=0.002, random_seed=SEED) X, Y = model.simulate(n_samples=4000) fig, axes = plt.subplots(ncols=3, nrows=1, figsize=(10, 3)) for i, noise_std in enumerate([0.0, 0.05, 0.2]): estimator1 = MixtureDensityNetwork('mdn%i' % i, 1, 1, n_centers=20, x_noise_std=noise_std, y_noise_std=noise_std, random_seed=SEED) estimator1.fit(X, Y) y = np.linspace(mu - 3 * std, mu + 3 * std, num=500) x_cond = np.ones(500) * 0.0 p_true = model.pdf(x_cond, y) p_est1 = estimator1.pdf(x_cond, y) line1 = axes[i].plot(y, p_true, label='true density', color='green') line2 = axes[i].plot(y, p_est1, label='est. density', color='red') axes[i].set_title('noise_std = %.2f' % noise_std) axes[0].set_xlabel('y') axes[2].set_xlabel('y') axes[0].set_ylabel('probability density')