def test_NF_adaptive_noise(self): adaptive_noise_fn = lambda n, d: 0.0 if n < 1000 else 5.0 X, Y = self.get_samples(mu=0, std=1, n_samples=999) est = NormalizingFlowEstimator("nf_999", 1, 1, y_noise_std=0.0, n_flows=2, hidden_sizes=(8, 8), x_noise_std=0.0, adaptive_noise_fn=adaptive_noise_fn, n_training_epochs=500) est.fit(X, Y) std_999 = est.std_(x_cond=np.array([[0.0]]))[0] X, Y = self.get_samples(mu=0, std=1, n_samples=1002) est = NormalizingFlowEstimator("nf_1002", 1, 1, y_noise_std=0.0, n_flows=2, hidden_sizes=(8, 8), x_noise_std=0.0, adaptive_noise_fn=adaptive_noise_fn, n_training_epochs=500) est.fit(X, Y) std_1002 = est.std_(x_cond=np.array([[0.0]]))[0] self.assertLess(std_999, std_1002) self.assertGreater(std_1002, 2)
def test_NF_l1_regularization(self): mu = 5 std = 5 X, Y = self.get_samples(mu=mu, std=std, n_samples=500) nf_no_reg = NormalizingFlowEstimator("nf_no_reg", 1, 1, hidden_sizes=(16, 16), n_flows=10, weight_normalization=False, l1_reg=0.0, l2_reg=0.0, n_training_epochs=500) nf_reg_l1 = NormalizingFlowEstimator("nf_reg_l1", 1, 1, hidden_sizes=(16, 16), n_flows=10, weight_normalization=False, l1_reg=10.0, l2_reg=0.0, n_training_epochs=500) nf_no_reg.fit(X, Y) nf_reg_l1.fit(X, Y) y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20) x = np.asarray([mu for i in range(y.shape[0])]) p_true = norm.pdf(y, loc=mu, scale=std) err_no_reg = np.mean(np.abs(nf_no_reg.pdf(x, y) - p_true)) err_reg_l1 = np.mean(np.abs(nf_reg_l1.pdf(x, y) - p_true)) print(err_no_reg, err_reg_l1) self.assertLessEqual(err_reg_l1, err_no_reg)
def test_dropout(self): with tf.Session() as sess: bimix_gauss = tf.contrib.distributions.Mixture( cat=tf.distributions.Categorical(probs=[0.5, 0.5]), components=[ tf.distributions.Normal(loc=-1., scale=0.5), tf.distributions.Normal(loc=+1., scale=0.5), ]) x = np.ones(5000) y = sess.run(bimix_gauss.sample([5000])) dropout_model = NormalizingFlowEstimator("nf_dropout_reasonable", 1, 1, flows_type=('affine', 'radial', 'radial'), data_normalization=True, dropout=0.5, random_seed=22) full_dropout = NormalizingFlowEstimator("nf_dropout_full", 1, 1, flows_type=('affine', 'radial', 'radial'), data_normalization=True, dropout=0.85, random_seed=22) dropout_model.fit(x, y) full_dropout.fit(x, y) p_est = dropout_model.pdf(x, y) p_est_trash = full_dropout.pdf(x, y) p_true = sess.run(bimix_gauss.prob(y)) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.02) self.assertGreater(np.mean(np.abs(p_true - p_est_trash)), 0.02)
def _test_flow_correct_dims_NN(self, flow_name): """ General structure: flow_params = MLP(x) pdf(y|x) = flow(y, flow_params) The tensor being transformed (=y) are of shape (batch_size, event_dims) - batch_size = len(x) == len(y) - event_dims = rank(y) For each element of x, the MLP outputs one parametrization for the flows for each of these parameters, the flow transforms one element of y therefore len(x) == len(y) the event dimension describes the rank of the base probability distribution that's being transformed Tensorflow's MultivariateNormal doesn't implement a CDF. Therefore we switch to a Normal for 1-D Problems Caveat: MultivariateNormal PDF output shape: (batch_size, ) UnivariateNormal PDF output shape: (batch_size, 1) Therefore we adapt the output shape of the ildj to be (batch_size, 1) for 1-D, (batch_size, ) for N-D The flows are transforming tensors (batch_size, event_size) Forward: (batch_size, event_size) -> (batch_size, event_size) Inverse: (batch_size, event_size) -> (batch_size, event_size) ILDJ: (batch_size, event_size) -> (batch_size, 1) [1-D] or (batch_size, ) [N-D] This forms a transformed distribution: Sample: -> (batch_size, event_size) PDF: (batch_size, event_size) -> (batch_size, 1) [1-D] or (batch_size, ) [N-D] CDF: (batch_size, event_size) -> (batch_size, 1) [EXISTS ONLY FOR 1-D!] """ tests = [ { 'x': [[1.], [0.], [2.], [4.], [1.]], 'y': [[1.], [0.], [2.], [3.], [1.]], 'ndim_x': 1, 'ndim_y': 1 }, { 'x': [[1., 1.], [0., 0.], [2., 2.], [4., 4.], [1., 1.]], 'y': [[1., 1.], [0., 0.], [2., 2.], [3., 3.], [1., 1.]], 'ndim_x': 2, 'ndim_y': 2 } ] with tf.Session() as sess: for test in tests: model = NormalizingFlowEstimator('nf_dimtest_' + flow_name + str(tests.index(test)), test['ndim_x'], test['ndim_y'], random_seed=22, n_training_epochs=2, flows_type=(flow_name,)) x, y = np.array(test['x']), np.array(test['y']) model.fit(x, y) p = model.pdf(x, y) self.assertEqual(p.shape, (len(y),)) # every test has equal first and last elements, theses are basic sanity tests self.assertAlmostEqual(p[0], p[-1], places=5) self.assertNotAlmostEqual(p[0], p[1], places=5)
def test_pickle_unpickle_NF_estimator(self): X, Y = self.get_samples() with tf.Session() as sess: model = NormalizingFlowEstimator('nf_pickle', 2, 2, ('affine', 'radial', 'radial'), data_normalization=True, random_seed=22, n_training_epochs=10) model.fit(X, Y) pdf_before = model.pdf(X, Y) dump_string = pickle.dumps(model) tf.reset_default_graph() with tf.Session() as sess: model_loaded = pickle.loads(dump_string) pdf_after = model_loaded.pdf(X, Y) diff = np.sum(np.abs(pdf_after - pdf_before)) self.assertAlmostEqual(diff, 0, places=2)
def test_NF_chain2_with_2d_gaussian2(self): mu = -5 std = 2.5 X, Y = self.get_samples(mu=mu, std=std) model = NormalizingFlowEstimator("nf_estimator_2d_chain2_2", 1, 1, flows_type=('radial', 'planar', 'radial'), n_training_epochs=500, random_seed=22) model.fit(X, Y) y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20) x = np.asarray([mu for i in range(y.shape[0])]) p_est = model.pdf(x, y) p_true = norm.pdf(y, loc=mu, scale=std) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_data_normalization(self): X, Y = self.get_samples(std=2, mean=20) with tf.Session() as sess: model = NormalizingFlowEstimator("nf_data_normalization", 1, 1, flows_type=('affine', 'radial', 'radial'), x_noise_std=None, y_noise_std=None, data_normalization=True, n_training_epochs=100) model.fit(X, Y) # test if data statistics were properly assigned to tf graph x_mean, x_std = model.sess.run([model.mean_x_sym, model.std_x_sym]) print(x_mean, x_std) mean_diff = float(np.abs(x_mean - 20)) std_diff = float(np.abs(x_std - 2)) self.assertLessEqual(mean_diff, 0.5) self.assertLessEqual(std_diff, 0.5)
def test_NF_identitiy_with_2d_gaussian(self): mu = 200 std = 23 X, Y = self.get_samples(mu=mu, std=std) model1 = NormalizingFlowEstimator("nf_estimator_2d_planar_no_id", 1, 1, flows_type=('planar',), n_training_epochs=50, random_seed=22) model2 = NormalizingFlowEstimator("nf_estimator_2d_planar_id", 1, 1, flows_type=('planar', 'identity'), n_training_epochs=50, random_seed=22) model1.fit(X, Y) model2.fit(X, Y) y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20) x = np.asarray([mu for i in range(y.shape[0])]) p = model1.pdf(x, y) p_id = model2.pdf(x, y) self.assertLessEqual(np.mean(np.abs(p - p_id)), 0.01)
def test_weight_decay(self): with tf.Session() as sess: bimix_gauss = tf.contrib.distributions.Mixture( cat=tf.distributions.Categorical(probs=[0.5, 0.5]), components=[ tf.distributions.Normal(loc=-1., scale=0.5), tf.distributions.Normal(loc=+1., scale=0.5), ]) x = np.ones(5000) y = sess.run(bimix_gauss.sample([5000])) model = NormalizingFlowEstimator("nf_estimator_weight_decay", 1, 1, flows_type=('affine', 'radial', 'radial'), data_normalization=True, weight_decay=0.0001, n_training_epochs=1000, random_seed=22) model.fit(x, y) p_est = model.pdf(x, y) p_true = sess.run(bimix_gauss.prob(y)) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.01)
def test_tri_modal_radial_chain(self): with tf.Session() as sess: bimix_gauss = tf.contrib.distributions.Mixture( cat=tf.distributions.Categorical(probs=[0.3, 0.4, 0.3]), components=[ tf.distributions.Normal(loc=-1., scale=0.4), tf.distributions.Normal(loc=0., scale=0.4), tf.distributions.Normal(loc=+1., scale=0.4), ]) x = np.ones(5000) y = sess.run(bimix_gauss.sample([5000])) model = NormalizingFlowEstimator("nf_estimator_bimodal_radial", 1, 1, flows_type=('radial', 'radial', 'radial'), n_training_epochs=1000, random_seed=22) model.fit(x, y) p_est = model.pdf(x, y) p_true = sess.run(bimix_gauss.prob(y)) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_bi_modal_planar_chain(self): with tf.Session() as sess: bimix_gauss = tf.contrib.distributions.Mixture( cat=tf.distributions.Categorical(probs=[0.5, 0.5]), components=[ tf.distributions.Normal(loc=-.4, scale=0.4), tf.distributions.Normal(loc=+.4, scale=0.4), ]) x = tf.distributions.Normal(loc=0., scale=1.).sample([5000]) y = bimix_gauss.sample([5000]) x, y = sess.run([x, y]) model = NormalizingFlowEstimator("nf_estimator_bimodal_planar", 1, 1, flows_type=('affine', 'planar', 'planar', 'planar'), n_training_epochs=1000, random_seed=22) model.fit(x, y) p_est = model.pdf(x, y) p_true = sess.run(bimix_gauss.prob(y)) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_NF_log_pdf(self): X, Y = np.random.normal(size=(1000, 3)), np.random.normal(size=(1000, 3)) with tf.Session() as sess: model = NormalizingFlowEstimator("nf_logprob", 3, 3, flows_type=('affine', 'planar'), n_training_epochs=10, random_seed=22) model.fit(X, Y) x, y = np.random.normal(size=(1000, 3)), np.random.normal(size=(1000, 3)) prob = model.pdf(x, y) log_prob = model.log_pdf(x, y) self.assertLessEqual(np.mean(np.abs(prob - np.exp(log_prob))), 0.001)
def test_NF_fit_by_crossval(self): X, Y = self.get_samples(std=1., mean=-4) param_grid = { 'n_training_epochs': [0, 500], 'data_normalization': [False] } model = NormalizingFlowEstimator('nf_crossval', 1, 1) model.fit_by_cv(X, Y, param_grid=param_grid) y = np.arange(-1, 5, 0.5) x = np.asarray([2 for _ in range(y.shape[0])]) p_est = model.pdf(x, y) p_true = norm.pdf(y, loc=2, scale=1) self.assertEqual(model.get_params()["n_training_epochs"], 500) self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.2)