def test_NF_l1_regularization(self):
mu = 5
std = 5
X, Y = self.get_samples(mu=mu, std=std, n_samples=500)
nf_no_reg = NormalizingFlowEstimator("nf_no_reg",
1,
1,
hidden_sizes=(16, 16),
n_flows=10,
weight_normalization=False,
l1_reg=0.0,
l2_reg=0.0,
n_training_epochs=500)
nf_reg_l1 = NormalizingFlowEstimator("nf_reg_l1",
1,
1,
hidden_sizes=(16, 16),
n_flows=10,
weight_normalization=False,
l1_reg=10.0,
l2_reg=0.0,
n_training_epochs=500)
nf_no_reg.fit(X, Y)
nf_reg_l1.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_true = norm.pdf(y, loc=mu, scale=std)
err_no_reg = np.mean(np.abs(nf_no_reg.pdf(x, y) - p_true))
err_reg_l1 = np.mean(np.abs(nf_reg_l1.pdf(x, y) - p_true))
print(err_no_reg, err_reg_l1)
self.assertLessEqual(err_reg_l1, err_no_reg)
def test_dropout(self):
with tf.Session() as sess:
bimix_gauss = tf.contrib.distributions.Mixture(
cat=tf.distributions.Categorical(probs=[0.5, 0.5]),
components=[
tf.distributions.Normal(loc=-1., scale=0.5),
tf.distributions.Normal(loc=+1., scale=0.5),
])
x = np.ones(5000)
y = sess.run(bimix_gauss.sample([5000]))
dropout_model = NormalizingFlowEstimator("nf_dropout_reasonable", 1, 1,
flows_type=('affine', 'radial', 'radial'),
data_normalization=True, dropout=0.5, random_seed=22)
full_dropout = NormalizingFlowEstimator("nf_dropout_full", 1, 1,
flows_type=('affine', 'radial', 'radial'),
data_normalization=True, dropout=0.85, random_seed=22)
dropout_model.fit(x, y)
full_dropout.fit(x, y)
p_est = dropout_model.pdf(x, y)
p_est_trash = full_dropout.pdf(x, y)
p_true = sess.run(bimix_gauss.prob(y))
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.02)
self.assertGreater(np.mean(np.abs(p_true - p_est_trash)), 0.02)
def test_NF_adaptive_noise(self):
adaptive_noise_fn = lambda n, d: 0.0 if n < 1000 else 5.0
X, Y = self.get_samples(mu=0, std=1, n_samples=999)
est = NormalizingFlowEstimator("nf_999",
1,
1,
y_noise_std=0.0,
n_flows=2,
hidden_sizes=(8, 8),
x_noise_std=0.0,
adaptive_noise_fn=adaptive_noise_fn,
n_training_epochs=500)
est.fit(X, Y)
std_999 = est.std_(x_cond=np.array([[0.0]]))[0]
X, Y = self.get_samples(mu=0, std=1, n_samples=1002)
est = NormalizingFlowEstimator("nf_1002",
1,
1,
y_noise_std=0.0,
n_flows=2,
hidden_sizes=(8, 8),
x_noise_std=0.0,
adaptive_noise_fn=adaptive_noise_fn,
n_training_epochs=500)
est.fit(X, Y)
std_1002 = est.std_(x_cond=np.array([[0.0]]))[0]
self.assertLess(std_999, std_1002)
self.assertGreater(std_1002, 2)
def _test_flow_correct_dims_NN(self, flow_name):
"""
General structure:
flow_params = MLP(x)
pdf(y|x) = flow(y, flow_params)
The tensor being transformed (=y) are of shape (batch_size, event_dims)
- batch_size = len(x) == len(y)
- event_dims = rank(y)
For each element of x, the MLP outputs one parametrization for the flows
for each of these parameters, the flow transforms one element of y
therefore len(x) == len(y)
the event dimension describes the rank of the base probability distribution that's being transformed
Tensorflow's MultivariateNormal doesn't implement a CDF. Therefore we switch to a Normal for 1-D Problems
Caveat:
MultivariateNormal PDF output shape: (batch_size, )
UnivariateNormal PDF output shape: (batch_size, 1)
Therefore we adapt the output shape of the ildj to be (batch_size, 1) for 1-D, (batch_size, ) for N-D
The flows are transforming tensors (batch_size, event_size)
Forward: (batch_size, event_size) -> (batch_size, event_size)
Inverse: (batch_size, event_size) -> (batch_size, event_size)
ILDJ: (batch_size, event_size) -> (batch_size, 1) [1-D] or (batch_size, ) [N-D]
This forms a transformed distribution:
Sample: -> (batch_size, event_size)
PDF: (batch_size, event_size) -> (batch_size, 1) [1-D] or (batch_size, ) [N-D]
CDF: (batch_size, event_size) -> (batch_size, 1) [EXISTS ONLY FOR 1-D!]
"""
tests = [
{
'x': [[1.], [0.], [2.], [4.], [1.]],
'y': [[1.], [0.], [2.], [3.], [1.]],
'ndim_x': 1,
'ndim_y': 1
},
{
'x': [[1., 1.], [0., 0.], [2., 2.], [4., 4.], [1., 1.]],
'y': [[1., 1.], [0., 0.], [2., 2.], [3., 3.], [1., 1.]],
'ndim_x': 2,
'ndim_y': 2
}
]
with tf.Session() as sess:
for test in tests:
model = NormalizingFlowEstimator('nf_dimtest_' + flow_name + str(tests.index(test)),
test['ndim_x'], test['ndim_y'],
random_seed=22, n_training_epochs=2,
flows_type=(flow_name,))
x, y = np.array(test['x']), np.array(test['y'])
model.fit(x, y)
p = model.pdf(x, y)
self.assertEqual(p.shape, (len(y),))
# every test has equal first and last elements, theses are basic sanity tests
self.assertAlmostEqual(p[0], p[-1], places=5)
self.assertNotAlmostEqual(p[0], p[1], places=5)
def test_NF_log_pdf(self):
X, Y = np.random.normal(size=(1000, 3)), np.random.normal(size=(1000, 3))
with tf.Session() as sess:
model = NormalizingFlowEstimator("nf_logprob", 3, 3, flows_type=('affine', 'planar'),
n_training_epochs=10, random_seed=22)
model.fit(X, Y)
x, y = np.random.normal(size=(1000, 3)), np.random.normal(size=(1000, 3))
prob = model.pdf(x, y)
log_prob = model.log_pdf(x, y)
self.assertLessEqual(np.mean(np.abs(prob - np.exp(log_prob))), 0.001)
def test_pickle_unpickle_NF_estimator(self):
X, Y = self.get_samples()
with tf.Session() as sess:
model = NormalizingFlowEstimator('nf_pickle', 2, 2, ('affine', 'radial', 'radial'),
data_normalization=True, random_seed=22, n_training_epochs=10)
model.fit(X, Y)
pdf_before = model.pdf(X, Y)
dump_string = pickle.dumps(model)
tf.reset_default_graph()
with tf.Session() as sess:
model_loaded = pickle.loads(dump_string)
pdf_after = model_loaded.pdf(X, Y)
diff = np.sum(np.abs(pdf_after - pdf_before))
self.assertAlmostEqual(diff, 0, places=2)
def test_NF_chain2_with_2d_gaussian2(self):
mu = -5
std = 2.5
X, Y = self.get_samples(mu=mu, std=std)
model = NormalizingFlowEstimator("nf_estimator_2d_chain2_2", 1, 1, flows_type=('radial', 'planar', 'radial'),
n_training_epochs=500, random_seed=22)
model.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p_est = model.pdf(x, y)
p_true = norm.pdf(y, loc=mu, scale=std)
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_data_normalization(self):
X, Y = self.get_samples(std=2, mean=20)
with tf.Session() as sess:
model = NormalizingFlowEstimator("nf_data_normalization", 1, 1, flows_type=('affine', 'radial', 'radial'),
x_noise_std=None, y_noise_std=None, data_normalization=True,
n_training_epochs=100)
model.fit(X, Y)
# test if data statistics were properly assigned to tf graph
x_mean, x_std = model.sess.run([model.mean_x_sym, model.std_x_sym])
print(x_mean, x_std)
mean_diff = float(np.abs(x_mean - 20))
std_diff = float(np.abs(x_std - 2))
self.assertLessEqual(mean_diff, 0.5)
self.assertLessEqual(std_diff, 0.5)
def test_NF_identitiy_with_2d_gaussian(self):
mu = 200
std = 23
X, Y = self.get_samples(mu=mu, std=std)
model1 = NormalizingFlowEstimator("nf_estimator_2d_planar_no_id", 1, 1, flows_type=('planar',),
n_training_epochs=50, random_seed=22)
model2 = NormalizingFlowEstimator("nf_estimator_2d_planar_id", 1, 1, flows_type=('planar', 'identity'),
n_training_epochs=50, random_seed=22)
model1.fit(X, Y)
model2.fit(X, Y)
y = np.arange(mu - 3 * std, mu + 3 * std, 6 * std / 20)
x = np.asarray([mu for i in range(y.shape[0])])
p = model1.pdf(x, y)
p_id = model2.pdf(x, y)
self.assertLessEqual(np.mean(np.abs(p - p_id)), 0.01)
def test_weight_decay(self):
with tf.Session() as sess:
bimix_gauss = tf.contrib.distributions.Mixture(
cat=tf.distributions.Categorical(probs=[0.5, 0.5]),
components=[
tf.distributions.Normal(loc=-1., scale=0.5),
tf.distributions.Normal(loc=+1., scale=0.5),
])
x = np.ones(5000)
y = sess.run(bimix_gauss.sample([5000]))
model = NormalizingFlowEstimator("nf_estimator_weight_decay", 1, 1,
flows_type=('affine', 'radial', 'radial'),
data_normalization=True, weight_decay=0.0001, n_training_epochs=1000,
random_seed=22)
model.fit(x, y)
p_est = model.pdf(x, y)
p_true = sess.run(bimix_gauss.prob(y))
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.01)
def test_tri_modal_radial_chain(self):
with tf.Session() as sess:
bimix_gauss = tf.contrib.distributions.Mixture(
cat=tf.distributions.Categorical(probs=[0.3, 0.4, 0.3]),
components=[
tf.distributions.Normal(loc=-1., scale=0.4),
tf.distributions.Normal(loc=0., scale=0.4),
tf.distributions.Normal(loc=+1., scale=0.4),
])
x = np.ones(5000)
y = sess.run(bimix_gauss.sample([5000]))
model = NormalizingFlowEstimator("nf_estimator_bimodal_radial", 1, 1,
flows_type=('radial', 'radial', 'radial'),
n_training_epochs=1000, random_seed=22)
model.fit(x, y)
p_est = model.pdf(x, y)
p_true = sess.run(bimix_gauss.prob(y))
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)
def test_bi_modal_planar_chain(self):
with tf.Session() as sess:
bimix_gauss = tf.contrib.distributions.Mixture(
cat=tf.distributions.Categorical(probs=[0.5, 0.5]),
components=[
tf.distributions.Normal(loc=-.4, scale=0.4),
tf.distributions.Normal(loc=+.4, scale=0.4),
])
x = tf.distributions.Normal(loc=0., scale=1.).sample([5000])
y = bimix_gauss.sample([5000])
x, y = sess.run([x, y])
model = NormalizingFlowEstimator("nf_estimator_bimodal_planar", 1, 1,
flows_type=('affine', 'planar', 'planar', 'planar'),
n_training_epochs=1000, random_seed=22)
model.fit(x, y)
p_est = model.pdf(x, y)
p_true = sess.run(bimix_gauss.prob(y))
self.assertLessEqual(np.mean(np.abs(p_true - p_est)), 0.1)