def test_ScheduledVariable(self):
v = ScheduledVariable('v', 123., dtype=tf.int32, model_var=True,
collections=['my_variables'])
assert_variables(['v'], trainable=False, collections=['my_variables'])
with TemporaryDirectory() as tmpdir:
saver = tf.train.Saver(var_list=[v.variable])
save_path = os.path.join(tmpdir, 'saved_var')
with self.test_session() as sess:
ensure_variables_initialized()
self.assertEqual(v.get(), 123)
self.assertEqual(sess.run(v), 123)
self.assertEqual(v.set(456), 456)
self.assertEqual(v.get(), 456)
saver.save(sess, save_path)
with self.test_session() as sess:
saver.restore(sess, save_path)
self.assertEqual(v.get(), 456)
sess.run(v.assign_op, feed_dict={v.assign_ph: 789})
self.assertEqual(v.get(), 789)
def test_model_variable(self):
a = model_variable('a', shape=(), dtype=tf.float32)
b = model_variable('b', shape=(), dtype=tf.float32, trainable=False,
collections=['my_collection'])
c = tf.get_variable('c', shape=(), dtype=tf.float32)
assert_variables(['a'], trainable=True,
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['b'], trainable=False,
collections=[tf.GraphKeys.MODEL_VARIABLES,
'my_collection'])
self.assertEqual(get_model_variables(), [a, b])
def test_act_norm_function(self):
x = np.random.normal(size=[2, 3, 4, 5, 6])
with self.test_session() as sess:
y = sess.run(act_norm(x, axis=[1, -1], initializing=True))
scale, bias, var_shape_aligned = \
naive_act_norm_initialize(x, axis=[1, -1])
y2 = naive_act_norm_transform(
x, var_shape_aligned=var_shape_aligned, value_ndims=4,
scale=scale, bias=bias
)[0]
np.testing.assert_allclose(y, y2)
# test variables creation when trainable
with tf.Graph().as_default():
_ = act_norm(tf.zeros([2, 3]), trainable=True, scale_type='linear')
assert_variables(['scale', 'bias'], trainable=True,
scope='act_norm',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['log_scale'], exist=False, scope='act_norm')
# test variables creation when non-trainable
with tf.Graph().as_default():
_ = act_norm(tf.zeros([2, 3]), trainable=False, scale_type='exp')
assert_variables(['log_scale', 'bias'], trainable=False,
scope='act_norm',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['scale'], exist=False, scope='act_norm')
def test_strict_mode(self):
assert_allclose = functools.partial(np.testing.assert_allclose,
atol=1e-6,
rtol=1e-5)
tf.set_random_seed(1234)
np.random.seed(1234)
VarScopeRandomState.set_global_seed(0)
with self.test_session() as sess:
shape = (5, 5)
m = InvertibleMatrix(shape, strict=True)
self.assertTupleEqual(m.shape, (5, 5))
assert_variables(['matrix'],
exist=False,
scope='invertible_matrix')
assert_variables(['pre_L', 'pre_U', 'log_s'],
trainable=True,
scope='invertible_matrix',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['P', 'sign'],
trainable=False,
scope='invertible_matrix',
collections=[tf.GraphKeys.MODEL_VARIABLES])
ensure_variables_initialized()
# check whether `P` is a permutation matrix
P = sess.run(m._P)
_ = PermutationMatrix(P)
# check `L` is a lower triangular matrix and has unit diags
pre_L, L = sess.run([m._pre_L, m._L])
assert_allclose(
pre_L * np.tril(np.ones(shape), k=-1) + np.eye(*shape), L)
# check `U` is an upper triangular matrix and has `exp(s)` diags
pre_U, sign, log_s, U = sess.run(
[m._pre_U, m._sign, m._log_s, m._U])
assert_allclose((pre_U * np.triu(np.ones(shape), k=1) +
np.diag(sign * np.exp(log_s))), U)
# check `matrix`, `inv_matrix` and `log_det`
matrix, inv_matrix, log_det = \
sess.run([m.matrix, m.inv_matrix, m.log_det])
assert_allclose(matrix, np.dot(P, np.dot(L, U)))
assert_allclose(inv_matrix, np.linalg.inv(matrix))
assert_allclose(log_det, np.sum(log_s))
# check whether or not `matrix` is orthogonal
assert_allclose(np.transpose(matrix), inv_matrix)
with tf.Graph().as_default():
# test non-trainable
_ = InvertibleMatrix(shape, strict=True, trainable=False)
assert_variables(['pre_L', 'pre_U', 'log_s', 'P', 'sign'],
trainable=False,
scope='invertible_matrix',
collections=[tf.GraphKeys.MODEL_VARIABLES])
def test_planar_nf_vars(self):
# test trainable
with tf.Graph().as_default():
_ = PlanarNormalizingFlow().apply(tf.zeros([2, 3]))
assert_variables(['w', 'b', 'u'],
trainable=True,
scope='planar_normalizing_flow',
collections=[tf.GraphKeys.MODEL_VARIABLES])
# test non-trainable
with tf.Graph().as_default():
_ = PlanarNormalizingFlow(trainable=False).apply(tf.zeros([2, 3]))
assert_variables(['w', 'b', 'u'],
trainable=False,
scope='planar_normalizing_flow',
collections=[tf.GraphKeys.MODEL_VARIABLES])
def test_feature_shuffling_flow(self):
np.random.seed(1234)
with self.test_session() as sess:
# axis = -1, value_ndims = 1
x = np.random.normal(size=[3, 4, 5, 6]).astype(np.float32)
x_ph = tf.placeholder(dtype=tf.float32, shape=[None, None, None, 6])
permutation = np.arange(6, dtype=np.int32)
np.random.shuffle(permutation)
y = x[..., permutation]
log_det = np.zeros([3, 4, 5]).astype(np.float32)
layer = FeatureShufflingFlow(axis=-1, value_ndims=1)
y_out, log_det_out = layer.transform(x_ph)
sess.run(tf.assign(layer._permutation, permutation))
y_out, log_det_out = sess.run(
[y_out, log_det_out], feed_dict={x_ph: x})
np.testing.assert_equal(y_out, y)
np.testing.assert_equal(log_det_out, log_det)
invertible_flow_standard_check(
self, layer, sess, x_ph, feed_dict={x_ph: x})
assert_variables(['permutation'], trainable=False,
scope='feature_shuffling_flow',
collections=[tf.GraphKeys.MODEL_VARIABLES])
# axis = -2, value_ndims = 3
x = np.random.normal(size=[3, 4, 5, 6]).astype(np.float32)
x_ph = tf.placeholder(dtype=tf.float32, shape=[None, None, 5, None])
permutation = np.arange(5, dtype=np.int32)
np.random.shuffle(permutation)
y = x[..., permutation, :]
log_det = np.zeros([3]).astype(np.float32)
layer = FeatureShufflingFlow(axis=-2, value_ndims=3)
y_out, log_det_out = layer.transform(x_ph)
sess.run(tf.assign(layer._permutation, permutation))
y_out, log_det_out = sess.run(
[y_out, log_det_out], feed_dict={x_ph: x})
np.testing.assert_equal(y_out, y)
np.testing.assert_equal(log_det_out, log_det)
invertible_flow_standard_check(
self, layer, sess, x_ph, feed_dict={x_ph: x})
def test_invertible_dense(self):
assert_allclose = functools.partial(np.testing.assert_allclose,
rtol=1e-5)
np.random.seed(1234)
with self.test_session() as sess:
x = np.random.normal(size=[3, 5, 7]).astype(np.float32)
x_ph = tf.placeholder(dtype=tf.float32, shape=[None, None, 7])
kernel = np.random.normal(size=(7, 7)).astype(x.dtype)
y, log_det = naive_invertible_linear(x, kernel, -1, 1)
layer = InvertibleDense(strict_invertible=False)
y_out, log_det_out = layer.transform(x_ph)
# The kernel is initialized as an orthogonal matrix. As a result,
# the log-det is zero (the computation result should be close to
# zero, but may not be zero). This is not good for testing.
# Thus we initialize it with an arbitrary matrix.
sess.run(tf.assign(layer._kernel_matrix._matrix, kernel))
y_out, log_det_out = sess.run([y_out, log_det_out],
feed_dict={x_ph: x})
assert_allclose(y_out, y)
assert_allclose(log_det_out, log_det)
invertible_flow_standard_check(self,
layer,
sess,
x_ph,
feed_dict={x_ph: x},
rtol=1e-5,
atol=1e-6)
assert_variables(['matrix'],
trainable=True,
scope='invertible_dense/kernel',
collections=[tf.GraphKeys.MODEL_VARIABLES])
# test non-trainable
with tf.Graph().as_default():
layer = InvertibleDense(strict_invertible=False, trainable=False)
layer.apply(tf.zeros([2, 3, 4, 5]))
assert_variables(['matrix'],
trainable=False,
scope='invertible_dense/kernel',
collections=[tf.GraphKeys.MODEL_VARIABLES])
def test_act_norm_vars(self):
# test trainable
with tf.Graph().as_default():
_ = act_norm(tf.zeros([2, 3]), trainable=True, scale_type='linear')
assert_variables(['scale', 'bias'],
trainable=True,
scope='act_norm',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['log_scale'], exist=False, scope='act_norm')
# test non-trainable
with tf.Graph().as_default():
_ = act_norm(tf.zeros([2, 3]), trainable=False, scale_type='exp')
assert_variables(['log_scale', 'bias'],
trainable=False,
scope='act_norm',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['scale'], exist=False, scope='act_norm')
def test_weight_norm_vars(self):
# test trainable
with tf.Graph().as_default():
_ = weight_norm(tf.zeros([2, 3]), -1)
assert_variables(['scale'],
trainable=True,
scope='weight_norm',
collections=[tf.GraphKeys.MODEL_VARIABLES])
# test non-trainable
with tf.Graph().as_default():
_ = weight_norm(tf.zeros([2, 3]), -1, trainable=False)
assert_variables(['scale'],
trainable=False,
scope='weight_norm',
collections=[tf.GraphKeys.MODEL_VARIABLES])
# test no scale
with tf.Graph().as_default():
_ = weight_norm(tf.zeros([2, 3]), -1, use_scale=False)
assert_variables(['scale'], exist=False, scope='weight_norm')
def test_non_strict_mode(self):
assert_allclose = functools.partial(np.testing.assert_allclose,
atol=1e-6,
rtol=1e-5)
tf.set_random_seed(1234)
np.random.seed(1234)
VarScopeRandomState.set_global_seed(0)
with self.test_session() as sess:
m = InvertibleMatrix(5, strict=False)
self.assertTupleEqual(m.shape, (5, 5))
assert_variables(['matrix'],
trainable=True,
scope='invertible_matrix',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['pre_L', 'pre_U', 'log_s', 'P', 'sign'],
exist=False,
scope='invertible_matrix')
ensure_variables_initialized()
# check `matrix`, `inv_matrix` and `log_det`
matrix, inv_matrix, log_det = \
sess.run([m.matrix, m.inv_matrix, m.log_det])
assert_allclose(inv_matrix, np.linalg.inv(matrix))
assert_allclose(log_det, np.linalg.slogdet(matrix)[1])
# check whether or not `matrix` is orthogonal
assert_allclose(np.transpose(matrix), inv_matrix)
# ensure m.log_det can compute grad
_ = tf.gradients(m.log_det, m.matrix)
with tf.Graph().as_default():
# test non-trainable
_ = InvertibleMatrix(5, strict=False, trainable=False)
assert_variables(['matrix'],
trainable=False,
scope='invertible_matrix',
collections=[tf.GraphKeys.MODEL_VARIABLES])
def test_deconv2d_vars(self):
np.random.seed(1234)
x = np.random.normal(size=[17, 11, 32, 31, 7]).astype(np.float32)
kernel = np.random.random(size=[3, 4, 5, 7]).astype(np.float32)
bias = np.random.random(size=[5]).astype(np.float32)
# test create variables
with tf.Graph().as_default():
# test NHWC
_ = deconv2d(x, 5, (3, 4), padding='same', channels_last=True)
assert_variables(['kernel', 'bias'],
trainable=True,
scope='deconv2d',
collections=[tf.GraphKeys.MODEL_VARIABLES])
kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2]
bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1]
self.assertEqual(kernel_var.shape, kernel.shape)
self.assertEqual(bias_var.shape, bias.shape)
# test NCHW
_ = deconv2d(np.transpose(x, [0, 1, -1, -3, -2]),
5, (3, 4),
padding='valid',
channels_last=False)
assert_variables(['kernel', 'bias'],
trainable=True,
scope='deconv2d_1',
collections=[tf.GraphKeys.MODEL_VARIABLES])
kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2]
bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1]
self.assertEqual(kernel_var.shape, kernel.shape)
self.assertEqual(bias_var.shape, bias.shape)
# test create variables, non-trainable
with tf.Graph().as_default():
# test NHWC
_ = deconv2d(x,
5, (3, 4),
padding='same',
channels_last=True,
trainable=False)
assert_variables(['kernel', 'bias'],
trainable=False,
scope='deconv2d',
collections=[tf.GraphKeys.MODEL_VARIABLES])
# test create variables with use_bias = False
with tf.Graph().as_default():
_ = deconv2d(x,
5, (3, 4),
padding='same',
channels_last=True,
use_bias=False)
assert_variables(['kernel'],
trainable=True,
scope='deconv2d',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['bias'], exist=False, scope='deconv2d')
def test_conv2d(self):
with mock.patch('tensorflow.nn.conv2d', patched_conv2d), \
self.test_session() as sess:
np.random.seed(1234)
x = np.random.normal(size=[17, 11, 32, 31, 5]).astype(np.float32)
kernel = np.random.random(size=[3, 4, 5, 7]).astype(np.float32)
bias = np.random.random(size=[7]).astype(np.float32)
# test strides 1, skip 1, same padding, NHWC
np.testing.assert_allclose(
self.run_conv2d(x,
7, (3, 4),
'same',
kernel,
bias,
1,
1,
channels_last=True),
self.conv2d_ans(x, 'same', kernel, bias, 1, 1))
# test strides 1, skip 1, valid padding, NCHW
np.testing.assert_allclose(
self.run_conv2d(x,
7, (3, 4),
'valid',
kernel,
bias, (1, 1),
1,
channels_last=False),
self.conv2d_ans(x, 'valid', kernel, bias, 1, 1))
# test strides (3, 2), skip 1, same padding, NHWC
np.testing.assert_allclose(
self.run_conv2d(x,
7, (3, 4),
'same',
kernel,
bias, (3, 2),
1,
channels_last=True),
self.conv2d_ans(x, 'same', kernel, bias, (3, 2), 1))
# test strides 1, skip 2, valid padding, NHWC
np.testing.assert_allclose(
self.run_conv2d(x,
7, (3, 4),
'valid',
kernel,
bias,
1,
2,
channels_last=True),
self.conv2d_ans(x, 'valid', kernel, bias, 1, 2))
# test dynamic shape, same padding, NHWC
ph = tf.placeholder(dtype=tf.float32,
shape=(None, None, None, None, 5))
np.testing.assert_allclose(
self.run_conv2d(x,
7, (3, 4),
'same',
kernel,
bias,
1,
1,
channels_last=True,
ph=ph),
self.conv2d_ans(x, 'same', kernel, bias, 1, 1))
# test dynamic shape, valid padding NCHW
ph = tf.placeholder(dtype=tf.float32,
shape=(None, None, 5, None, None))
np.testing.assert_allclose(
self.run_conv2d(x,
7, (3, 4),
'valid',
kernel,
bias,
1,
1,
channels_last=False,
ph=ph),
self.conv2d_ans(x, 'valid', kernel, bias, 1, 1))
# test errors
with pytest.raises(ValueError,
match='Invalid value for argument `strides`: '
'expected to be one or two positive '
'integers'):
_ = self.run_conv2d(x,
7, (3, 4),
'valid',
kernel,
bias,
0,
2,
channels_last=False)
with pytest.raises(
ValueError,
match='`channels_last` == False is incompatible '
'with `dilations` > 1'):
_ = self.run_conv2d(x,
7, (3, 4),
'valid',
kernel,
bias,
1,
2,
channels_last=False)
with pytest.raises(ValueError,
match='`strides` > 1 is incompatible with '
'`dilations` > 1'):
_ = self.run_conv2d(x,
7, (3, 4),
'valid',
kernel,
bias,
2,
2,
channels_last=True)
# test create variables
with tf.Graph().as_default():
# test NHWC
_ = conv2d(x, 7, (3, 4), padding='same', channels_last=True)
assert_variables(['kernel', 'bias'],
trainable=True,
scope='conv2d',
collections=[tf.GraphKeys.MODEL_VARIABLES])
kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2]
bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1]
self.assertEqual(kernel_var.shape, kernel.shape)
self.assertEqual(bias_var.shape, bias.shape)
# test NCHW
_ = conv2d(np.transpose(x, [0, 1, -1, -3, -2]),
7, (3, 4),
padding='valid',
channels_last=False)
assert_variables(['kernel', 'bias'],
trainable=True,
scope='conv2d_1',
collections=[tf.GraphKeys.MODEL_VARIABLES])
kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2]
bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1]
self.assertEqual(kernel_var.shape, kernel.shape)
self.assertEqual(bias_var.shape, bias.shape)
# test create variables, non-trainable
with tf.Graph().as_default():
# test NHWC
_ = conv2d(x,
7, (3, 4),
padding='same',
channels_last=True,
trainable=False)
assert_variables(['kernel', 'bias'],
trainable=False,
scope='conv2d',
collections=[tf.GraphKeys.MODEL_VARIABLES])
# test create variables with use_bias = False
with tf.Graph().as_default():
_ = conv2d(x,
7, (3, 4),
padding='same',
channels_last=True,
use_bias=False)
assert_variables(['kernel'],
trainable=True,
scope='conv2d',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['bias'], exist=False, scope='conv2d')
def test_linear(self):
np.random.seed(1234)
kernel = np.random.normal(size=(5, 3)).astype(np.float64)
bias = np.random.normal(size=(3,)).astype(np.float64)
x = np.random.normal(size=(11, 7, 5)).astype(np.float64)
with self.test_session() as sess:
# test 2d input
np.testing.assert_allclose(
sess.run(
dense(
tf.constant(x[0]), 3,
kernel=tf.constant(kernel),
bias=tf.constant(bias)
)
),
np.dot(x[0], kernel) + bias,
rtol=1e-5
)
# test 3d input
ans = np.dot(x, kernel) + bias
self.assertEqual(ans.shape, (11, 7, 3))
np.testing.assert_allclose(
sess.run(
dense(
tf.constant(x, dtype=tf.float64), 3,
kernel=tf.constant(kernel),
bias=tf.constant(bias)
)
),
ans,
rtol=1e-5
)
# test dynamic batch and sampling size
ph = tf.placeholder(dtype=tf.float64, shape=(None, None, 5))
np.testing.assert_allclose(
sess.run(
dense(
ph, 3,
kernel=tf.constant(kernel),
bias=tf.constant(bias)
),
feed_dict={ph: x}
),
ans,
rtol=1e-5
)
# test use_bias is False
ans = np.dot(x, kernel)
self.assertEqual(ans.shape, (11, 7, 3))
np.testing.assert_allclose(
sess.run(
dense(
tf.constant(x, dtype=tf.float64), 3,
kernel=tf.constant(kernel),
bias=tf.constant(bias),
use_bias=False
)
),
ans,
rtol=1e-5
)
# test create variables
with tf.Graph().as_default():
_ = dense(tf.constant(x, dtype=tf.float64), 3)
assert_variables(['kernel', 'bias'], trainable=True, scope='dense',
collections=[tf.GraphKeys.MODEL_VARIABLES])
kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2]
bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1]
self.assertEqual(get_static_shape(kernel_var), kernel.shape)
self.assertEqual(get_static_shape(bias_var), bias.shape)
# test create variables, non-trainable
with tf.Graph().as_default():
_ = dense(tf.constant(x, dtype=tf.float64), 3, trainable=False)
assert_variables(['kernel', 'bias'], trainable=False, scope='dense',
collections=[tf.GraphKeys.MODEL_VARIABLES])
# test create variables, use_bias is False
with tf.Graph().as_default():
_ = dense(tf.constant(x, dtype=tf.float64), 3, use_bias=False)
assert_variables(['kernel'], trainable=True, scope='dense',
collections=[tf.GraphKeys.MODEL_VARIABLES])
assert_variables(['bias'], exist=False, scope='dense')
