def test_ScheduledVariable(self): v = ScheduledVariable('v', 123., dtype=tf.int32, model_var=True, collections=['my_variables']) assert_variables(['v'], trainable=False, collections=['my_variables']) with TemporaryDirectory() as tmpdir: saver = tf.train.Saver(var_list=[v.variable]) save_path = os.path.join(tmpdir, 'saved_var') with self.test_session() as sess: ensure_variables_initialized() self.assertEqual(v.get(), 123) self.assertEqual(sess.run(v), 123) self.assertEqual(v.set(456), 456) self.assertEqual(v.get(), 456) saver.save(sess, save_path) with self.test_session() as sess: saver.restore(sess, save_path) self.assertEqual(v.get(), 456) sess.run(v.assign_op, feed_dict={v.assign_ph: 789}) self.assertEqual(v.get(), 789)
def test_model_variable(self): a = model_variable('a', shape=(), dtype=tf.float32) b = model_variable('b', shape=(), dtype=tf.float32, trainable=False, collections=['my_collection']) c = tf.get_variable('c', shape=(), dtype=tf.float32) assert_variables(['a'], trainable=True, collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['b'], trainable=False, collections=[tf.GraphKeys.MODEL_VARIABLES, 'my_collection']) self.assertEqual(get_model_variables(), [a, b])
def test_act_norm_function(self): x = np.random.normal(size=[2, 3, 4, 5, 6]) with self.test_session() as sess: y = sess.run(act_norm(x, axis=[1, -1], initializing=True)) scale, bias, var_shape_aligned = \ naive_act_norm_initialize(x, axis=[1, -1]) y2 = naive_act_norm_transform( x, var_shape_aligned=var_shape_aligned, value_ndims=4, scale=scale, bias=bias )[0] np.testing.assert_allclose(y, y2) # test variables creation when trainable with tf.Graph().as_default(): _ = act_norm(tf.zeros([2, 3]), trainable=True, scale_type='linear') assert_variables(['scale', 'bias'], trainable=True, scope='act_norm', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['log_scale'], exist=False, scope='act_norm') # test variables creation when non-trainable with tf.Graph().as_default(): _ = act_norm(tf.zeros([2, 3]), trainable=False, scale_type='exp') assert_variables(['log_scale', 'bias'], trainable=False, scope='act_norm', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['scale'], exist=False, scope='act_norm')
def test_strict_mode(self): assert_allclose = functools.partial(np.testing.assert_allclose, atol=1e-6, rtol=1e-5) tf.set_random_seed(1234) np.random.seed(1234) VarScopeRandomState.set_global_seed(0) with self.test_session() as sess: shape = (5, 5) m = InvertibleMatrix(shape, strict=True) self.assertTupleEqual(m.shape, (5, 5)) assert_variables(['matrix'], exist=False, scope='invertible_matrix') assert_variables(['pre_L', 'pre_U', 'log_s'], trainable=True, scope='invertible_matrix', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['P', 'sign'], trainable=False, scope='invertible_matrix', collections=[tf.GraphKeys.MODEL_VARIABLES]) ensure_variables_initialized() # check whether `P` is a permutation matrix P = sess.run(m._P) _ = PermutationMatrix(P) # check `L` is a lower triangular matrix and has unit diags pre_L, L = sess.run([m._pre_L, m._L]) assert_allclose( pre_L * np.tril(np.ones(shape), k=-1) + np.eye(*shape), L) # check `U` is an upper triangular matrix and has `exp(s)` diags pre_U, sign, log_s, U = sess.run( [m._pre_U, m._sign, m._log_s, m._U]) assert_allclose((pre_U * np.triu(np.ones(shape), k=1) + np.diag(sign * np.exp(log_s))), U) # check `matrix`, `inv_matrix` and `log_det` matrix, inv_matrix, log_det = \ sess.run([m.matrix, m.inv_matrix, m.log_det]) assert_allclose(matrix, np.dot(P, np.dot(L, U))) assert_allclose(inv_matrix, np.linalg.inv(matrix)) assert_allclose(log_det, np.sum(log_s)) # check whether or not `matrix` is orthogonal assert_allclose(np.transpose(matrix), inv_matrix) with tf.Graph().as_default(): # test non-trainable _ = InvertibleMatrix(shape, strict=True, trainable=False) assert_variables(['pre_L', 'pre_U', 'log_s', 'P', 'sign'], trainable=False, scope='invertible_matrix', collections=[tf.GraphKeys.MODEL_VARIABLES])
def test_planar_nf_vars(self): # test trainable with tf.Graph().as_default(): _ = PlanarNormalizingFlow().apply(tf.zeros([2, 3])) assert_variables(['w', 'b', 'u'], trainable=True, scope='planar_normalizing_flow', collections=[tf.GraphKeys.MODEL_VARIABLES]) # test non-trainable with tf.Graph().as_default(): _ = PlanarNormalizingFlow(trainable=False).apply(tf.zeros([2, 3])) assert_variables(['w', 'b', 'u'], trainable=False, scope='planar_normalizing_flow', collections=[tf.GraphKeys.MODEL_VARIABLES])
def test_feature_shuffling_flow(self): np.random.seed(1234) with self.test_session() as sess: # axis = -1, value_ndims = 1 x = np.random.normal(size=[3, 4, 5, 6]).astype(np.float32) x_ph = tf.placeholder(dtype=tf.float32, shape=[None, None, None, 6]) permutation = np.arange(6, dtype=np.int32) np.random.shuffle(permutation) y = x[..., permutation] log_det = np.zeros([3, 4, 5]).astype(np.float32) layer = FeatureShufflingFlow(axis=-1, value_ndims=1) y_out, log_det_out = layer.transform(x_ph) sess.run(tf.assign(layer._permutation, permutation)) y_out, log_det_out = sess.run( [y_out, log_det_out], feed_dict={x_ph: x}) np.testing.assert_equal(y_out, y) np.testing.assert_equal(log_det_out, log_det) invertible_flow_standard_check( self, layer, sess, x_ph, feed_dict={x_ph: x}) assert_variables(['permutation'], trainable=False, scope='feature_shuffling_flow', collections=[tf.GraphKeys.MODEL_VARIABLES]) # axis = -2, value_ndims = 3 x = np.random.normal(size=[3, 4, 5, 6]).astype(np.float32) x_ph = tf.placeholder(dtype=tf.float32, shape=[None, None, 5, None]) permutation = np.arange(5, dtype=np.int32) np.random.shuffle(permutation) y = x[..., permutation, :] log_det = np.zeros([3]).astype(np.float32) layer = FeatureShufflingFlow(axis=-2, value_ndims=3) y_out, log_det_out = layer.transform(x_ph) sess.run(tf.assign(layer._permutation, permutation)) y_out, log_det_out = sess.run( [y_out, log_det_out], feed_dict={x_ph: x}) np.testing.assert_equal(y_out, y) np.testing.assert_equal(log_det_out, log_det) invertible_flow_standard_check( self, layer, sess, x_ph, feed_dict={x_ph: x})
def test_invertible_dense(self): assert_allclose = functools.partial(np.testing.assert_allclose, rtol=1e-5) np.random.seed(1234) with self.test_session() as sess: x = np.random.normal(size=[3, 5, 7]).astype(np.float32) x_ph = tf.placeholder(dtype=tf.float32, shape=[None, None, 7]) kernel = np.random.normal(size=(7, 7)).astype(x.dtype) y, log_det = naive_invertible_linear(x, kernel, -1, 1) layer = InvertibleDense(strict_invertible=False) y_out, log_det_out = layer.transform(x_ph) # The kernel is initialized as an orthogonal matrix. As a result, # the log-det is zero (the computation result should be close to # zero, but may not be zero). This is not good for testing. # Thus we initialize it with an arbitrary matrix. sess.run(tf.assign(layer._kernel_matrix._matrix, kernel)) y_out, log_det_out = sess.run([y_out, log_det_out], feed_dict={x_ph: x}) assert_allclose(y_out, y) assert_allclose(log_det_out, log_det) invertible_flow_standard_check(self, layer, sess, x_ph, feed_dict={x_ph: x}, rtol=1e-5, atol=1e-6) assert_variables(['matrix'], trainable=True, scope='invertible_dense/kernel', collections=[tf.GraphKeys.MODEL_VARIABLES]) # test non-trainable with tf.Graph().as_default(): layer = InvertibleDense(strict_invertible=False, trainable=False) layer.apply(tf.zeros([2, 3, 4, 5])) assert_variables(['matrix'], trainable=False, scope='invertible_dense/kernel', collections=[tf.GraphKeys.MODEL_VARIABLES])
def test_act_norm_vars(self): # test trainable with tf.Graph().as_default(): _ = act_norm(tf.zeros([2, 3]), trainable=True, scale_type='linear') assert_variables(['scale', 'bias'], trainable=True, scope='act_norm', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['log_scale'], exist=False, scope='act_norm') # test non-trainable with tf.Graph().as_default(): _ = act_norm(tf.zeros([2, 3]), trainable=False, scale_type='exp') assert_variables(['log_scale', 'bias'], trainable=False, scope='act_norm', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['scale'], exist=False, scope='act_norm')
def test_weight_norm_vars(self): # test trainable with tf.Graph().as_default(): _ = weight_norm(tf.zeros([2, 3]), -1) assert_variables(['scale'], trainable=True, scope='weight_norm', collections=[tf.GraphKeys.MODEL_VARIABLES]) # test non-trainable with tf.Graph().as_default(): _ = weight_norm(tf.zeros([2, 3]), -1, trainable=False) assert_variables(['scale'], trainable=False, scope='weight_norm', collections=[tf.GraphKeys.MODEL_VARIABLES]) # test no scale with tf.Graph().as_default(): _ = weight_norm(tf.zeros([2, 3]), -1, use_scale=False) assert_variables(['scale'], exist=False, scope='weight_norm')
def test_non_strict_mode(self): assert_allclose = functools.partial(np.testing.assert_allclose, atol=1e-6, rtol=1e-5) tf.set_random_seed(1234) np.random.seed(1234) VarScopeRandomState.set_global_seed(0) with self.test_session() as sess: m = InvertibleMatrix(5, strict=False) self.assertTupleEqual(m.shape, (5, 5)) assert_variables(['matrix'], trainable=True, scope='invertible_matrix', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['pre_L', 'pre_U', 'log_s', 'P', 'sign'], exist=False, scope='invertible_matrix') ensure_variables_initialized() # check `matrix`, `inv_matrix` and `log_det` matrix, inv_matrix, log_det = \ sess.run([m.matrix, m.inv_matrix, m.log_det]) assert_allclose(inv_matrix, np.linalg.inv(matrix)) assert_allclose(log_det, np.linalg.slogdet(matrix)[1]) # check whether or not `matrix` is orthogonal assert_allclose(np.transpose(matrix), inv_matrix) # ensure m.log_det can compute grad _ = tf.gradients(m.log_det, m.matrix) with tf.Graph().as_default(): # test non-trainable _ = InvertibleMatrix(5, strict=False, trainable=False) assert_variables(['matrix'], trainable=False, scope='invertible_matrix', collections=[tf.GraphKeys.MODEL_VARIABLES])
def test_deconv2d_vars(self): np.random.seed(1234) x = np.random.normal(size=[17, 11, 32, 31, 7]).astype(np.float32) kernel = np.random.random(size=[3, 4, 5, 7]).astype(np.float32) bias = np.random.random(size=[5]).astype(np.float32) # test create variables with tf.Graph().as_default(): # test NHWC _ = deconv2d(x, 5, (3, 4), padding='same', channels_last=True) assert_variables(['kernel', 'bias'], trainable=True, scope='deconv2d', collections=[tf.GraphKeys.MODEL_VARIABLES]) kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2] bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1] self.assertEqual(kernel_var.shape, kernel.shape) self.assertEqual(bias_var.shape, bias.shape) # test NCHW _ = deconv2d(np.transpose(x, [0, 1, -1, -3, -2]), 5, (3, 4), padding='valid', channels_last=False) assert_variables(['kernel', 'bias'], trainable=True, scope='deconv2d_1', collections=[tf.GraphKeys.MODEL_VARIABLES]) kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2] bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1] self.assertEqual(kernel_var.shape, kernel.shape) self.assertEqual(bias_var.shape, bias.shape) # test create variables, non-trainable with tf.Graph().as_default(): # test NHWC _ = deconv2d(x, 5, (3, 4), padding='same', channels_last=True, trainable=False) assert_variables(['kernel', 'bias'], trainable=False, scope='deconv2d', collections=[tf.GraphKeys.MODEL_VARIABLES]) # test create variables with use_bias = False with tf.Graph().as_default(): _ = deconv2d(x, 5, (3, 4), padding='same', channels_last=True, use_bias=False) assert_variables(['kernel'], trainable=True, scope='deconv2d', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['bias'], exist=False, scope='deconv2d')
def test_conv2d(self): with mock.patch('tensorflow.nn.conv2d', patched_conv2d), \ self.test_session() as sess: np.random.seed(1234) x = np.random.normal(size=[17, 11, 32, 31, 5]).astype(np.float32) kernel = np.random.random(size=[3, 4, 5, 7]).astype(np.float32) bias = np.random.random(size=[7]).astype(np.float32) # test strides 1, skip 1, same padding, NHWC np.testing.assert_allclose( self.run_conv2d(x, 7, (3, 4), 'same', kernel, bias, 1, 1, channels_last=True), self.conv2d_ans(x, 'same', kernel, bias, 1, 1)) # test strides 1, skip 1, valid padding, NCHW np.testing.assert_allclose( self.run_conv2d(x, 7, (3, 4), 'valid', kernel, bias, (1, 1), 1, channels_last=False), self.conv2d_ans(x, 'valid', kernel, bias, 1, 1)) # test strides (3, 2), skip 1, same padding, NHWC np.testing.assert_allclose( self.run_conv2d(x, 7, (3, 4), 'same', kernel, bias, (3, 2), 1, channels_last=True), self.conv2d_ans(x, 'same', kernel, bias, (3, 2), 1)) # test strides 1, skip 2, valid padding, NHWC np.testing.assert_allclose( self.run_conv2d(x, 7, (3, 4), 'valid', kernel, bias, 1, 2, channels_last=True), self.conv2d_ans(x, 'valid', kernel, bias, 1, 2)) # test dynamic shape, same padding, NHWC ph = tf.placeholder(dtype=tf.float32, shape=(None, None, None, None, 5)) np.testing.assert_allclose( self.run_conv2d(x, 7, (3, 4), 'same', kernel, bias, 1, 1, channels_last=True, ph=ph), self.conv2d_ans(x, 'same', kernel, bias, 1, 1)) # test dynamic shape, valid padding NCHW ph = tf.placeholder(dtype=tf.float32, shape=(None, None, 5, None, None)) np.testing.assert_allclose( self.run_conv2d(x, 7, (3, 4), 'valid', kernel, bias, 1, 1, channels_last=False, ph=ph), self.conv2d_ans(x, 'valid', kernel, bias, 1, 1)) # test errors with pytest.raises(ValueError, match='Invalid value for argument `strides`: ' 'expected to be one or two positive ' 'integers'): _ = self.run_conv2d(x, 7, (3, 4), 'valid', kernel, bias, 0, 2, channels_last=False) with pytest.raises( ValueError, match='`channels_last` == False is incompatible ' 'with `dilations` > 1'): _ = self.run_conv2d(x, 7, (3, 4), 'valid', kernel, bias, 1, 2, channels_last=False) with pytest.raises(ValueError, match='`strides` > 1 is incompatible with ' '`dilations` > 1'): _ = self.run_conv2d(x, 7, (3, 4), 'valid', kernel, bias, 2, 2, channels_last=True) # test create variables with tf.Graph().as_default(): # test NHWC _ = conv2d(x, 7, (3, 4), padding='same', channels_last=True) assert_variables(['kernel', 'bias'], trainable=True, scope='conv2d', collections=[tf.GraphKeys.MODEL_VARIABLES]) kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2] bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1] self.assertEqual(kernel_var.shape, kernel.shape) self.assertEqual(bias_var.shape, bias.shape) # test NCHW _ = conv2d(np.transpose(x, [0, 1, -1, -3, -2]), 7, (3, 4), padding='valid', channels_last=False) assert_variables(['kernel', 'bias'], trainable=True, scope='conv2d_1', collections=[tf.GraphKeys.MODEL_VARIABLES]) kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2] bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1] self.assertEqual(kernel_var.shape, kernel.shape) self.assertEqual(bias_var.shape, bias.shape) # test create variables, non-trainable with tf.Graph().as_default(): # test NHWC _ = conv2d(x, 7, (3, 4), padding='same', channels_last=True, trainable=False) assert_variables(['kernel', 'bias'], trainable=False, scope='conv2d', collections=[tf.GraphKeys.MODEL_VARIABLES]) # test create variables with use_bias = False with tf.Graph().as_default(): _ = conv2d(x, 7, (3, 4), padding='same', channels_last=True, use_bias=False) assert_variables(['kernel'], trainable=True, scope='conv2d', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['bias'], exist=False, scope='conv2d')
def test_linear(self): np.random.seed(1234) kernel = np.random.normal(size=(5, 3)).astype(np.float64) bias = np.random.normal(size=(3,)).astype(np.float64) x = np.random.normal(size=(11, 7, 5)).astype(np.float64) with self.test_session() as sess: # test 2d input np.testing.assert_allclose( sess.run( dense( tf.constant(x[0]), 3, kernel=tf.constant(kernel), bias=tf.constant(bias) ) ), np.dot(x[0], kernel) + bias, rtol=1e-5 ) # test 3d input ans = np.dot(x, kernel) + bias self.assertEqual(ans.shape, (11, 7, 3)) np.testing.assert_allclose( sess.run( dense( tf.constant(x, dtype=tf.float64), 3, kernel=tf.constant(kernel), bias=tf.constant(bias) ) ), ans, rtol=1e-5 ) # test dynamic batch and sampling size ph = tf.placeholder(dtype=tf.float64, shape=(None, None, 5)) np.testing.assert_allclose( sess.run( dense( ph, 3, kernel=tf.constant(kernel), bias=tf.constant(bias) ), feed_dict={ph: x} ), ans, rtol=1e-5 ) # test use_bias is False ans = np.dot(x, kernel) self.assertEqual(ans.shape, (11, 7, 3)) np.testing.assert_allclose( sess.run( dense( tf.constant(x, dtype=tf.float64), 3, kernel=tf.constant(kernel), bias=tf.constant(bias), use_bias=False ) ), ans, rtol=1e-5 ) # test create variables with tf.Graph().as_default(): _ = dense(tf.constant(x, dtype=tf.float64), 3) assert_variables(['kernel', 'bias'], trainable=True, scope='dense', collections=[tf.GraphKeys.MODEL_VARIABLES]) kernel_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-2] bias_var = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[-1] self.assertEqual(get_static_shape(kernel_var), kernel.shape) self.assertEqual(get_static_shape(bias_var), bias.shape) # test create variables, non-trainable with tf.Graph().as_default(): _ = dense(tf.constant(x, dtype=tf.float64), 3, trainable=False) assert_variables(['kernel', 'bias'], trainable=False, scope='dense', collections=[tf.GraphKeys.MODEL_VARIABLES]) # test create variables, use_bias is False with tf.Graph().as_default(): _ = dense(tf.constant(x, dtype=tf.float64), 3, use_bias=False) assert_variables(['kernel'], trainable=True, scope='dense', collections=[tf.GraphKeys.MODEL_VARIABLES]) assert_variables(['bias'], exist=False, scope='dense')