def testUnknownLastDim(self): with tf.Graph().as_default() as g, self.test_session(g): inputs = tf.placeholder(dtype=tf.float32) inputs.set_shape(tf.TensorShape((5, 3, 3, None))) with self.assertRaisesRegexp(ValueError, 'undefined last dimension'): layer_norm_custom(inputs)
def testCreateVariables(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height, width, 3), seed=1) layer_norm_custom(images) beta = tf.contrib.framework.get_variables_by_name('beta')[0] gamma = tf.contrib.framework.get_variables_by_name('gamma')[0] self.assertEqual(beta.op.name, 'LayerNorm/beta') self.assertEqual(gamma.op.name, 'LayerNorm/gamma')
def testReuseVariables(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height, width, 3), seed=1) layer_norm_custom(images, scope='ln') layer_norm_custom(images, scope='ln', reuse=True) beta = tf.contrib.framework.get_variables_by_name('beta') gamma = tf.contrib.framework.get_variables_by_name('gamma') self.assertEqual(len(beta), 1) self.assertEqual(len(gamma), 1)
def doOutputTest(self, input_shape, center=False, scale=False): with self.test_session() as sess: input_values = np.random.rand(*input_shape) inputs = tf.constant(input_values, shape=input_shape, dtype=tf.float32) output_op = layer_norm_custom(inputs, center=center, scale=scale, scope='LN') # reshape input to 2D so the built-in layer_norm layer # will generate equivalent results g_inputs = tf.reshape(inputs, [-1, input_shape[-1]]) gold_op = tf.contrib.layers.layer_norm(g_inputs, center=center, scale=scale, scope="gold_LN") # Initialize all variables sess.run(tf.initialize_all_variables()) # sess.run(tf.global_variables_initializer()) outputs = sess.run(output_op) golds = sess.run(gold_op) tol = 1e-4 self.assertAllClose(outputs.ravel(), golds.ravel(), rtol=tol, atol=tol)
def testReuseVars(self): height, width = 3, 3 with self.test_session() as sess: image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) output_train = layer_norm_custom(images, scope='LN') output_eval = layer_norm_custom(images, scope='LN', reuse=True) # Initialize all variables # sess.run(tf.global_variables_initializer()) sess.run(tf.initialize_all_variables()) # output_train and output_eval should be the same. self.assertAllClose(sess.run([output_train]), sess.run([output_eval]))
def doGradientTest(self, input_shape, center=False, scale=False): with self.test_session() as sess: # np.random.seed(5) input_values = np.random.rand(*input_shape) inputs = tf.constant( input_values, shape=input_shape, dtype=tf.float32) output_op = layer_norm_custom( inputs, center=center, scale=scale, variables_collections=["_var_cust"], scope='LN') # reshape input to 2D so the built-in layer_norm layer # will generate equivalent results g_inputs = tf.reshape(inputs, [-1, input_shape[-1]]) gold_op = tf.contrib.layers.layer_norm( g_inputs, center=center, scale=scale, variables_collections=["_var_gold"], scope="gold_LN") cust_vars = tf.get_collection("_var_cust") gold_vars = tf.get_collection("_var_gold") cust_grads = tf.gradients(tf.nn.relu( output_op), [inputs] + cust_vars) gold_grads = tf.gradients( tf.nn.relu(gold_op), [inputs] + gold_vars) # Initialize all variables # sess.run(tf.initialize_all_variables()) sess.run(tf.global_variables_initializer()) _c_grads = sess.run(cust_grads) _g_grads = sess.run(gold_grads) # _g_grads = relu_gradients(input_values) for _c, _g in zip(_c_grads, _g_grads): tol = 1e-4 * _g.max() self.assertAllClose( _c.ravel(), _g.ravel(), rtol=tol, atol=tol)
def testCreateOp(self): height, width = 3, 3 with self.test_session(): images = np.random.uniform(size=(5, height, width, 3)) output = layer_norm_custom(images) self.assertTrue( output.op.name.startswith('LayerNorm/LayerNormFusedCustom')) self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3])
def testUnknownShape(self): with tf.Graph().as_default() as g, self.test_session(g): inputs = tf.placeholder(dtype=tf.float32) with self.assertRaisesRegexp(ValueError, 'undefined rank'): layer_norm_custom(inputs)