def testGradientInference(self, data_format): # TODO(b/64270657): Use gradient_checker here in addition to comparing with # this reference implementation. channel = 3 x_shape = [2, 2, 6, channel] scale_shape = [channel] grad_val = np.random.random_sample(x_shape).astype(np.float32) x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) mean_val = np.random.random_sample(scale_shape).astype(np.float32) var_val = np.random.random_sample(scale_shape).astype(np.float32) data_format_src = "NHWC" with self.cached_session() as sess, self.test_scope(): grad_val_converted = test_utils.ConvertBetweenDataFormats( grad_val, data_format_src, data_format) x_val_converted = test_utils.ConvertBetweenDataFormats( x_val, data_format_src, data_format) grad = array_ops.placeholder( np.float32, shape=x_val_converted.shape, name="grad") x = array_ops.placeholder( np.float32, shape=x_val_converted.shape, name="x") mean = array_ops.placeholder(np.float32, shape=scale_shape, name="mean") var = array_ops.placeholder(np.float32, shape=scale_shape, name="var") scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale") with self.test_scope(): out = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, mean, var, data_format=data_format, is_training=False) grad_x, grad_scale, grad_offset, _, _ = out ref_x, ref_scale, ref_offset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, mean, var, data_format=data_format, is_training=False) grad_x_val, grad_scale_val, grad_offset_val, = sess.run( [grad_x, grad_scale, grad_offset], { grad: grad_val_converted, x: x_val_converted, mean: mean_val, var: var_val, scale: scale_val }) grad_x_ref, grad_scale_ref, grad_offset_ref, = sess.run( [ref_x, ref_scale, ref_offset], { grad: grad_val_converted, x: x_val_converted, mean: mean_val, var: var_val, scale: scale_val }) self.assertAllClose(grad_x_val, grad_x_ref, atol=1e-2) self.assertAllClose(grad_scale_val, grad_scale_ref, atol=1e-2) self.assertAllClose(grad_offset_val, grad_offset_ref, atol=1e-3)
def testGradientInference(self, data_format): # TODO(b/64270657): Use gradient_checker here in addition to comparing with # this reference implementation. channel = 3 x_shape = [2, 2, 6, channel] scale_shape = [channel] grad_val = np.random.random_sample(x_shape).astype(np.float32) x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) mean_val = np.random.random_sample(scale_shape).astype(np.float32) var_val = np.random.random_sample(scale_shape).astype(np.float32) data_format_src = "NHWC" with self.session() as sess, self.test_scope(): grad_val_converted = test_utils.ConvertBetweenDataFormats( grad_val, data_format_src, data_format) x_val_converted = test_utils.ConvertBetweenDataFormats( x_val, data_format_src, data_format) grad = array_ops.placeholder( np.float32, shape=x_val_converted.shape, name="grad") x = array_ops.placeholder( np.float32, shape=x_val_converted.shape, name="x") mean = array_ops.placeholder(np.float32, shape=scale_shape, name="mean") var = array_ops.placeholder(np.float32, shape=scale_shape, name="var") scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale") with self.test_scope(): out = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, mean, var, data_format=data_format, is_training=False) grad_x, grad_scale, grad_offset, _, _ = out ref_x, ref_scale, ref_offset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, mean, var, data_format=data_format, is_training=False) grad_x_val, grad_scale_val, grad_offset_val, = sess.run( [grad_x, grad_scale, grad_offset], { grad: grad_val_converted, x: x_val_converted, mean: mean_val, var: var_val, scale: scale_val }) grad_x_ref, grad_scale_ref, grad_offset_ref, = sess.run( [ref_x, ref_scale, ref_offset], { grad: grad_val_converted, x: x_val_converted, mean: mean_val, var: var_val, scale: scale_val }) self.assertAllClose(grad_x_val, grad_x_ref, atol=1e-2) self.assertAllClose(grad_scale_val, grad_scale_ref, atol=1e-2) self.assertAllClose(grad_offset_val, grad_offset_ref, atol=1e-3)
def _FusedBatchNormGrad(op, *grad): """Return the gradients for the 3 inputs of BatchNorm. Args: op: The BatchNormOp for which we need to compute gradients. *grad: An argument list for tensors of gradients wrt the outputs with grad[0] as grad_y. Returns: grad_x: gradient for x, which is scale * rsqrt(variance + epsilon) * [grad_y - mean(grad_y) - (x - mean(x)) * mean(grad_y * (x - mean(x))) / (variance + epsilon)] in training mode; grad_y * scale * rsqrt(pop_variance + epsilon) in freeze mode. grad_scale: gradient for scale, which is sum(grad_y * (x - mean(x)) * rsqrt(variance + epsilon)) in training mode; sum(grad_y * (x - pop_mean) * rsqrt(pop_variance + epsilon)) in freeze mode. grad_offset: gradient for offset, which is sum(grad_y) in training mode; sum(grad_y) in freeze mode. """ x = op.inputs[0] grad_y = grad[0] scale = op.inputs[1] epsilon = op.get_attr("epsilon") data_format = op.get_attr("data_format") is_training = op.get_attr("is_training") if is_training: return gen_nn_ops.fused_batch_norm_grad( grad_y, x, scale, op.outputs[3], op.outputs[4], epsilon=epsilon, data_format=data_format, is_training=is_training) else: pop_mean = op.inputs[3] pop_var = op.inputs[4] if data_format == b"NCHW": x = array_ops.transpose(x, [0, 2, 3, 1]) grad_y = array_ops.transpose(grad_y, [0, 2, 3, 1]) dx, dscale, doffset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad_y, x, scale, pop_mean, pop_var, epsilon=epsilon, data_format='NHWC', is_training=is_training) if data_format == b"NCHW": dx = array_ops.transpose(dx, [0, 3, 1, 2]) return dx, dscale, doffset, None, None
def _FusedBatchNormGrad(op, *grad): """Return the gradients for the 3 inputs of BatchNorm. Args: op: The BatchNormOp for which we need to compute gradients. *grad: An argument list for tensors of gradients wrt the outputs with grad[0] as grad_y. Returns: grad_x: gradient for x, which is scale * rsqrt(variance + epsilon) * [grad_y - mean(grad_y) - (x - mean(x)) * mean(grad_y * (x - mean(x))) / (variance + epsilon)] in training mode; grad_y * scale * rsqrt(pop_variance + epsilon) in freeze mode. grad_scale: gradient for scale, which is sum(grad_y * (x - mean(x)) * rsqrt(variance + epsilon)) in training mode; sum(grad_y * (x - pop_mean) * rsqrt(pop_variance + epsilon)) in freeze mode. grad_offset: gradient for offset, which is sum(grad_y) in training mode; sum(grad_y) in freeze mode. """ x = op.inputs[0] grad_y = grad[0] scale = op.inputs[1] epsilon = op.get_attr("epsilon") data_format = op.get_attr("data_format") is_training = op.get_attr("is_training") if is_training: return gen_nn_ops.fused_batch_norm_grad(grad_y, x, scale, op.outputs[3], op.outputs[4], epsilon=epsilon, data_format=data_format, is_training=is_training) else: pop_mean = op.inputs[3] pop_var = op.inputs[4] if data_format == b"NCHW": x = array_ops.transpose(x, [0, 2, 3, 1]) grad_y = array_ops.transpose(grad_y, [0, 2, 3, 1]) dx, dscale, doffset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad_y, x, scale, pop_mean, pop_var, epsilon=epsilon, data_format='NHWC', is_training=is_training) if data_format == b"NCHW": dx = array_ops.transpose(dx, [0, 3, 1, 2]) return dx, dscale, doffset, None, None
def _FusedBatchNormGrad(op, *grad): """Return the gradients for the 3 inputs of BatchNorm. Args: op: The BatchNormOp for which we need to compute gradients. *grad: An argument list for tensors of gradients wrt the outputs with grad[0] as grad_y. Returns: grad_x: gradient for x, which is scale * rsqrt(variance + epsilon) * [grad_y - mean(grad_y) - (x - mean(x)) * mean(grad_y * (x - mean(x))) / (variance + epsilon)] grad_scale: gradient for scale, which is sum(grad_y * (x - mean(x)) * rsqrt(variance + epsilon)) grad_offset: gradient for offset, which is sum(grad_y) """ return gen_nn_ops.fused_batch_norm_grad( grad[0], op.inputs[0], op.inputs[1], op.outputs[3], op.outputs[4], epsilon=op.get_attr("epsilon"), data_format=op.get_attr("data_format"), is_training=op.get_attr("is_training"))
def _FusedBatchNormGrad(op, *grad): """Return the gradients for the 3 inputs of BatchNorm. Args: op: The BatchNormOp for which we need to compute gradients. *grad: An argument list for tensors of gradients wrt the outputs with grad[0] as grad_y. Returns: grad_x: gradient for x, which is scale * rsqrt(variance + epsilon) * [grad_y - mean(grad_y) - (x - mean(x)) * mean(grad_y * (x - mean(x))) / (variance + epsilon)] grad_scale: gradient for scale, which is sum(grad_y * (x - mean(x)) * rsqrt(variance + epsilon)) grad_offset: gradient for offset, which is sum(grad_y) """ return gen_nn_ops.fused_batch_norm_grad( grad[0], op.inputs[0], op.inputs[1], op.outputs[3], op.outputs[4], epsilon=op.get_attr("epsilon"), data_format=op.get_attr("data_format"), is_training=op.get_attr("is_training"))
def testGradient(self): # TODO(b/64270657): Use gradient_checker here in addition to comparing with # this reference implementation. x_shape = [2, 2, 6, 2] scale_shape = [2] grad_val = np.random.random_sample(x_shape).astype(np.float32) x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) mean_val = np.random.random_sample(scale_shape).astype(np.float32) var_val = np.random.random_sample(scale_shape).astype(np.float32) epsilon = 0.001 with self.test_session() as sess, self.test_scope(): grad = array_ops.placeholder(np.float32, shape=x_shape, name="grad") x = array_ops.placeholder(np.float32, shape=x_shape, name="x") mean = array_ops.placeholder(np.float32, shape=scale_shape, name="mean") var = array_ops.placeholder(np.float32, shape=scale_shape, name="var") scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale") grad_x, grad_scale, grad_offset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, mean, var, data_format="NHWC") grad_x_val, grad_scale_val, grad_offset_val = sess.run( [grad_x, grad_scale, grad_offset], { grad: grad_val, x: x_val, mean: mean_val, var: var_val, scale: scale_val }) grad_x_ref, grad_scale_ref, grad_offset_ref = self._reference_grad( x_val, grad_val, scale_val, mean_val, var_val, epsilon, "NHWC") self.assertAllClose(grad_x_val, grad_x_ref, atol=1e-2) self.assertAllClose(grad_scale_val, grad_scale_ref, atol=1e-2) self.assertAllClose(grad_offset_val, grad_offset_ref, atol=1e-3)
def _FusedBatchNormGrad(op, *grad): """Return the gradients for the 3 inputs of BatchNorm. Args: op: The BatchNormOp for which we need to compute gradients. *grad: An argument list for tensors of gradients wrt the outputs with grad[0] as grad_y. Returns: grad_x: gradient for x, which is scale * rsqrt(variance + epsilon) * [grad_y - mean(grad_y) - (x - mean(x)) * mean(grad_y * (x - mean(x))) / (variance + epsilon)] in training mode; grad_y * scale * rsqrt(pop_variance + epsilon) in freeze mode. grad_scale: gradient for scale, which is sum(grad_y * (x - mean(x)) * rsqrt(variance + epsilon)) in training mode; sum(grad_y * (x - pop_mean) * rsqrt(pop_variance + epsilon)) in freeze mode. grad_offset: gradient for offset, which is sum(grad_y) in training mode; sum(grad_y) in freeze mode. """ x = op.inputs[0] grad_y = grad[0] scale = op.inputs[1] epsilon = op.get_attr("epsilon") data_format = op.get_attr("data_format") is_training = op.get_attr("is_training") if is_training: return gen_nn_ops.fused_batch_norm_grad( grad_y, x, scale, op.outputs[3], op.outputs[4], epsilon=epsilon, data_format=data_format, is_training=is_training) else: pop_mean = op.inputs[3] pop_var = op.inputs[4] if data_format == b"NHWC": reduce_axis = [0, 1, 2] else: reduce_axis = [0, 2, 3] shape = [1, array_ops.size(pop_mean), 1, 1] pop_mean = array_ops.reshape(pop_mean, shape) pop_var = array_ops.reshape(pop_var, shape) scale = array_ops.reshape(scale, shape) grad_offset = math_ops.reduce_sum(grad_y, axis=reduce_axis) var_rsqrt = math_ops.rsqrt(pop_var + epsilon) grad_scale = math_ops.reduce_sum( grad_y * (x - pop_mean) * var_rsqrt, axis=reduce_axis) grad_x = grad_y * scale * var_rsqrt return grad_x, grad_scale, grad_offset, None, None
def _FusedBatchNormGrad(op, *grad): """Return the gradients for the 3 inputs of BatchNorm. Args: op: The BatchNormOp for which we need to compute gradients. *grad: An argument list for tensors of gradients wrt the outputs with grad[0] as grad_y. Returns: grad_x: gradient for x, which is scale * rsqrt(variance + epsilon) * [grad_y - mean(grad_y) - (x - mean(x)) * mean(grad_y * (x - mean(x))) / (variance + epsilon)] in training mode; grad_y * scale * rsqrt(pop_variance + epsilon) in freeze mode. grad_scale: gradient for scale, which is sum(grad_y * (x - mean(x)) * rsqrt(variance + epsilon)) in training mode; sum(grad_y * (x - pop_mean) * rsqrt(pop_variance + epsilon)) in freeze mode. grad_offset: gradient for offset, which is sum(grad_y) in training mode; sum(grad_y) in freeze mode. """ x = op.inputs[0] grad_y = grad[0] scale = op.inputs[1] epsilon = op.get_attr("epsilon") data_format = op.get_attr("data_format") is_training = op.get_attr("is_training") if is_training: return gen_nn_ops.fused_batch_norm_grad(grad_y, x, scale, op.outputs[3], op.outputs[4], epsilon=epsilon, data_format=data_format, is_training=is_training) else: pop_mean = op.inputs[3] pop_var = op.inputs[4] if data_format == b"NHWC": reduce_axis = [0, 1, 2] else: reduce_axis = [0, 2, 3] shape = [1, array_ops.size(pop_mean), 1, 1] pop_mean = array_ops.reshape(pop_mean, shape) pop_var = array_ops.reshape(pop_var, shape) scale = array_ops.reshape(scale, shape) grad_offset = math_ops.reduce_sum(grad_y, axis=reduce_axis) var_rsqrt = math_ops.rsqrt(pop_var + epsilon) grad_scale = math_ops.reduce_sum(grad_y * (x - pop_mean) * var_rsqrt, axis=reduce_axis) grad_x = grad_y * scale * var_rsqrt return grad_x, grad_scale, grad_offset, None, None
def testGradientTraining(self, data_format): # TODO(b/64270657): Use gradient_checker here in addition to comparing with # this reference implementation. channel = 3 x_shape = [2, 2, 6, channel] scale_shape = [channel] grad_val = np.random.random_sample(x_shape).astype(np.float32) x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) mean_val = np.random.random_sample(scale_shape).astype(np.float32) var_val = np.random.random_sample(scale_shape).astype(np.float32) epsilon = 0.001 data_format_src = "NHWC" grad_x_ref, grad_scale_ref, grad_offset_ref = self._reference_grad( x_val, grad_val, scale_val, mean_val, var_val, epsilon, data_format_src) # TODO(b/110530713): Support data format HWCN on GPU if self.device == "XLA_GPU" and data_format == "HWCN": self.skipTest("GPU does not support data format HWCN.") with self.test_session() as sess, self.test_scope(): grad_val_converted = test_utils.ConvertBetweenDataFormats( grad_val, data_format_src, data_format) x_val_converted = test_utils.ConvertBetweenDataFormats( x_val, data_format_src, data_format) grad_x_ref_converted = test_utils.ConvertBetweenDataFormats( grad_x_ref, data_format_src, data_format) grad = array_ops.placeholder( np.float32, shape=x_val_converted.shape, name="grad") x = array_ops.placeholder( np.float32, shape=x_val_converted.shape, name="x") mean = array_ops.placeholder(np.float32, shape=scale_shape, name="mean") var = array_ops.placeholder(np.float32, shape=scale_shape, name="var") scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale") grad_x, grad_scale, grad_offset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, mean, var, data_format=data_format, is_training=True) grad_x_val, grad_scale_val, grad_offset_val = sess.run( [grad_x, grad_scale, grad_offset], { grad: grad_val_converted, x: x_val_converted, mean: mean_val, var: var_val, scale: scale_val }) self.assertAllClose(grad_x_val, grad_x_ref_converted, atol=1e-2) self.assertAllClose(grad_scale_val, grad_scale_ref, atol=1e-2) self.assertAllClose(grad_offset_val, grad_offset_ref, atol=1e-3)
def test_fused_batch_norm(): import tensorflow as tf from tensorflow.python.ops import gen_nn_ops from dace.frontend.tensorflow import TFSession num_channels = 3 size = [8, 224, 224, num_channels] config = tf.ConfigProto() config.gpu_options.allow_growth = True inp = tf.placeholder(tf.float32, size) scale = tf.placeholder(tf.float32, [num_channels]) offset = tf.placeholder(tf.float32, [num_channels]) populationMean = tf.placeholder(tf.float32, [num_channels]) populationVariance = tf.placeholder(tf.float32, [num_channels]) y, mean, var, _, var_sqrt = gen_nn_ops._fused_batch_norm(inp, scale, offset, [], [], epsilon=0.1, is_training=True) outputs = [y, mean, var] test_in = np.random.uniform(size=size).astype(np.float32) test_scale = np.random.uniform(size=[num_channels]).astype(np.float32) test_offset = np.random.uniform(size=[num_channels]).astype(np.float32) sess_tf = tf.Session(config=config) sess_dace = TFSession() outputs_dace = sess_dace.run( outputs, feed_dict={ inp: test_in, scale: test_scale, offset: test_offset, }, ) outputs_tf = sess_tf.run( outputs, feed_dict={ inp: test_in, scale: test_scale, offset: test_offset, }, ) try: assert (tf.linalg.norm(outputs_tf[0] - outputs_dace[0]).eval(session=sess_tf) < 1e-1 and tf.linalg.norm(outputs_dace[2] - outputs_tf[2]).eval(session=sess_tf) < 1e-4 and tf.linalg.norm(outputs_dace[1] - outputs_tf[1]).eval(session=sess_tf) < 1e-4) except: print("FBN test failed") print( tf.linalg.norm(outputs_tf[0] - outputs_dace[0]).eval(session=sess_tf)) print( tf.linalg.norm(outputs_tf[1] - outputs_dace[1]).eval(session=sess_tf)) print( tf.linalg.norm(outputs_tf[2] - outputs_dace[2]).eval(session=sess_tf)) ################# FBN GRADIENT TEST ############################### outputGrad = tf.placeholder(tf.float32, size) x_grad, gamma_grad, beta_grad, _, _ = gen_nn_ops.fused_batch_norm_grad( outputGrad, inp, scale, outputs[1], var_sqrt, epsilon=0.1, is_training=True) gradients = [x_grad, gamma_grad, beta_grad] test_outputgrad = np.random.uniform(size=size).astype(np.float32) outputs_dace = sess_dace.run( gradients, feed_dict={ inp: test_in, outputGrad: test_outputgrad, scale: test_scale, offset: test_offset, }, ) # TF x_grad, gamma_grad, beta_grad, _, _ = gen_nn_ops.fused_batch_norm_grad( outputGrad, inp, scale, outputs[1], tf.math.rsqrt(outputs[2] + float(0.1)) if tf.test.is_built_with_cuda() else outputs[2], epsilon=0.1, is_training=True, ) gradients = [x_grad, gamma_grad, beta_grad] # writer = tf.summary.FileWriter("./", sess_tf.graph) outputs_tf = sess_tf.run( gradients, feed_dict={ inp: test_in, outputGrad: test_outputgrad, scale: test_scale, offset: test_offset, }, ) try: assert (tf.linalg.norm(outputs_tf[0] - outputs_dace[0]).eval(session=sess_tf) < 1e-1 and tf.linalg.norm(outputs_dace[2] - outputs_tf[2]).eval(session=sess_tf) < 10 and tf.linalg.norm(outputs_dace[1] - outputs_tf[1]).eval(session=sess_tf) < 10) except: print("FBN Gradient test failed") print( tf.linalg.norm(outputs_tf[0] - outputs_dace[0]).eval(session=sess_tf)) print( tf.linalg.norm(outputs_tf[1] - outputs_dace[1]).eval(session=sess_tf)) print( tf.linalg.norm(outputs_tf[2] - outputs_dace[2]).eval(session=sess_tf)) print( tf.linalg.norm(outputs_tf[2] - np.sum(test_outputgrad, axis=(0, 1, 2))).eval( session=sess_tf))
print( tf.linalg.norm(outputs_tf[0] - outputs_dace[0]).eval(session=sess_tf)) print( tf.linalg.norm(outputs_tf[1] - outputs_dace[1]).eval(session=sess_tf)) print( tf.linalg.norm(outputs_tf[2] - outputs_dace[2]).eval(session=sess_tf)) ################# FBN GRADIENT TEST ############################### outputGrad = tf.placeholder(tf.float32, size) x_grad, gamma_grad, beta_grad, _, _ = gen_nn_ops.fused_batch_norm_grad( outputGrad, inp, scale, outputs[1], var_sqrt, epsilon=0.1, is_training=True) gradients = [x_grad, gamma_grad, beta_grad] test_outputgrad = np.random.uniform(size=size).astype(np.float32) outputs_dace = sess_dace.run( gradients, feed_dict={ inp: test_in, outputGrad: test_outputgrad, scale: test_scale, offset: test_offset, }, ) # TF
def testGradientTraining(self, data_format): # TODO(b/64270657): Use gradient_checker here in addition to comparing with # this reference implementation. channel = 3 x_shape = [2, 2, 6, channel] scale_shape = [channel] grad_val = np.random.random_sample(x_shape).astype(np.float32) x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) mean_val = np.random.random_sample(scale_shape).astype(np.float32) var_val = np.random.random_sample(scale_shape).astype(np.float32) epsilon = 0.001 # The TensorFlow FusedBatchNormGrad training operation takes two inputs with # implementation defined values. In theory the only correct value these # inputs are the corresponding reserve_space_{1|2} outputs from the # FusedBatchNorm training operation. However, in practice, we rely on the # first one being mean on {C|G}PU, and the second one being variance on CPU # and inverse(sqrt(variance + epsilon)) on GPU (we test this assumption # separately). reserve_space_1_val = mean_val if self.device == "XLA_GPU": reserve_space_2_val = np.reciprocal(np.sqrt(var_val + epsilon)) else: reserve_space_2_val = var_val data_format_src = "NHWC" grad_x_ref, grad_scale_ref, grad_offset_ref = self._reference_grad( x_val, grad_val, scale_val, mean_val, var_val, epsilon, data_format_src) with self.cached_session() as sess, self.test_scope(): grad_val_converted = test_utils.ConvertBetweenDataFormats( grad_val, data_format_src, data_format) x_val_converted = test_utils.ConvertBetweenDataFormats( x_val, data_format_src, data_format) grad_x_ref_converted = test_utils.ConvertBetweenDataFormats( grad_x_ref, data_format_src, data_format) grad = array_ops.placeholder( np.float32, shape=x_val_converted.shape, name="grad") x = array_ops.placeholder( np.float32, shape=x_val_converted.shape, name="x") reserve_space_1 = array_ops.placeholder( np.float32, shape=scale_shape, name="reserve_space_1") reserve_space_2 = array_ops.placeholder( np.float32, shape=scale_shape, name="reserve_space_2") scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale") grad_x, grad_scale, grad_offset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, reserve_space_1, reserve_space_2, data_format=data_format, is_training=True) grad_x_val, grad_scale_val, grad_offset_val = sess.run( [grad_x, grad_scale, grad_offset], { grad: grad_val_converted, x: x_val_converted, reserve_space_1: reserve_space_1_val, reserve_space_2: reserve_space_2_val, scale: scale_val }) self.assertAllClose(grad_x_val, grad_x_ref_converted, atol=1e-2) self.assertAllClose(grad_scale_val, grad_scale_ref, atol=1e-2) self.assertAllClose(grad_offset_val, grad_offset_ref, atol=1e-3)
def testGradientTraining(self, data_format): # TODO(b/64270657): Use gradient_checker here in addition to comparing with # this reference implementation. channel = 3 x_shape = [2, 2, 6, channel] scale_shape = [channel] grad_val = np.random.random_sample(x_shape).astype(np.float32) x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) mean_val = np.random.random_sample(scale_shape).astype(np.float32) var_val = np.random.random_sample(scale_shape).astype(np.float32) epsilon = 0.001 # The TensorFlow FusedBatchNormGrad training operation takes two inputs with # implementation defined values. In theory the only correct value these # inputs are the corresponding reserve_space_{1|2} outputs from the # FusedBatchNorm training operation. However, in practice, we rely on the # first one being mean on {C|G}PU, and the second one being variance on CPU # and inverse(sqrt(variance + epsilon)) on GPU (we test this assumption # separately). reserve_space_1_val = mean_val if self.device == "XLA_GPU": reserve_space_2_val = np.reciprocal(np.sqrt(var_val + epsilon)) else: reserve_space_2_val = var_val data_format_src = "NHWC" grad_x_ref, grad_scale_ref, grad_offset_ref = self._reference_grad( x_val, grad_val, scale_val, mean_val, var_val, epsilon, data_format_src) with self.cached_session() as sess, self.test_scope(): grad_val_converted = test_utils.ConvertBetweenDataFormats( grad_val, data_format_src, data_format) x_val_converted = test_utils.ConvertBetweenDataFormats( x_val, data_format_src, data_format) grad_x_ref_converted = test_utils.ConvertBetweenDataFormats( grad_x_ref, data_format_src, data_format) grad = array_ops.placeholder(np.float32, shape=x_val_converted.shape, name="grad") x = array_ops.placeholder(np.float32, shape=x_val_converted.shape, name="x") reserve_space_1 = array_ops.placeholder(np.float32, shape=scale_shape, name="reserve_space_1") reserve_space_2 = array_ops.placeholder(np.float32, shape=scale_shape, name="reserve_space_2") scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale") grad_x, grad_scale, grad_offset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, reserve_space_1, reserve_space_2, data_format=data_format, is_training=True) grad_x_val, grad_scale_val, grad_offset_val = sess.run( [grad_x, grad_scale, grad_offset], { grad: grad_val_converted, x: x_val_converted, reserve_space_1: reserve_space_1_val, reserve_space_2: reserve_space_2_val, scale: scale_val }) self.assertAllClose(grad_x_val, grad_x_ref_converted, atol=1e-2) self.assertAllClose(grad_scale_val, grad_scale_ref, atol=1e-2) self.assertAllClose(grad_offset_val, grad_offset_ref, atol=1e-3)
def testGradientTraining(self, data_format): # TODO(b/64270657): Use gradient_checker here in addition to comparing with # this reference implementation. channel = 3 x_shape = [2, 2, 6, channel] scale_shape = [channel] grad_val = np.random.random_sample(x_shape).astype(np.float32) x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) mean_val = np.random.random_sample(scale_shape).astype(np.float32) var_val = np.random.random_sample(scale_shape).astype(np.float32) epsilon = 0.001 data_format_src = "NHWC" grad_x_ref, grad_scale_ref, grad_offset_ref = self._reference_grad( x_val, grad_val, scale_val, mean_val, var_val, epsilon, data_format_src) # TODO(b/110530713): Support data format HWCN on GPU if self.device == "XLA_GPU" and data_format == "HWCN": self.skipTest("GPU does not support data format HWCN.") with self.test_session() as sess, self.test_scope(): grad_val_converted = self._convertBetweenDataFormats( grad_val, data_format_src, data_format) x_val_converted = self._convertBetweenDataFormats( x_val, data_format_src, data_format) grad_x_ref_converted = self._convertBetweenDataFormats( grad_x_ref, data_format_src, data_format) grad = array_ops.placeholder(np.float32, shape=x_val_converted.shape, name="grad") x = array_ops.placeholder(np.float32, shape=x_val_converted.shape, name="x") mean = array_ops.placeholder(np.float32, shape=scale_shape, name="mean") var = array_ops.placeholder(np.float32, shape=scale_shape, name="var") scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale") grad_x, grad_scale, grad_offset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, mean, var, data_format=data_format, is_training=True) grad_x_val, grad_scale_val, grad_offset_val = sess.run( [grad_x, grad_scale, grad_offset], { grad: grad_val_converted, x: x_val_converted, mean: mean_val, var: var_val, scale: scale_val }) self.assertAllClose(grad_x_val, grad_x_ref_converted, atol=1e-2) self.assertAllClose(grad_scale_val, grad_scale_ref, atol=1e-2) self.assertAllClose(grad_offset_val, grad_offset_ref, atol=1e-3)
def testGradientInference(self): # TODO (b/64270657): Use gradient_checker here in addition to comparing with id:149 # https://github.com/imdone/tensorflow/issues/150 # this reference implementation. channel = 3 x_shape = [2, 2, 6, channel] scale_shape = [channel] grad_val = np.random.random_sample(x_shape).astype(np.float32) x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) mean_val = np.random.random_sample(scale_shape).astype(np.float32) var_val = np.random.random_sample(scale_shape).astype(np.float32) with self.test_session() as sess, self.test_scope(): grad = array_ops.placeholder(np.float32, shape=x_shape, name="grad") x = array_ops.placeholder(np.float32, shape=x_shape, name="x") mean = array_ops.placeholder(np.float32, shape=scale_shape, name="mean") var = array_ops.placeholder(np.float32, shape=scale_shape, name="var") scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale") with self.test_scope(): out = gen_nn_ops.fused_batch_norm_grad(grad, x, scale, mean, var, data_format="NHWC", is_training=False) grad_x, grad_scale, grad_offset, _, _ = out ref_x, ref_scale, ref_offset, _, _ = gen_nn_ops.fused_batch_norm_grad( grad, x, scale, mean, var, data_format="NHWC", is_training=False) grad_x_val, grad_scale_val, grad_offset_val, = sess.run( [grad_x, grad_scale, grad_offset], { grad: grad_val, x: x_val, mean: mean_val, var: var_val, scale: scale_val }) grad_x_ref, grad_scale_ref, grad_offset_ref, = sess.run( [ref_x, ref_scale, ref_offset], { grad: grad_val, x: x_val, mean: mean_val, var: var_val, scale: scale_val }) self.assertAllClose(grad_x_val, grad_x_ref, atol=1e-2) self.assertAllClose(grad_scale_val, grad_scale_ref, atol=1e-2) self.assertAllClose(grad_offset_val, grad_offset_ref, atol=1e-3)