def testInvalidLabel(self):
features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.],
[1., 2., 3., 4.]]
labels = [4, 3, 0, -1]
if test.is_built_with_gpu_support() and test.is_gpu_available():
with self.session(use_gpu=True) as sess:
loss, backprop = (
gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
features, labels))
tf_loss, tf_backprop = self.evaluate([loss, backprop])
self.assertAllClose(
[[np.nan] * 4, [0.25, 0.25, 0.25, -0.75],
[-0.968, 0.087, 0.237, 0.6439], [np.nan] * 4],
tf_backprop,
rtol=1e-3,
atol=1e-3)
self.assertAllClose([np.nan, 1.3862, 3.4420, np.nan],
tf_loss,
rtol=1e-3,
atol=1e-3)
with self.session(use_gpu=False) as sess:
loss, backprop = (
gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
features, labels))
with self.assertRaisesOpError("Received a label value of"):
self.evaluate([loss, backprop])
def testInvalidLabel(self):
features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.],
[1., 2., 3., 4.]]
labels = [4, 3, 0, -1]
if test.is_built_with_cuda() and test.is_gpu_available():
with self.session(use_gpu=True) as sess:
loss, backprop = (
gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
features, labels))
tf_loss, tf_backprop = sess.run([loss, backprop])
self.assertAllClose(
[[np.nan] * 4, [0.25, 0.25, 0.25, -0.75],
[-0.968, 0.087, 0.237, 0.6439], [np.nan] * 4],
tf_backprop,
rtol=1e-3,
atol=1e-3)
self.assertAllClose(
[np.nan, 1.3862, 3.4420, np.nan], tf_loss, rtol=1e-3, atol=1e-3)
with self.session(use_gpu=False) as sess:
loss, backprop = (
gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features, labels))
with self.assertRaisesOpError("Received a label value of"):
sess.run([loss, backprop])
def test_sparse_softmax_cross_entropy_with_logits_2d(self):
num_classes = 10
batch_size = 1000
total_size = num_classes * batch_size
labels = constant_op.constant(self.generate_random_numbers(
batch_size, 0, num_classes - 1, "DTYPE_INT"),
shape=[batch_size])
features = constant_op.constant(self.generate_random_numbers(
total_size, 0.0, 1.0),
shape=[batch_size, num_classes])
# Run on CPU
with self.cpu_device:
out_cpu = sparse_softmax_cross_entropy_with_logits(
features, labels)
with self.session as sess:
expected = sess.run(out_cpu)
# Run on nGraph
with self.device:
out = sparse_softmax_cross_entropy_with_logits(features, labels)
with self.session as sess:
result = sess.run(out)
assert np.allclose(result[0], expected[0])
assert np.allclose(result[1], expected[1])
def _testXent(self, np_features, np_labels):
np_loss, np_backprop = self._npXent(np_features, np_labels)
with self.cached_session(use_gpu=True) as sess:
loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
np_features, np_labels)
tf_loss, tf_backprop = self.evaluate([loss, backprop])
self.assertAllCloseAccordingToType(np_loss, tf_loss)
self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
def _testXent(self, np_features, np_labels):
np_loss, np_backprop = self._npXent(np_features, np_labels)
with self.cached_session(use_gpu=True) as sess:
loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
np_features, np_labels)
tf_loss, tf_backprop = sess.run([loss, backprop])
self.assertAllCloseAccordingToType(np_loss, tf_loss)
self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
def testSingleClass(self):
for label_dtype in np.int32, np.int64:
with self.cached_session(use_gpu=True) as sess:
loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
np.array([[1.], [-1.], [0.]]).astype(np.float32),
np.array([0, 0, 0]).astype(label_dtype))
tf_loss, tf_backprop = self.evaluate([loss, backprop])
self.assertAllClose([0.0, 0.0, 0.0], tf_loss)
self.assertAllClose([[0.0], [0.0], [0.0]], tf_backprop)
def testSingleClass(self):
for label_dtype in np.int32, np.int64:
with self.cached_session(use_gpu=True) as sess:
loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
np.array([[1.], [-1.], [0.]]).astype(np.float32),
np.array([0, 0, 0]).astype(label_dtype))
tf_loss, tf_backprop = sess.run([loss, backprop])
self.assertAllClose([0.0, 0.0, 0.0], tf_loss)
self.assertAllClose([[0.0], [0.0], [0.0]], tf_backprop)
def testInvalidLabelCPU(self):
features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.],
[1., 2., 3., 4.]]
labels = [4, 3, 0, -1]
with self.assertRaisesRegex(
(errors_impl.InvalidArgumentError, errors_impl.UnknownError),
"Received a label value of"):
self.evaluate(
gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features, labels))
def testInvalidLabelGPU(self):
features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.],
[1., 2., 3., 4.]]
labels = [4, 3, 0, -1]
loss, backprop = self.evaluate(
gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features, labels))
self.assertAllClose([[np.nan] * 4, [0.25, 0.25, 0.25, -0.75],
[-0.968, 0.087, 0.237, 0.6439], [np.nan] * 4],
backprop,
rtol=1e-3,
atol=1e-3)
self.assertAllClose([np.nan, 1.3862, 3.4420, np.nan],
loss,
rtol=1e-3,
atol=1e-3)
def testBfloat16(self):
for label_dtype in np.int32, np.int64:
np_features = np.array([[1., 1., 1., 1.], [1., 2., 3.,
4.]]).astype(np.float32)
np_labels = np.array([0, 3]).astype(label_dtype)
np_loss, np_backprop = self._npXent(np_features, np_labels)
np_features_bf16 = math_ops.cast(np_features, dtypes.bfloat16)
np_loss_bf16 = math_ops.cast(np_loss, dtypes.bfloat16)
np_backprop_bf16 = math_ops.cast(np_backprop, dtypes.bfloat16)
with self.cached_session(use_gpu=False):
loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
np_features_bf16, np_labels)
tf_loss, tf_backprop = self.evaluate([loss, backprop])
self.assertAllCloseAccordingToType(np_loss_bf16, tf_loss)
self.assertAllCloseAccordingToType(np_backprop_bf16, tf_backprop)
def test_sparse_softmax_cross_entropy():
tf.reset_default_graph()
num_batches = 100
num_classes = 10
v_logits = tf.Variable(tf.truncated_normal([num_batches, num_classes], mean=0, stddev=5), name="logits")
v_labels = tf.Variable(tf.random_uniform([num_batches], minval=0, maxval=num_classes, dtype=tf.int64), name="labels")
v_grad = tf.Variable(tf.truncated_normal([num_batches], mean=0, stddev=1), name="grad")
t_init = tf.global_variables_initializer()
t_debug = False
iters = 100
widgets = ["Sparse softmax cross entropy int test: ", pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA()]
pbar = pb.ProgressBar(widgets=widgets, maxval=iters)
pbar.start()
for i in range(iters):
pbar.update(i)
with tf.Session('') as sess:
t_init.run()
z_op, z_back = waveflow.wavecomp_ops_module.wave_sparse_softmax_cross_entropy_with_logits_int(features=v_logits, labels=v_labels)
z_op2, z_back2 = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features=v_logits, labels=v_labels)
z_grad = tf.gradients(z_op, v_logits, v_grad)[0]
z_grad2 = tf.gradients(z_op2, v_logits, v_grad)[0]
z, zb, zg, z2, zb2, zg2, labels, logits, grad = sess.run([z_op, z_back, z_grad, z_op2, z_back2, z_grad2, v_labels, v_logits, v_grad])
if t_debug:
print("labels: %s" % (labels))
print("logits: %s" % (logits))
print("grad: %s" % (grad))
print("z: %s" % (z))
print("z2: %s" % (z2))
print("zb: %s" % (zb))
print("zb2: %s" % (zb2))
print("zg: %s" % (zg))
print("zg2: %s" % (zg2))
assert compare_tensor(z, z2, "Output mismatch", 1e-3)
assert compare_tensor(zb, zb2, "Backprop mismatch", 1e-3)
assert compare_tensor(zg, zg2, "Gradient mismatch", 1e-3)
pbar.finish()
return True
def test_sparse_softmax_cross_entropy_with_logits_2d(self):
num_classes = 10
batch_size = 1000
total_size = num_classes * batch_size
labels = constant_op.constant(
self.generate_random_numbers(batch_size, 0, num_classes - 1,
"DTYPE_INT"),
shape=[batch_size])
features = constant_op.constant(
self.generate_random_numbers(total_size, 0.0, 1.0),
shape=[batch_size, num_classes])
out = sparse_softmax_cross_entropy_with_logits(features, labels)
sess_fn = lambda sess: sess.run(out)
expected = self.without_ngraph(sess_fn)
result = self.with_ngraph(sess_fn)
assert np.allclose(result[0], expected[0], rtol=0, atol=1e-02)
assert np.allclose(result[1], expected[1], rtol=0, atol=1e-02)
def testExceptionThrowing(self):
with self.session(), test_util.force_gpu():
for features_dtype in [dtypes.float16, dtypes.float32]:
for labels_dtype in [dtypes.int32, dtypes.int64]:
features = constant_op.constant([[0.3, 0.5], [0.2, 0.6]],
dtype=features_dtype)
labels = constant_op.constant([1, 0], dtype=labels_dtype)
with self.assertRaisesRegex(
errors_impl.UnimplementedError,
"The GPU implementation of SparseSoftmaxCrossEntropyWithLogits "
+
"that would have been executed is not deterministic. Note that "
+
"the Python API uses an alternative, deterministic, "
+
"GPU-accelerated path when determinsim is enabled."
):
result = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
features=features, labels=labels)
self.evaluate(result)
def sparse_softmax_crossent_with_logits(logits=None, labels=None, name=None):
"""docstring."""
# TODO(jamesqin): merge with tf.nn.sparse_softmax_cross_entropy_with_logits
# Basically forks the tf lib function, only that the result isn't casted
# back to tf.float16 if the input is tf.float16
# TODO(jamesqin): implement a fused kernel to reduce memory footprint.
# Reshape logits and labels to rank 2.
with tf.name_scope(
name, "SparseSoftmaxCrossEntropyWithLogits", [labels, logits]
):
labels = tf.convert_to_tensor(labels)
logits = tf.convert_to_tensor(logits)
precise_logits = tf.cast(logits, tf.float32) if (
tf.as_dtype(logits.dtype) == tf.float16
) else logits
# Store label shape for result later.
labels_static_shape = labels.get_shape()
labels_shape = tf.shape(labels)
static_shapes_fully_defined = (
labels_static_shape.is_fully_defined() and
logits.get_shape()[:-1].is_fully_defined()
)
if logits.get_shape().ndims is not None and logits.get_shape(
).ndims == 0:
raise ValueError(
"Logits cannot be scalars - received shape %s." %
logits.get_shape()
)
if logits.get_shape().ndims is not None and (
labels_static_shape.ndims is not None and
labels_static_shape.ndims != logits.get_shape().ndims - 1
):
raise ValueError(
"Rank mismatch: Rank of labels (received %s) should "
"equal rank of logits minus 1 (received %s)." %
(labels_static_shape.ndims, logits.get_shape().ndims)
)
if (
static_shapes_fully_defined and
labels_static_shape != logits.get_shape()[:-1]
):
raise ValueError(
"Shape mismatch: The shape of labels (received %s) "
"should equal the shape of logits except for the last "
"dimension (received %s)." %
(labels_static_shape, logits.get_shape())
)
# Check if no reshapes are required.
if logits.get_shape().ndims == 2:
cost, _ = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
precise_logits, labels, name=name
)
# cost.dtype is always fp32
return cost
# Perform a check of the dynamic shapes if the static shapes are not fully
# defined.
shape_checks = []
if not static_shapes_fully_defined:
xla_compile = (os.environ.get("xla_compile") == "true")
use_xla = (os.environ.get("use_xla") == "true")
if not (xla_compile or use_xla):
# Assert isn't registered w/ GPU, not working w/ xla.compile()
shape_checks.append(
tf.assert_equal(tf.shape(labels),
tf.shape(logits)[:-1])
)
with tf.control_dependencies(shape_checks):
# Reshape logits to 2 dim, labels to 1 dim.
num_classes = tf.shape(logits)[tf.rank(logits) - 1]
precise_logits = tf.reshape(precise_logits, [-1, num_classes])
labels = tf.reshape(labels, [-1])
# The second output tensor contains the gradients. We use it in
# _CrossEntropyGrad() in nn_grad but not here.
cost, _ = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
precise_logits, labels, name=name
)
cost = tf.reshape(cost, labels_shape)
cost.set_shape(labels_static_shape)
# cost is always fp32
return cost
