def testInvalidLabel(self): features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.], [1., 2., 3., 4.]] labels = [4, 3, 0, -1] if test.is_built_with_gpu_support() and test.is_gpu_available(): with self.session(use_gpu=True) as sess: loss, backprop = ( gen_nn_ops.sparse_softmax_cross_entropy_with_logits( features, labels)) tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllClose( [[np.nan] * 4, [0.25, 0.25, 0.25, -0.75], [-0.968, 0.087, 0.237, 0.6439], [np.nan] * 4], tf_backprop, rtol=1e-3, atol=1e-3) self.assertAllClose([np.nan, 1.3862, 3.4420, np.nan], tf_loss, rtol=1e-3, atol=1e-3) with self.session(use_gpu=False) as sess: loss, backprop = ( gen_nn_ops.sparse_softmax_cross_entropy_with_logits( features, labels)) with self.assertRaisesOpError("Received a label value of"): self.evaluate([loss, backprop])
def testInvalidLabel(self): features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.], [1., 2., 3., 4.]] labels = [4, 3, 0, -1] if test.is_built_with_cuda() and test.is_gpu_available(): with self.session(use_gpu=True) as sess: loss, backprop = ( gen_nn_ops.sparse_softmax_cross_entropy_with_logits( features, labels)) tf_loss, tf_backprop = sess.run([loss, backprop]) self.assertAllClose( [[np.nan] * 4, [0.25, 0.25, 0.25, -0.75], [-0.968, 0.087, 0.237, 0.6439], [np.nan] * 4], tf_backprop, rtol=1e-3, atol=1e-3) self.assertAllClose( [np.nan, 1.3862, 3.4420, np.nan], tf_loss, rtol=1e-3, atol=1e-3) with self.session(use_gpu=False) as sess: loss, backprop = ( gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features, labels)) with self.assertRaisesOpError("Received a label value of"): sess.run([loss, backprop])
def test_sparse_softmax_cross_entropy_with_logits_2d(self): num_classes = 10 batch_size = 1000 total_size = num_classes * batch_size labels = constant_op.constant(self.generate_random_numbers( batch_size, 0, num_classes - 1, "DTYPE_INT"), shape=[batch_size]) features = constant_op.constant(self.generate_random_numbers( total_size, 0.0, 1.0), shape=[batch_size, num_classes]) # Run on CPU with self.cpu_device: out_cpu = sparse_softmax_cross_entropy_with_logits( features, labels) with self.session as sess: expected = sess.run(out_cpu) # Run on nGraph with self.device: out = sparse_softmax_cross_entropy_with_logits(features, labels) with self.session as sess: result = sess.run(out) assert np.allclose(result[0], expected[0]) assert np.allclose(result[1], expected[1])
def _testXent(self, np_features, np_labels): np_loss, np_backprop = self._npXent(np_features, np_labels) with self.cached_session(use_gpu=True) as sess: loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( np_features, np_labels) tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllCloseAccordingToType(np_loss, tf_loss) self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
def _testXent(self, np_features, np_labels): np_loss, np_backprop = self._npXent(np_features, np_labels) with self.cached_session(use_gpu=True) as sess: loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( np_features, np_labels) tf_loss, tf_backprop = sess.run([loss, backprop]) self.assertAllCloseAccordingToType(np_loss, tf_loss) self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
def testSingleClass(self): for label_dtype in np.int32, np.int64: with self.cached_session(use_gpu=True) as sess: loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( np.array([[1.], [-1.], [0.]]).astype(np.float32), np.array([0, 0, 0]).astype(label_dtype)) tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllClose([0.0, 0.0, 0.0], tf_loss) self.assertAllClose([[0.0], [0.0], [0.0]], tf_backprop)
def testSingleClass(self): for label_dtype in np.int32, np.int64: with self.cached_session(use_gpu=True) as sess: loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( np.array([[1.], [-1.], [0.]]).astype(np.float32), np.array([0, 0, 0]).astype(label_dtype)) tf_loss, tf_backprop = sess.run([loss, backprop]) self.assertAllClose([0.0, 0.0, 0.0], tf_loss) self.assertAllClose([[0.0], [0.0], [0.0]], tf_backprop)
def testInvalidLabelCPU(self): features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.], [1., 2., 3., 4.]] labels = [4, 3, 0, -1] with self.assertRaisesRegex( (errors_impl.InvalidArgumentError, errors_impl.UnknownError), "Received a label value of"): self.evaluate( gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features, labels))
def testInvalidLabelGPU(self): features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.], [1., 2., 3., 4.]] labels = [4, 3, 0, -1] loss, backprop = self.evaluate( gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features, labels)) self.assertAllClose([[np.nan] * 4, [0.25, 0.25, 0.25, -0.75], [-0.968, 0.087, 0.237, 0.6439], [np.nan] * 4], backprop, rtol=1e-3, atol=1e-3) self.assertAllClose([np.nan, 1.3862, 3.4420, np.nan], loss, rtol=1e-3, atol=1e-3)
def testBfloat16(self): for label_dtype in np.int32, np.int64: np_features = np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float32) np_labels = np.array([0, 3]).astype(label_dtype) np_loss, np_backprop = self._npXent(np_features, np_labels) np_features_bf16 = math_ops.cast(np_features, dtypes.bfloat16) np_loss_bf16 = math_ops.cast(np_loss, dtypes.bfloat16) np_backprop_bf16 = math_ops.cast(np_backprop, dtypes.bfloat16) with self.cached_session(use_gpu=False): loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( np_features_bf16, np_labels) tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllCloseAccordingToType(np_loss_bf16, tf_loss) self.assertAllCloseAccordingToType(np_backprop_bf16, tf_backprop)
def test_sparse_softmax_cross_entropy(): tf.reset_default_graph() num_batches = 100 num_classes = 10 v_logits = tf.Variable(tf.truncated_normal([num_batches, num_classes], mean=0, stddev=5), name="logits") v_labels = tf.Variable(tf.random_uniform([num_batches], minval=0, maxval=num_classes, dtype=tf.int64), name="labels") v_grad = tf.Variable(tf.truncated_normal([num_batches], mean=0, stddev=1), name="grad") t_init = tf.global_variables_initializer() t_debug = False iters = 100 widgets = ["Sparse softmax cross entropy int test: ", pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA()] pbar = pb.ProgressBar(widgets=widgets, maxval=iters) pbar.start() for i in range(iters): pbar.update(i) with tf.Session('') as sess: t_init.run() z_op, z_back = waveflow.wavecomp_ops_module.wave_sparse_softmax_cross_entropy_with_logits_int(features=v_logits, labels=v_labels) z_op2, z_back2 = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features=v_logits, labels=v_labels) z_grad = tf.gradients(z_op, v_logits, v_grad)[0] z_grad2 = tf.gradients(z_op2, v_logits, v_grad)[0] z, zb, zg, z2, zb2, zg2, labels, logits, grad = sess.run([z_op, z_back, z_grad, z_op2, z_back2, z_grad2, v_labels, v_logits, v_grad]) if t_debug: print("labels: %s" % (labels)) print("logits: %s" % (logits)) print("grad: %s" % (grad)) print("z: %s" % (z)) print("z2: %s" % (z2)) print("zb: %s" % (zb)) print("zb2: %s" % (zb2)) print("zg: %s" % (zg)) print("zg2: %s" % (zg2)) assert compare_tensor(z, z2, "Output mismatch", 1e-3) assert compare_tensor(zb, zb2, "Backprop mismatch", 1e-3) assert compare_tensor(zg, zg2, "Gradient mismatch", 1e-3) pbar.finish() return True
def test_sparse_softmax_cross_entropy_with_logits_2d(self): num_classes = 10 batch_size = 1000 total_size = num_classes * batch_size labels = constant_op.constant( self.generate_random_numbers(batch_size, 0, num_classes - 1, "DTYPE_INT"), shape=[batch_size]) features = constant_op.constant( self.generate_random_numbers(total_size, 0.0, 1.0), shape=[batch_size, num_classes]) out = sparse_softmax_cross_entropy_with_logits(features, labels) sess_fn = lambda sess: sess.run(out) expected = self.without_ngraph(sess_fn) result = self.with_ngraph(sess_fn) assert np.allclose(result[0], expected[0], rtol=0, atol=1e-02) assert np.allclose(result[1], expected[1], rtol=0, atol=1e-02)
def testExceptionThrowing(self): with self.session(), test_util.force_gpu(): for features_dtype in [dtypes.float16, dtypes.float32]: for labels_dtype in [dtypes.int32, dtypes.int64]: features = constant_op.constant([[0.3, 0.5], [0.2, 0.6]], dtype=features_dtype) labels = constant_op.constant([1, 0], dtype=labels_dtype) with self.assertRaisesRegex( errors_impl.UnimplementedError, "The GPU implementation of SparseSoftmaxCrossEntropyWithLogits " + "that would have been executed is not deterministic. Note that " + "the Python API uses an alternative, deterministic, " + "GPU-accelerated path when determinsim is enabled." ): result = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( features=features, labels=labels) self.evaluate(result)
def sparse_softmax_crossent_with_logits(logits=None, labels=None, name=None): """docstring.""" # TODO(jamesqin): merge with tf.nn.sparse_softmax_cross_entropy_with_logits # Basically forks the tf lib function, only that the result isn't casted # back to tf.float16 if the input is tf.float16 # TODO(jamesqin): implement a fused kernel to reduce memory footprint. # Reshape logits and labels to rank 2. with tf.name_scope( name, "SparseSoftmaxCrossEntropyWithLogits", [labels, logits] ): labels = tf.convert_to_tensor(labels) logits = tf.convert_to_tensor(logits) precise_logits = tf.cast(logits, tf.float32) if ( tf.as_dtype(logits.dtype) == tf.float16 ) else logits # Store label shape for result later. labels_static_shape = labels.get_shape() labels_shape = tf.shape(labels) static_shapes_fully_defined = ( labels_static_shape.is_fully_defined() and logits.get_shape()[:-1].is_fully_defined() ) if logits.get_shape().ndims is not None and logits.get_shape( ).ndims == 0: raise ValueError( "Logits cannot be scalars - received shape %s." % logits.get_shape() ) if logits.get_shape().ndims is not None and ( labels_static_shape.ndims is not None and labels_static_shape.ndims != logits.get_shape().ndims - 1 ): raise ValueError( "Rank mismatch: Rank of labels (received %s) should " "equal rank of logits minus 1 (received %s)." % (labels_static_shape.ndims, logits.get_shape().ndims) ) if ( static_shapes_fully_defined and labels_static_shape != logits.get_shape()[:-1] ): raise ValueError( "Shape mismatch: The shape of labels (received %s) " "should equal the shape of logits except for the last " "dimension (received %s)." % (labels_static_shape, logits.get_shape()) ) # Check if no reshapes are required. if logits.get_shape().ndims == 2: cost, _ = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( precise_logits, labels, name=name ) # cost.dtype is always fp32 return cost # Perform a check of the dynamic shapes if the static shapes are not fully # defined. shape_checks = [] if not static_shapes_fully_defined: xla_compile = (os.environ.get("xla_compile") == "true") use_xla = (os.environ.get("use_xla") == "true") if not (xla_compile or use_xla): # Assert isn't registered w/ GPU, not working w/ xla.compile() shape_checks.append( tf.assert_equal(tf.shape(labels), tf.shape(logits)[:-1]) ) with tf.control_dependencies(shape_checks): # Reshape logits to 2 dim, labels to 1 dim. num_classes = tf.shape(logits)[tf.rank(logits) - 1] precise_logits = tf.reshape(precise_logits, [-1, num_classes]) labels = tf.reshape(labels, [-1]) # The second output tensor contains the gradients. We use it in # _CrossEntropyGrad() in nn_grad but not here. cost, _ = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( precise_logits, labels, name=name ) cost = tf.reshape(cost, labels_shape) cost.set_shape(labels_static_shape) # cost is always fp32 return cost