def testIncompatibleShapes(self): with self.test_session(): x, _, _ = _EmbeddingParams(1, 10, dtype=dtypes.float32) sp_ids = sparse_tensor_lib.SparseTensor( constant_op.constant([[0, 0], [0, 1], [1, 0]], dtypes.int64), constant_op.constant([0, 1, 2], dtypes.int32), constant_op.constant([2, 2], dtypes.int64)) sp_weights = sparse_tensor_lib.SparseTensor( constant_op.constant([[0, 0], [0, 1]], dtypes.int64), constant_op.constant([12.0, 5.0], dtypes.float32), constant_op.constant([1, 2], dtypes.int64)) with self.assertRaises(ValueError): embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( x, sp_ids, sp_weights, combiner="mean")
def testIncompatibleShapes(self): with self.cached_session(): x, _, _ = _EmbeddingParams(1, 10, dtype=dtypes.float32) sp_ids = sparse_tensor_lib.SparseTensor( constant_op.constant([[0, 0], [0, 1], [1, 0]], dtypes.int64), constant_op.constant([0, 1, 2], dtypes.int32), constant_op.constant([2, 2], dtypes.int64)) sp_weights = sparse_tensor_lib.SparseTensor( constant_op.constant([[0, 0], [0, 1]], dtypes.int64), constant_op.constant([12.0, 5.0], dtypes.float32), constant_op.constant([1, 2], dtypes.int64)) with self.assertRaises(ValueError): embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( x, sp_ids, sp_weights, combiner="mean")
def testGradientsEmbeddingLookupSparse(self): vocab_size = 12 batch_size = 4 param_shape = [2, 3] sp_ids, sp_weights, _, _, _ = (self._RandomIdsAndWeights( batch_size, vocab_size)) for num_shards, combiner, dtype, ignore_weights in itertools.product( [1, 3], ["sum", "mean", "sqrtn"], [dtypes.float32, dtypes.float64], [True, False]): with self.test_session(): x, params, _ = _EmbeddingParams(num_shards, vocab_size, shape=param_shape, dtype=dtype) y = embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( x, sp_ids, None if ignore_weights else sp_weights, combiner=combiner) x_name = [_PName(i) for i in range(num_shards)] x_init_value = [params[x_n + ":0"] for x_n in x_name] x_shape = [i.shape for i in x_init_value] y_shape = [batch_size] + list( params[_PName(0) + ":0"].shape[1:]) err = gradient_checker.compute_gradient_error( x, x_shape, y, y_shape, x_init_value=x_init_value) self.assertLess(err, 1e-5 if dtype == dtypes.float64 else 2e-3)
def testGradientsEmbeddingLookupSparse(self): vocab_size = 12 batch_size = 4 param_shape = [2, 3] sp_ids, sp_weights, _, _, _ = (self._RandomIdsAndWeights( batch_size, vocab_size)) for num_shards, combiner, dtype, ignore_weights in itertools.product( [1, 3], ["sum", "mean", "sqrtn"], [dtypes.float32, dtypes.float64], [True, False]): with self.cached_session(): x, params, _ = _EmbeddingParams( num_shards, vocab_size, shape=param_shape, dtype=dtype) y = embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( x, sp_ids, None if ignore_weights else sp_weights, combiner=combiner) x_name = [_PName(i) for i in range(num_shards)] x_init_value = [params[x_n + ":0"] for x_n in x_name] x_shape = [i.shape for i in x_init_value] y_shape = [batch_size] + list(params[_PName(0) + ":0"].shape[1:]) err = gradient_checker.compute_gradient_error( x, x_shape, y, y_shape, x_init_value=x_init_value) self.assertLess(err, 1e-5 if dtype == dtypes.float64 else 2e-3)
def testEmbeddingLookupSparse(self): vocab_size = 13 batch_size = 10 param_shape = [2, 5] expected_lookup_result_shape = [None] + param_shape sp_ids, sp_weights, ids, weights, vals_per_batch_entry = ( self._RandomIdsAndWeights(batch_size, vocab_size)) grouped_ids = self._GroupByBatchEntry(ids, vals_per_batch_entry) grouped_weights = self._GroupByBatchEntry(weights, vals_per_batch_entry) grouped_ignored_weights = self._GroupByBatchEntry( np.ones(np.sum(vals_per_batch_entry)), vals_per_batch_entry) for num_shards, combiner, dtype, ignore_weights in itertools.product( [1, 5], ["sum", "mean", "sqrtn"], [dtypes.float32, dtypes.float64], [True, False]): with self.test_session(): p, params, feed_dict = _EmbeddingParams(num_shards, vocab_size, shape=param_shape, dtype=dtype) embedding_sum = \ embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( p, sp_ids, None if ignore_weights else sp_weights, combiner=combiner) self.assertEqual(embedding_sum.get_shape().as_list(), expected_lookup_result_shape) tf_embedding_sum = embedding_sum.eval(feed_dict=feed_dict) np_embedding_sum, np_weight_sum, np_weight_sq_sum = _EmbeddingResult( params, grouped_ids, num_shards, vocab_size, weight_vals=grouped_ignored_weights if ignore_weights else grouped_weights) if combiner == "mean": np_embedding_sum /= np.reshape(np_weight_sum, (batch_size, 1, 1)) if combiner == "sqrtn": np_embedding_sum /= np.reshape(np.sqrt(np_weight_sq_sum), (batch_size, 1, 1)) self.assertAllClose(np_embedding_sum, tf_embedding_sum)
def testEmbeddingLookupSparse(self): vocab_size = 13 batch_size = 10 param_shape = [2, 5] expected_lookup_result_shape = param_shape sp_ids, sp_weights, ids, weights, vals_per_batch_entry = ( self._RandomIdsAndWeights(batch_size, vocab_size)) grouped_ids = self._GroupByBatchEntry(ids, vals_per_batch_entry) grouped_weights = self._GroupByBatchEntry(weights, vals_per_batch_entry) grouped_ignored_weights = self._GroupByBatchEntry( np.ones(np.sum(vals_per_batch_entry)), vals_per_batch_entry) for num_shards, combiner, dtype, ignore_weights in itertools.product( [1, 5], ["sum", "mean", "sqrtn"], [dtypes.float32, dtypes.float64], [True, False]): with self.cached_session(): p, params, feed_dict = _EmbeddingParams( num_shards, vocab_size, shape=param_shape, dtype=dtype) embedding_sum = \ embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( p, sp_ids, None if ignore_weights else sp_weights, combiner=combiner) self.assertEqual(embedding_sum.get_shape().as_list()[1:], expected_lookup_result_shape) tf_embedding_sum = embedding_sum.eval(feed_dict=feed_dict) np_embedding_sum, np_weight_sum, np_weight_sq_sum = _EmbeddingResult( params, grouped_ids, num_shards, vocab_size, weight_vals=grouped_ignored_weights if ignore_weights else grouped_weights) if combiner == "mean": np_embedding_sum /= np.reshape(np_weight_sum, (batch_size, 1, 1)) if combiner == "sqrtn": np_embedding_sum /= np.reshape( np.sqrt(np_weight_sq_sum), (batch_size, 1, 1)) self.assertAllClose(np_embedding_sum, tf_embedding_sum)