def __init__(self, input_dataset, batch_size, row_shape):
"""See `Dataset.dense_to_sparse_batch()` for more details."""
if not isinstance(dataset_ops.get_legacy_output_types(input_dataset),
dtypes.DType):
raise TypeError(
"DenseToSparseDataset requires an input whose elements "
"have a single component, whereas the input has %r." %
dataset_ops.get_legacy_output_types(input_dataset))
self._input_dataset = input_dataset
self._batch_size = batch_size
self._row_shape = row_shape
self._structure = structure.SparseTensorStructure(
dataset_ops.get_legacy_output_types(input_dataset),
tensor_shape.vector(None).concatenate(self._row_shape))
if compat.forward_compatible(2019, 8, 3):
variant_tensor = ged_ops.dense_to_sparse_batch_dataset(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._batch_size,
row_shape=convert.partial_shape_to_tensor(self._row_shape),
**self._flat_structure)
else:
variant_tensor = ged_ops.experimental_dense_to_sparse_batch_dataset(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._batch_size,
row_shape=convert.partial_shape_to_tensor(self._row_shape),
**self._flat_structure)
super(_DenseToSparseBatchDataset,
self).__init__(input_dataset, variant_tensor)
def testPartialShapeToTensorKnownDimension(self):
with self.test_session() as sess:
self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor(
tensor_shape.TensorShape([1]))))
self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor((1,))))
self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor([1])))
self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor(
constant_op.constant([1], dtype=dtypes.int64))))
def testPartialShapeToTensorScalar(self):
with self.cached_session() as sess:
self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(
tensor_shape.TensorShape([]))))
self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(())))
self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor([])))
self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(
constant_op.constant([], dtype=dtypes.int64))))
def testPartialShapeToTensorUnknownDimension(self):
self.assertAllEqual([-1],
self.evaluate(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([None]))))
self.assertAllEqual([-1],
self.evaluate(convert.partial_shape_to_tensor((None,))))
self.assertAllEqual([-1],
self.evaluate(convert.partial_shape_to_tensor([None])))
self.assertAllEqual([-1],
self.evaluate(convert.partial_shape_to_tensor([-1])))
self.assertAllEqual([-1],
self.evaluate(
convert.partial_shape_to_tensor(
constant_op.constant([-1],
dtype=dtypes.int64))))
with self.assertRaisesRegexp(
ValueError, r"The given shape .* must be a 1-D tensor of tf.int64 "
r"values, but the shape was \(2, 2\)."):
convert.partial_shape_to_tensor(constant_op.constant(
[[1, 1], [1, 1]], dtype=dtypes.int64))
with self.assertRaisesRegexp(
TypeError, r"The given shape .* must be a 1-D tensor of tf.int64 "
r"values, but the element type was float32."):
convert.partial_shape_to_tensor(constant_op.constant([1., 1.]))
def testPartialShapeToTensorUnknownDimension(self):
with self.test_session() as sess:
self.assertAllEqual([-1],
sess.run(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([None]))))
self.assertAllEqual([-1],
sess.run(
convert.partial_shape_to_tensor((None, ))))
self.assertAllEqual([-1],
sess.run(
convert.partial_shape_to_tensor([None])))
self.assertAllEqual([-1],
sess.run(convert.partial_shape_to_tensor([-1
])))
self.assertAllEqual([-1],
sess.run(
convert.partial_shape_to_tensor(
constant_op.constant(
[-1], dtype=dtypes.int64))))
with self.assertRaisesRegexp(
ValueError,
r"The given shape .* must be a 1-D tensor of tf.int64 "
r"values, but the shape was \(2, 2\)."):
convert.partial_shape_to_tensor(
constant_op.constant([[1, 1], [1, 1]], dtype=dtypes.int64))
with self.assertRaisesRegexp(
TypeError,
r"The given shape .* must be a 1-D tensor of tf.int64 "
r"values, but the element type was float32."):
convert.partial_shape_to_tensor(constant_op.constant([1., 1.]))
def testPartialShapeToTensorScalar(self):
self.assertAllEqual([],
self.evaluate(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([]))))
self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor(())))
self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor([])))
self.assertAllEqual([],
self.evaluate(
convert.partial_shape_to_tensor(
constant_op.constant([], dtype=dtypes.int64))))
def testPartialShapeToTensorKnownDimension(self):
self.assertAllEqual([1],
self.evaluate(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([1]))))
self.assertAllEqual([1], self.evaluate(
convert.partial_shape_to_tensor((1,))))
self.assertAllEqual([1], self.evaluate(
convert.partial_shape_to_tensor([1])))
self.assertAllEqual([1],
self.evaluate(
convert.partial_shape_to_tensor(
constant_op.constant([1], dtype=dtypes.int64))))
def testPartialShapeToTensorScalar(self):
self.assertAllEqual([],
self.evaluate(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([]))))
self.assertAllEqual([],
self.evaluate(convert.partial_shape_to_tensor(())))
self.assertAllEqual([],
self.evaluate(convert.partial_shape_to_tensor([])))
self.assertAllEqual([],
self.evaluate(
convert.partial_shape_to_tensor(
constant_op.constant([],
dtype=dtypes.int64))))
def testPartialShapeToTensorScalar(self):
with self.test_session() as sess:
self.assertAllEqual([],
sess.run(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([]))))
self.assertAllEqual([],
sess.run(convert.partial_shape_to_tensor(())))
self.assertAllEqual([],
sess.run(convert.partial_shape_to_tensor([])))
self.assertAllEqual([],
sess.run(
convert.partial_shape_to_tensor(
constant_op.constant(
[], dtype=dtypes.int64))))
def testPartialShapeToTensorKnownDimension(self):
with self.cached_session() as sess:
self.assertAllEqual([1],
sess.run(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([1]))))
self.assertAllEqual([1],
sess.run(convert.partial_shape_to_tensor(
(1, ))))
self.assertAllEqual([1],
sess.run(convert.partial_shape_to_tensor([1])))
self.assertAllEqual([1],
sess.run(
convert.partial_shape_to_tensor(
constant_op.constant(
[1], dtype=dtypes.int64))))
def testPartialShapeToTensorKnownDimension(self):
self.assertAllEqual([1],
self.evaluate(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([1]))))
self.assertAllEqual([1],
self.evaluate(
convert.partial_shape_to_tensor((1, ))))
self.assertAllEqual([1],
self.evaluate(convert.partial_shape_to_tensor([1
])))
self.assertAllEqual([1],
self.evaluate(
convert.partial_shape_to_tensor(
constant_op.constant([1],
dtype=dtypes.int64))))
def _as_variant_tensor(self):
return gen_dataset_ops.dense_to_sparse_batch_dataset(
self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access
self._batch_size,
row_shape=convert.partial_shape_to_tensor(self._row_shape),
output_shapes=nest.flatten(
sparse.as_dense_shapes(self.output_shapes, self.output_classes)),
output_types=nest.flatten(
sparse.as_dense_types(self.output_types, self.output_classes)))
def __init__(self, input_dataset, batch_size, row_shape):
"""See `Dataset.dense_to_sparse_batch()` for more details."""
if not isinstance(input_dataset.output_types, dtypes.DType):
raise TypeError("DenseToSparseDataset requires an input whose elements "
"have a single component, whereas the input has %r." %
input_dataset.output_types)
self._input_dataset = input_dataset
self._batch_size = batch_size
self._row_shape = row_shape
self._structure = structure.SparseTensorStructure(
input_dataset.output_types,
tensor_shape.vector(None).concatenate(self._row_shape))
variant_tensor = ged_ops.experimental_dense_to_sparse_batch_dataset(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._batch_size,
row_shape=convert.partial_shape_to_tensor(self._row_shape),
**dataset_ops.flat_structure(self))
super(_DenseToSparseBatchDataset, self).__init__(input_dataset,
variant_tensor)
def __init__(self, input_dataset, batch_size, row_shape):
"""See `Dataset.dense_to_sparse_batch()` for more details."""
if not isinstance(
dataset_ops.get_legacy_output_types(input_dataset), dtypes.DType):
raise TypeError("`dense_to_sparse_batch` requires an input dataset whose "
"elements have a single component, but the given dataset "
"has the following component types: "
f"{dataset_ops.get_legacy_output_types(input_dataset)}.")
self._input_dataset = input_dataset
self._batch_size = batch_size
self._row_shape = row_shape
self._element_spec = sparse_tensor.SparseTensorSpec(
tensor_shape.TensorShape([None]).concatenate(self._row_shape),
dataset_ops.get_legacy_output_types(input_dataset))
variant_tensor = ged_ops.dense_to_sparse_batch_dataset(
self._input_dataset._variant_tensor, # pylint: disable=protected-access
self._batch_size,
row_shape=convert.partial_shape_to_tensor(self._row_shape),
**self._flat_structure)
super(_DenseToSparseBatchDataset, self).__init__(input_dataset,
variant_tensor)
def _as_variant_tensor(self):
return gen_dataset_ops.dense_to_sparse_batch_dataset(
self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access
self._batch_size,
row_shape=convert.partial_shape_to_tensor(self._row_shape),
**dataset_ops.flat_structure(self))
def max_init_fn(_): return convert.partial_shape_to_tensor(padded_shape)
def _padded_batch_dense_window(dataset, padded_shape, padding_value=None):
"""Batches a window of dense tensors with padding."""
padded_shape = math_ops.cast(
convert.partial_shape_to_tensor(padded_shape), dtypes.int32)
def key_fn(_):
return np.int64(0)
def max_init_fn(_):
return padded_shape
def max_reduce_fn(state, value):
"""Computes the maximum shape to pad to."""
condition = math_ops.reduce_all(
math_ops.logical_or(
math_ops.less_equal(array_ops.shape(value), padded_shape),
math_ops.equal(padded_shape, -1)))
assert_op = control_flow_ops.Assert(condition, [
"Actual shape greater than padded shape: ",
array_ops.shape(value), padded_shape
])
with ops.control_dependencies([assert_op]):
return math_ops.maximum(state, array_ops.shape(value))
def finalize_fn(state):
return state
# Compute the padded shape.
max_reducer = grouping.Reducer(max_init_fn, max_reduce_fn, finalize_fn)
padded_shape = get_single_element.get_single_element(
dataset.apply(grouping.group_by_reducer(key_fn, max_reducer)))
if padding_value is None:
if dataset.output_types == dtypes.string:
padding_value = ""
elif dataset.output_types == dtypes.bool:
padding_value = False
elif dataset.output_types == dtypes.variant:
raise TypeError("Unable to create padding for field of type 'variant'")
else:
padding_value = 0
def batch_init_fn(_):
return array_ops.fill(
array_ops.concat([np.array([0], dtype=np.int32), padded_shape], 0),
constant_op.constant(padding_value, dtype=dataset.output_types))
def batch_reduce_fn(state, value):
return array_ops.concat([state, [value]], 0)
def pad_fn(value):
shape = array_ops.shape(value)
left = array_ops.zeros_like(shape)
right = padded_shape - shape
return array_ops.pad(
value, array_ops.stack([left, right], 1), constant_values=padding_value)
batch_reducer = grouping.Reducer(batch_init_fn, batch_reduce_fn, finalize_fn)
return get_single_element.get_single_element(
dataset.map(pad_fn).apply(
grouping.group_by_reducer(key_fn, batch_reducer)))
def max_init_fn(_): return convert.partial_shape_to_tensor(padded_shape)
def _padded_batch_dense_window(dataset, padded_shape, padding_value=None):
"""Batches a window of dense tensors with padding."""
padded_shape = math_ops.cast(
convert.partial_shape_to_tensor(padded_shape), dtypes.int32)
def key_fn(_):
return np.int64(0)
def max_init_fn(_):
return padded_shape
def max_reduce_fn(state, value):
"""Computes the maximum shape to pad to."""
condition = math_ops.reduce_all(
math_ops.logical_or(
math_ops.less_equal(array_ops.shape(value), padded_shape),
math_ops.equal(padded_shape, -1)))
assert_op = control_flow_ops.Assert(condition, [
"Actual shape greater than padded shape: ",
array_ops.shape(value), padded_shape
])
with ops.control_dependencies([assert_op]):
return math_ops.maximum(state, array_ops.shape(value))
def finalize_fn(state):
return state
# Compute the padded shape.
max_reducer = grouping.Reducer(max_init_fn, max_reduce_fn, finalize_fn)
padded_shape = get_single_element.get_single_element(
dataset.apply(grouping.group_by_reducer(key_fn, max_reducer)))
dataset_output_types = dataset_ops.get_legacy_output_types(dataset)
if padding_value is None:
if dataset_output_types == dtypes.string:
padding_value = ""
elif dataset_output_types == dtypes.bool:
padding_value = False
elif dataset_output_types == dtypes.variant:
raise TypeError("Unable to create padding for field of type 'variant'")
else:
padding_value = 0
def batch_init_fn(_):
batch_shape = array_ops.concat(
[np.array([0], dtype=np.int32), padded_shape], 0)
return gen_array_ops.empty(batch_shape, dtype=dataset_output_types)
def batch_reduce_fn(state, value):
return array_ops.concat([state, [value]], 0)
def pad_fn(value):
shape = array_ops.shape(value)
left = array_ops.zeros_like(shape)
right = padded_shape - shape
return array_ops.pad(
value, array_ops.stack([left, right], 1), constant_values=padding_value)
batch_reducer = grouping.Reducer(batch_init_fn, batch_reduce_fn, finalize_fn)
return get_single_element.get_single_element(
dataset.map(pad_fn).apply(
grouping.group_by_reducer(key_fn, batch_reducer)))
def _as_variant_tensor(self):
return gen_dataset_ops.dense_to_sparse_batch_dataset(
self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access
self._batch_size,
row_shape=convert.partial_shape_to_tensor(self._row_shape),
**dataset_ops.flat_structure(self))
def testPartialShapeToTensorMultipleDimensions(self):
self.assertAllEqual([3, 6],
self.evaluate(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([3, 6]))))
self.assertAllEqual([3, 6],
self.evaluate(convert.partial_shape_to_tensor((3, 6))))
self.assertAllEqual([3, 6],
self.evaluate(convert.partial_shape_to_tensor([3, 6])))
self.assertAllEqual([3, 6],
self.evaluate(
convert.partial_shape_to_tensor(
constant_op.constant([3, 6],
dtype=dtypes.int64))))
self.assertAllEqual([3, -1],
self.evaluate(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([3, None]))))
self.assertAllEqual([3, -1],
self.evaluate(
convert.partial_shape_to_tensor((3, None))))
self.assertAllEqual([3, -1],
self.evaluate(
convert.partial_shape_to_tensor([3, None])))
self.assertAllEqual([3, -1],
self.evaluate(
convert.partial_shape_to_tensor(
constant_op.constant([3, -1],
dtype=dtypes.int64))))
self.assertAllEqual([-1, -1],
self.evaluate(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([None, None]))))
self.assertAllEqual([-1, -1],
self.evaluate(
convert.partial_shape_to_tensor((None, None))))
self.assertAllEqual([-1, -1],
self.evaluate(
convert.partial_shape_to_tensor([None, None])))
self.assertAllEqual([-1, -1],
self.evaluate(
convert.partial_shape_to_tensor(
constant_op.constant([-1, -1],
dtype=dtypes.int64))))
def testPartialShapeToTensorMultipleDimensions(self):
with self.test_session() as sess:
self.assertAllEqual([3, 6],
sess.run(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([3, 6]))))
self.assertAllEqual([3, 6],
sess.run(
convert.partial_shape_to_tensor((3, 6))))
self.assertAllEqual([3, 6],
sess.run(
convert.partial_shape_to_tensor([3, 6])))
self.assertAllEqual([3, 6],
sess.run(
convert.partial_shape_to_tensor(
constant_op.constant(
[3, 6], dtype=dtypes.int64))))
self.assertAllEqual([3, -1],
sess.run(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([3, None]))))
self.assertAllEqual([3, -1],
sess.run(
convert.partial_shape_to_tensor(
(3, None))))
self.assertAllEqual([3, -1],
sess.run(
convert.partial_shape_to_tensor([3,
None])))
self.assertAllEqual([3, -1],
sess.run(
convert.partial_shape_to_tensor(
constant_op.constant(
[3, -1], dtype=dtypes.int64))))
self.assertAllEqual([-1, -1],
sess.run(
convert.partial_shape_to_tensor(
tensor_shape.TensorShape([None,
None]))))
self.assertAllEqual([-1, -1],
sess.run(
convert.partial_shape_to_tensor(
(None, None))))
self.assertAllEqual(
[-1, -1],
sess.run(convert.partial_shape_to_tensor([None, None])))
self.assertAllEqual([-1, -1],
sess.run(
convert.partial_shape_to_tensor(
constant_op.constant(
[-1, -1], dtype=dtypes.int64))))
