def __init__(self, input_dataset, batch_size, row_shape): """See `Dataset.dense_to_sparse_batch()` for more details.""" if not isinstance(dataset_ops.get_legacy_output_types(input_dataset), dtypes.DType): raise TypeError( "DenseToSparseDataset requires an input whose elements " "have a single component, whereas the input has %r." % dataset_ops.get_legacy_output_types(input_dataset)) self._input_dataset = input_dataset self._batch_size = batch_size self._row_shape = row_shape self._structure = structure.SparseTensorStructure( dataset_ops.get_legacy_output_types(input_dataset), tensor_shape.vector(None).concatenate(self._row_shape)) if compat.forward_compatible(2019, 8, 3): variant_tensor = ged_ops.dense_to_sparse_batch_dataset( self._input_dataset._variant_tensor, # pylint: disable=protected-access self._batch_size, row_shape=convert.partial_shape_to_tensor(self._row_shape), **self._flat_structure) else: variant_tensor = ged_ops.experimental_dense_to_sparse_batch_dataset( self._input_dataset._variant_tensor, # pylint: disable=protected-access self._batch_size, row_shape=convert.partial_shape_to_tensor(self._row_shape), **self._flat_structure) super(_DenseToSparseBatchDataset, self).__init__(input_dataset, variant_tensor)
def testPartialShapeToTensorKnownDimension(self): with self.test_session() as sess: self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor( tensor_shape.TensorShape([1])))) self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor((1,)))) self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor([1]))) self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor( constant_op.constant([1], dtype=dtypes.int64))))
def testPartialShapeToTensorScalar(self): with self.cached_session() as sess: self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor( tensor_shape.TensorShape([])))) self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(()))) self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor([]))) self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor( constant_op.constant([], dtype=dtypes.int64))))
def testPartialShapeToTensorUnknownDimension(self): self.assertAllEqual([-1], self.evaluate( convert.partial_shape_to_tensor( tensor_shape.TensorShape([None])))) self.assertAllEqual([-1], self.evaluate(convert.partial_shape_to_tensor((None,)))) self.assertAllEqual([-1], self.evaluate(convert.partial_shape_to_tensor([None]))) self.assertAllEqual([-1], self.evaluate(convert.partial_shape_to_tensor([-1]))) self.assertAllEqual([-1], self.evaluate( convert.partial_shape_to_tensor( constant_op.constant([-1], dtype=dtypes.int64)))) with self.assertRaisesRegexp( ValueError, r"The given shape .* must be a 1-D tensor of tf.int64 " r"values, but the shape was \(2, 2\)."): convert.partial_shape_to_tensor(constant_op.constant( [[1, 1], [1, 1]], dtype=dtypes.int64)) with self.assertRaisesRegexp( TypeError, r"The given shape .* must be a 1-D tensor of tf.int64 " r"values, but the element type was float32."): convert.partial_shape_to_tensor(constant_op.constant([1., 1.]))
def testPartialShapeToTensorUnknownDimension(self): with self.test_session() as sess: self.assertAllEqual([-1], sess.run( convert.partial_shape_to_tensor( tensor_shape.TensorShape([None])))) self.assertAllEqual([-1], sess.run( convert.partial_shape_to_tensor((None, )))) self.assertAllEqual([-1], sess.run( convert.partial_shape_to_tensor([None]))) self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor([-1 ]))) self.assertAllEqual([-1], sess.run( convert.partial_shape_to_tensor( constant_op.constant( [-1], dtype=dtypes.int64)))) with self.assertRaisesRegexp( ValueError, r"The given shape .* must be a 1-D tensor of tf.int64 " r"values, but the shape was \(2, 2\)."): convert.partial_shape_to_tensor( constant_op.constant([[1, 1], [1, 1]], dtype=dtypes.int64)) with self.assertRaisesRegexp( TypeError, r"The given shape .* must be a 1-D tensor of tf.int64 " r"values, but the element type was float32."): convert.partial_shape_to_tensor(constant_op.constant([1., 1.]))
def testPartialShapeToTensorScalar(self): self.assertAllEqual([], self.evaluate( convert.partial_shape_to_tensor( tensor_shape.TensorShape([])))) self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor(()))) self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor([]))) self.assertAllEqual([], self.evaluate( convert.partial_shape_to_tensor( constant_op.constant([], dtype=dtypes.int64))))
def testPartialShapeToTensorKnownDimension(self): self.assertAllEqual([1], self.evaluate( convert.partial_shape_to_tensor( tensor_shape.TensorShape([1])))) self.assertAllEqual([1], self.evaluate( convert.partial_shape_to_tensor((1,)))) self.assertAllEqual([1], self.evaluate( convert.partial_shape_to_tensor([1]))) self.assertAllEqual([1], self.evaluate( convert.partial_shape_to_tensor( constant_op.constant([1], dtype=dtypes.int64))))
def testPartialShapeToTensorScalar(self): self.assertAllEqual([], self.evaluate( convert.partial_shape_to_tensor( tensor_shape.TensorShape([])))) self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor(()))) self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor([]))) self.assertAllEqual([], self.evaluate( convert.partial_shape_to_tensor( constant_op.constant([], dtype=dtypes.int64))))
def testPartialShapeToTensorScalar(self): with self.test_session() as sess: self.assertAllEqual([], sess.run( convert.partial_shape_to_tensor( tensor_shape.TensorShape([])))) self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(()))) self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor([]))) self.assertAllEqual([], sess.run( convert.partial_shape_to_tensor( constant_op.constant( [], dtype=dtypes.int64))))
def testPartialShapeToTensorKnownDimension(self): with self.cached_session() as sess: self.assertAllEqual([1], sess.run( convert.partial_shape_to_tensor( tensor_shape.TensorShape([1])))) self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor( (1, )))) self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor([1]))) self.assertAllEqual([1], sess.run( convert.partial_shape_to_tensor( constant_op.constant( [1], dtype=dtypes.int64))))
def testPartialShapeToTensorKnownDimension(self): self.assertAllEqual([1], self.evaluate( convert.partial_shape_to_tensor( tensor_shape.TensorShape([1])))) self.assertAllEqual([1], self.evaluate( convert.partial_shape_to_tensor((1, )))) self.assertAllEqual([1], self.evaluate(convert.partial_shape_to_tensor([1 ]))) self.assertAllEqual([1], self.evaluate( convert.partial_shape_to_tensor( constant_op.constant([1], dtype=dtypes.int64))))
def _as_variant_tensor(self): return gen_dataset_ops.dense_to_sparse_batch_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._batch_size, row_shape=convert.partial_shape_to_tensor(self._row_shape), output_shapes=nest.flatten( sparse.as_dense_shapes(self.output_shapes, self.output_classes)), output_types=nest.flatten( sparse.as_dense_types(self.output_types, self.output_classes)))
def __init__(self, input_dataset, batch_size, row_shape): """See `Dataset.dense_to_sparse_batch()` for more details.""" if not isinstance(input_dataset.output_types, dtypes.DType): raise TypeError("DenseToSparseDataset requires an input whose elements " "have a single component, whereas the input has %r." % input_dataset.output_types) self._input_dataset = input_dataset self._batch_size = batch_size self._row_shape = row_shape self._structure = structure.SparseTensorStructure( input_dataset.output_types, tensor_shape.vector(None).concatenate(self._row_shape)) variant_tensor = ged_ops.experimental_dense_to_sparse_batch_dataset( self._input_dataset._variant_tensor, # pylint: disable=protected-access self._batch_size, row_shape=convert.partial_shape_to_tensor(self._row_shape), **dataset_ops.flat_structure(self)) super(_DenseToSparseBatchDataset, self).__init__(input_dataset, variant_tensor)
def __init__(self, input_dataset, batch_size, row_shape): """See `Dataset.dense_to_sparse_batch()` for more details.""" if not isinstance( dataset_ops.get_legacy_output_types(input_dataset), dtypes.DType): raise TypeError("`dense_to_sparse_batch` requires an input dataset whose " "elements have a single component, but the given dataset " "has the following component types: " f"{dataset_ops.get_legacy_output_types(input_dataset)}.") self._input_dataset = input_dataset self._batch_size = batch_size self._row_shape = row_shape self._element_spec = sparse_tensor.SparseTensorSpec( tensor_shape.TensorShape([None]).concatenate(self._row_shape), dataset_ops.get_legacy_output_types(input_dataset)) variant_tensor = ged_ops.dense_to_sparse_batch_dataset( self._input_dataset._variant_tensor, # pylint: disable=protected-access self._batch_size, row_shape=convert.partial_shape_to_tensor(self._row_shape), **self._flat_structure) super(_DenseToSparseBatchDataset, self).__init__(input_dataset, variant_tensor)
def _as_variant_tensor(self): return gen_dataset_ops.dense_to_sparse_batch_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._batch_size, row_shape=convert.partial_shape_to_tensor(self._row_shape), **dataset_ops.flat_structure(self))
def max_init_fn(_): return convert.partial_shape_to_tensor(padded_shape)
def _padded_batch_dense_window(dataset, padded_shape, padding_value=None): """Batches a window of dense tensors with padding.""" padded_shape = math_ops.cast( convert.partial_shape_to_tensor(padded_shape), dtypes.int32) def key_fn(_): return np.int64(0) def max_init_fn(_): return padded_shape def max_reduce_fn(state, value): """Computes the maximum shape to pad to.""" condition = math_ops.reduce_all( math_ops.logical_or( math_ops.less_equal(array_ops.shape(value), padded_shape), math_ops.equal(padded_shape, -1))) assert_op = control_flow_ops.Assert(condition, [ "Actual shape greater than padded shape: ", array_ops.shape(value), padded_shape ]) with ops.control_dependencies([assert_op]): return math_ops.maximum(state, array_ops.shape(value)) def finalize_fn(state): return state # Compute the padded shape. max_reducer = grouping.Reducer(max_init_fn, max_reduce_fn, finalize_fn) padded_shape = get_single_element.get_single_element( dataset.apply(grouping.group_by_reducer(key_fn, max_reducer))) if padding_value is None: if dataset.output_types == dtypes.string: padding_value = "" elif dataset.output_types == dtypes.bool: padding_value = False elif dataset.output_types == dtypes.variant: raise TypeError("Unable to create padding for field of type 'variant'") else: padding_value = 0 def batch_init_fn(_): return array_ops.fill( array_ops.concat([np.array([0], dtype=np.int32), padded_shape], 0), constant_op.constant(padding_value, dtype=dataset.output_types)) def batch_reduce_fn(state, value): return array_ops.concat([state, [value]], 0) def pad_fn(value): shape = array_ops.shape(value) left = array_ops.zeros_like(shape) right = padded_shape - shape return array_ops.pad( value, array_ops.stack([left, right], 1), constant_values=padding_value) batch_reducer = grouping.Reducer(batch_init_fn, batch_reduce_fn, finalize_fn) return get_single_element.get_single_element( dataset.map(pad_fn).apply( grouping.group_by_reducer(key_fn, batch_reducer)))
def max_init_fn(_): return convert.partial_shape_to_tensor(padded_shape)
def _padded_batch_dense_window(dataset, padded_shape, padding_value=None): """Batches a window of dense tensors with padding.""" padded_shape = math_ops.cast( convert.partial_shape_to_tensor(padded_shape), dtypes.int32) def key_fn(_): return np.int64(0) def max_init_fn(_): return padded_shape def max_reduce_fn(state, value): """Computes the maximum shape to pad to.""" condition = math_ops.reduce_all( math_ops.logical_or( math_ops.less_equal(array_ops.shape(value), padded_shape), math_ops.equal(padded_shape, -1))) assert_op = control_flow_ops.Assert(condition, [ "Actual shape greater than padded shape: ", array_ops.shape(value), padded_shape ]) with ops.control_dependencies([assert_op]): return math_ops.maximum(state, array_ops.shape(value)) def finalize_fn(state): return state # Compute the padded shape. max_reducer = grouping.Reducer(max_init_fn, max_reduce_fn, finalize_fn) padded_shape = get_single_element.get_single_element( dataset.apply(grouping.group_by_reducer(key_fn, max_reducer))) dataset_output_types = dataset_ops.get_legacy_output_types(dataset) if padding_value is None: if dataset_output_types == dtypes.string: padding_value = "" elif dataset_output_types == dtypes.bool: padding_value = False elif dataset_output_types == dtypes.variant: raise TypeError("Unable to create padding for field of type 'variant'") else: padding_value = 0 def batch_init_fn(_): batch_shape = array_ops.concat( [np.array([0], dtype=np.int32), padded_shape], 0) return gen_array_ops.empty(batch_shape, dtype=dataset_output_types) def batch_reduce_fn(state, value): return array_ops.concat([state, [value]], 0) def pad_fn(value): shape = array_ops.shape(value) left = array_ops.zeros_like(shape) right = padded_shape - shape return array_ops.pad( value, array_ops.stack([left, right], 1), constant_values=padding_value) batch_reducer = grouping.Reducer(batch_init_fn, batch_reduce_fn, finalize_fn) return get_single_element.get_single_element( dataset.map(pad_fn).apply( grouping.group_by_reducer(key_fn, batch_reducer)))
def _as_variant_tensor(self): return gen_dataset_ops.dense_to_sparse_batch_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._batch_size, row_shape=convert.partial_shape_to_tensor(self._row_shape), **dataset_ops.flat_structure(self))
def testPartialShapeToTensorMultipleDimensions(self): self.assertAllEqual([3, 6], self.evaluate( convert.partial_shape_to_tensor( tensor_shape.TensorShape([3, 6])))) self.assertAllEqual([3, 6], self.evaluate(convert.partial_shape_to_tensor((3, 6)))) self.assertAllEqual([3, 6], self.evaluate(convert.partial_shape_to_tensor([3, 6]))) self.assertAllEqual([3, 6], self.evaluate( convert.partial_shape_to_tensor( constant_op.constant([3, 6], dtype=dtypes.int64)))) self.assertAllEqual([3, -1], self.evaluate( convert.partial_shape_to_tensor( tensor_shape.TensorShape([3, None])))) self.assertAllEqual([3, -1], self.evaluate( convert.partial_shape_to_tensor((3, None)))) self.assertAllEqual([3, -1], self.evaluate( convert.partial_shape_to_tensor([3, None]))) self.assertAllEqual([3, -1], self.evaluate( convert.partial_shape_to_tensor( constant_op.constant([3, -1], dtype=dtypes.int64)))) self.assertAllEqual([-1, -1], self.evaluate( convert.partial_shape_to_tensor( tensor_shape.TensorShape([None, None])))) self.assertAllEqual([-1, -1], self.evaluate( convert.partial_shape_to_tensor((None, None)))) self.assertAllEqual([-1, -1], self.evaluate( convert.partial_shape_to_tensor([None, None]))) self.assertAllEqual([-1, -1], self.evaluate( convert.partial_shape_to_tensor( constant_op.constant([-1, -1], dtype=dtypes.int64))))
def testPartialShapeToTensorMultipleDimensions(self): with self.test_session() as sess: self.assertAllEqual([3, 6], sess.run( convert.partial_shape_to_tensor( tensor_shape.TensorShape([3, 6])))) self.assertAllEqual([3, 6], sess.run( convert.partial_shape_to_tensor((3, 6)))) self.assertAllEqual([3, 6], sess.run( convert.partial_shape_to_tensor([3, 6]))) self.assertAllEqual([3, 6], sess.run( convert.partial_shape_to_tensor( constant_op.constant( [3, 6], dtype=dtypes.int64)))) self.assertAllEqual([3, -1], sess.run( convert.partial_shape_to_tensor( tensor_shape.TensorShape([3, None])))) self.assertAllEqual([3, -1], sess.run( convert.partial_shape_to_tensor( (3, None)))) self.assertAllEqual([3, -1], sess.run( convert.partial_shape_to_tensor([3, None]))) self.assertAllEqual([3, -1], sess.run( convert.partial_shape_to_tensor( constant_op.constant( [3, -1], dtype=dtypes.int64)))) self.assertAllEqual([-1, -1], sess.run( convert.partial_shape_to_tensor( tensor_shape.TensorShape([None, None])))) self.assertAllEqual([-1, -1], sess.run( convert.partial_shape_to_tensor( (None, None)))) self.assertAllEqual( [-1, -1], sess.run(convert.partial_shape_to_tensor([None, None]))) self.assertAllEqual([-1, -1], sess.run( convert.partial_shape_to_tensor( constant_op.constant( [-1, -1], dtype=dtypes.int64))))