def parser_fn(proto):
serialized = tf.io.parse_single_example(proto, feature_description)
deserialized = {
k: (tf.sparse.to_dense(sparse_ops.deserialize_sparse(v, K.floatx()))
if k != 'eventId' else v)
for k, v in serialized.items()
}
[deserialized[k].set_shape(feature_shape[k]) for k in deserialized.keys()]
x = deserialized['image']
x = tf.expand_dims(x, axis=0)
x = K.pool2d(x, pool_size=(2, 2), strides=(2, 2), pool_mode='avg')
# sum instead of avg
x = 4. * x
deserialized['image'] = tf.squeeze(x)
deserialized['S_image'] = deserialized['image'][:, :, 0]
deserialized['C_image'] = deserialized['image'][:, :, 1]
deserialized['tseed'] = tf.constant(4., tf.float32)
deserialized['tneighbour'] = tf.constant(2., tf.float32)
deserialized['tcell'] = tf.constant(0., tf.float32)
deserialized['tenergy'] = tf.constant(1000., tf.float32)
deserialized['tlocmax'] = tf.constant(500., tf.float32)
deserialized['tnum'] = tf.constant(3, tf.int32)
return deserialized
def _from_tensor_list(self, flat_value):
if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or
not flat_value[0].shape.is_compatible_with(tensor_shape.vector(3))):
raise ValueError("SparseTensorStructure corresponds to a single "
"tf.variant vector of length 3.")
return sparse_ops.deserialize_sparse(
flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)
def testMapDefunWithVariantTensor(self):
@function.defun(
input_signature=[tensor_spec.TensorSpec([], dtypes.variant)])
def fn(x):
return x
st = sparse_tensor.SparseTensor(indices=[[0, 0], [1, 2]],
values=[1, 2],
dense_shape=[3, 4])
serialized = sparse_ops.serialize_sparse_v2(st,
out_type=dtypes.variant)
serialized = array_ops.stack([serialized, serialized])
map_defun_op = map_defun.map_defun(fn, [serialized], [dtypes.variant],
[None])[0]
deserialized = sparse_ops.deserialize_sparse(map_defun_op,
dtypes.int32)
expected = sparse_tensor.SparseTensorValue(indices=[[0, 0,
0], [0, 1, 2],
[1, 0, 0],
[1, 1, 2]],
values=[1, 2, 1, 2],
dense_shape=[2, 3, 4])
actual = self.evaluate(deserialized)
self.assertValuesEqual(expected, actual)
def _from_compatible_tensor_list(self, flat_value):
ret = sparse_ops.deserialize_sparse(flat_value[0],
dtype=self._dtype,
rank=self._dense_shape.ndims)
ret.indices.set_shape([None, self._dense_shape.ndims])
ret.dense_shape.set_shape([self._dense_shape.ndims])
return ret
def testMapDefunWithVariantTensorAsCaptured(self):
st = sparse_tensor.SparseTensor(indices=[[0, 0], [1, 2]],
values=[1, 2],
dense_shape=[3, 4])
serialized = sparse_ops.serialize_sparse_v2(st,
out_type=dtypes.variant)
@function.defun(
input_signature=[tensor_spec.TensorSpec([], dtypes.int32)])
def fn(x):
del x
return serialized
x = constant_op.constant([0, 0])
map_defun_op = map_defun.map_defun(fn, [x], [dtypes.variant],
[None])[0]
deserialized = sparse_ops.deserialize_sparse(map_defun_op,
dtypes.int32)
expected = sparse_tensor.SparseTensorValue(indices=[[0, 0,
0], [0, 1, 2],
[1, 0, 0],
[1, 1, 2]],
values=[1, 2, 1, 2],
dense_shape=[2, 3, 4])
actual = self.evaluate(deserialized)
self.assertSparseValuesEqual(expected, actual)
def testVariantSerializeDeserializeScalarBatch(self):
with self.test_session(use_gpu=False) as sess:
indices_value = np.array([[]], dtype=np.int64)
values_value = np.array([37], dtype=np.int32)
shape_value = np.array([], dtype=np.int64)
sparse_tensor = self._SparseTensorPlaceholder()
serialized = sparse_ops.serialize_sparse(sparse_tensor,
out_type=dtypes.variant)
stacked = array_ops.stack([serialized, serialized])
deserialized = sparse_ops.deserialize_sparse(stacked,
dtype=dtypes.int32)
deserialized_value = sess.run(deserialized,
feed_dict={
sparse_tensor.indices:
indices_value,
sparse_tensor.values:
values_value,
sparse_tensor.dense_shape:
shape_value
})
self.assertAllEqual(deserialized_value.indices,
np.array([[0], [1]], dtype=np.int64))
self.assertAllEqual(deserialized_value.values,
np.array([37, 37], dtype=np.int32))
self.assertAllEqual(deserialized_value.dense_shape,
np.array([2], dtype=np.int64))
def testSerializeDeserializeNestedBatch(self):
with self.test_session(use_gpu=False) as sess:
sp_input = self._SparseTensorValue_5x6(np.arange(6))
serialized = sparse_ops.serialize_sparse(sp_input)
serialized = array_ops.stack([serialized, serialized])
serialized = array_ops.stack([serialized, serialized])
sp_deserialized = sparse_ops.deserialize_sparse(serialized,
dtype=dtypes.int32)
combined_indices, combined_values, combined_shape = sess.run(
sp_deserialized)
# minibatch 0
self.assertAllEqual(combined_indices[:6, :2], [[0, 0]] * 6)
self.assertAllEqual(combined_indices[:6, 2:], sp_input[0])
self.assertAllEqual(combined_values[:6], sp_input[1])
# minibatch 1
self.assertAllEqual(combined_indices[6:12, :2], [[0, 1]] * 6)
self.assertAllEqual(combined_indices[6:12, 2:], sp_input[0])
self.assertAllEqual(combined_values[6:12], sp_input[1])
# minibatch 2
self.assertAllEqual(combined_indices[12:18, :2], [[1, 0]] * 6)
self.assertAllEqual(combined_indices[12:18, 2:], sp_input[0])
self.assertAllEqual(combined_values[12:18], sp_input[1])
# minibatch 3
self.assertAllEqual(combined_indices[18:, :2], [[1, 1]] * 6)
self.assertAllEqual(combined_indices[18:, 2:], sp_input[0])
self.assertAllEqual(combined_values[18:], sp_input[1])
self.assertAllEqual(combined_shape, [2, 2, 5, 6])
def testSerializeDeserializeNestedBatch(self):
with self.test_session(use_gpu=False) as sess:
sp_input = self._SparseTensorValue_5x6(np.arange(6))
serialized = sparse_ops.serialize_sparse(sp_input)
serialized = array_ops.stack([serialized, serialized])
serialized = array_ops.stack([serialized, serialized])
sp_deserialized = sparse_ops.deserialize_sparse(
serialized, dtype=dtypes.int32)
combined_indices, combined_values, combined_shape = sess.run(
sp_deserialized)
# minibatch 0
self.assertAllEqual(combined_indices[:6, :2], [[0, 0]] * 6)
self.assertAllEqual(combined_indices[:6, 2:], sp_input[0])
self.assertAllEqual(combined_values[:6], sp_input[1])
# minibatch 1
self.assertAllEqual(combined_indices[6:12, :2], [[0, 1]] * 6)
self.assertAllEqual(combined_indices[6:12, 2:], sp_input[0])
self.assertAllEqual(combined_values[6:12], sp_input[1])
# minibatch 2
self.assertAllEqual(combined_indices[12:18, :2], [[1, 0]] * 6)
self.assertAllEqual(combined_indices[12:18, 2:], sp_input[0])
self.assertAllEqual(combined_values[12:18], sp_input[1])
# minibatch 3
self.assertAllEqual(combined_indices[18:, :2], [[1, 1]] * 6)
self.assertAllEqual(combined_indices[18:, 2:], sp_input[0])
self.assertAllEqual(combined_values[18:], sp_input[1])
self.assertAllEqual(combined_shape, [2, 2, 5, 6])
def testSerializeDeserialize(self):
with self.test_session(use_gpu=False) as sess:
sp_input = self._SparseTensorValue_5x6(np.arange(6))
serialized = sparse_ops.serialize_sparse(sp_input)
sp_deserialized = sparse_ops.deserialize_sparse(serialized,
dtype=dtypes.int32)
indices, values, shape = sess.run(sp_deserialized)
self.assertAllEqual(indices, sp_input[0])
self.assertAllEqual(values, sp_input[1])
self.assertAllEqual(shape, sp_input[2])
def testSerializeDeserialize(self):
with self.test_session(use_gpu=False) as sess:
sp_input = self._SparseTensorValue_5x6(np.arange(6))
serialized = sparse_ops.serialize_sparse(sp_input)
sp_deserialized = sparse_ops.deserialize_sparse(
serialized, dtype=dtypes.int32)
indices, values, shape = sess.run(sp_deserialized)
self.assertAllEqual(indices, sp_input[0])
self.assertAllEqual(values, sp_input[1])
self.assertAllEqual(shape, sp_input[2])
def testSerializeDeserializeBatchInconsistentShape(self):
with self.test_session(use_gpu=False) as sess:
sp_input0 = self._SparseTensorValue_5x6(np.arange(6))
sp_input1 = self._SparseTensorValue_3x4(np.arange(6))
serialized0 = sparse_ops.serialize_sparse(sp_input0)
serialized1 = sparse_ops.serialize_sparse(sp_input1)
serialized = array_ops.stack([serialized0, serialized1])
sp_deserialized = sparse_ops.deserialize_sparse(serialized,
dtype=dtypes.int32)
with self.assertRaisesOpError(
r"Inconsistent shape across SparseTensors: dimension 0 prior to "
r"SparseTensor\[1\] was: 5 but rank of SparseTensor\[1\] is: 3"
):
sess.run(sp_deserialized)
def deserialize_sparse_tensors(tensors, types):
"""Deserializes sparse tensors.
Args:
tensors: a structure of tensors to deserialize.
types: a structure object the holds information about which tensors in
`tensors` represent serialized sparse tensors
Returns:
`tensors` with any serialized sparse tensors replaced by their deserialized
version.
"""
# TODO(b/63669786): support batching of sparse tensors
ret = nest.pack_sequence_as(types, [
sparse_ops.deserialize_sparse(tensor, ty.dtype)
if isinstance(ty, SparseType) else tensor
for (tensor, ty) in zip(nest.flatten(tensors), nest.flatten(types))
])
return ret
def testSerializeDeserializeBatch(self):
with self.test_session(use_gpu=False) as sess:
sp_input = self._SparseTensorValue_5x6(np.arange(6))
serialized = sparse_ops.serialize_sparse(sp_input)
serialized = array_ops.stack([serialized, serialized])
sp_deserialized = sparse_ops.deserialize_sparse(
serialized, dtype=dtypes.int32)
combined_indices, combined_values, combined_shape = sess.run(
sp_deserialized)
self.assertAllEqual(combined_indices[:6, 0], [0] * 6) # minibatch 0
self.assertAllEqual(combined_indices[:6, 1:], sp_input[0])
self.assertAllEqual(combined_indices[6:, 0], [1] * 6) # minibatch 1
self.assertAllEqual(combined_indices[6:, 1:], sp_input[0])
self.assertAllEqual(combined_values[:6], sp_input[1])
self.assertAllEqual(combined_values[6:], sp_input[1])
self.assertAllEqual(combined_shape, [2, 5, 6])
def testSerializeDeserializeBatch(self):
with self.test_session(use_gpu=False) as sess:
sp_input = self._SparseTensorValue_5x6(np.arange(6))
serialized = sparse_ops.serialize_sparse(sp_input)
serialized = array_ops.stack([serialized, serialized])
sp_deserialized = sparse_ops.deserialize_sparse(
serialized, dtype=dtypes.int32)
combined_indices, combined_values, combined_shape = sess.run(
sp_deserialized)
self.assertAllEqual(combined_indices[:6, 0], [0] * 6) # minibatch 0
self.assertAllEqual(combined_indices[:6, 1:], sp_input[0])
self.assertAllEqual(combined_indices[6:, 0], [1] * 6) # minibatch 1
self.assertAllEqual(combined_indices[6:, 1:], sp_input[0])
self.assertAllEqual(combined_values[:6], sp_input[1])
self.assertAllEqual(combined_values[6:], sp_input[1])
self.assertAllEqual(combined_shape, [2, 5, 6])
def testVariantSerializeDeserializeScalar(self):
with self.session(use_gpu=False) as sess:
indices_value = np.array([[]], dtype=np.int64)
values_value = np.array([37], dtype=np.int32)
shape_value = np.array([], dtype=np.int64)
sparse_tensor = self._SparseTensorPlaceholder()
serialized = sparse_ops.serialize_sparse(
sparse_tensor, out_type=dtypes.variant)
deserialized = sparse_ops.deserialize_sparse(
serialized, dtype=dtypes.int32)
deserialized_value = sess.run(
deserialized,
feed_dict={
sparse_tensor.indices: indices_value,
sparse_tensor.values: values_value,
sparse_tensor.dense_shape: shape_value
})
self.assertAllEqual(deserialized_value.indices, indices_value)
self.assertAllEqual(deserialized_value.values, values_value)
self.assertAllEqual(deserialized_value.dense_shape, shape_value)
def testMapDefunWithVariantTensorAsCaptured(self):
st = sparse_tensor.SparseTensor(
indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])
serialized = sparse_ops.serialize_sparse_v2(st, out_type=dtypes.variant)
@function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.int32)])
def fn(x):
del x
return serialized
x = constant_op.constant([0, 0])
map_defun_op = map_defun.map_defun(fn, [x], [dtypes.variant], [None])[0]
deserialized = sparse_ops.deserialize_sparse(map_defun_op, dtypes.int32)
expected = sparse_tensor.SparseTensorValue(
indices=[[0, 0, 0], [0, 1, 2], [1, 0, 0], [1, 1, 2]],
values=[1, 2, 1, 2],
dense_shape=[2, 3, 4])
actual = self.evaluate(deserialized)
self.assertSparseValuesEqual(expected, actual)
def testMapDefunWithStrTensor(self):
@function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)])
def fn(x):
return x
st = sparse_tensor.SparseTensor(
indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])
serialized = sparse_ops.serialize_sparse_v2(st, out_type=dtypes.string)
serialized = array_ops.stack([serialized, serialized])
map_defun_op = map_defun.map_defun(fn, [serialized], [dtypes.string],
[None])[0]
deserialized = sparse_ops.deserialize_sparse(map_defun_op, dtypes.int32)
expected = sparse_tensor.SparseTensorValue(
indices=[[0, 0, 0], [0, 1, 2], [1, 0, 0], [1, 1, 2]],
values=[1, 2, 1, 2],
dense_shape=[2, 3, 4])
actual = self.evaluate(deserialized)
self.assertSparseValuesEqual(expected, actual)
def deserialize_sparse_tensors(tensors, types, shapes, classes):
"""Deserializes sparse tensors.
Args:
tensors: a structure of tensors to deserialize.
types: a structure that holds information about types of `tensors`
shapes: a structure that holds information about shapes of `tensors`
classes: a structure of objects that identify the dataset item classes
Returns:
`tensors` with any serialized sparse tensors replaced by their deserialized
version.
"""
ret = nest.pack_sequence_as(types, [
sparse_ops.deserialize_sparse(tensor, dtype=ty, rank=shape.ndims)
if c is sparse_tensor.SparseTensor else tensor
for (tensor, ty, shape, c) in zip(
nest.flatten(tensors), nest.flatten(types), nest.flatten(shapes),
nest.flatten(classes))
])
return ret
def deserialize_sparse_tensors(tensors, types, shapes, classes):
"""Deserializes sparse tensors.
Args:
tensors: a structure of tensors to deserialize.
types: a structure that holds information about types of `tensors`
shapes: a structure that holds information about shapes of `tensors`
classes: a structure of objects that identify the dataset item classes
Returns:
`tensors` with any serialized sparse tensors replaced by their deserialized
version.
"""
ret = nest.pack_sequence_as(types, [
sparse_ops.deserialize_sparse(tensor, dtype=ty, rank=shape.ndims)
if c is sparse_tensor.SparseTensor else tensor
for (tensor, ty, shape, c) in zip(
nest.flatten(tensors), nest.flatten(types), nest.flatten(shapes),
nest.flatten(classes))
])
return ret
def _from_compatible_tensor_list(self, flat_value):
ret = sparse_ops.deserialize_sparse(
flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)
ret.indices.set_shape([None, self._dense_shape.ndims])
ret.dense_shape.set_shape([self._dense_shape.ndims])
return ret
def _from_compatible_tensor_list(self, flat_value):
return sparse_ops.deserialize_sparse(flat_value[0],
dtype=self._dtype,
rank=self._dense_shape.ndims)
def f():
return sparse_ops.deserialize_sparse(
serialized_sparse=mu_lock, dtype=dtypes.int32)
def _from_compatible_tensor_list(self, flat_value):
return sparse_ops.deserialize_sparse(
flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)
