def parser_fn(proto): serialized = tf.io.parse_single_example(proto, feature_description) deserialized = { k: (tf.sparse.to_dense(sparse_ops.deserialize_sparse(v, K.floatx())) if k != 'eventId' else v) for k, v in serialized.items() } [deserialized[k].set_shape(feature_shape[k]) for k in deserialized.keys()] x = deserialized['image'] x = tf.expand_dims(x, axis=0) x = K.pool2d(x, pool_size=(2, 2), strides=(2, 2), pool_mode='avg') # sum instead of avg x = 4. * x deserialized['image'] = tf.squeeze(x) deserialized['S_image'] = deserialized['image'][:, :, 0] deserialized['C_image'] = deserialized['image'][:, :, 1] deserialized['tseed'] = tf.constant(4., tf.float32) deserialized['tneighbour'] = tf.constant(2., tf.float32) deserialized['tcell'] = tf.constant(0., tf.float32) deserialized['tenergy'] = tf.constant(1000., tf.float32) deserialized['tlocmax'] = tf.constant(500., tf.float32) deserialized['tnum'] = tf.constant(3, tf.int32) return deserialized
def _from_tensor_list(self, flat_value): if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or not flat_value[0].shape.is_compatible_with(tensor_shape.vector(3))): raise ValueError("SparseTensorStructure corresponds to a single " "tf.variant vector of length 3.") return sparse_ops.deserialize_sparse( flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)
def testMapDefunWithVariantTensor(self): @function.defun( input_signature=[tensor_spec.TensorSpec([], dtypes.variant)]) def fn(x): return x st = sparse_tensor.SparseTensor(indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]) serialized = sparse_ops.serialize_sparse_v2(st, out_type=dtypes.variant) serialized = array_ops.stack([serialized, serialized]) map_defun_op = map_defun.map_defun(fn, [serialized], [dtypes.variant], [None])[0] deserialized = sparse_ops.deserialize_sparse(map_defun_op, dtypes.int32) expected = sparse_tensor.SparseTensorValue(indices=[[0, 0, 0], [0, 1, 2], [1, 0, 0], [1, 1, 2]], values=[1, 2, 1, 2], dense_shape=[2, 3, 4]) actual = self.evaluate(deserialized) self.assertValuesEqual(expected, actual)
def _from_compatible_tensor_list(self, flat_value): ret = sparse_ops.deserialize_sparse(flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims) ret.indices.set_shape([None, self._dense_shape.ndims]) ret.dense_shape.set_shape([self._dense_shape.ndims]) return ret
def testMapDefunWithVariantTensorAsCaptured(self): st = sparse_tensor.SparseTensor(indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]) serialized = sparse_ops.serialize_sparse_v2(st, out_type=dtypes.variant) @function.defun( input_signature=[tensor_spec.TensorSpec([], dtypes.int32)]) def fn(x): del x return serialized x = constant_op.constant([0, 0]) map_defun_op = map_defun.map_defun(fn, [x], [dtypes.variant], [None])[0] deserialized = sparse_ops.deserialize_sparse(map_defun_op, dtypes.int32) expected = sparse_tensor.SparseTensorValue(indices=[[0, 0, 0], [0, 1, 2], [1, 0, 0], [1, 1, 2]], values=[1, 2, 1, 2], dense_shape=[2, 3, 4]) actual = self.evaluate(deserialized) self.assertSparseValuesEqual(expected, actual)
def testVariantSerializeDeserializeScalarBatch(self): with self.test_session(use_gpu=False) as sess: indices_value = np.array([[]], dtype=np.int64) values_value = np.array([37], dtype=np.int32) shape_value = np.array([], dtype=np.int64) sparse_tensor = self._SparseTensorPlaceholder() serialized = sparse_ops.serialize_sparse(sparse_tensor, out_type=dtypes.variant) stacked = array_ops.stack([serialized, serialized]) deserialized = sparse_ops.deserialize_sparse(stacked, dtype=dtypes.int32) deserialized_value = sess.run(deserialized, feed_dict={ sparse_tensor.indices: indices_value, sparse_tensor.values: values_value, sparse_tensor.dense_shape: shape_value }) self.assertAllEqual(deserialized_value.indices, np.array([[0], [1]], dtype=np.int64)) self.assertAllEqual(deserialized_value.values, np.array([37, 37], dtype=np.int32)) self.assertAllEqual(deserialized_value.dense_shape, np.array([2], dtype=np.int64))
def testSerializeDeserializeNestedBatch(self): with self.test_session(use_gpu=False) as sess: sp_input = self._SparseTensorValue_5x6(np.arange(6)) serialized = sparse_ops.serialize_sparse(sp_input) serialized = array_ops.stack([serialized, serialized]) serialized = array_ops.stack([serialized, serialized]) sp_deserialized = sparse_ops.deserialize_sparse(serialized, dtype=dtypes.int32) combined_indices, combined_values, combined_shape = sess.run( sp_deserialized) # minibatch 0 self.assertAllEqual(combined_indices[:6, :2], [[0, 0]] * 6) self.assertAllEqual(combined_indices[:6, 2:], sp_input[0]) self.assertAllEqual(combined_values[:6], sp_input[1]) # minibatch 1 self.assertAllEqual(combined_indices[6:12, :2], [[0, 1]] * 6) self.assertAllEqual(combined_indices[6:12, 2:], sp_input[0]) self.assertAllEqual(combined_values[6:12], sp_input[1]) # minibatch 2 self.assertAllEqual(combined_indices[12:18, :2], [[1, 0]] * 6) self.assertAllEqual(combined_indices[12:18, 2:], sp_input[0]) self.assertAllEqual(combined_values[12:18], sp_input[1]) # minibatch 3 self.assertAllEqual(combined_indices[18:, :2], [[1, 1]] * 6) self.assertAllEqual(combined_indices[18:, 2:], sp_input[0]) self.assertAllEqual(combined_values[18:], sp_input[1]) self.assertAllEqual(combined_shape, [2, 2, 5, 6])
def testSerializeDeserializeNestedBatch(self): with self.test_session(use_gpu=False) as sess: sp_input = self._SparseTensorValue_5x6(np.arange(6)) serialized = sparse_ops.serialize_sparse(sp_input) serialized = array_ops.stack([serialized, serialized]) serialized = array_ops.stack([serialized, serialized]) sp_deserialized = sparse_ops.deserialize_sparse( serialized, dtype=dtypes.int32) combined_indices, combined_values, combined_shape = sess.run( sp_deserialized) # minibatch 0 self.assertAllEqual(combined_indices[:6, :2], [[0, 0]] * 6) self.assertAllEqual(combined_indices[:6, 2:], sp_input[0]) self.assertAllEqual(combined_values[:6], sp_input[1]) # minibatch 1 self.assertAllEqual(combined_indices[6:12, :2], [[0, 1]] * 6) self.assertAllEqual(combined_indices[6:12, 2:], sp_input[0]) self.assertAllEqual(combined_values[6:12], sp_input[1]) # minibatch 2 self.assertAllEqual(combined_indices[12:18, :2], [[1, 0]] * 6) self.assertAllEqual(combined_indices[12:18, 2:], sp_input[0]) self.assertAllEqual(combined_values[12:18], sp_input[1]) # minibatch 3 self.assertAllEqual(combined_indices[18:, :2], [[1, 1]] * 6) self.assertAllEqual(combined_indices[18:, 2:], sp_input[0]) self.assertAllEqual(combined_values[18:], sp_input[1]) self.assertAllEqual(combined_shape, [2, 2, 5, 6])
def testSerializeDeserialize(self): with self.test_session(use_gpu=False) as sess: sp_input = self._SparseTensorValue_5x6(np.arange(6)) serialized = sparse_ops.serialize_sparse(sp_input) sp_deserialized = sparse_ops.deserialize_sparse(serialized, dtype=dtypes.int32) indices, values, shape = sess.run(sp_deserialized) self.assertAllEqual(indices, sp_input[0]) self.assertAllEqual(values, sp_input[1]) self.assertAllEqual(shape, sp_input[2])
def testSerializeDeserialize(self): with self.test_session(use_gpu=False) as sess: sp_input = self._SparseTensorValue_5x6(np.arange(6)) serialized = sparse_ops.serialize_sparse(sp_input) sp_deserialized = sparse_ops.deserialize_sparse( serialized, dtype=dtypes.int32) indices, values, shape = sess.run(sp_deserialized) self.assertAllEqual(indices, sp_input[0]) self.assertAllEqual(values, sp_input[1]) self.assertAllEqual(shape, sp_input[2])
def testSerializeDeserializeBatchInconsistentShape(self): with self.test_session(use_gpu=False) as sess: sp_input0 = self._SparseTensorValue_5x6(np.arange(6)) sp_input1 = self._SparseTensorValue_3x4(np.arange(6)) serialized0 = sparse_ops.serialize_sparse(sp_input0) serialized1 = sparse_ops.serialize_sparse(sp_input1) serialized = array_ops.stack([serialized0, serialized1]) sp_deserialized = sparse_ops.deserialize_sparse(serialized, dtype=dtypes.int32) with self.assertRaisesOpError( r"Inconsistent shape across SparseTensors: dimension 0 prior to " r"SparseTensor\[1\] was: 5 but rank of SparseTensor\[1\] is: 3" ): sess.run(sp_deserialized)
def deserialize_sparse_tensors(tensors, types): """Deserializes sparse tensors. Args: tensors: a structure of tensors to deserialize. types: a structure object the holds information about which tensors in `tensors` represent serialized sparse tensors Returns: `tensors` with any serialized sparse tensors replaced by their deserialized version. """ # TODO(b/63669786): support batching of sparse tensors ret = nest.pack_sequence_as(types, [ sparse_ops.deserialize_sparse(tensor, ty.dtype) if isinstance(ty, SparseType) else tensor for (tensor, ty) in zip(nest.flatten(tensors), nest.flatten(types)) ]) return ret
def testSerializeDeserializeBatch(self): with self.test_session(use_gpu=False) as sess: sp_input = self._SparseTensorValue_5x6(np.arange(6)) serialized = sparse_ops.serialize_sparse(sp_input) serialized = array_ops.stack([serialized, serialized]) sp_deserialized = sparse_ops.deserialize_sparse( serialized, dtype=dtypes.int32) combined_indices, combined_values, combined_shape = sess.run( sp_deserialized) self.assertAllEqual(combined_indices[:6, 0], [0] * 6) # minibatch 0 self.assertAllEqual(combined_indices[:6, 1:], sp_input[0]) self.assertAllEqual(combined_indices[6:, 0], [1] * 6) # minibatch 1 self.assertAllEqual(combined_indices[6:, 1:], sp_input[0]) self.assertAllEqual(combined_values[:6], sp_input[1]) self.assertAllEqual(combined_values[6:], sp_input[1]) self.assertAllEqual(combined_shape, [2, 5, 6])
def testSerializeDeserializeBatch(self): with self.test_session(use_gpu=False) as sess: sp_input = self._SparseTensorValue_5x6(np.arange(6)) serialized = sparse_ops.serialize_sparse(sp_input) serialized = array_ops.stack([serialized, serialized]) sp_deserialized = sparse_ops.deserialize_sparse( serialized, dtype=dtypes.int32) combined_indices, combined_values, combined_shape = sess.run( sp_deserialized) self.assertAllEqual(combined_indices[:6, 0], [0] * 6) # minibatch 0 self.assertAllEqual(combined_indices[:6, 1:], sp_input[0]) self.assertAllEqual(combined_indices[6:, 0], [1] * 6) # minibatch 1 self.assertAllEqual(combined_indices[6:, 1:], sp_input[0]) self.assertAllEqual(combined_values[:6], sp_input[1]) self.assertAllEqual(combined_values[6:], sp_input[1]) self.assertAllEqual(combined_shape, [2, 5, 6])
def testVariantSerializeDeserializeScalar(self): with self.session(use_gpu=False) as sess: indices_value = np.array([[]], dtype=np.int64) values_value = np.array([37], dtype=np.int32) shape_value = np.array([], dtype=np.int64) sparse_tensor = self._SparseTensorPlaceholder() serialized = sparse_ops.serialize_sparse( sparse_tensor, out_type=dtypes.variant) deserialized = sparse_ops.deserialize_sparse( serialized, dtype=dtypes.int32) deserialized_value = sess.run( deserialized, feed_dict={ sparse_tensor.indices: indices_value, sparse_tensor.values: values_value, sparse_tensor.dense_shape: shape_value }) self.assertAllEqual(deserialized_value.indices, indices_value) self.assertAllEqual(deserialized_value.values, values_value) self.assertAllEqual(deserialized_value.dense_shape, shape_value)
def testMapDefunWithVariantTensorAsCaptured(self): st = sparse_tensor.SparseTensor( indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]) serialized = sparse_ops.serialize_sparse_v2(st, out_type=dtypes.variant) @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.int32)]) def fn(x): del x return serialized x = constant_op.constant([0, 0]) map_defun_op = map_defun.map_defun(fn, [x], [dtypes.variant], [None])[0] deserialized = sparse_ops.deserialize_sparse(map_defun_op, dtypes.int32) expected = sparse_tensor.SparseTensorValue( indices=[[0, 0, 0], [0, 1, 2], [1, 0, 0], [1, 1, 2]], values=[1, 2, 1, 2], dense_shape=[2, 3, 4]) actual = self.evaluate(deserialized) self.assertSparseValuesEqual(expected, actual)
def testMapDefunWithStrTensor(self): @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)]) def fn(x): return x st = sparse_tensor.SparseTensor( indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]) serialized = sparse_ops.serialize_sparse_v2(st, out_type=dtypes.string) serialized = array_ops.stack([serialized, serialized]) map_defun_op = map_defun.map_defun(fn, [serialized], [dtypes.string], [None])[0] deserialized = sparse_ops.deserialize_sparse(map_defun_op, dtypes.int32) expected = sparse_tensor.SparseTensorValue( indices=[[0, 0, 0], [0, 1, 2], [1, 0, 0], [1, 1, 2]], values=[1, 2, 1, 2], dense_shape=[2, 3, 4]) actual = self.evaluate(deserialized) self.assertSparseValuesEqual(expected, actual)
def deserialize_sparse_tensors(tensors, types, shapes, classes): """Deserializes sparse tensors. Args: tensors: a structure of tensors to deserialize. types: a structure that holds information about types of `tensors` shapes: a structure that holds information about shapes of `tensors` classes: a structure of objects that identify the dataset item classes Returns: `tensors` with any serialized sparse tensors replaced by their deserialized version. """ ret = nest.pack_sequence_as(types, [ sparse_ops.deserialize_sparse(tensor, dtype=ty, rank=shape.ndims) if c is sparse_tensor.SparseTensor else tensor for (tensor, ty, shape, c) in zip( nest.flatten(tensors), nest.flatten(types), nest.flatten(shapes), nest.flatten(classes)) ]) return ret
def deserialize_sparse_tensors(tensors, types, shapes, classes): """Deserializes sparse tensors. Args: tensors: a structure of tensors to deserialize. types: a structure that holds information about types of `tensors` shapes: a structure that holds information about shapes of `tensors` classes: a structure of objects that identify the dataset item classes Returns: `tensors` with any serialized sparse tensors replaced by their deserialized version. """ ret = nest.pack_sequence_as(types, [ sparse_ops.deserialize_sparse(tensor, dtype=ty, rank=shape.ndims) if c is sparse_tensor.SparseTensor else tensor for (tensor, ty, shape, c) in zip( nest.flatten(tensors), nest.flatten(types), nest.flatten(shapes), nest.flatten(classes)) ]) return ret
def _from_compatible_tensor_list(self, flat_value): ret = sparse_ops.deserialize_sparse( flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims) ret.indices.set_shape([None, self._dense_shape.ndims]) ret.dense_shape.set_shape([self._dense_shape.ndims]) return ret
def _from_compatible_tensor_list(self, flat_value): return sparse_ops.deserialize_sparse(flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)
def f(): return sparse_ops.deserialize_sparse( serialized_sparse=mu_lock, dtype=dtypes.int32)
def _from_compatible_tensor_list(self, flat_value): return sparse_ops.deserialize_sparse( flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)