def testtensor_to_ndarray(self): t = elasticdl_pb2.Tensor() # No dim defined, should raise. self.assertRaises(ValueError, tensor_to_ndarray, t) # Empty array, should be ok. t.dim.append(0) t.content = b"" arr = tensor_to_ndarray(t) np.testing.assert_array_equal(np.array([], dtype=np.float32), arr) # Pathological case, one of the dimensions is 0. del t.dim[:] t.dim.extend([2, 0, 1, 9]) t.content = b"" arr = tensor_to_ndarray(t) np.testing.assert_array_equal( np.ndarray(shape=[2, 0, 1, 9], dtype=np.float32), arr) t.content = b"\0" * (4 * 12) # Wrong content size, should raise del t.dim[:] t.dim.extend([11]) self.assertRaises(ValueError, tensor_to_ndarray, t) # Compatible dimensions, should be ok. for m in (1, 2, 3, 4, 6, 12): del t.dim[:] t.content = b"\0" * (4 * 12) t.dim.extend([m, 12 // m]) arr = tensor_to_ndarray(t)
def _create_tensor_pb(self, values, indices=None): pb = elasticdl_pb2.Tensor() pb.dim.extend(values.shape) pb.dtype = dtype_numpy_to_tensor(values.dtype) pb.content = values.tobytes() if indices is not None: pb.indices.extend(tuple(indices)) return pb
def pull_embedding_vector(self, request, _): ret = elasticdl_pb2.Tensor() if not request.ids: return ret embedding_vectors = self._parameters.get_embedding_param( request.name, request.ids) tensor = Tensor(values=embedding_vectors) serialize_tensor(tensor, ret) return ret
def verify(values, name=None, indices=None): tensor = Tensor(values, indices, name) pb = elasticdl_pb2.Tensor() serialize_tensor(tensor, pb) tensor_new = Tensor() deserialize_tensor_pb(pb, tensor_new) np.testing.assert_array_equal(values, tensor_new.values) if indices is not None: np.testing.assert_array_equal(indices, tensor_new.indices) if name: self.assertEqual(name, tensor.name)
def ndarray_to_tensor(arr, indices=None): """Convert ndarray to Tensor PB""" if not is_numpy_dtype_allowed(arr.dtype): raise ValueError("Dtype of ndarray %s is not supported", arr.dtype) tensor = elasticdl_pb2.Tensor() tensor.dim.extend(arr.shape) tensor.content = arr.tobytes() if indices: tensor.indices.extend(indices) tensor.dtype = dtype_numpy_to_tensor(arr.dtype) return tensor
def test_deserialize_tensor_pb(self): pb = elasticdl_pb2.Tensor() tensor = Tensor() # No dim defined, should raise. self.assertRaises(ValueError, deserialize_tensor_pb, pb, tensor) # Empty array, should be ok. pb.dim.append(0) pb.content = b"" pb.dtype = tensor_dtype_pb2.DT_FLOAT32 deserialize_tensor_pb(pb, tensor) np.testing.assert_array_equal(np.array([], dtype=np.float32), tensor.values) # Wrong type, should raise del pb.dim[:] pb.dim.append(0) pb.content = b"" pb.dtype = tensor_dtype_pb2.DT_INVALID self.assertRaises(ValueError, deserialize_tensor_pb, pb, tensor) # Pathological case, one of the dimensions is 0. del pb.dim[:] pb.dim.extend([2, 0, 1, 9]) pb.content = b"" pb.dtype = tensor_dtype_pb2.DT_FLOAT32 deserialize_tensor_pb(pb, tensor) np.testing.assert_array_equal( np.ndarray(shape=[2, 0, 1, 9], dtype=np.float32), tensor.values) # Wrong content size, should raise del pb.dim[:] pb.dim.append(11) pb.content = b"\0" * (4 * 12) pb.dtype = tensor_dtype_pb2.DT_FLOAT32 self.assertRaises(ValueError, deserialize_tensor_pb, pb, tensor) # Compatible dimensions, should be ok. for m in (1, 2, 3, 4, 6, 12): for with_inidices in [True, False]: del pb.dim[:] pb.content = b"\0" * (4 * 12) pb.dim.extend([m, 12 // m]) if with_inidices: pb.indices.extend([0] * m) pb.dtype = tensor_dtype_pb2.DT_FLOAT32 deserialize_tensor_pb(pb, tensor) self.assertEqual((m, 12 // m), tensor.values.shape) self.assertTrue(isinstance(tensor.values, np.ndarray)) if tensor.indices is not None: self.assertTrue(isinstance(tensor.indices, np.ndarray))
def ndarray_to_tensor(arr, indices=None): """Convert ndarray to Tensor PB""" if arr.dtype != np.float32: raise ValueError( "expected ndarray to be of float32 type, got %s type", arr.dtype ) tensor = elasticdl_pb2.Tensor() tensor.dim.extend(arr.shape) tensor.content = arr.tobytes() if indices: tensor.indices.extend(indices) return tensor
def to_tensor_pb(self): tensor_pb = elasticdl_pb2.Tensor() serialize_tensor(self, tensor_pb) return tensor_pb