def testtensor_to_ndarray(self):
t = elasticdl_pb2.Tensor()
# No dim defined, should raise.
self.assertRaises(ValueError, tensor_to_ndarray, t)
# Empty array, should be ok.
t.dim.append(0)
t.content = b""
arr = tensor_to_ndarray(t)
np.testing.assert_array_equal(np.array([], dtype=np.float32), arr)
# Pathological case, one of the dimensions is 0.
del t.dim[:]
t.dim.extend([2, 0, 1, 9])
t.content = b""
arr = tensor_to_ndarray(t)
np.testing.assert_array_equal(
np.ndarray(shape=[2, 0, 1, 9], dtype=np.float32), arr)
t.content = b"\0" * (4 * 12)
# Wrong content size, should raise
del t.dim[:]
t.dim.extend([11])
self.assertRaises(ValueError, tensor_to_ndarray, t)
# Compatible dimensions, should be ok.
for m in (1, 2, 3, 4, 6, 12):
del t.dim[:]
t.content = b"\0" * (4 * 12)
t.dim.extend([m, 12 // m])
arr = tensor_to_ndarray(t)
def _create_tensor_pb(self, values, indices=None):
pb = elasticdl_pb2.Tensor()
pb.dim.extend(values.shape)
pb.dtype = dtype_numpy_to_tensor(values.dtype)
pb.content = values.tobytes()
if indices is not None:
pb.indices.extend(tuple(indices))
return pb
def pull_embedding_vector(self, request, _):
ret = elasticdl_pb2.Tensor()
if not request.ids:
return ret
embedding_vectors = self._parameters.get_embedding_param(
request.name, request.ids)
tensor = Tensor(values=embedding_vectors)
serialize_tensor(tensor, ret)
return ret
def verify(values, name=None, indices=None):
tensor = Tensor(values, indices, name)
pb = elasticdl_pb2.Tensor()
serialize_tensor(tensor, pb)
tensor_new = Tensor()
deserialize_tensor_pb(pb, tensor_new)
np.testing.assert_array_equal(values, tensor_new.values)
if indices is not None:
np.testing.assert_array_equal(indices, tensor_new.indices)
if name:
self.assertEqual(name, tensor.name)
def ndarray_to_tensor(arr, indices=None):
"""Convert ndarray to Tensor PB"""
if not is_numpy_dtype_allowed(arr.dtype):
raise ValueError("Dtype of ndarray %s is not supported", arr.dtype)
tensor = elasticdl_pb2.Tensor()
tensor.dim.extend(arr.shape)
tensor.content = arr.tobytes()
if indices:
tensor.indices.extend(indices)
tensor.dtype = dtype_numpy_to_tensor(arr.dtype)
return tensor
def test_deserialize_tensor_pb(self):
pb = elasticdl_pb2.Tensor()
tensor = Tensor()
# No dim defined, should raise.
self.assertRaises(ValueError, deserialize_tensor_pb, pb, tensor)
# Empty array, should be ok.
pb.dim.append(0)
pb.content = b""
pb.dtype = tensor_dtype_pb2.DT_FLOAT32
deserialize_tensor_pb(pb, tensor)
np.testing.assert_array_equal(np.array([], dtype=np.float32),
tensor.values)
# Wrong type, should raise
del pb.dim[:]
pb.dim.append(0)
pb.content = b""
pb.dtype = tensor_dtype_pb2.DT_INVALID
self.assertRaises(ValueError, deserialize_tensor_pb, pb, tensor)
# Pathological case, one of the dimensions is 0.
del pb.dim[:]
pb.dim.extend([2, 0, 1, 9])
pb.content = b""
pb.dtype = tensor_dtype_pb2.DT_FLOAT32
deserialize_tensor_pb(pb, tensor)
np.testing.assert_array_equal(
np.ndarray(shape=[2, 0, 1, 9], dtype=np.float32), tensor.values)
# Wrong content size, should raise
del pb.dim[:]
pb.dim.append(11)
pb.content = b"\0" * (4 * 12)
pb.dtype = tensor_dtype_pb2.DT_FLOAT32
self.assertRaises(ValueError, deserialize_tensor_pb, pb, tensor)
# Compatible dimensions, should be ok.
for m in (1, 2, 3, 4, 6, 12):
for with_inidices in [True, False]:
del pb.dim[:]
pb.content = b"\0" * (4 * 12)
pb.dim.extend([m, 12 // m])
if with_inidices:
pb.indices.extend([0] * m)
pb.dtype = tensor_dtype_pb2.DT_FLOAT32
deserialize_tensor_pb(pb, tensor)
self.assertEqual((m, 12 // m), tensor.values.shape)
self.assertTrue(isinstance(tensor.values, np.ndarray))
if tensor.indices is not None:
self.assertTrue(isinstance(tensor.indices, np.ndarray))
def ndarray_to_tensor(arr, indices=None):
"""Convert ndarray to Tensor PB"""
if arr.dtype != np.float32:
raise ValueError(
"expected ndarray to be of float32 type, got %s type", arr.dtype
)
tensor = elasticdl_pb2.Tensor()
tensor.dim.extend(arr.shape)
tensor.content = arr.tobytes()
if indices:
tensor.indices.extend(indices)
return tensor
def to_tensor_pb(self):
tensor_pb = elasticdl_pb2.Tensor()
serialize_tensor(self, tensor_pb)
return tensor_pb
