def test_map_int(self):
int_dtypes = [
tf.int8, tf.uint8, tf.uint16, tf.uint32, tf.uint64, tf.int16,
tf.int32, tf.int64
]
for id in int_dtypes:
assert tf.int64 == DatasetMetadata.map_int(id)
assert tf.float32 == DatasetMetadata.map_int(tf.float32)
assert tf.float16 == DatasetMetadata.map_int(tf.float16)
assert tf.float64 == DatasetMetadata.map_int(tf.float64)
assert tf.string == DatasetMetadata.map_int(tf.string)
def build_features(tensors, entity_name):
"""
Create features from metadata, used to deserialize the tfrecord.
:param tensors: list of metadata for all tensors.
:param entity_name: entity by which the records are grouped.
:return: a tuple of context_features and sequence_features
"""
sequence_features = dict()
context_features = dict()
for tensor in tensors:
tensor_dtype = DatasetMetadata.map_int(tensor.dtype)
if tensor.name == entity_name:
# entity_name column is a scalar
context_features[entity_name] = tf.io.FixedLenFeature(
shape=[], dtype=tensor_dtype)
else:
if tensor.isSparse:
# If this is a sparse tensor, we process indices and values separately.
# Note in the metadata, we don't see _indices and _values,
# only the feature name.
indices_name = f"{tensor.name}_{DatasetMetadata.INDICES}"
values_name = f"{tensor.name}_{DatasetMetadata.VALUES}"
sequence_features[indices_name] = tf.io.VarLenFeature(
dtype=tf.int64)
sequence_features[values_name] = tf.io.VarLenFeature(
dtype=tensor_dtype)
else:
context_features[tensor.name] = tf.io.VarLenFeature(
dtype=tensor_dtype)
if len(sequence_features) == 0:
sequence_features = None
if len(context_features) == 0:
context_features = None
return context_features, sequence_features
def build_features(tensors):
"""
Create features from metadata, used to deserialize the tfrecord.
:param tensors: list of metadata for all tensors.
:return: tfrecord features
"""
tf_features = {}
for feature in tensors:
if feature.isSparse:
# If this is a sparse tensor, we process indices and values separately.
# Note in the metadata, we don't see _indices and _values,
# only the feature name.
tf_features[feature.name] = tf.io.SparseFeature(
index_key=f"{feature.name}_{DatasetMetadata.INDICES}",
value_key=f"{feature.name}_{DatasetMetadata.VALUES}",
dtype=DatasetMetadata.map_int(feature.dtype),
size=_unpack_one_element_list(feature.shape))
else:
tf_features[feature.name] = tf.io.FixedLenFeature(
shape=feature.shape,
dtype=DatasetMetadata.map_int(feature.dtype))
return tf_features