def testDecodeExampleWithRepeatedImages(self):
image_shape = (2, 3, 3)
image_format = 'png'
image, _ = self.GenerateImage(
image_format=image_format, image_shape=image_shape)
tf_encoded = self._Encoder(image, image_format)
with self.cached_session():
tf_string = tf_encoded.eval()
example = example_pb2.Example(
features=feature_pb2.Features(
feature={
'image/encoded':
feature_pb2.Feature(
bytes_list=feature_pb2.BytesList(
value=[tf_string, tf_string])),
'image/format':
self._StringFeature(image_format),
}))
serialized_example = example.SerializeToString()
with self.cached_session():
serialized_example = array_ops.reshape(serialized_example, shape=[])
decoder = tfexample_decoder.TFExampleDecoder(
keys_to_features={
'image/encoded':
parsing_ops.FixedLenFeature((2,), dtypes.string),
'image/format':
parsing_ops.FixedLenFeature(
(), dtypes.string, default_value=image_format),
},
items_to_handlers={'image': tfexample_decoder.Image(repeated=True)})
[tf_image] = decoder.decode(serialized_example, ['image'])
output_image = tf_image.eval()
self.assertEqual(output_image.shape, (2, 2, 3, 3))
self.assertAllEqual(np.squeeze(output_image[0, :, :, :]), image)
self.assertAllEqual(np.squeeze(output_image[1, :, :, :]), image)
def testDecodeExampleWithItemHandlerCallback(self):
np.random.seed(0)
tensor_shape = (2, 3, 1)
np_array = np.random.rand(2, 3, 1)
example = example_pb2.Example(
features=feature_pb2.Features(feature={
'image/depth_map': self._EncodedFloatFeature(np_array),
}))
serialized_example = example.SerializeToString()
with self.cached_session():
serialized_example = array_ops.reshape(serialized_example, shape=[])
keys_to_features = {
'image/depth_map':
parsing_ops.FixedLenFeature(
tensor_shape,
dtypes.float32,
default_value=array_ops.zeros(tensor_shape))
}
def HandleDepth(keys_to_tensors):
depth = list(keys_to_tensors.values())[0]
depth += 1
return depth
items_to_handlers = {
'depth':
tfexample_decoder.ItemHandlerCallback('image/depth_map',
HandleDepth)
}
decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
[tf_depth] = decoder.decode(serialized_example, ['depth'])
depth = tf_depth.eval()
self.assertAllClose(np_array, depth - 1)
def testDecodeExampleWithBoundingBoxSparse(self):
num_bboxes = 10
np_ymin = np.random.rand(num_bboxes, 1)
np_xmin = np.random.rand(num_bboxes, 1)
np_ymax = np.random.rand(num_bboxes, 1)
np_xmax = np.random.rand(num_bboxes, 1)
np_bboxes = np.hstack([np_ymin, np_xmin, np_ymax, np_xmax])
example = example_pb2.Example(
features=feature_pb2.Features(
feature={
'image/object/bbox/ymin': self._EncodedFloatFeature(np_ymin),
'image/object/bbox/xmin': self._EncodedFloatFeature(np_xmin),
'image/object/bbox/ymax': self._EncodedFloatFeature(np_ymax),
'image/object/bbox/xmax': self._EncodedFloatFeature(np_xmax),
}))
serialized_example = example.SerializeToString()
with self.cached_session():
serialized_example = array_ops.reshape(serialized_example, shape=[])
keys_to_features = {
'image/object/bbox/ymin': parsing_ops.VarLenFeature(dtypes.float32),
'image/object/bbox/xmin': parsing_ops.VarLenFeature(dtypes.float32),
'image/object/bbox/ymax': parsing_ops.VarLenFeature(dtypes.float32),
'image/object/bbox/xmax': parsing_ops.VarLenFeature(dtypes.float32),
}
items_to_handlers = {
'object/bbox':
tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'),
}
decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
[tf_bboxes] = decoder.decode(serialized_example, ['object/bbox'])
bboxes = tf_bboxes.eval()
self.assertAllClose(np_bboxes, bboxes)
def testDecodeExampleWithInt64Tensor(self):
np_array = np.random.randint(1, 10, size=(2, 3, 1))
example = example_pb2.Example(
features=feature_pb2.Features(feature={
'array': self._EncodedInt64Feature(np_array),
}))
serialized_example = example.SerializeToString()
with self.cached_session():
serialized_example = array_ops.reshape(serialized_example, shape=[])
keys_to_features = {
'array': parsing_ops.FixedLenFeature(np_array.shape, dtypes.int64)
}
items_to_handlers = {
'array': tfexample_decoder.Tensor('array'),
}
decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
[tf_array] = decoder.decode(serialized_example, ['array'])
self.assertAllEqual(tf_array.eval(), np_array)
def testDecodeExampleWithVarLenTensorToDense(self):
np_array = np.array([[1, 2, 3], [4, 5, 6]])
example = example_pb2.Example(
features=feature_pb2.Features(feature={
'labels': self._EncodedInt64Feature(np_array),
}))
serialized_example = example.SerializeToString()
with self.cached_session():
serialized_example = array_ops.reshape(serialized_example, shape=[])
keys_to_features = {
'labels': parsing_ops.VarLenFeature(dtype=dtypes.int64),
}
items_to_handlers = {
'labels': tfexample_decoder.Tensor('labels', shape=np_array.shape),
}
decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
[tf_labels] = decoder.decode(serialized_example, ['labels'])
labels = tf_labels.eval()
self.assertAllEqual(labels, np_array)
def parse_example_factory():
"""Parse example factory."""
def _int64_feature(*values):
return feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=values))
def _bytes_feature(*values):
return feature_pb2.Feature(
bytes_list=feature_pb2.BytesList(
value=[v.encode("utf-8") for v in values]))
return dataset_ops.Dataset.from_tensor_slices(
constant_op.constant([
example_pb2.Example(
features=feature_pb2.Features(
feature={
"dense_int": _int64_feature(i),
"dense_str": _bytes_feature(str(i)),
"sparse_int": _int64_feature(i, i * 2, i * 4, i * 8),
"sparse_str": _bytes_feature(*["abc"] * i)
})).SerializeToString() for i in range(10)
]))
def generate_image(self, image_format, image_shape):
"""Generates an image and an example containing the encoded image.
Args:
image_format: the encoding format of the image.
image_shape: the shape of the image to generate.
Returns:
image: the generated image.
example: a TF-example with a feature key 'image/encoded' set to the
serialized image and a feature key 'image/format' set to the image
encoding format ['jpeg', 'JPEG', 'png', 'PNG', 'raw'].
"""
num_pixels = image_shape[0] * image_shape[1] * image_shape[2]
image = np.linspace(0, num_pixels - 1, num=num_pixels).reshape(image_shape).astype(np.uint8)
tf_encoded = self._encode(image, image_format)
example = example_pb2.Example(features=feature_pb2.Features(feature={
'image/encoded': self._encode_bytes_feature(tf_encoded),
'image/format': self._string_feature(image_format)
}))
return image, example.SerializeToString()
def testDecodeExampleShapeKeyTensor(self):
np_image = np.random.rand(2, 3, 1).astype('f')
np_labels = np.array([[[1], [2], [3]], [[4], [5], [6]]])
example = example_pb2.Example(features=feature_pb2.Features(
feature={
'image':
self._EncodedFloatFeature(np_image),
'image/shape':
self._EncodedInt64Feature(np.array(np_image.shape)),
'labels':
self._EncodedInt64Feature(np_labels),
'labels/shape':
self._EncodedInt64Feature(np.array(np_labels.shape)),
}))
serialized_example = example.SerializeToString()
with self.test_session():
serialized_example = array_ops.reshape(serialized_example,
shape=[])
keys_to_features = {
'image': parsing_ops.VarLenFeature(dtype=dtypes.float32),
'image/shape': parsing_ops.VarLenFeature(dtype=dtypes.int64),
'labels': parsing_ops.VarLenFeature(dtype=dtypes.int64),
'labels/shape': parsing_ops.VarLenFeature(dtype=dtypes.int64),
}
items_to_handlers = {
'image':
tfexample_decoder.Tensor('image', shape_keys='image/shape'),
'labels':
tfexample_decoder.Tensor('labels', shape_keys='labels/shape'),
}
decoder = tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
[tf_image, tf_labels] = decoder.decode(serialized_example,
['image', 'labels'])
self.assertAllEqual(tf_image.eval(), np_image)
self.assertAllEqual(tf_labels.eval(), np_labels)
def _create_tf_example(feature_dict):
"""
Creates a tf example protobuf message given a feature dict. The protobuf message is defined here
https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/input.proto#L19
Args:
feature_dict (dict of str -> feature): feature can be any of the following:
int, strings, unicode object, float, or list of any of the previous types.
Returns:
a tf.train.Example including the features
"""
def _create_feature(feature):
feature_list = feature if isinstance(feature, list) else [feature]
# Each feature can be exactly one kind:
# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto#L76
feature_type = type(feature_list[0])
if feature_type == int:
return feature_pb2.Feature(int64_list=feature_pb2.Int64List(
value=feature_list))
elif feature_type == str:
return feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
value=feature_list))
elif feature_type == unicode:
return feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
value=map(lambda x: str(x), feature_list)))
elif feature_type == float:
return feature_pb2.Feature(float_list=feature_pb2.FloatList(
value=feature_list))
else:
message = """Unsupported request data format: {}, {}.
Valid formats: float, int, str any object that implements __iter__
or classification_pb2.ClassificationRequest"""
raise ValueError(message.format(feature, type(feature)))
features = {k: _create_feature(v) for k, v in feature_dict.items()}
return example_pb2.Example(features=feature_pb2.Features(feature=features))
def test_decode_example_multi_shape_key_tensor(self):
np_image = np.random.rand(2, 3, 1).astype('f')
np_labels = np.array([[[1], [2], [3]], [[4], [5], [6]]])
height, width, depth = np_labels.shape
example = example_pb2.Example(features=feature_pb2.Features(feature={
'image': self._encode_float_feature(np_image),
'image/shape': self._encode_int64_feature(np.array(np_image.shape)),
'labels': self._encode_int64_feature(np_labels),
'labels/height': self._encode_int64_feature(np.array([height])),
'labels/width': self._encode_int64_feature(np.array([width])),
'labels/depth': self._encode_int64_feature(np.array([depth])),
}))
serialized_example = example.SerializeToString()
with self.test_session():
serialized_example = array_ops.reshape(serialized_example, shape=[])
keys_to_features = {
'image': parsing_ops.VarLenFeature(dtype=dtypes.float32),
'image/shape': parsing_ops.VarLenFeature(dtype=dtypes.int64),
'labels': parsing_ops.VarLenFeature(dtype=dtypes.int64),
'labels/height': parsing_ops.VarLenFeature(dtype=dtypes.int64),
'labels/width': parsing_ops.VarLenFeature(dtype=dtypes.int64),
'labels/depth': parsing_ops.VarLenFeature(dtype=dtypes.int64),
}
items_to_handlers = {
'image':
tfexample_decoder.Tensor('image', shape_keys='image/shape'),
'labels':
tfexample_decoder.Tensor(
'labels',
shape_keys=['labels/height', 'labels/width', 'labels/depth']),
}
decoder = TFExampleDecoder(keys_to_features, items_to_handlers)
[tf_image, tf_labels] = decoder.decode(serialized_example, ['image', 'labels'])
self.assertAllEqual(tf_image.eval(), np_image)
self.assertAllEqual(tf_labels.eval(), np_labels)
def test_decode_example_with_sparse_tensor(self):
np_indices = np.array([[1], [2], [5]])
np_values = np.array([0.1, 0.2, 0.6]).astype('f')
example = example_pb2.Example(features=feature_pb2.Features(feature={
'indices': self._encode_int64_feature(np_indices),
'values': self._encode_float_feature(np_values),
}))
serialized_example = example.SerializeToString()
with self.test_session():
serialized_example = array_ops.reshape(serialized_example, shape=[])
keys_to_features = {
'indices': parsing_ops.VarLenFeature(dtype=dtypes.int64),
'values': parsing_ops.VarLenFeature(dtype=dtypes.float32),
}
items_to_handlers = {'labels': tfexample_decoder.SparseTensor()}
decoder = TFExampleDecoder(keys_to_features, items_to_handlers)
[tf_labels] = decoder.decode(serialized_example, ['labels'])
labels = tf_labels.eval()
self.assertAllEqual(labels.indices, np_indices)
self.assertAllEqual(labels.values, np_values)
self.assertAllEqual(labels.dense_shape, np_values.shape)
def test_decode_example_with_var_len_tensor(self):
np_array = np.array([[[1], [2], [3]], [[4], [5], [6]]])
example = example_pb2.Example(features=feature_pb2.Features(
feature={
'labels': self._encode_int64_feature(np_array),
}))
serialized_example = example.SerializeToString()
with self.test_session():
serialized_example = array_ops.reshape(serialized_example,
shape=[])
keys_to_features = {
'labels': parsing_ops.VarLenFeature(dtype=dtypes.int64),
}
items_to_handlers = {
'labels': tfexample_decoder.Tensor('labels'),
}
decoder = TFExampleDecoder(keys_to_features, items_to_handlers)
[tf_labels] = decoder.decode(serialized_example, ['labels'])
labels = tf_labels.eval()
self.assertAllEqual(labels, np_array.flatten())
def test_decode_example_with_float_tensor(self):
np_array = np.random.rand(2, 3, 1).astype('f')
example = example_pb2.Example(features=feature_pb2.Features(
feature={
'array': self._encode_float_feature(np_array),
}))
serialized_example = example.SerializeToString()
with self.test_session():
serialized_example = array_ops.reshape(serialized_example,
shape=[])
keys_to_features = {
'array':
parsing_ops.FixedLenFeature(np_array.shape, dtypes.float32)
}
items_to_handlers = {
'array': tfexample_decoder.Tensor('array'),
}
decoder = TFExampleDecoder(keys_to_features, items_to_handlers)
[tf_array] = decoder.decode(serialized_example, ['array'])
self.assertAllEqual(tf_array.eval(), np_array)
def test_parse_single_example(self):
def _int64_feature(*values):
return feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=values))
def _bytes_feature(*values):
return feature_pb2.Feature(
bytes_list=feature_pb2.BytesList(
value=[v.encode("utf-8") for v in values]))
examples = constant_op.constant([
example_pb2.Example(
features=feature_pb2.Features(
feature={
"dense_int": _int64_feature(i),
"dense_str": _bytes_feature(str(i)),
"sparse_int": _int64_feature(i, i * 2, i * 4, i * 8),
"sparse_str": _bytes_feature(*["abc"] * i)
})).SerializeToString() for i in range(10)
])
features = {
"dense_int": parsing_ops.FixedLenFeature((), dtypes.int64, 0),
"dense_str": parsing_ops.FixedLenFeature((), dtypes.string, ""),
"sparse_int": parsing_ops.VarLenFeature(dtypes.int64),
"sparse_str": parsing_ops.VarLenFeature(dtypes.string),
}
def loop_fn(i):
example_proto = array_ops.gather(examples, i)
f = parsing_ops.parse_single_example(example_proto, features)
return f
pfor = pfor_control_flow_ops.pfor(loop_fn, iters=10)
manual = parsing_ops.parse_example(examples, features)
self.run_and_assert_equal(pfor, manual)
def test_decode_example_with_string_tensor(self):
tensor_shape = (2, 3, 1)
np_array = np.array([[['ab'], ['cd'], ['ef']], [['ghi'], ['jkl'], ['mnop']]])
example = example_pb2.Example(features=feature_pb2.Features(feature={
'labels': self._bytes_feature(np_array),
}))
serialized_example = example.SerializeToString()
with self.test_session():
serialized_example = array_ops.reshape(serialized_example, shape=[])
keys_to_features = {
'labels': parsing_ops.FixedLenFeature(
tensor_shape, dtypes.string,
default_value=constant_op.constant('', shape=tensor_shape, dtype=dtypes.string))
}
items_to_handlers = {'labels': tfexample_decoder.Tensor('labels')}
decoder = TFExampleDecoder(keys_to_features, items_to_handlers)
[tf_labels] = decoder.decode(serialized_example, ['labels'])
labels = tf_labels.eval()
labels = labels.astype(np_array.dtype)
self.assertTrue(np.array_equal(np_array, labels))
def create_tf_record(self):
path = os.path.join(self.get_temp_dir(), 'tfrecord')
writer = tf.python_io.TFRecordWriter(path)
image_tensor = np.random.randint(255, size=(4, 5, 3)).astype(np.uint8)
with self.test_session():
encoded_jpeg = tf.image.encode_jpeg(
tf.constant(image_tensor)).eval()
example = example_pb2.Example(features=feature_pb2.Features(
feature={
'image/encoded':
feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
value=[encoded_jpeg])),
'image/format':
feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
value=['jpeg'.encode('utf-8')])),
'image/transcript':
feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
value=['hello'.encode('utf-8')]))
}))
writer.write(example.SerializeToString())
writer.close()
return path
def create_tf_record(self):
path = os.path.join(self.get_temp_dir(), 'tfrecord')
writer = tf.python_io.TFRecordWriter(path)
image_tensor = np.random.randint(255, size=(4, 5, 3)).astype(np.uint8)
with self.test_session():
encoded_jpeg = tf.image.encode_jpeg(
tf.constant(image_tensor)).eval()
example = example_pb2.Example(features=feature_pb2.Features(
feature={
'image/encoded':
feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
value=[encoded_jpeg])),
'image/format':
feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
value=['jpeg'.encode('utf-8')])),
'image/object/bbox/xmin':
feature_pb2.Feature(float_list=feature_pb2.FloatList(
value=[0.0])),
'image/object/bbox/xmax':
feature_pb2.Feature(float_list=feature_pb2.FloatList(
value=[1.0])),
'image/object/bbox/ymin':
feature_pb2.Feature(float_list=feature_pb2.FloatList(
value=[0.0])),
'image/object/bbox/ymax':
feature_pb2.Feature(float_list=feature_pb2.FloatList(
value=[1.0])),
'image/object/class/label':
feature_pb2.Feature(int64_list=feature_pb2.Int64List(
value=[2])),
}))
writer.write(example.SerializeToString())
writer.close()
return path
def testSaveWithSignatures(self):
model = keras.models.Sequential()
model.add(
keras.layers.Dense(5,
input_shape=(3, ),
kernel_regularizer=regularizers.get('l2')))
model.add(keras.layers.Dropout(0.5))
model.add(
keras.layers.Dense(4, kernel_regularizer=regularizers.get('l2')))
input_arr = np.random.random((2, 3))
target_arr = np.random.random((2, 4))
model.compile(loss='mse', optimizer='rmsprop')
model.train_on_batch(input_arr, target_arr)
@def_function.function(
input_signature=[tensor_spec.TensorSpec((None, 3))])
def predict(inputs):
return {'predictions': model(inputs)}
feature_configs = {
'inputs':
parsing_ops.FixedLenFeature(shape=[2, 3], dtype=dtypes.float32)
}
@def_function.function(
input_signature=[tensor_spec.TensorSpec([None], dtypes.string)])
def parse_and_predict(examples):
features = parsing_ops.parse_single_example(
examples[0], feature_configs)
return {
'predictions': model(features['inputs']),
'layer_1_outputs': model.layers[0](features['inputs'])
}
saved_model_dir = self._save_model_dir()
model.save(saved_model_dir,
save_format='tf',
signatures={
'predict': predict,
'parse_and_predict': parse_and_predict
})
model.save('/tmp/saved',
save_format='tf',
signatures={
'predict': predict,
'parse_and_predict': parse_and_predict
})
loaded = keras_load.load(saved_model_dir)
self.assertAllClose(
model.predict(input_arr),
loaded.signatures['predict'](ops.convert_to_tensor_v2(
input_arr.astype('float32')))['predictions'])
feature = {
'inputs':
feature_pb2.Feature(float_list=feature_pb2.FloatList(
value=input_arr.astype('float32').flatten()))
}
example = example_pb2.Example(features=feature_pb2.Features(
feature=feature))
outputs = loaded.signatures['parse_and_predict'](
ops.convert_to_tensor_v2([example.SerializeToString()]))
self.assertAllClose(model.predict(input_arr), outputs['predictions'])
self.assertAllClose(model.layers[0](input_arr),
outputs['layer_1_outputs'])
from tensorflow.python.data.experimental.ops import parsing_ops as contrib_parsing_ops
from tensorflow.python.data.kernel_tests import test_base
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.util import nest
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.ops import parsing_ops
from tensorflow.python.platform import test
from tensorflow.python.platform import tf_logging
# Helpers for creating Example objects
example = example_pb2.Example
feature = feature_pb2.Feature
features = lambda d: feature_pb2.Features(feature=d)
bytes_feature = lambda v: feature(bytes_list=feature_pb2.BytesList(value=v))
int64_feature = lambda v: feature(int64_list=feature_pb2.Int64List(value=v))
float_feature = lambda v: feature(float_list=feature_pb2.FloatList(value=v))
# Helpers for creating SequenceExample objects
feature_list = lambda l: feature_pb2.FeatureList(feature=l)
feature_lists = lambda d: feature_pb2.FeatureLists(feature_list=d)
sequence_example = example_pb2.SequenceExample
def _compare_output_to_expected(tester, dict_tensors, expected_tensors,
flat_output):
tester.assertEqual(set(dict_tensors.keys()), set(expected_tensors.keys()))
i = 0 # Index into the flattened output of session.run()
for k, v in sorted(dict_tensors.items()):
def get_example_proto(cls):
return [example_pb2.Example(features=feature_pb2.Features(feature={
k: feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=[v]))
for k, v in d.items()
})) for d in cls.values]
def test_input_fn_from_parse_example(self):
"""Tests complete flow with input_fn constructed from parse_example."""
label_dimension = 2
input_dimension = label_dimension
batch_size = 10
prediction_length = batch_size
data = np.linspace(0.,
2.,
batch_size * label_dimension,
dtype=np.float32)
data = data.reshape(batch_size, label_dimension)
serialized_examples = []
for datum in data:
example = example_pb2.Example(features=feature_pb2.Features(
feature={
'x':
feature_pb2.Feature(float_list=feature_pb2.FloatList(
value=datum)),
'y':
feature_pb2.Feature(float_list=feature_pb2.FloatList(
value=datum[:label_dimension])),
}))
serialized_examples.append(example.SerializeToString())
feature_spec = {
'x': parsing_ops.FixedLenFeature([input_dimension],
dtypes.float32),
'y': parsing_ops.FixedLenFeature([label_dimension],
dtypes.float32),
}
def _train_input_fn():
feature_map = parsing_ops.parse_example(serialized_examples,
feature_spec)
features = queue_parsed_features(feature_map)
labels = features.pop('y')
return features, labels
def _eval_input_fn():
feature_map = parsing_ops.parse_example(
input_lib.limit_epochs(serialized_examples, num_epochs=1),
feature_spec)
features = queue_parsed_features(feature_map)
labels = features.pop('y')
return features, labels
def _predict_input_fn():
feature_map = parsing_ops.parse_example(
input_lib.limit_epochs(serialized_examples, num_epochs=1),
feature_spec)
features = queue_parsed_features(feature_map)
features.pop('y')
return features, None
self._test_complete_flow(train_input_fn=_train_input_fn,
eval_input_fn=_eval_input_fn,
predict_input_fn=_predict_input_fn,
input_dimension=input_dimension,
label_dimension=label_dimension,
prediction_length=prediction_length)
def testParseExampleInputFn(self):
"""Tests complete flow with input_fn constructed from parse_example."""
n_classes = 3
batch_size = 10
words = [
b'dog', b'cat', b'bird', b'the', b'a', b'sat', b'flew', b'slept'
]
_, examples_file = tempfile.mkstemp()
writer = python_io.TFRecordWriter(examples_file)
for _ in range(batch_size):
sequence_length = random.randint(1, len(words))
sentence = random.sample(words, sequence_length)
label = random.randint(0, n_classes - 1)
example = example_pb2.Example(features=feature_pb2.Features(
feature={
'tokens':
feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
value=sentence)),
'label':
feature_pb2.Feature(int64_list=feature_pb2.Int64List(
value=[label])),
}))
writer.write(example.SerializeToString())
writer.close()
col = seq_fc.sequence_categorical_column_with_hash_bucket(
'tokens', hash_bucket_size=10)
embed = fc.embedding_column(col, dimension=2)
feature_columns = [embed]
feature_spec = parsing_utils.classifier_parse_example_spec(
feature_columns, label_key='label', label_dtype=dtypes.int64)
def _train_input_fn():
dataset = readers.make_batched_features_dataset(
examples_file, batch_size, feature_spec)
return dataset.map(lambda features:
(features, features.pop('label')))
def _eval_input_fn():
dataset = readers.make_batched_features_dataset(examples_file,
batch_size,
feature_spec,
num_epochs=1)
return dataset.map(lambda features:
(features, features.pop('label')))
def _predict_input_fn():
dataset = readers.make_batched_features_dataset(examples_file,
batch_size,
feature_spec,
num_epochs=1)
def features_fn(features):
features.pop('label')
return features
return dataset.map(features_fn)
self._test_complete_flow(feature_columns=feature_columns,
train_input_fn=_train_input_fn,
eval_input_fn=_eval_input_fn,
predict_input_fn=_predict_input_fn,
n_classes=n_classes,
batch_size=batch_size)
def test_label_from_example(self, truth_label):
feature = {'label': _int_feature([truth_label])}
example = example_pb2.Example(features=feature_pb2.Features(
feature=feature))
output = vis.label_from_example(example)
self.assertEqual(truth_label, output)