def test_encode_decode(self):
nonunicode_text = 'hello world'
unicode_text = u'你好，世界'
expectations = [
# Non-unicode
test_utils.FeatureExpectation(
name='text',
feature=features.Text(),
value=nonunicode_text,
expected=tf.compat.as_bytes(nonunicode_text)),
# Unicode
test_utils.FeatureExpectation(
name='text_unicode',
feature=features.Text(),
value=unicode_text,
expected=tf.compat.as_bytes(unicode_text)),
]
specs = features.SpecDict(
{exp.name: exp.feature
for exp in expectations})
decoded_sample = test_utils.features_encode_decode(
specs, dict([(exp.name, exp.value) for exp in expectations]))
for exp in expectations:
self.assertAllEqual(decoded_sample[exp.name], exp.expected)
def test_encode_decode(self):
specs = features.SpecDict({
'img':
features.Image(),
'img_shaped':
features.Image(shape=(32, 64, 3)),
})
img = np.random.randint(256, size=(128, 100, 1), dtype=np.uint8)
img_shaped = np.random.randint(256, size=(32, 64, 3), dtype=np.uint8)
decoded_sample = _encode_decode(specs, {
'img': img,
'img_shaped': img_shaped,
})
self.assertAllEqual(decoded_sample['img'], img)
self.assertAllEqual(decoded_sample['img_shaped'], img_shaped)
# 'img' shape can be dynamic
img2 = np.random.randint(256, size=(64, 200, 1), dtype=np.uint8)
decoded_sample = _encode_decode(specs, {
'img': img2,
'img_shaped': img_shaped,
})
self.assertAllEqual(decoded_sample['img'], img2)
# 'img_shaped' shape should be static
img_shaped2 = np.random.randint(256, size=(31, 64, 3), dtype=np.uint8)
with self.assertRaises(ValueError) as err:
_encode_decode(specs, {
'img': img2,
'img_shaped': img_shaped2,
})
self.assertIn('Shape (31, 64, 3) do not match', str(err.exception))
def _info(self):
mnist_shape = (_MNIST_IMAGE_SIZE, _MNIST_IMAGE_SIZE, 1)
return dataset_builder.DatasetInfo(specs=features.SpecDict({
"input":
features.Image(shape=mnist_shape),
"target":
tf.int64,
}), )
def _info(self):
cifar_shape = (_CIFAR_IMAGE_SIZE, _CIFAR_IMAGE_SIZE, 3)
return dataset_builder.DatasetInfo(
specs=features.SpecDict({
"input": features.Image(shape=cifar_shape),
"target": tf.int64, # Could replace by features.Label()
}),
)
def _info(self):
return dataset_builder.DatasetInfo(
specs=features.SpecDict({
"x": tf.int64,
"y": tf.int64,
"z": tf.string,
}),
)
def setUp(self):
# Create the spec dict used for all tests
self._specs = features.SpecDict({
'input':
features.OneOf(
choice='choice2',
feature_dict={
'choice1': tf.float32,
'choice2': TestInputConnector(),
},
),
})
def _info(self):
cifar_shape = (_CIFAR_IMAGE_SIZE, _CIFAR_IMAGE_SIZE, 3)
label_to_use = "coarse_labels" if self._use_coarse_labels else "fine_labels"
return dataset_builder.DatasetInfo(
specs=features.SpecDict({
"input": features.Image(shape=cifar_shape),
"target": features.OneOf(choice=label_to_use, feature_dict={
"coarse_labels": tf.int64,
"fine_labels": tf.int64,
}),
}),
)
def test_encode_decode(self):
specs = features.SpecDict({
'img':
features.Image(),
'img_shaped':
features.Image(shape=(32, 64, 3)),
'img_file':
features.Image(),
})
img = np.random.randint(256, size=(128, 100, 3), dtype=np.uint8)
img_shaped = np.random.randint(256, size=(32, 64, 3), dtype=np.uint8)
img_file_path = os.path.join(os.path.dirname(__file__),
'../test_data/6pixels.png')
img_file_expected_content = [ # see tests_data/README.md
[[0, 255, 0], [255, 0, 0], [255, 0, 255]],
[[0, 0, 255], [255, 255, 0], [126, 127, 128]],
]
decoded_sample = test_utils.features_encode_decode(
specs, {
'img': img,
'img_shaped': img_shaped,
'img_file': img_file_path,
})
self.assertAllEqual(decoded_sample['img'], img)
self.assertAllEqual(decoded_sample['img_shaped'], img_shaped)
self.assertAllEqual(decoded_sample['img_file'],
img_file_expected_content)
# 'img' shape can be dynamic
img2 = np.random.randint(256, size=(64, 200, 3), dtype=np.uint8)
decoded_sample = test_utils.features_encode_decode(
specs, {
'img': img2,
'img_shaped': img_shaped,
'img_file': img_file_path,
})
self.assertAllEqual(decoded_sample['img'], img2)
# 'img_shaped' shape should be static
img_shaped2 = np.random.randint(256, size=(31, 64, 3), dtype=np.uint8)
with self.assertRaisesWithPredicateMatch(ValueError,
'are incompatible'):
test_utils.features_encode_decode(
specs, {
'img': img2,
'img_shaped': img_shaped2,
'img_file': img_file_path,
})
def setUp(self):
# Create the spec dict used for all tests
self._specs = features.SpecDict({
'input': TestInputConnector(),
'output': TestOutputConnector(),
'img': {
'size': {
'height': tf.int32,
'width': tf.int32,
},
'metadata/path': tf.string,
}
})
def test_shapes_dynamic(self):
specs = features.SpecDict({
'input':
features.Tensor(shape=(None, None, 1), dtype=tf.int32),
})
# Numpy array
np_input = np.random.randint(256, size=(2, 3, 1), dtype=np.int32)
tf_output = _encode_decode(specs, {
'input': np_input,
})
self.assertAllEqual(tf_output['input'], np_input)
# Invalid shape
with self.assertRaises(ValueError) as err:
_encode_decode(specs, {
'input':
np.random.randint(256, size=(2, 3, 2), dtype=np.int32),
})
self.assertIn('Shape (2, 3, 2) do not match', str(err.exception))
def test_shapes_dynamic(self):
specs = features.SpecDict({
'input':
features.Tensor(shape=(None, None, 1), dtype=tf.int32),
})
# Numpy array
np_input = np.random.randint(256, size=(2, 3, 1), dtype=np.int32)
tf_output = _encode_decode(specs, {
'input': np_input,
})
self.assertAllEqual(tf_output['input'], np_input)
# Invalid shape
with self.assertRaisesWithPredicateMatch(ValueError,
'are incompatible'):
_encode_decode(specs, {
'input':
np.random.randint(256, size=(2, 3, 2), dtype=np.int32),
})
def test_wrong_input(self):
specs = features.SpecDict({
'img': features.Image(),
})
# Correct shape/type should succeed
_encode_decode(specs, {
'img':
np.random.randint(256, size=(128, 128, 1), dtype=np.uint8),
})
_encode_decode(specs, {
'img':
np.random.randint(256, size=(64, 64, 1), dtype=np.uint8),
})
# Invalid type
with self.assertRaises(ValueError) as err:
_encode_decode(
specs, {
'img':
np.random.randint(256, size=(128, 128, 1),
dtype=np.uint32),
})
self.assertIn('Image should be uint8', str(err.exception))
# Invalid number of dimensions
with self.assertRaises(ValueError) as err:
_encode_decode(specs, {
'img':
np.random.randint(256, size=(128, 128), dtype=np.uint8),
})
self.assertIn('Shapes should have same length', str(err.exception))
# Invalid number of channels
with self.assertRaises(ValueError) as err:
_encode_decode(
specs, {
'img':
np.random.randint(256, size=(128, 128, 3), dtype=np.uint8),
})
self.assertIn('Shape (128, 128, 3) do not match', str(err.exception))
def test_wrong_input(self):
specs = features.SpecDict({
'img': features.Image(),
})
# Correct shape/type should succeed
test_utils.features_encode_decode(specs, {
'img':
np.random.randint(256, size=(128, 128, 3), dtype=np.uint8),
})
test_utils.features_encode_decode(specs, {
'img':
np.random.randint(256, size=(64, 64, 3), dtype=np.uint8),
})
# Invalid type
with self.assertRaisesWithPredicateMatch(ValueError,
'should be uint8'):
test_utils.features_encode_decode(
specs, {
'img':
np.random.randint(256, size=(128, 128, 3),
dtype=np.uint32),
})
# Invalid number of dimensions
with self.assertRaisesWithPredicateMatch(ValueError,
'must have the same rank'):
test_utils.features_encode_decode(specs, {
'img':
np.random.randint(256, size=(128, 128), dtype=np.uint8),
})
# Invalid number of channels
with self.assertRaisesWithPredicateMatch(ValueError,
'are incompatible'):
test_utils.features_encode_decode(
specs, {
'img':
np.random.randint(256, size=(128, 128, 1), dtype=np.uint8),
})
def test_shapes_static(self):
specs = features.SpecDict({
'input':
features.Tensor(shape=(2, 3), dtype=tf.float32),
})
# Numpy array
np_input = np.random.rand(2, 3).astype(np.float32)
tf_output = _encode_decode(specs, {
'input': np_input,
})
self.assertAllEqual(tf_output['input'], np_input)
# Python array
array_input = [
[1, 2, 3],
[4, 5, 6],
]
tf_output = _encode_decode(specs, {
'input': array_input,
})
self.assertAllEqual(tf_output['input'], array_input)
# Invalid type
with self.assertRaises(ValueError) as err:
_encode_decode(specs, {
'input': np.random.randint(256, size=(2, 3)),
})
self.assertIn('Dtype int64 do not match', str(err.exception))
# Invalid shape
with self.assertRaises(ValueError) as err:
_encode_decode(specs, {
'input': np.random.rand(2, 4).astype(np.float32),
})
self.assertIn('Shape (2, 4) do not match', str(err.exception))
def test_shapes_static(self):
specs = features.SpecDict({
'input':
features.Tensor(shape=(2, 3), dtype=tf.float32),
})
# Numpy array
np_input = np.random.rand(2, 3).astype(np.float32)
tf_output = _encode_decode(specs, {
'input': np_input,
})
self.assertAllEqual(tf_output['input'], np_input)
# Python array
array_input = [
[1, 2, 3],
[4, 5, 6],
]
tf_output = _encode_decode(specs, {
'input': array_input,
})
self.assertAllEqual(tf_output['input'], array_input)
# Invalid type
with self.assertRaisesWithPredicateMatch(ValueError,
'int64 do not match'):
_encode_decode(specs, {
'input': np.random.randint(256, size=(2, 3)),
})
# Invalid shape
with self.assertRaisesWithPredicateMatch(ValueError,
'are incompatible'):
_encode_decode(specs, {
'input': np.random.rand(2, 4).astype(np.float32),
})
def _info(self):
return dataset_info.DatasetInfo(
specs=features.SpecDict({"x": tf.int64}),
)