def testEndianness(self):
self.assertAllEqual([[0x04030201]],
parsing_ops.decode_raw(["\x01\x02\x03\x04"],
dtypes.int32,
little_endian=True))
self.assertAllEqual([[0x01020304]],
parsing_ops.decode_raw(["\x01\x02\x03\x04"],
dtypes.int32,
little_endian=False))
def testToInt16(self):
self.assertAllEqual(
[[ord("A") + ord("a") * 256, ord("B") + ord("C") * 256]],
parsing_ops.decode_raw(["AaBC"], dtypes.uint16))
with self.assertRaisesOpError(
"Input to DecodeRaw has length 3 that is not a multiple of 2, the "
"size of int16"):
self.evaluate(parsing_ops.decode_raw(["123", "456"], dtypes.int16))
def testToUInt16(self):
# Use FF/EE/DD/CC so that decoded value is higher than 32768 for uint16
self.assertAllEqual(
[[0xFF + 0xEE * 256, 0xDD + 0xCC * 256]],
parsing_ops.decode_raw([b"\xFF\xEE\xDD\xCC"], dtypes.uint16))
with self.assertRaisesOpError(
"Input to DecodeRaw has length 3 that is not a multiple of 2, the "
"size of uint16"):
self.evaluate(parsing_ops.decode_raw(["123", "456"], dtypes.uint16))
def testEndianness(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode_le = parsing_ops.decode_raw(
in_bytes, out_type=dtypes.int32, little_endian=True)
decode_be = parsing_ops.decode_raw(
in_bytes, out_type=dtypes.int32, little_endian=False)
result = decode_le.eval(feed_dict={in_bytes: ["\x01\x02\x03\x04"]})
self.assertAllEqual([[0x04030201]], result)
result = decode_be.eval(feed_dict={in_bytes: ["\x01\x02\x03\x04"]})
self.assertAllEqual([[0x01020304]], result)
def testEndianness(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode_le = parsing_ops.decode_raw(
in_bytes, out_type=dtypes.int32, little_endian=True)
decode_be = parsing_ops.decode_raw(
in_bytes, out_type=dtypes.int32, little_endian=False)
result = decode_le.eval(feed_dict={in_bytes: ["\x01\x02\x03\x04"]})
self.assertAllEqual([[0x04030201]], result)
result = decode_be.eval(feed_dict={in_bytes: ["\x01\x02\x03\x04"]})
self.assertAllEqual([[0x01020304]], result)
def testToUint8(self):
self.assertAllEqual(
[[ord("A")], [ord("a")]],
parsing_ops.decode_raw(["A", "a"], dtypes.uint8))
self.assertAllEqual(
[[ord("w"), ord("e"), ord("r")], [ord("X"), ord("Y"), ord("Z")]],
parsing_ops.decode_raw(["wer", "XYZ"], dtypes.uint8))
with self.assertRaisesOpError(
"DecodeRaw requires input strings to all be the same size, but "
"element 1 has size 5 != 6"):
self.evaluate(parsing_ops.decode_raw(["short", "longer"], dtypes.uint8))
def testEmptyStringInput(self):
with self.test_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.float16)
result = decode.eval(feed_dict={in_bytes: [""]})
self.assertEqual(len(result), 1)
def testEmptyStringInput(self):
with self.test_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.float16)
result = decode.eval(feed_dict={in_bytes: [""]})
self.assertEqual(len(result), 1)
def testEmptyStringInput(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.float16)
for num_inputs in range(3):
result = decode.eval(feed_dict={in_bytes: [""] * num_inputs})
self.assertEqual((num_inputs, 0), result.shape)
def decode(self, data, items):
example = parsing_ops.parse_single_example(data,
self._keys_to_features)
# Reshape non-sparse elements just once:
for k in self._keys_to_features:
v = self._keys_to_features[k]
if isinstance(v, parsing_ops.FixedLenFeature):
example[k] = tf.reshape(example[k], v.shape)
# example['image'] = tf.reshape(parsing_ops.decode_raw(example['image'], tf.uint8), [32, 32, 1])
outputs = []
for item in items:
if item == 'image':
outputs.append(
tf.reshape(
parsing_ops.decode_raw(example['image'], tf.uint8),
[32, 32]))
elif item == 'label':
outputs.append(example['label'])
elif item == 'uid':
outputs.append(example['uid'])
elif item == 'file_path':
outputs.append(example['file_path'])
elif item == 'user_set/labels':
st_labels = example['user_set/labels']
outputs.append(
tf.sparse_to_dense(st_labels.indices,
st_labels.dense_shape,
st_labels.values))
elif item == 'user_set/images':
st_labels = example['user_set/labels']
st_images = example['user_set/images']
num_examples = tf.shape(st_labels)[0]
outputs.append(
tf.reshape(
parsing_ops.decode_raw(st_images.values, tf.uint8),
[num_examples, 32, 32]))
elif item == 'user_set/file_paths':
st_paths = example['user_set/file_paths']
outputs.append(
tf.sparse_to_dense(st_paths.indices,
st_paths.dense_shape,
st_paths.values,
default_value=''))
return outputs
def testEndianness(self):
self.assertAllEqual(
[[0x04030201]],
parsing_ops.decode_raw(
["\x01\x02\x03\x04"], dtypes.int32, little_endian=True))
self.assertAllEqual(
[[0x01020304]],
parsing_ops.decode_raw(
["\x01\x02\x03\x04"], dtypes.int32, little_endian=False))
self.assertAllEqual([[1 + 2j]],
parsing_ops.decode_raw([b"\x00\x00\x80?\x00\x00\x00@"],
dtypes.complex64,
little_endian=True))
self.assertAllEqual([[1 + 2j]],
parsing_ops.decode_raw([b"?\x80\x00\x00@\x00\x00\x00"],
dtypes.complex64,
little_endian=False))
def testShapeInference(self):
for dtype in [dtypes.bool, dtypes.int8, dtypes.uint8, dtypes.int16,
dtypes.uint16, dtypes.int32, dtypes.int64, dtypes.float16,
dtypes.float32, dtypes.float64, dtypes.complex64,
dtypes.complex128]:
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, dtype)
self.assertEqual([None, None], decode.get_shape().as_list())
def testEmptyStringInput(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.float16)
for num_inputs in range(3):
result = decode.eval(feed_dict={in_bytes: [""] * num_inputs})
self.assertEqual((num_inputs, 0), result.shape)
def testToFloat16(self):
with self.test_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.float16)
self.assertEqual([None, None], decode.get_shape().as_list())
expected_result = np.matrix([[1, -2, -3, 4]], dtype=np.float16)
result = decode.eval(feed_dict={in_bytes: [expected_result.tostring()]})
self.assertAllEqual(expected_result, result)
def testToFloat16(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.float16)
self.assertEqual([None, None], decode.get_shape().as_list())
expected_result = np.matrix([[1, -2, -3, 4]], dtype="<f2")
result = decode.eval(feed_dict={in_bytes: [expected_result.tostring()]})
self.assertAllEqual(expected_result, result)
def testToBool(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.bool)
self.assertEqual([None, None], decode.get_shape().as_list())
expected_result = np.matrix([[True, False, False, True]], dtype="<b1")
result = decode.eval(feed_dict={in_bytes: [expected_result.tostring()]})
self.assertAllEqual(expected_result, result)
def testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU(self):
if not test_util.is_gpu_available():
self.skipTest("No GPU available")
with ops.device("/job:localhost/replica:0/task:0/cpu:0"):
dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
iterator_3_handle = iterator_3.string_handle()
def _encode_raw(byte_array):
return bytes(bytearray(byte_array))
@function.Defun(dtypes.uint8)
def _remote_fn(h):
handle = script_ops.py_func(_encode_raw, [h], dtypes.string)
remote_iterator = iterator_ops.Iterator.from_string_handle(
handle, dataset_ops.get_legacy_output_types(dataset_3),
dataset_ops.get_legacy_output_shapes(dataset_3))
return remote_iterator.get_next()
with ops.device("/job:localhost/replica:0/task:0/device:GPU:0"):
target_placeholder = array_ops.placeholder(dtypes.string, shape=[])
iterator_3_handle_uint8 = parsing_ops.decode_raw(
input_bytes=iterator_3_handle, out_type=dtypes.uint8)
remote_op = functional_ops.remote_call(
args=[iterator_3_handle_uint8],
Tout=[dtypes.int32],
f=_remote_fn,
target=target_placeholder)
with self.cached_session() as sess:
elem = sess.run(remote_op,
feed_dict={
target_placeholder:
"/job:localhost/replica:0/task:0/cpu:0"
})
self.assertEqual(elem, [1])
elem = sess.run(remote_op,
feed_dict={
target_placeholder:
"/job:localhost/replica:0/task:0/cpu:0"
})
self.assertEqual(elem, [2])
elem = sess.run(remote_op,
feed_dict={
target_placeholder:
"/job:localhost/replica:0/task:0/cpu:0"
})
self.assertEqual(elem, [3])
with self.assertRaises(errors.OutOfRangeError):
sess.run(remote_op,
feed_dict={
target_placeholder:
"/job:localhost/replica:0/task:0/cpu:0"
})
def testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU(self):
if not test_util.is_gpu_available():
self.skipTest("No GPU available")
with ops.device("/job:localhost/replica:0/task:0/cpu:0"):
dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
iterator_3_handle = iterator_3.string_handle()
def _encode_raw(byte_array):
return bytes(bytearray(byte_array))
@function.Defun(dtypes.uint8)
def _remote_fn(h):
handle = script_ops.py_func(_encode_raw, [h], dtypes.string)
remote_iterator = iterator_ops.Iterator.from_string_handle(
handle, dataset_ops.get_legacy_output_types(dataset_3),
dataset_ops.get_legacy_output_shapes(dataset_3))
return remote_iterator.get_next()
with ops.device("/job:localhost/replica:0/task:0/device:GPU:0"):
target_placeholder = array_ops.placeholder(dtypes.string, shape=[])
iterator_3_handle_uint8 = parsing_ops.decode_raw(
bytes=iterator_3_handle, out_type=dtypes.uint8)
remote_op = functional_ops.remote_call(
args=[iterator_3_handle_uint8],
Tout=[dtypes.int32],
f=_remote_fn,
target=target_placeholder)
with self.cached_session() as sess:
elem = sess.run(
remote_op,
feed_dict={
target_placeholder: "/job:localhost/replica:0/task:0/cpu:0"
})
self.assertEqual(elem, [1])
elem = sess.run(
remote_op,
feed_dict={
target_placeholder: "/job:localhost/replica:0/task:0/cpu:0"
})
self.assertEqual(elem, [2])
elem = sess.run(
remote_op,
feed_dict={
target_placeholder: "/job:localhost/replica:0/task:0/cpu:0"
})
self.assertEqual(elem, [3])
with self.assertRaises(errors.OutOfRangeError):
sess.run(
remote_op,
feed_dict={
target_placeholder: "/job:localhost/replica:0/task:0/cpu:0"
})
def testToComplex64(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.complex64)
self.assertEqual([None, None], decode.get_shape().as_list())
expected_result = np.matrix([[1 + 1j, 2 - 2j, -3 + 3j, -4 - 4j]],
dtype="<c8")
result = decode.eval(feed_dict={in_bytes: [expected_result.tostring()]})
self.assertAllEqual(expected_result, result)
def testToComplex128(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.complex128)
self.assertEqual([None, None], decode.get_shape().as_list())
expected_result = np.matrix([[1 + 1j, 2 - 2j, -3 + 3j, -4 - 4j]],
dtype="<c16")
result = decode.eval(feed_dict={in_bytes: [expected_result.tostring()]})
self.assertAllEqual(expected_result, result)
def testDecompressWithRaw(self):
for compression_type in ["ZLIB", "GZIP", ""]:
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decompressed = parsing_ops.decode_compressed(
in_bytes, compression_type=compression_type)
decode = parsing_ops.decode_raw(decompressed, out_type=dtypes.int16)
result = decode.eval(
feed_dict={in_bytes: [self._compress(b"AaBC", compression_type)]})
self.assertAllEqual(
[[ord("A") + ord("a") * 256, ord("B") + ord("C") * 256]], result)
def testDecompressWithRaw(self):
for compression_type in ["ZLIB", "GZIP", ""]:
with self.test_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decompressed = parsing_ops.decode_compressed(
in_bytes, compression_type=compression_type)
decode = parsing_ops.decode_raw(decompressed, out_type=dtypes.int16)
result = decode.eval(
feed_dict={in_bytes: [self._compress(b"AaBC", compression_type)]})
self.assertAllEqual(
[[ord("A") + ord("a") * 256, ord("B") + ord("C") * 256]], result)
def testToInt16(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.int16)
self.assertEqual([None, None], decode.get_shape().as_list())
result = decode.eval(feed_dict={in_bytes: ["AaBC"]})
self.assertAllEqual(
[[ord("A") + ord("a") * 256, ord("B") + ord("C") * 256]], result)
with self.assertRaisesOpError(
"Input to DecodeRaw has length 3 that is not a multiple of 2, the "
"size of int16"):
decode.eval(feed_dict={in_bytes: ["123", "456"]})
def testToInt16(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.int16)
self.assertEqual([None, None], decode.get_shape().as_list())
result = decode.eval(feed_dict={in_bytes: ["AaBC"]})
self.assertAllEqual(
[[ord("A") + ord("a") * 256, ord("B") + ord("C") * 256]], result)
with self.assertRaisesOpError(
"Input to DecodeRaw has length 3 that is not a multiple of 2, the "
"size of int16"):
decode.eval(feed_dict={in_bytes: ["123", "456"]})
def testToUInt16(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.uint16)
self.assertEqual([None, None], decode.get_shape().as_list())
# Use FF/EE/DD/CC so that decoded value is higher than 32768 for uint16
result = decode.eval(feed_dict={in_bytes: [b"\xFF\xEE\xDD\xCC"]})
self.assertAllEqual(
[[0xFF + 0xEE * 256, 0xDD + 0xCC * 256]], result)
with self.assertRaisesOpError(
"Input to DecodeRaw has length 3 that is not a multiple of 2, the "
"size of uint16"):
decode.eval(feed_dict={in_bytes: ["123", "456"]})
def testToUInt16(self):
with self.test_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.uint16)
self.assertEqual([None, None], decode.get_shape().as_list())
# Use FF/EE/DD/CC so that decoded value is higher than 32768 for uint16
result = decode.eval(feed_dict={in_bytes: [b"\xFF\xEE\xDD\xCC"]})
self.assertAllEqual([[0xFF + 0xEE * 256, 0xDD + 0xCC * 256]],
result)
with self.assertRaisesOpError(
"Input to DecodeRaw has length 3 that is not a multiple of 2, the "
"size of uint16"):
decode.eval(feed_dict={in_bytes: ["123", "456"]})
def testToUint8(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[2])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.uint8)
self.assertEqual([2, None], decode.get_shape().as_list())
result = decode.eval(feed_dict={in_bytes: ["A", "a"]})
self.assertAllEqual([[ord("A")], [ord("a")]], result)
result = decode.eval(feed_dict={in_bytes: ["wer", "XYZ"]})
self.assertAllEqual([[ord("w"), ord("e"), ord("r")],
[ord("X"), ord("Y"), ord("Z")]], result)
with self.assertRaisesOpError(
"DecodeRaw requires input strings to all be the same size, but "
"element 1 has size 5 != 6"):
decode.eval(feed_dict={in_bytes: ["short", "longer"]})
def testToUint8(self):
with self.cached_session():
in_bytes = array_ops.placeholder(dtypes.string, shape=[2])
decode = parsing_ops.decode_raw(in_bytes, out_type=dtypes.uint8)
self.assertEqual([2, None], decode.get_shape().as_list())
result = decode.eval(feed_dict={in_bytes: ["A", "a"]})
self.assertAllEqual([[ord("A")], [ord("a")]], result)
result = decode.eval(feed_dict={in_bytes: ["wer", "XYZ"]})
self.assertAllEqual([[ord("w"), ord("e"), ord("r")],
[ord("X"), ord("Y"), ord("Z")]], result)
with self.assertRaisesOpError(
"DecodeRaw requires input strings to all be the same size, but "
"element 1 has size 5 != 6"):
decode.eval(feed_dict={in_bytes: ["short", "longer"]})
def decode_raw():
return parsing_ops.decode_raw(image_buffer, out_type=tf.uint8)
def decode_raw(): return parsing_ops.decode_raw(image_buffer, out_type=self._dtype)
def decode_raw(): return parsing_ops.decode_raw(image_buffer, dtypes.uint8)
def decode_raw(): """Decodes a raw image.""" return parsing_ops.decode_raw(image_buffer, out_type=self._dtype)
def decode_raw():
return parsing_ops.decode_raw(image_buffer, dtypes.uint8)
def testToComplex128(self):
result = np.matrix([[1 + 1j, 2 - 2j, -3 + 3j, -4 - 4j]], dtype="<c16")
self.assertAllEqual(
result,
parsing_ops.decode_raw([result.tostring()], dtypes.complex128))
def decode_raw(): """Decodes a raw image.""" return parsing_ops.decode_raw(image_buffer, out_type=self._dtype)
def decode_raw(): return parsing_ops.decode_raw(image_buffer, out_type=self._dtype)
def decode(in_bytes, compression_type=compression_type):
decompressed = parsing_ops.decode_compressed(
in_bytes, compression_type)
return parsing_ops.decode_raw(decompressed,
out_type=dtypes.int16)
def testToComplex64(self):
result = np.matrix([[1 + 1j, 2 - 2j, -3 + 3j, -4 - 4j]], dtype="<c8")
self.assertAllEqual(
result, parsing_ops.decode_raw([result.tobytes()],
dtypes.complex64))
def testToBool(self):
result = np.matrix([[True, False, False, True]], dtype="<b1")
self.assertAllEqual(
result, parsing_ops.decode_raw([result.tobytes()], dtypes.bool))
def testToFloat16(self):
result = np.matrix([[1, -2, -3, 4]], dtype="<f2")
self.assertAllEqual(
result, parsing_ops.decode_raw([result.tobytes()], dtypes.float16))
def testEmptyStringInput(self):
for num_inputs in range(3):
result = parsing_ops.decode_raw([""] * num_inputs, dtypes.float16)
self.assertEqual((num_inputs, 0), self.evaluate(result).shape)
