def _create_tfrecord_dataset(tmpdir):
if not gfile.Exists(tmpdir):
gfile.MakeDirs(tmpdir)
data_sources = test_utils.create_tfrecord_files(tmpdir, num_files=1)
keys_to_features = {
'image/encoded':
parsing_ops.FixedLenFeature(
shape=(), dtype=dtypes.string, default_value=''),
'image/format':
parsing_ops.FixedLenFeature(
shape=(), dtype=dtypes.string, default_value='jpeg'),
'image/class/label':
parsing_ops.FixedLenFeature(
shape=[1],
dtype=dtypes.int64,
default_value=array_ops.zeros(
[1], dtype=dtypes.int64))
}
items_to_handlers = {
'image': tfexample_decoder.Image(),
'label': tfexample_decoder.Tensor('image/class/label'),
}
decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
return dataset.Dataset(
data_sources=data_sources,
reader=io_ops.TFRecordReader,
decoder=decoder,
num_samples=100,
items_to_descriptions=None)
def _create_tfrecord_dataset(tmpdir):
data_sources = test_utils.create_tfrecord_files(
tmpdir,
num_files=1)
keys_to_features = {
'image/encoded': tf.FixedLenFeature(
shape=(), dtype=tf.string, default_value=''),
'image/format': tf.FixedLenFeature(
shape=(), dtype=tf.string, default_value='jpeg'),
'image/class/label': tf.FixedLenFeature(
shape=[1], dtype=tf.int64,
default_value=tf.zeros([1], dtype=tf.int64))
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
return slim.dataset.Dataset(
data_sources=data_sources,
reader=tf.TFRecordReader,
decoder=decoder,
num_samples=100,
items_to_descriptions=None)
def _verify_all_data_sources_read(self, shared_queue):
with self.test_session():
tfrecord_paths = test_utils.create_tfrecord_files(self.get_temp_dir(), num_files=3)
num_readers = len(tfrecord_paths)
p_reader = slim.parallel_reader.ParallelReader(tf.TFRecordReader, shared_queue, num_readers=num_readers)
data_files = slim.parallel_reader.get_data_files(tfrecord_paths)
filename_queue = tf.train.string_input_producer(data_files)
key, value = p_reader.read(filename_queue)
count0 = 0
count1 = 0
count2 = 0
num_reads = 50
sv = tf.train.Supervisor(logdir=self.get_temp_dir())
with sv.prepare_or_wait_for_session() as sess:
sv.start_queue_runners(sess)
for _ in range(num_reads):
current_key, _ = sess.run([key, value])
if "0-of-3" in str(current_key):
count0 += 1
if "1-of-3" in str(current_key):
count1 += 1
if "2-of-3" in str(current_key):
count2 += 1
self.assertGreater(count0, 0)
self.assertGreater(count1, 0)
self.assertGreater(count2, 0)
self.assertEquals(count0 + count1 + count2, num_reads)
def _create_tfrecord_dataset(tmpdir):
if not gfile.Exists(tmpdir):
gfile.MakeDirs(tmpdir)
data_sources = test_utils.create_tfrecord_files(tmpdir, num_files=1)
keys_to_features = {
'image/encoded':
parsing_ops.FixedLenFeature(
shape=(), dtype=dtypes.string, default_value=''),
'image/format':
parsing_ops.FixedLenFeature(
shape=(), dtype=dtypes.string, default_value='jpeg'),
'image/class/label':
parsing_ops.FixedLenFeature(
shape=[1],
dtype=dtypes.int64,
default_value=array_ops.zeros(
[1], dtype=dtypes.int64))
}
items_to_handlers = {
'image': tfexample_decoder.Image(),
'label': tfexample_decoder.Tensor('image/class/label'),
}
decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
return dataset.Dataset(
data_sources=data_sources,
reader=io_ops.TFRecordReader,
decoder=decoder,
num_samples=100,
items_to_descriptions=None)
def _verify_read_up_to_out(self, shared_queue):
with self.test_session():
num_files = 3
num_records_per_file = 7
tfrecord_paths = test_utils.create_tfrecord_files(
self.get_temp_dir(),
num_files=num_files,
num_records_per_file=num_records_per_file)
p_reader = parallel_reader.ParallelReader(
io_ops.TFRecordReader, shared_queue, num_readers=5)
data_files = parallel_reader.get_data_files(tfrecord_paths)
filename_queue = input_lib.string_input_producer(data_files, num_epochs=1)
key, value = p_reader.read_up_to(filename_queue, 4)
count0 = 0
count1 = 0
count2 = 0
all_keys_count = 0
all_values_count = 0
sv = supervisor.Supervisor(logdir=self.get_temp_dir())
with sv.prepare_or_wait_for_session() as sess:
sv.start_queue_runners(sess)
while True:
try:
current_keys, current_values = sess.run([key, value])
self.assertEquals(len(current_keys), len(current_values))
all_keys_count += len(current_keys)
all_values_count += len(current_values)
for current_key in current_keys:
if '0-of-3' in str(current_key):
count0 += 1
if '1-of-3' in str(current_key):
count1 += 1
if '2-of-3' in str(current_key):
count2 += 1
except errors_impl.OutOfRangeError:
break
self.assertEquals(count0, num_records_per_file)
self.assertEquals(count1, num_records_per_file)
self.assertEquals(count2, num_records_per_file)
self.assertEquals(
all_keys_count,
num_files * num_records_per_file)
self.assertEquals(all_values_count, all_keys_count)
self.assertEquals(
count0 + count1 + count2,
all_keys_count)
def testOutOfRangeError(self):
with self.test_session():
[tfrecord_path] = test_utils.create_tfrecord_files(self.get_temp_dir(), num_files=1)
key, value = slim.parallel_reader.single_pass_read(tfrecord_path, reader_class=tf.TFRecordReader)
init_op = tf.initialize_local_variables()
with self.test_session() as sess:
sess.run(init_op)
with tf.contrib.slim.queues.QueueRunners(sess):
num_reads = 11
with self.assertRaises(tf.errors.OutOfRangeError):
for _ in range(num_reads):
sess.run([key, value])
def _verify_read_up_to_out(self, shared_queue):
with self.test_session():
num_files = 3
num_records_per_file = 7
tfrecord_paths = test_utils.create_tfrecord_files(
self.get_temp_dir(),
num_files=num_files,
num_records_per_file=num_records_per_file)
p_reader = parallel_reader.ParallelReader(io_ops.TFRecordReader,
shared_queue,
num_readers=5)
data_files = parallel_reader.get_data_files(tfrecord_paths)
filename_queue = input_lib.string_input_producer(data_files,
num_epochs=1)
key, value = p_reader.read_up_to(filename_queue, 4)
count0 = 0
count1 = 0
count2 = 0
all_keys_count = 0
all_values_count = 0
sv = supervisor.Supervisor(logdir=self.get_temp_dir())
with sv.prepare_or_wait_for_session() as sess:
sv.start_queue_runners(sess)
while True:
try:
current_keys, current_values = sess.run([key, value])
self.assertEquals(len(current_keys), len(current_values))
all_keys_count += len(current_keys)
all_values_count += len(current_values)
for current_key in current_keys:
if '0-of-3' in str(current_key):
count0 += 1
if '1-of-3' in str(current_key):
count1 += 1
if '2-of-3' in str(current_key):
count2 += 1
except errors_impl.OutOfRangeError:
break
self.assertEquals(count0, num_records_per_file)
self.assertEquals(count1, num_records_per_file)
self.assertEquals(count2, num_records_per_file)
self.assertEquals(all_keys_count, num_files * num_records_per_file)
self.assertEquals(all_values_count, all_keys_count)
self.assertEquals(count0 + count1 + count2, all_keys_count)
def testOutOfRangeError(self):
with self.test_session():
[tfrecord_path
] = test_utils.create_tfrecord_files(self.get_temp_dir(),
num_files=1)
key, value = slim.parallel_reader.single_pass_read(
tfrecord_path, reader_class=tf.TFRecordReader)
init_op = tf.initialize_local_variables()
with self.test_session() as sess:
sess.run(init_op)
with tf.contrib.slim.queues.QueueRunners(sess):
num_reads = 11
with self.assertRaises(tf.errors.OutOfRangeError):
for _ in range(num_reads):
sess.run([key, value])
def testOutOfRangeError(self):
with self.cached_session():
[tfrecord_path
] = test_utils.create_tfrecord_files(self.get_temp_dir(),
num_files=1)
key, value = parallel_reader.single_pass_read(
tfrecord_path, reader_class=io_ops.TFRecordReader)
init_op = variables.local_variables_initializer()
with self.cached_session() as sess:
sess.run(init_op)
with queues.QueueRunners(sess):
num_reads = 11
with self.assertRaises(errors_impl.OutOfRangeError):
for _ in range(num_reads):
sess.run([key, value])
def testTFRecordReader(self):
with self.test_session():
[tfrecord_path] = test_utils.create_tfrecord_files(self.get_temp_dir(), num_files=1)
key, value = slim.parallel_reader.single_pass_read(tfrecord_path, reader_class=tf.TFRecordReader)
init_op = tf.initialize_local_variables()
with self.test_session() as sess:
sess.run(init_op)
with tf.contrib.slim.queues.QueueRunners(sess):
flowers = 0
num_reads = 9
for _ in range(num_reads):
current_key, _ = sess.run([key, value])
if "flowers" in str(current_key):
flowers += 1
self.assertGreater(flowers, 0)
self.assertEquals(flowers, num_reads)
def testTFRecordReader(self):
with self.test_session():
self._tfrecord_paths = test_utils.create_tfrecord_files(self.get_temp_dir(), num_files=3)
key, value = slim.parallel_reader.parallel_read(
self._tfrecord_paths, reader_class=tf.TFRecordReader, num_readers=3
)
sv = tf.train.Supervisor(logdir=self.get_temp_dir())
with sv.prepare_or_wait_for_session() as sess:
sv.start_queue_runners(sess)
flowers = 0
num_reads = 100
for _ in range(num_reads):
current_key, _ = sess.run([key, value])
if "flowers" in str(current_key):
flowers += 1
self.assertGreater(flowers, 0)
self.assertEquals(flowers, num_reads)
def testTFRecordReader(self):
with self.test_session():
[tfrecord_path
] = test_utils.create_tfrecord_files(self.get_temp_dir(),
num_files=1)
key, value = slim.parallel_reader.single_pass_read(
tfrecord_path, reader_class=tf.TFRecordReader)
init_op = tf.initialize_local_variables()
with self.test_session() as sess:
sess.run(init_op)
with tf.contrib.slim.queues.QueueRunners(sess):
flowers = 0
num_reads = 9
for _ in range(num_reads):
current_key, _ = sess.run([key, value])
if 'flowers' in str(current_key):
flowers += 1
self.assertGreater(flowers, 0)
self.assertEquals(flowers, num_reads)
def testTFRecordReader(self):
with self.test_session():
self._tfrecord_paths = test_utils.create_tfrecord_files(
self.get_temp_dir(), num_files=3)
key, value = parallel_reader.parallel_read(
self._tfrecord_paths,
reader_class=io_ops.TFRecordReader,
num_readers=3)
sv = supervisor.Supervisor(logdir=self.get_temp_dir())
with sv.managed_session() as sess:
flowers = 0
num_reads = 100
for _ in range(num_reads):
current_key, _ = sess.run([key, value])
if 'flowers' in str(current_key):
flowers += 1
self.assertGreater(flowers, 0)
self.assertEquals(flowers, num_reads)
def _verify_all_data_sources_read(self, shared_queue):
with self.test_session():
tfrecord_paths = test_utils.create_tfrecord_files(
self.get_temp_dir(),
num_files=3)
num_readers = len(tfrecord_paths)
p_reader = slim.parallel_reader.ParallelReader(
tf.TFRecordReader,
shared_queue,
num_readers=num_readers)
data_files = slim.parallel_reader.get_data_files(
tfrecord_paths)
filename_queue = tf.train.string_input_producer(data_files)
key, value = p_reader.read(filename_queue)
count0 = 0
count1 = 0
count2 = 0
num_reads = 50
sv = tf.train.Supervisor(logdir=self.get_temp_dir())
with sv.prepare_or_wait_for_session() as sess:
sv.start_queue_runners(sess)
for _ in range(num_reads):
current_key, _ = sess.run([key, value])
if '0-of-3' in str(current_key):
count0 += 1
if '1-of-3' in str(current_key):
count1 += 1
if '2-of-3' in str(current_key):
count2 += 1
self.assertGreater(count0, 0)
self.assertGreater(count1, 0)
self.assertGreater(count2, 0)
self.assertEquals(count0 + count1 + count2, num_reads)
def testTFRecordReader(self):
with self.test_session():
self._tfrecord_paths = test_utils.create_tfrecord_files(
self.get_temp_dir(), num_files=3)
key, value = slim.parallel_reader.parallel_read(
self._tfrecord_paths,
reader_class=tf.TFRecordReader,
num_readers=3)
sv = tf.train.Supervisor(logdir=self.get_temp_dir())
with sv.prepare_or_wait_for_session() as sess:
sv.start_queue_runners(sess)
flowers = 0
num_reads = 100
for _ in range(num_reads):
current_key, _ = sess.run([key, value])
if 'flowers' in str(current_key):
flowers += 1
self.assertGreater(flowers, 0)
self.assertEquals(flowers, num_reads)
