def test_multiple_workers_with_shared_queue(self):
gfile.Glob = self._orig_glob
filenames = self._create_sorted_temp_files([
"ABC\n", "DEF\n", "GHI\n", "JKL\n", "MNO\n", "PQR\n", "STU\n", "VWX\n",
"YZ\n"
])
batch_size = 1
queue_capacity = 5
name = "my_batch"
shared_file_name_queue_name = "%s/file_name_queue" % name
example_queue_name = "%s/fifo_queue" % name
worker_file_name_queue_name = "%s/file_name_queue/fifo_queue" % name
server = tf.train.Server.create_local_server()
with tf.Graph().as_default() as g1, tf.Session(
server.target, graph=g1) as session:
_, inputs = _read_keyed_batch_examples_shared_queue(
filenames,
batch_size,
reader=tf.TextLineReader,
randomize_input=False,
num_epochs=1,
queue_capacity=queue_capacity,
name=name)
session.run(tf.initialize_local_variables())
# Run the three queues once manually.
self._run_queue(shared_file_name_queue_name, session)
self._run_queue(worker_file_name_queue_name, session)
self._run_queue(example_queue_name, session)
self.assertAllEqual(session.run(inputs), [b"ABC"])
# Run the worker and the example queue.
self._run_queue(worker_file_name_queue_name, session)
self._run_queue(example_queue_name, session)
self.assertAllEqual(session.run(inputs), [b"DEF"])
with tf.Graph().as_default() as g2, tf.Session(
server.target, graph=g2) as session:
_, inputs = _read_keyed_batch_examples_shared_queue(
filenames,
batch_size,
reader=tf.TextLineReader,
randomize_input=False,
num_epochs=1,
queue_capacity=queue_capacity,
name=name)
# Run the worker and the example queue.
self._run_queue(worker_file_name_queue_name, session)
self._run_queue(example_queue_name, session)
self.assertAllEqual(session.run(inputs), [b"GHI"])
self.assertTrue(g1 is not g2)
def test_multiple_workers_with_shared_queue(self):
gfile.Glob = self._orig_glob
filenames = self._create_sorted_temp_files([
"ABC\n", "DEF\n", "GHI\n", "JKL\n", "MNO\n", "PQR\n", "STU\n",
"VWX\n", "YZ\n"
])
batch_size = 1
queue_capacity = 5
name = "my_batch"
shared_file_name_queue_name = "%s/file_name_queue" % name
example_queue_name = "%s/fifo_queue" % name
worker_file_name_queue_name = "%s/file_name_queue/fifo_queue" % name
server = tf.train.Server.create_local_server()
with tf.Graph().as_default() as g1, tf.Session(server.target,
graph=g1) as session:
_, inputs = _read_keyed_batch_examples_shared_queue(
filenames,
batch_size,
reader=tf.TextLineReader,
randomize_input=False,
num_epochs=1,
queue_capacity=queue_capacity,
name=name)
session.run(tf.initialize_local_variables())
# Run the three queues once manually.
self._run_queue(shared_file_name_queue_name, session)
self._run_queue(worker_file_name_queue_name, session)
self._run_queue(example_queue_name, session)
self.assertAllEqual(session.run(inputs), [b"ABC"])
# Run the worker and the example queue.
self._run_queue(worker_file_name_queue_name, session)
self._run_queue(example_queue_name, session)
self.assertAllEqual(session.run(inputs), [b"DEF"])
with tf.Graph().as_default() as g2, tf.Session(server.target,
graph=g2) as session:
_, inputs = _read_keyed_batch_examples_shared_queue(
filenames,
batch_size,
reader=tf.TextLineReader,
randomize_input=False,
num_epochs=1,
queue_capacity=queue_capacity,
name=name)
# Run the worker and the example queue.
self._run_queue(worker_file_name_queue_name, session)
self._run_queue(example_queue_name, session)
self.assertAllEqual(session.run(inputs), [b"GHI"])
self.assertTrue(g1 is not g2)
def test_read_text_lines_multifile_with_shared_queue(self):
gfile.Glob = self._orig_glob
filenames = self._create_sorted_temp_files(["ABC\n", "DEF\nGHK\n"])
batch_size = 1
queue_capacity = 5
name = "my_batch"
with ops.Graph().as_default() as g, self.test_session(
graph=g) as session:
keys, inputs = _read_keyed_batch_examples_shared_queue(
filenames,
batch_size,
reader=io_ops.TextLineReader,
randomize_input=False,
num_epochs=1,
queue_capacity=queue_capacity,
name=name)
self.assertAllEqual((None, ), keys.get_shape().as_list())
self.assertAllEqual((None, ), inputs.get_shape().as_list())
session.run([
variables.local_variables_initializer(),
variables.global_variables_initializer()
])
coord = coordinator.Coordinator()
threads = queue_runner_impl.start_queue_runners(session,
coord=coord)
self.assertEqual("%s:1" % name, inputs.name)
example_queue_name = "%s/fifo_queue" % name
worker_file_name_queue_name = "%s/file_name_queue/fifo_queue" % name
test_util.assert_ops_in_graph(
{
"%s/read/TextLineReaderV2" % name: "TextLineReaderV2",
example_queue_name: "FIFOQueueV2",
worker_file_name_queue_name: "FIFOQueueV2",
name: "QueueDequeueUpToV2"
}, g)
self.assertAllEqual(session.run(inputs), [b"ABC"])
self.assertAllEqual(session.run(inputs), [b"DEF"])
self.assertAllEqual(session.run(inputs), [b"GHK"])
with self.assertRaises(errors.OutOfRangeError):
session.run(inputs)
coord.request_stop()
coord.join(threads)
def test_read_text_lines_multifile_with_shared_queue(self):
gfile.Glob = self._orig_glob
filenames = self._create_sorted_temp_files(["ABC\n", "DEF\nGHK\n"])
batch_size = 1
queue_capacity = 5
name = "my_batch"
with tf.Graph().as_default() as g, self.test_session(
graph=g) as session:
_, inputs = _read_keyed_batch_examples_shared_queue(
filenames,
batch_size,
reader=tf.TextLineReader,
randomize_input=False,
num_epochs=1,
queue_capacity=queue_capacity,
name=name)
session.run(tf.initialize_local_variables())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(session, coord=coord)
self.assertEqual("%s:1" % name, inputs.name)
shared_file_name_queue_name = "%s/file_name_queue" % name
file_names_name = "%s/input" % shared_file_name_queue_name
example_queue_name = "%s/fifo_queue" % name
worker_file_name_queue_name = "%s/file_name_queue/fifo_queue" % name
test_util.assert_ops_in_graph(
{
file_names_name: "Const",
shared_file_name_queue_name: "FIFOQueue",
"%s/read/TextLineReader" % name: "TextLineReader",
example_queue_name: "FIFOQueue",
worker_file_name_queue_name: "FIFOQueue",
name: "QueueDequeueUpTo"
}, g)
self.assertAllEqual(session.run(inputs), [b"ABC"])
self.assertAllEqual(session.run(inputs), [b"DEF"])
self.assertAllEqual(session.run(inputs), [b"GHK"])
with self.assertRaises(errors.OutOfRangeError):
session.run(inputs)
coord.request_stop()
coord.join(threads)
def test_read_text_lines_multifile_with_shared_queue(self):
gfile.Glob = self._orig_glob
filenames = self._create_sorted_temp_files(["ABC\n", "DEF\nGHK\n"])
batch_size = 1
queue_capacity = 5
name = "my_batch"
with tf.Graph().as_default() as g, self.test_session(graph=g) as session:
keys, inputs = _read_keyed_batch_examples_shared_queue(
filenames,
batch_size,
reader=tf.TextLineReader,
randomize_input=False,
num_epochs=1,
queue_capacity=queue_capacity,
name=name)
self.assertAllEqual((None,), keys.get_shape().as_list())
self.assertAllEqual((None,), inputs.get_shape().as_list())
session.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(session, coord=coord)
self.assertEqual("%s:1" % name, inputs.name)
shared_file_name_queue_name = "%s/file_name_queue" % name
file_names_name = "%s/input" % shared_file_name_queue_name
example_queue_name = "%s/fifo_queue" % name
worker_file_name_queue_name = "%s/file_name_queue/fifo_queue" % name
test_util.assert_ops_in_graph({
file_names_name: "Const",
shared_file_name_queue_name: "FIFOQueue",
"%s/read/TextLineReader" % name: "TextLineReader",
example_queue_name: "FIFOQueue",
worker_file_name_queue_name: "FIFOQueue",
name: "QueueDequeueUpTo"
}, g)
self.assertAllEqual(session.run(inputs), [b"ABC"])
self.assertAllEqual(session.run(inputs), [b"DEF"])
self.assertAllEqual(session.run(inputs), [b"GHK"])
with self.assertRaises(errors.OutOfRangeError):
session.run(inputs)
coord.request_stop()
coord.join(threads)
