def setUp(self):
self.queue = queue.Queue()
self.io_thread = IOWriterThread(self.queue)
self.temp_dir = tempfile.mkdtemp()
self.filename = os.path.join(self.temp_dir, 'foo')
# Create the file, since IOWriterThread expects
# files to exist, we need to first creat the file.
open(self.filename, 'w').close()
class TestIOWriterThread(unittest.TestCase):
def setUp(self):
self.queue = queue.Queue()
self.io_thread = IOWriterThread(self.queue)
self.temp_dir = tempfile.mkdtemp()
self.filename = os.path.join(self.temp_dir, 'foo')
# Create the file, since IOWriterThread expects
# files to exist, we need to first creat the file.
open(self.filename, 'w').close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_handles_io_request(self):
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
def test_out_of_order_io_requests(self):
self.queue.put(IORequest(self.filename, 6, b'morestuff', False))
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobarmorestuff')
def test_multiple_files_in_queue(self):
second_file = os.path.join(self.temp_dir, 'bar')
open(second_file, 'w').close()
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IORequest(second_file, 0, b'otherstuff', False))
self.queue.put(IOCloseRequest(second_file))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
with open(second_file, 'rb') as f:
self.assertEqual(f.read(), b'otherstuff')
def test_stream_requests(self):
# Test that offset has no affect on the order in which requests
# are written to stdout. The order of requests for a stream are
# first in first out.
self.queue.put(IORequest('nonexistant-file', 10, b'foobar', True))
self.queue.put(IORequest('nonexistant-file', 6, b'otherstuff', True))
# The thread should not try to close the file name because it is
# writing to stdout. If it does, the thread will fail because
# the file does not exist.
self.queue.put(ShutdownThreadRequest())
with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout:
self.io_thread.run()
self.assertEqual(mock_stdout.getvalue(), 'foobarotherstuff')
class TestIOWriterThread(unittest.TestCase):
def setUp(self):
self.queue = queue.Queue()
self.io_thread = IOWriterThread(self.queue)
self.temp_dir = tempfile.mkdtemp()
self.filename = os.path.join(self.temp_dir, 'foo')
# Create the file, since IOWriterThread expects
# files to exist, we need to first creat the file.
open(self.filename, 'w').close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_handles_io_request(self):
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
def test_out_of_order_io_requests(self):
self.queue.put(IORequest(self.filename, 6, b'morestuff', False))
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobarmorestuff')
def test_multiple_files_in_queue(self):
second_file = os.path.join(self.temp_dir, 'bar')
open(second_file, 'w').close()
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IORequest(second_file, 0, b'otherstuff', False))
self.queue.put(IOCloseRequest(second_file))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
with open(second_file, 'rb') as f:
self.assertEqual(f.read(), b'otherstuff')
def test_stream_requests(self):
# Test that offset has no affect on the order in which requests
# are written to stdout. The order of requests for a stream are
# first in first out.
self.queue.put(IORequest('nonexistant-file', 10, b'foobar', True))
self.queue.put(IORequest('nonexistant-file', 6, b'otherstuff', True))
# The thread should not try to close the file name because it is
# writing to stdout. If it does, the thread will fail because
# the file does not exist.
self.queue.put(ShutdownThreadRequest())
with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout:
self.io_thread.run()
self.assertEqual(mock_stdout.getvalue(), 'foobarotherstuff')
def setUp(self):
self.queue = queue.Queue()
self.io_thread = IOWriterThread(self.queue)
self.temp_dir = tempfile.mkdtemp()
self.filename = os.path.join(self.temp_dir, 'foo')
# Create the file, since IOWriterThread expects
# files to exist, we need to first creat the file.
open(self.filename, 'w').close()
class TestIOWriterThread(unittest.TestCase):
def setUp(self):
self.queue = queue.Queue()
self.io_thread = IOWriterThread(self.queue)
self.temp_dir = tempfile.mkdtemp()
self.filename = os.path.join(self.temp_dir, 'foo')
# Create the file, since IOWriterThread expects
# files to exist, we need to first creat the file.
open(self.filename, 'w').close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_handles_io_request(self):
self.queue.put(IORequest(self.filename, 0, b'foobar'))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
def test_out_of_order_io_requests(self):
self.queue.put(IORequest(self.filename, 6, b'morestuff'))
self.queue.put(IORequest(self.filename, 0, b'foobar'))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobarmorestuff')
def test_multiple_files_in_queue(self):
second_file = os.path.join(self.temp_dir, 'bar')
open(second_file, 'w').close()
self.queue.put(IORequest(self.filename, 0, b'foobar'))
self.queue.put(IORequest(second_file, 0, b'otherstuff'))
self.queue.put(IOCloseRequest(second_file))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
with open(second_file, 'rb') as f:
self.assertEqual(f.read(), b'otherstuff')
class TestIOWriterThread(unittest.TestCase):
def setUp(self):
self.queue = queue.Queue()
self.io_thread = IOWriterThread(self.queue)
self.temp_dir = tempfile.mkdtemp()
self.filename = os.path.join(self.temp_dir, 'foo')
# Create the file, since IOWriterThread expects
# files to exist, we need to first creat the file.
open(self.filename, 'w').close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_handles_io_request(self):
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
def test_out_of_order_io_requests(self):
self.queue.put(IORequest(self.filename, 6, b'morestuff', False))
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobarmorestuff')
def test_multiple_files_in_queue(self):
second_file = os.path.join(self.temp_dir, 'bar')
open(second_file, 'w').close()
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IORequest(second_file, 0, b'otherstuff', False))
self.queue.put(IOCloseRequest(second_file))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
with open(second_file, 'rb') as f:
self.assertEqual(f.read(), b'otherstuff')
def test_mtime_set_at_file_close_time(self):
# We're picking something other than the close time so that can verify
# that the IOCloseRequest can specify what the mtime should be.
now_time = int(time.time() - 100)
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename, now_time))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
actual_mtime = int(os.stat(self.filename).st_mtime)
self.assertEqual(actual_mtime, now_time)
def test_stream_requests(self):
# Test that offset has no affect on the order in which requests
# are written to stdout. The order of requests for a stream are
# first in first out.
self.queue.put(IORequest('nonexistant-file', 10, b'foobar', True))
self.queue.put(IORequest('nonexistant-file', 6, b'otherstuff', True))
# The thread should not try to close the file name because it is
# writing to stdout. If it does, the thread will fail because
# the file does not exist.
self.queue.put(ShutdownThreadRequest())
with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout:
self.io_thread.run()
self.assertEqual(mock_stdout.getvalue(), 'foobarotherstuff')
def test_io_thread_moves_on_after_failed_task(self):
# This first request will fail because 'unknown-file' does not exist.
self.queue.put(IORequest('unknown-file', 0, b'foobar', False))
# But the IO thread should move on to these requests and not
# exit it's run loop.
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
class TestIOWriterThread(unittest.TestCase):
def setUp(self):
self.queue = queue.Queue()
self.io_thread = IOWriterThread(self.queue)
self.temp_dir = tempfile.mkdtemp()
self.filename = os.path.join(self.temp_dir, 'foo')
# Create the file, since IOWriterThread expects
# files to exist, we need to first creat the file.
open(self.filename, 'w').close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_handles_io_request(self):
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
def test_out_of_order_io_requests(self):
self.queue.put(IORequest(self.filename, 6, b'morestuff', False))
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobarmorestuff')
def test_multiple_files_in_queue(self):
second_file = os.path.join(self.temp_dir, 'bar')
open(second_file, 'w').close()
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IORequest(second_file, 0, b'otherstuff', False))
self.queue.put(IOCloseRequest(second_file))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
with open(second_file, 'rb') as f:
self.assertEqual(f.read(), b'otherstuff')
def test_mtime_set_at_file_close_time(self):
# We're picking something other than the close time so that can verify
# that the IOCloseRequest can specify what the mtime should be.
now_time = int(time.time() - 100)
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename, now_time))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
actual_mtime = int(os.stat(self.filename).st_mtime)
self.assertEqual(actual_mtime, now_time)
def test_stream_requests(self):
# Test that offset has no affect on the order in which requests
# are written to stdout. The order of requests for a stream are
# first in first out.
self.queue.put(IORequest('nonexistant-file', 10, b'foobar', True))
self.queue.put(IORequest('nonexistant-file', 6, b'otherstuff', True))
# The thread should not try to close the file name because it is
# writing to stdout. If it does, the thread will fail because
# the file does not exist.
self.queue.put(ShutdownThreadRequest())
with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout:
self.io_thread.run()
self.assertEqual(mock_stdout.getvalue(), 'foobarotherstuff')
def test_io_thread_moves_on_after_failed_task(self):
# This first request will fail because 'unknown-file' does not exist.
self.queue.put(IORequest('unknown-file', 0, b'foobar', False))
# But the IO thread should move on to these requests and not
# exit it's run loop.
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')