Example #1
0
 def setUp(self):
     self.queue = queue.Queue()
     self.io_thread = IOWriterThread(self.queue)
     self.temp_dir = tempfile.mkdtemp()
     self.filename = os.path.join(self.temp_dir, 'foo')
     # Create the file, since IOWriterThread expects
     # files to exist, we need to first creat the file.
     open(self.filename, 'w').close()
Example #2
0
class TestIOWriterThread(unittest.TestCase):

    def setUp(self):
        self.queue = queue.Queue()
        self.io_thread = IOWriterThread(self.queue)
        self.temp_dir = tempfile.mkdtemp()
        self.filename = os.path.join(self.temp_dir, 'foo')
        # Create the file, since IOWriterThread expects
        # files to exist, we need to first creat the file.
        open(self.filename, 'w').close()

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_handles_io_request(self):
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')

    def test_out_of_order_io_requests(self):
        self.queue.put(IORequest(self.filename, 6, b'morestuff', False))
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobarmorestuff')

    def test_multiple_files_in_queue(self):
        second_file = os.path.join(self.temp_dir, 'bar')
        open(second_file, 'w').close()
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IORequest(second_file, 0, b'otherstuff', False))
        self.queue.put(IOCloseRequest(second_file))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())

        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')
        with open(second_file, 'rb') as f:
            self.assertEqual(f.read(), b'otherstuff')

    def test_stream_requests(self):
        # Test that offset has no affect on the order in which requests
        # are written to stdout. The order of requests for a stream are
        # first in first out.
        self.queue.put(IORequest('nonexistant-file', 10, b'foobar', True))
        self.queue.put(IORequest('nonexistant-file', 6, b'otherstuff', True))
        # The thread should not try to close the file name because it is
        # writing to stdout.  If it does, the thread will fail because
        # the file does not exist.
        self.queue.put(ShutdownThreadRequest())
        with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout:
            self.io_thread.run()
            self.assertEqual(mock_stdout.getvalue(), 'foobarotherstuff')
Example #3
0
class TestIOWriterThread(unittest.TestCase):

    def setUp(self):
        self.queue = queue.Queue()
        self.io_thread = IOWriterThread(self.queue)
        self.temp_dir = tempfile.mkdtemp()
        self.filename = os.path.join(self.temp_dir, 'foo')
        # Create the file, since IOWriterThread expects
        # files to exist, we need to first creat the file.
        open(self.filename, 'w').close()

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_handles_io_request(self):
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')

    def test_out_of_order_io_requests(self):
        self.queue.put(IORequest(self.filename, 6, b'morestuff', False))
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobarmorestuff')

    def test_multiple_files_in_queue(self):
        second_file = os.path.join(self.temp_dir, 'bar')
        open(second_file, 'w').close()
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IORequest(second_file, 0, b'otherstuff', False))
        self.queue.put(IOCloseRequest(second_file))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())

        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')
        with open(second_file, 'rb') as f:
            self.assertEqual(f.read(), b'otherstuff')

    def test_stream_requests(self):
        # Test that offset has no affect on the order in which requests
        # are written to stdout. The order of requests for a stream are
        # first in first out.
        self.queue.put(IORequest('nonexistant-file', 10, b'foobar', True))
        self.queue.put(IORequest('nonexistant-file', 6, b'otherstuff', True))
        # The thread should not try to close the file name because it is
        # writing to stdout.  If it does, the thread will fail because
        # the file does not exist.
        self.queue.put(ShutdownThreadRequest())
        with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout:
            self.io_thread.run()
            self.assertEqual(mock_stdout.getvalue(), 'foobarotherstuff')
Example #4
0
 def setUp(self):
     self.queue = queue.Queue()
     self.io_thread = IOWriterThread(self.queue)
     self.temp_dir = tempfile.mkdtemp()
     self.filename = os.path.join(self.temp_dir, 'foo')
     # Create the file, since IOWriterThread expects
     # files to exist, we need to first creat the file.
     open(self.filename, 'w').close()
Example #5
0
class TestIOWriterThread(unittest.TestCase):

    def setUp(self):
        self.queue = queue.Queue()
        self.io_thread = IOWriterThread(self.queue)
        self.temp_dir = tempfile.mkdtemp()
        self.filename = os.path.join(self.temp_dir, 'foo')
        # Create the file, since IOWriterThread expects
        # files to exist, we need to first creat the file.
        open(self.filename, 'w').close()

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_handles_io_request(self):
        self.queue.put(IORequest(self.filename, 0, b'foobar'))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')

    def test_out_of_order_io_requests(self):
        self.queue.put(IORequest(self.filename, 6, b'morestuff'))
        self.queue.put(IORequest(self.filename, 0, b'foobar'))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobarmorestuff')

    def test_multiple_files_in_queue(self):
        second_file = os.path.join(self.temp_dir, 'bar')
        open(second_file, 'w').close()
        self.queue.put(IORequest(self.filename, 0, b'foobar'))
        self.queue.put(IORequest(second_file, 0, b'otherstuff'))
        self.queue.put(IOCloseRequest(second_file))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())

        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')
        with open(second_file, 'rb') as f:
            self.assertEqual(f.read(), b'otherstuff')
Example #6
0
class TestIOWriterThread(unittest.TestCase):

    def setUp(self):
        self.queue = queue.Queue()
        self.io_thread = IOWriterThread(self.queue)
        self.temp_dir = tempfile.mkdtemp()
        self.filename = os.path.join(self.temp_dir, 'foo')
        # Create the file, since IOWriterThread expects
        # files to exist, we need to first creat the file.
        open(self.filename, 'w').close()

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_handles_io_request(self):
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')

    def test_out_of_order_io_requests(self):
        self.queue.put(IORequest(self.filename, 6, b'morestuff', False))
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobarmorestuff')

    def test_multiple_files_in_queue(self):
        second_file = os.path.join(self.temp_dir, 'bar')
        open(second_file, 'w').close()
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IORequest(second_file, 0, b'otherstuff', False))
        self.queue.put(IOCloseRequest(second_file))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())

        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')
        with open(second_file, 'rb') as f:
            self.assertEqual(f.read(), b'otherstuff')

    def test_mtime_set_at_file_close_time(self):
        # We're picking something other than the close time so that can verify
        # that the IOCloseRequest can specify what the mtime should be.
        now_time = int(time.time() - 100)
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename, now_time))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        actual_mtime = int(os.stat(self.filename).st_mtime)
        self.assertEqual(actual_mtime, now_time)

    def test_stream_requests(self):
        # Test that offset has no affect on the order in which requests
        # are written to stdout. The order of requests for a stream are
        # first in first out.
        self.queue.put(IORequest('nonexistant-file', 10, b'foobar', True))
        self.queue.put(IORequest('nonexistant-file', 6, b'otherstuff', True))
        # The thread should not try to close the file name because it is
        # writing to stdout.  If it does, the thread will fail because
        # the file does not exist.
        self.queue.put(ShutdownThreadRequest())
        with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout:
            self.io_thread.run()
            self.assertEqual(mock_stdout.getvalue(), 'foobarotherstuff')

    def test_io_thread_moves_on_after_failed_task(self):
        # This first request will fail because 'unknown-file' does not exist.
        self.queue.put(IORequest('unknown-file', 0, b'foobar', False))
        # But the IO thread should move on to these requests and not
        # exit it's run loop.
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')
Example #7
0
class TestIOWriterThread(unittest.TestCase):
    def setUp(self):
        self.queue = queue.Queue()
        self.io_thread = IOWriterThread(self.queue)
        self.temp_dir = tempfile.mkdtemp()
        self.filename = os.path.join(self.temp_dir, 'foo')
        # Create the file, since IOWriterThread expects
        # files to exist, we need to first creat the file.
        open(self.filename, 'w').close()

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_handles_io_request(self):
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')

    def test_out_of_order_io_requests(self):
        self.queue.put(IORequest(self.filename, 6, b'morestuff', False))
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobarmorestuff')

    def test_multiple_files_in_queue(self):
        second_file = os.path.join(self.temp_dir, 'bar')
        open(second_file, 'w').close()
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IORequest(second_file, 0, b'otherstuff', False))
        self.queue.put(IOCloseRequest(second_file))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())

        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')
        with open(second_file, 'rb') as f:
            self.assertEqual(f.read(), b'otherstuff')

    def test_mtime_set_at_file_close_time(self):
        # We're picking something other than the close time so that can verify
        # that the IOCloseRequest can specify what the mtime should be.
        now_time = int(time.time() - 100)
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename, now_time))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        actual_mtime = int(os.stat(self.filename).st_mtime)
        self.assertEqual(actual_mtime, now_time)

    def test_stream_requests(self):
        # Test that offset has no affect on the order in which requests
        # are written to stdout. The order of requests for a stream are
        # first in first out.
        self.queue.put(IORequest('nonexistant-file', 10, b'foobar', True))
        self.queue.put(IORequest('nonexistant-file', 6, b'otherstuff', True))
        # The thread should not try to close the file name because it is
        # writing to stdout.  If it does, the thread will fail because
        # the file does not exist.
        self.queue.put(ShutdownThreadRequest())
        with mock.patch('sys.stdout', new=six.StringIO()) as mock_stdout:
            self.io_thread.run()
            self.assertEqual(mock_stdout.getvalue(), 'foobarotherstuff')

    def test_io_thread_moves_on_after_failed_task(self):
        # This first request will fail because 'unknown-file' does not exist.
        self.queue.put(IORequest('unknown-file', 0, b'foobar', False))
        # But the IO thread should move on to these requests and not
        # exit it's run loop.
        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
        self.queue.put(IOCloseRequest(self.filename))
        self.queue.put(ShutdownThreadRequest())
        self.io_thread.run()
        with open(self.filename, 'rb') as f:
            self.assertEqual(f.read(), b'foobar')