def test_multiple_files_in_queue(self):
second_file = os.path.join(self.temp_dir, 'bar')
open(second_file, 'w').close()
self.queue.put(IORequest(self.filename, 0, b'foobar'))
self.queue.put(IORequest(second_file, 0, b'otherstuff'))
self.queue.put(IOCloseRequest(second_file))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
with open(second_file, 'rb') as f:
self.assertEqual(f.read(), b'otherstuff')
def test_handles_io_request(self):
self.queue.put(IORequest(self.filename, 0, b'foobar'))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
def test_handles_io_request(self):
self.queue.put(IORequest(self.filename, 0, b'foobar'))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(QUEUE_END_SENTINEL)
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
def test_out_of_order_io_requests(self):
self.queue.put(IORequest(self.filename, 6, b'morestuff'))
self.queue.put(IORequest(self.filename, 0, b'foobar'))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobarmorestuff')
def test_mtime_set_at_file_close_time(self):
# We're picking something other than the close time so that can verify
# that the IOCloseRequest can specify what the mtime should be.
now_time = int(time.time() - 100)
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename, now_time))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
actual_mtime = int(os.stat(self.filename).st_mtime)
self.assertEqual(actual_mtime, now_time)
def test_io_thread_moves_on_after_failed_task(self):
# This first request will fail because 'unknown-file' does not exist.
self.queue.put(IORequest('unknown-file', 0, b'foobar', False))
# But the IO thread should move on to these requests and not
# exit it's run loop.
self.queue.put(IORequest(self.filename, 0, b'foobar', False))
self.queue.put(IOCloseRequest(self.filename))
self.queue.put(ShutdownThreadRequest())
self.io_thread.run()
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'foobar')
def __call__(self):
# When the file is downloading, we have a few things we need to do:
# 1) Fix up the last modified time to match s3.
# 2) Tell the result_queue we're done.
# 3) Queue an IO request to the IO thread letting it know we're
# done with the file.
self._context.wait_for_completion()
last_update_tuple = self._filename.last_update.timetuple()
mod_timestamp = time.mktime(last_update_tuple)
desired_mtime = int(mod_timestamp)
message = print_operation(self._filename, False,
self._parameters['dryrun'])
print_task = {'message': message, 'error': False}
self._result_queue.put(PrintTask(**print_task))
self._io_queue.put(IOCloseRequest(self._filename.dest, desired_mtime))