def multi_download(self):
"""
This performs the multipart download. It assigns ranges to get from
s3 of particular object to a task.It creates a queue ``part_queue``
which is directly responsible with controlling the progress of the
multipart download. It then creates ``DownloadPartTasks`` for
threads to run via the ``executer``. This fucntion waits
for all of the parts in the multipart download to finish, and then
the last modification time is changed to the last modified time
of the s3 object. This method waits on its parts to finish.
So, threads are required to process the parts for this function
to complete.
"""
part_queue = NoBlockQueue(self.interrupt)
dest_queue = NoBlockQueue(self.interrupt)
part_counter = MultiCounter()
write_lock = threading.Lock()
counter_lock = threading.Lock()
d = os.path.dirname(self.dest)
try:
if not os.path.exists(d):
os.makedirs(d)
except Exception:
pass
size_uploads = self.chunksize
num_uploads = int(self.size / size_uploads)
with open(self.dest, 'wb') as f:
for i in range(num_uploads):
part = (self, i, size_uploads)
part_queue.put(part)
task = DownloadPartTask(session=self.session,
executer=self.executer,
part_queue=part_queue,
dest_queue=dest_queue,
f=f,
region=self.region,
printQueue=self.printQueue,
write_lock=write_lock,
part_counter=part_counter,
counter_lock=counter_lock)
self.executer.submit(task)
part_queue.join()
# The following ensures that if the multipart download is
# in progress, all part uploads finish before releasing the
# the file handle. This really only applies when an interrupt
# signal is sent because the ``part_queue.join()`` ensures this
# if the process is not interrupted.
while part_counter.count:
time.sleep(0.1)
part_list = []
while not dest_queue.empty():
part = dest_queue.get()
part_list.append(part)
if len(part_list) != num_uploads:
raise Exception()
last_update_tuple = self.last_update.timetuple()
mod_timestamp = time.mktime(last_update_tuple)
os.utime(self.dest, (int(mod_timestamp), int(mod_timestamp)))
def test_incomplete_read_is_retried(self):
self.client.get_object.side_effect = \
IncompleteReadError(actual_bytes=1, expected_bytes=2)
task = DownloadPartTask(0, 1024 * 1024, self.result_queue,
self.filename, self.context, self.io_queue)
with self.assertRaises(RetriesExeededError):
task()
self.context.cancel.assert_called_with()
self.assertEqual(DownloadPartTask.TOTAL_ATTEMPTS,
self.client.get_object.call_count)
def test_readtimeout_is_retried(self):
self.client.get_object.side_effect = \
ReadTimeoutError(None, None, None)
task = DownloadPartTask(0, 1024 * 1024, self.result_queue,
self.filename, self.context, self.io_queue)
with self.assertRaises(RetriesExeededError):
task()
self.context.cancel.assert_called_with()
self.assertEqual(DownloadPartTask.TOTAL_ATTEMPTS,
self.client.get_object.call_count)
def test_download_succeeds(self):
body = mock.Mock()
body.read.return_value = b''
self.client.get_object.side_effect = [socket.error, {'Body': body}]
task = DownloadPartTask(0, 1024 * 1024, self.result_queue,
self.filename, self.context, self.io_queue)
task()
self.assertEqual(self.result_queue.put.call_count, 1)
# And we tried twice, the first one failed, the second one
# succeeded.
self.assertEqual(self.client.get_object.call_count, 2)
def test_socket_timeout_is_retried(self):
self.client.get_object.side_effect = socket.error
task = DownloadPartTask(0, 1024 * 1024, self.result_queue,
self.filename, self.context, self.io_queue)
# The mock is configured to keep raising a socket.error
# so we should cancel the download.
with self.assertRaises(RetriesExeededError):
task()
self.context.cancel.assert_called_with()
# And we retried the request multiple times.
self.assertEqual(DownloadPartTask.TOTAL_ATTEMPTS,
self.client.get_object.call_count)
def test_download_queues_io_properly(self):
body = mock.Mock()
body.read.side_effect = [b'foobar', b'morefoobar', b'']
self.client.get_object.side_effect = [{'Body': body}]
task = DownloadPartTask(0, 1024 * 1024, self.result_queue,
self.filename, self.context, self.io_queue)
task()
call_args_list = self.io_queue.put.call_args_list
self.assertEqual(len(call_args_list), 2)
self.assertEqual(call_args_list[0],
mock.call(('local/file', 0, b'foobar', False)))
self.assertEqual(call_args_list[1],
mock.call(('local/file', 6, b'morefoobar', False)))
def test_failed_downloads_add_error_to_result_queue(self):
self.client.get_object.side_effect = ClientError(
{'Error': {
'Code': 'BadRequest'
}}, 'get_object')
task = DownloadPartTask(0, 1024 * 1024, self.result_queue,
self.filename, self.context, self.io_queue,
self.params)
with self.assertRaises(ClientError):
task()
# Should have queued a task marked as an error.
self.assertEqual(self.result_queue.put.call_count, 1)
print_task = self.result_queue.put.call_args[0][0]
self.assertTrue(print_task.error)
def test_download_queues_io_properly(self):
body = mock.Mock()
body.read.side_effect = [b'foobar', b'morefoobar', b'']
self.service.get_operation.return_value.call.side_effect = [
(mock.Mock(), {'Body': body}),
]
task = DownloadPartTask(0, 1024 * 1024, self.result_queue,
self.service, self.filename, self.context,
self.io_queue)
task()
call_args_list = self.io_queue.put.call_args_list
self.assertEqual(len(call_args_list), 2)
self.assertEqual(call_args_list[0],
mock.call(('local/file', 0, b'foobar')))
self.assertEqual(call_args_list[1],
mock.call(('local/file', 6, b'morefoobar')))
def test_retried_requests_dont_enqueue_writes_twice(self):
error_body = mock.Mock()
error_body.read.side_effect = socket.timeout
success_body = mock.Mock()
success_body.read.side_effect = [b'foobar', b'']
incomplete_read = (mock.Mock(), {'Body': error_body})
success_read = (mock.Mock(), {'Body': success_body})
self.service.get_operation.return_value.call.side_effect = [
# The first request results in an error when reading the request.
incomplete_read,
success_read,
]
self.filename.is_stream = True
task = DownloadPartTask(0, constants.CHUNKSIZE, self.result_queue,
self.service, self.filename, self.context,
self.io_queue)
task()
call_args_list = self.io_queue.put.call_args_list
self.assertEqual(len(call_args_list), 1)
self.assertEqual(call_args_list[0],
mock.call(('local/file', 0, b'foobar', True)))
success_body.read.assert_called_with()
def test_retried_requests_dont_enqueue_writes_twice(self):
error_body = mock.Mock()
error_body.read.side_effect = socket.timeout
success_body = mock.Mock()
success_body.read.side_effect = [b'foobar', b'']
incomplete_read = {'Body': error_body}
success_read = {'Body': success_body}
self.client.get_object.side_effect = [
# The first request results in an error when reading the request.
incomplete_read,
success_read,
]
self.filename.is_stream = True
task = DownloadPartTask(0,
transferconfig.DEFAULTS['multipart_chunksize'],
self.result_queue, self.filename, self.context,
self.io_queue)
task()
call_args_list = self.io_queue.put.call_args_list
self.assertEqual(len(call_args_list), 1)
self.assertEqual(call_args_list[0],
mock.call(('local/file', 0, b'foobar', True)))
success_body.read.assert_called_with()
def download_task(self):
return DownloadPartTask(None, None, None, None, mock.Mock(), None,
None)