def _enqueue_range_download_tasks(self,
filename,
remove_remote_file=False):
num_downloads = int(filename.size / self.chunksize)
context = tasks.MultipartDownloadContext(num_downloads)
create_file_task = tasks.CreateLocalFileTask(
context=context, filename=filename, result_queue=self.result_queue)
self.executor.submit(create_file_task)
self._do_enqueue_range_download_tasks(
filename=filename,
chunksize=self.chunksize,
num_downloads=num_downloads,
context=context,
remove_remote_file=remove_remote_file)
complete_file_task = tasks.CompleteDownloadTask(
context=context,
filename=filename,
result_queue=self.result_queue,
params=self.params,
io_queue=self.write_queue)
self.executor.submit(complete_file_task)
self._multipart_downloads.append((context, filename.dest))
if remove_remote_file:
remove_task = tasks.RemoveRemoteObjectTask(filename=filename,
context=context)
self.executor.submit(remove_task)
return num_downloads
def _enqueue_range_download_tasks(self,
filename,
remove_remote_file=False):
chunksize = find_chunksize(filename.size, self.chunksize)
num_downloads = int(filename.size / chunksize)
context = tasks.MultipartDownloadContext(num_downloads)
create_file_task = tasks.CreateLocalFileTask(context=context,
filename=filename)
self.executor.submit(create_file_task)
for i in range(num_downloads):
task = tasks.DownloadPartTask(part_number=i,
chunk_size=chunksize,
result_queue=self.result_queue,
service=filename.service,
filename=filename,
context=context,
io_queue=self.write_queue)
self.executor.submit(task)
complete_file_task = tasks.CompleteDownloadTask(
context=context,
filename=filename,
result_queue=self.result_queue,
params=self.params,
io_queue=self.write_queue)
self.executor.submit(complete_file_task)
self._multipart_downloads.append((context, filename.dest))
if remove_remote_file:
remove_task = tasks.RemoveRemoteObjectTask(filename=filename,
context=context)
self.executor.submit(remove_task)
return num_downloads
def _enqueue_multipart_copy_tasks(self, filename,
remove_remote_file=False):
chunksize = find_chunksize(filename.size, self.chunksize)
num_uploads = int(math.ceil(filename.size / float(chunksize)))
upload_context = self._enqueue_upload_start_task(
chunksize, num_uploads, filename)
self._enqueue_upload_tasks(
num_uploads, chunksize, upload_context, filename, tasks.CopyPartTask)
self._enqueue_upload_end_task(filename, upload_context)
if remove_remote_file:
remove_task = tasks.RemoveRemoteObjectTask(
filename=filename, context=upload_context)
self.executor.submit(remove_task)
return num_uploads
