def setUp(self):
title = test_util.CreateRandomFileName()
file_list = []
for x in range(0, 10):
file1 = self.drive.CreateFile()
file1['title'] = title
pydrive_retry(file1.Upload)
file_list.append(file1)
self.title = title
self.file_list = file_list
def _parallel_uploader(self,
num_of_uploads,
num_of_workers,
use_per_thread_http=False):
drive = GoogleDrive(self.ga)
thread_pool = ThreadPoolExecutor(max_workers=num_of_workers)
# Create list of gdrive_files.
upload_files = []
remote_name = test_util.CreateRandomFileName()
for i in range(num_of_uploads):
file_name = self.first_file if i % 2 == 0 else self.second_file
up_file = drive.CreateFile()
up_file["title"] = remote_name
up_file.SetContentFile(file_name)
upload_files.append(up_file)
# Ensure there are no files with the random file name.
files = pydrive_retry(lambda: drive.ListFile(
param={
"q": "title = '%s' and trashed = false" % remote_name
}).GetList())
self.assertTrue(len(files) == 0)
# Submit upload jobs to ThreadPoolExecutor.
futures = []
for i in range(num_of_uploads):
upload_worker = self.UploadWorker(upload_files[i],
use_per_thread_http)
futures.append(thread_pool.submit(upload_worker.run))
# Ensure that all threads a) return, and b) encountered no exceptions.
for future in as_completed(futures):
self.assertIsNone(future.exception())
thread_pool.shutdown()
# Ensure all files were uploaded.
files = pydrive_retry(lambda: drive.ListFile(
param={
"q": "title = '%s' and trashed = false" % remote_name
}).GetList())
self.assertTrue(len(files) == self.FILE_UPLOAD_COUNT)
# Remove uploaded files.
self.DeleteUploadedFiles(drive, [fi["id"] for fi in upload_files])
def _parallel_uploader(self, num_of_uploads, num_of_workers):
"""
:returns: list[str] of file IDs
"""
drive = GoogleDrive(self.ga)
thread_pool = ThreadPoolExecutor(max_workers=num_of_workers)
first_file = self.getTempFile("first_file", "some string")
second_file = self.getTempFile("second_file", "another string")
# Create list of gdrive_files.
upload_files = []
remote_name = test_util.CreateRandomFileName()
for i in range(num_of_uploads):
file_name = first_file if i % 2 == 0 else second_file
up_file = drive.CreateFile()
up_file["title"] = remote_name
up_file.SetContentFile(file_name)
upload_files.append(up_file)
# Ensure there are no files with the random file name.
files = pydrive_retry(lambda: drive.ListFile(
param={
"q": "title = '%s' and trashed = false" % remote_name
}).GetList())
self.assertTrue(len(files) == 0)
# Submit upload jobs to ThreadPoolExecutor.
futures = []
for up_file in upload_files:
futures.append(thread_pool.submit(pydrive_retry, up_file.Upload))
# Ensure that all threads a) return, and b) encountered no exceptions.
for future in as_completed(futures):
self.assertIsNone(future.exception())
thread_pool.shutdown()
# Ensure all files were uploaded.
files = pydrive_retry(lambda: drive.ListFile(
param={
"q": "title = '%s' and trashed = false" % remote_name
}).GetList())
self.assertTrue(len(files) == self.FILE_UPLOAD_COUNT)
return [fi["id"] for fi in upload_files]